fix(husk): template artifacts stay restorable by a non-root husk (mandatory CoW + group-read contract)

cmd/forkd/jailer.go

@@ -87,6 +87,13 @@ func parseUIDRange(s string) (uint32, uint32, error) {
 	if low > high {
 		return 0, 0, fmt.Errorf("--uid-range %q: low bound above high bound", s)
 	}
+	// The shared kvm gid owns the group-readable template artifacts so a husk
+	// reads them through the group class. A per-VM jailer uid/gid drawn from a
+	// range that covers it would collide with that shared group, so a jailed
+	// build VM could carry the template-read group. Refuse such a range.
+	if low <= firecracker.SharedKVMGID && firecracker.SharedKVMGID <= high {
+		return 0, 0, fmt.Errorf("--uid-range %q: range covers the shared kvm gid %d, reserved for template-read group ownership; choose a range that excludes it", s, firecracker.SharedKVMGID)
+	}
 	return uint32(low), uint32(high), nil
 }
 

cmd/forkd/jailer_test.go

@@ -22,6 +22,8 @@ func TestParseUIDRange(t *testing.T) {
 		{in: "-64999", wantErr: true},
 		{in: "64999-64000", wantErr: true}, // low above high
 		{in: "0-100", wantErr: true},       // uid 0 is root; never jail as root
+		{in: "64500-65500", wantErr: true}, // covers the shared kvm gid 65000
+		{in: "65000-65000", wantErr: true}, // exactly the shared kvm gid
 		{in: "", wantErr: true},
 	}
 	for _, c := range cases {

docs/threat-model.md

@@ -348,6 +348,33 @@ tenant's filesystem state into another. The clone is removed on pod teardown
 (`Stub.Close`). Fully pod-native snapshot delivery (a CAS pull into the pod,
 removing the shared read-only mem/vmstate hostPath) remains a documented follow-up.
 
+Template artifact FILE PERMISSION posture (#597). The jailed build leaves the
+canonical template artifacts (`rootfs.ext4`, `snapshot/mem`, `snapshot/vmstate`)
+owned by the per-VM jailer build uid (`firecracker.JailerBuildUID`, the low end
+of `--uid-range`), which a husk VMM that is neither that uid nor privileged
+cannot open. `internal/firecracker` `normalizeTemplateArtifacts` now hands them
+back to `root` and the shared kvm group (`firecracker.SharedKVMGID`, a fixed gid
+OUTSIDE the per-VM jailer range so it can never collide with a per-VM jailer
+gid), with group-readable, NOT world-writable modes (files `0o640`, dirs
+`0o750`); the prior posture was world-readable `0o644`/`0o755`. The husk pod
+carries `SharedKVMGID` as a supplemental group (`huskpod.go` PodSecurityContext),
+so the current uid-0 husk reads the artifacts as the file OWNER and a future
+non-root husk (#585) reads them through the file GROUP class. `internal/fork`
+`checkTemplateArtifactInvariants` is the read-side reuse gate: it refuses to
+reuse an on-disk template whose ownership regressed off `root:SharedKVMGID`,
+group-readable. The artifacts stay read-only to the husk (its per-activation
+reflink clone is the only writable copy) and the template hostPath stays a
+read-write MOUNT only because Firecracker opens the baked rootfs O_RDWR at load
+(above); the FILES themselves are never world-writable. The husk now
+FAILS CLOSED when a template rootfs is configured without a per-activation CoW
+clone rather than restoring the shared template in place (`Stub.Activate`,
+mandatory CoW), so a misconfiguration can never silently reopen the shared
+`rootfs.ext4` O_RDWR. Residual, stated honestly: because the baked rootfs load is
+O_RDWR, a FUTURE non-root husk (#585) reading through the group class will still
+need the load itself resolved (group-write on the template, or a read-only baked
+rootfs drive); this PR keeps the current uid-0 husk correct as the file owner and
+lands the group-read contract the non-root husk will build on.
+
 ### Warm-pool autoscaling (no integrity-gate move)
 
 Demand-driven warm-pool autoscaling changes only WHEN and HOW MANY dormant husk

internal/controller/huskpod.go

@@ -14,6 +14,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	v1 "mitos.run/mitos/api/v1"
+	"mitos.run/mitos/internal/firecracker"
 	"mitos.run/mitos/internal/tenant"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -885,6 +886,17 @@ func (r *SandboxPoolReconciler) buildHuskPod(pool *v1.SandboxPool, template *v1.
 				SeccompProfile: &corev1.SeccompProfile{
 					Type: corev1.SeccompProfileTypeRuntimeDefault,
 				},
+				// The shared kvm group the template artifacts are group-owned by
+				// (forkd's normalizeTemplateArtifacts writes them root:SharedKVMGID,
+				// group-readable). Carrying it as a supplemental group means every
+				// stub process is in the file group class of the template rootfs,
+				// snapshot mem, and vmstate, so a future non-root husk (issue #585)
+				// reads the shared template through the group without owning it. The
+				// current uid-0 husk already reads them as owner; adding the group is
+				// a no-op for it and does not widen anything (issue #597). Not applied
+				// as fsGroup: hostPath volumes are exempt from fsGroup ownership
+				// management, so it would neither help nor take effect there.
+				SupplementalGroups: []int64{firecracker.SharedKVMGID},
 			},
 			// Pin to a KVM node: the dormant VMM needs /dev/kvm AND the pod must
 			// land where the template snapshot hostPath exists. The nodeAffinity

internal/controller/huskpod_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"mitos.run/mitos/internal/controller"
+	"mitos.run/mitos/internal/firecracker"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -316,6 +317,19 @@ func TestBuildHuskPodPSARestricted(t *testing.T) {
 	if psc.RunAsNonRoot == nil || *psc.RunAsNonRoot {
 		t.Error("pod RunAsNonRoot must be explicitly false (the documented /dev/kvm device exception)")
 	}
+	// The husk pod carries the shared kvm gid as a supplemental group so a future
+	// non-root husk (issue #585) reads the group-owned template artifacts through
+	// the group class (issue #597). The current uid-0 husk reads them as owner;
+	// the group is additive and never widens access.
+	foundKVMGroup := false
+	for _, g := range psc.SupplementalGroups {
+		if g == firecracker.SharedKVMGID {
+			foundKVMGroup = true
+		}
+	}
+	if !foundKVMGroup {
+		t.Errorf("pod SupplementalGroups = %v, want to include the shared kvm gid %d so the husk can read the group-owned template artifacts", psc.SupplementalGroups, firecracker.SharedKVMGID)
+	}
 
 	// CONTAINER-LEVEL securityContext: every other restricted control IS satisfied,
 	// so the husk pod's securityContext is restricted-clean and only the hostPath +

internal/controller/pool_rebuild.go

@@ -140,6 +140,16 @@ func templateRestoreFailing(pods []corev1.Pod, digest string) bool {
 func (r *SandboxPoolReconciler) driveTemplateHealth(ctx context.Context, pool *v1.SandboxPool, template *v1.PoolTemplateSpec, templateID string, nodeFilter map[string]bool, dormantPods []corev1.Pod, warmReady bool, now metav1.Time) {
 	logger := log.FromContext(ctx)
 	digest := pool.Status.TemplateDigest
+	// TODO(#578, #579): templateRestoreFailing detects husk pods that CRASHLOOP
+	// (e.g. a Prepare-time reflink clone or snapshot verify that fails closed), so
+	// a template that is unreadable at Prepare is already reflected here. It does
+	// NOT yet cover a pod that stays dormant+Ready but fails every ACTIVATE (e.g.
+	// the mandatory-CoW fail-closed on a missing clone, or a filesystem
+	// permission error surfaced only at load): the pool can still report Ready
+	// while 100% of activations fail. Surfacing per-activation failures on the
+	// SandboxPool status is the #578 honesty gap and is deferred to that issue;
+	// #597 closes the read/CoW permission root cause so this path fails at Prepare
+	// (visible) rather than silently at Activate wherever possible.
 	failing := templateRestoreFailing(dormantPods, digest)
 
 	if !failing {

internal/firecracker/jailer.go

@@ -27,6 +27,27 @@ const jailerExecFileName = "firecracker"
 // binds the socket there.
 const jailedAPISocketRelPath = "/run/firecracker.socket"
 
+// JailerBuildUID is the low end of the default per-VM jailer uid/gid range
+// (cmd/forkd --uid-range "64000-64999"). The jailed template build VM runs as
+// this uid, and the build hardlinks the canonical template artifacts into its
+// chroot; a freshly built, un-normalized template therefore comes out owned by
+// this uid, which a husk VMM that is neither this uid nor privileged cannot
+// open (issues #583, #597). It is named here so the ownership normalize
+// (normalizeTemplateArtifacts) and the read-side reuse invariant
+// (internal/fork.checkTemplateArtifactInvariants) refer to the same value.
+const JailerBuildUID = 64000
+
+// SharedKVMGID is the group the normalized template artifacts are owned by so a
+// husk VMM can READ them (rootfs.ext4 for the per-activation reflink clone,
+// snapshot mem and vmstate for the load) through the file group class, without
+// being the file owner. It is chosen OUTSIDE the per-VM jailer uid/gid range
+// [64000, 64999] so it can never collide with a per-VM jailer gid. The husk pod
+// carries it as a supplemental group (see the controller husk pod builder), so
+// once the husk runs a non-root uid (issue #585) it still reads the shared
+// template artifacts through this group. The current uid-0 husk owns the
+// artifacts directly (they are root-owned after normalize) and is unaffected.
+const SharedKVMGID = 65000
+
 // JailerConfig configures launching Firecracker through the jailer
 // binary. The zero value disables the jailer: StartVM execs the
 // firecracker binary directly, exactly as before.

internal/firecracker/template.go

@@ -673,36 +673,74 @@ func (tm *TemplateManager) CreateTemplate(id string, cfg VMConfig, initCommands
 }
 
 // normalizeTemplateArtifacts hands every file under the canonical template
-// tree back to the daemon's own uid/gid with world-readable modes (files
-// 0o644, dirs 0o755) after the jailed build VM is done with it.
+// tree back to the daemon's own uid and the shared kvm group with
+// group-readable modes (files 0o640, dirs 0o750) after the jailed build VM is
+// done with it.
 //
-// Why this exists (#583): in jailer mode the build hardlinks the canonical
-// rootfs into the per-VM chroot and chownIntoJail flips the SHARED inode to
-// the jailed uid; it must, because the deprivileged build VM writes the
-// rootfs. The snapshot mem and vmstate are likewise created inside the jail
-// and hardlinked out still owned by the jailed uid. Left that way, the husk
-// VMM (uid 0 with ALL capabilities dropped, so no CAP_DAC_OVERRIDE) fails
-// EACCES opening the rootfs O_RDWR at /snapshot/load and the warm pool
-// crashloops. The bug was latent while the data dir and the chroot base sat
-// on different filesystems: prepareChroot's EXDEV fallback copied instead of
-// hardlinking, so the chown hit the jail-private copy. Consolidating both
-// onto one filesystem made the hardlink succeed and unmasked it.
+// Why this exists (#583, #597): in jailer mode the build hardlinks the
+// canonical rootfs into the per-VM chroot and chownIntoJail flips the SHARED
+// inode to the jailed uid (JailerBuildUID); it must, because the deprivileged
+// build VM writes the rootfs. The snapshot mem and vmstate are likewise created
+// inside the jail and hardlinked out still owned by the jailed uid. Left that
+// way, the husk VMM (uid 0 with ALL capabilities dropped, so no
+// CAP_DAC_OVERRIDE) fails EACCES opening the rootfs O_RDWR at /snapshot/load
+// and the warm pool crashloops. The bug was latent while the data dir and the
+// chroot base sat on different filesystems: prepareChroot's EXDEV fallback
+// copied instead of hardlinking, so the chown hit the jail-private copy.
+// Consolidating both onto one filesystem made the hardlink succeed and
+// unmasked it.
 //
-// Chowning to the daemon's own euid keeps this a no-op in non-jailed and
-// non-root paths (mock engine, unit tests) while restoring root ownership on
-// a production forkd.
+// The owner is the daemon's own euid (root:0 on a production forkd), which
+// keeps the current uid-0 husk the file OWNER so its O_RDWR snapshot load still
+// works. The GROUP is set to SharedKVMGID and the mode made group-readable so a
+// FUTURE non-root husk (issue #585) that carries SharedKVMGID as a supplemental
+// group can still read the shared template (its per-activation rootfs reflink
+// clone reads rootfs.ext4; the load reads mem and vmstate) without owning the
+// files. The artifacts stay NOT world-writable. This requires the ability to
+// chgrp to SharedKVMGID, i.e. it runs on a production root forkd; the mock
+// engine never calls it.
 func normalizeTemplateArtifacts(root string) error {
-	uid, gid := os.Geteuid(), os.Getegid()
+	uid := os.Geteuid()
+	// The group is the shared kvm gid so a future non-root husk (issue #585)
+	// carrying it as a supplemental group reads the template through the group
+	// class. Only a builder able to chgrp to a group it does not belong to (a
+	// root forkd) can set it. A non-root builder (standalone sandbox-server,
+	// self-host) both builds and restores the template as the SAME uid, so it
+	// needs no shared-group contract and cannot set it. Attempt the shared gid
+	// and fall back to the builder's own gid on the first permission error; the
+	// reuse invariant (internal/fork.checkTemplateArtifactInvariants) accepts
+	// either group so the two stay in lockstep.
+	gid := SharedKVMGID
+	fellBack := false
 	return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		mode := os.FileMode(0o644)
+		// Never follow symlinks: os.Chown and os.Chmod dereference them, so a
+		// symlink planted in the build output could redirect this ownership or
+		// mode change onto a target outside the template tree. Template
+		// artifacts are regular files and directories; skip anything else, and
+		// use Lchown so the link inode itself is retargeted, never its target.
+		if d.Type()&os.ModeSymlink != 0 {
+			return nil
+		}
+		mode := os.FileMode(0o640)
 		if d.IsDir() {
-			mode = 0o755
+			mode = 0o750
 		}
-		if err := os.Chown(path, uid, gid); err != nil {
-			return fmt.Errorf("chown %s: %w", path, err)
+		if err := os.Lchown(path, uid, gid); err != nil {
+			if !fellBack && errors.Is(err, os.ErrPermission) {
+				// Not privileged enough to hand the artifacts to the shared kvm
+				// group: the non-root builder path. Keep the builder's own gid,
+				// which it can always set, for this and every remaining entry.
+				gid = os.Getegid()
+				fellBack = true
+				if err := os.Lchown(path, uid, gid); err != nil {
+					return fmt.Errorf("chown %s: %w", path, err)
+				}
+			} else {
+				return fmt.Errorf("chown %s: %w", path, err)
+			}
 		}
 		if err := os.Chmod(path, mode); err != nil {
 			return fmt.Errorf("chmod %s: %w", path, err)