fix(clone): replace removed --on-demand with --restore-mode

docs/configuration.md

@@ -13,6 +13,7 @@ systemd unit reads them from `/etc/cocoon/vk-cocoon.env`.
 | `VK_LEASES_PATH` | `/var/lib/cocoon/net/leases.json` | cocoon-net JSON lease file. |
 | `VK_COCOON_BIN` | `/usr/local/bin/cocoon` | Path to the cocoon CLI binary. |
 | `VK_ORPHAN_POLICY` | `destroy` | `destroy` (auto-clean), `alert`, or `keep`. |
+| `VK_RESTORE_MODE` | `ondemand` | Guest-memory restore mode for Cloud Hypervisor clones: `copy`, `ondemand`, or `mmap`. Windows VMs always use `copy` (lazy restore stalls DHCP boot); Firecracker has no restore mode. `mmap` shares page cache across clones of one snapshot — the fastest fan-out — but requires a Cloud Hypervisor build with mmap restore support (cocoonstack/cloud-hypervisor `dev`); on other CH builds clones fail, so it is opt-in. Invalid values abort startup. |
 | `VK_NODE_IP` | auto-detected | Override the virtual node's InternalIP address (first non-loopback IPv4 used otherwise). |
 | `VK_NODE_POOL` | `default` | Cocoon pool label stamped onto the registered node. |
 | `VK_PROVIDER_ID` | unset | Cloud-provider ProviderID for the virtual node (e.g. `gce://<project>/<zone>/<instance>`). Prevents cloud node lifecycle controllers from deleting the virtual node. |

main.go

@@ -51,6 +51,7 @@ const (
 	defaultNodeName     = "cocoon-pool"
 	defaultMetricsAddr  = ":9091"
 	defaultOrphanPolicy = string(provider.OrphanDestroy)
+	defaultRestoreMode  = string(vm.RestoreOnDemand)
 
 	defaultTLSCert        = "/etc/cocoon/vk/tls/vk-kubelet.crt"
 	defaultTLSKey         = "/etc/cocoon/vk/tls/vk-kubelet.key"
@@ -77,6 +78,7 @@ func main() {
 	leasesPath := commonk8s.EnvOrDefault("VK_LEASES_PATH", network.DefaultLeasesPath)
 	cocoonBin := commonk8s.EnvOrDefault("VK_COCOON_BIN", "")
 	orphanPolicy := commonk8s.EnvOrDefault("VK_ORPHAN_POLICY", defaultOrphanPolicy)
+	restoreMode := commonk8s.EnvOrDefault("VK_RESTORE_MODE", defaultRestoreMode)
 	nodeIP := commonk8s.EnvOrDefault("VK_NODE_IP", "")
 	nodePool := commonk8s.EnvOrDefault("VK_NODE_POOL", meta.DefaultNodePool)
 	providerID := os.Getenv("VK_PROVIDER_ID")
@@ -124,6 +126,7 @@ func main() {
 		leasesPath:   leasesPath,
 		cocoonBin:    cocoonBin,
 		orphanPolicy: orphanPolicy,
+		restoreMode:  restoreMode,
 		clientset:    clientset,
 		recorder:     recorder,
 	})
@@ -219,6 +222,7 @@ type buildOpts struct {
 	leasesPath   string
 	cocoonBin    string
 	orphanPolicy string
+	restoreMode  string
 	clientset    kubernetes.Interface
 	recorder     record.EventRecorder
 }
@@ -235,6 +239,10 @@ func buildRegistry(opts buildOpts) (oci.Registry, error) {
 
 func buildProvider(ctx context.Context, opts buildOpts) (*cocoon.Provider, error) {
 	logger := log.WithFunc("buildProvider")
+	restoreMode, err := vm.ParseRestoreMode(opts.restoreMode)
+	if err != nil {
+		return nil, fmt.Errorf("parse VK_RESTORE_MODE: %w", err)
+	}
 	registry, err := buildRegistry(opts)
 	if err != nil {
 		return nil, fmt.Errorf("construct registry client: %w", err)
@@ -261,6 +269,7 @@ func buildProvider(ctx context.Context, opts buildOpts) (*cocoon.Provider, error
 	p.GuestSAC = &sac.Dialer{}
 	p.Probes = probes.NewManager(ctx)
 	p.OrphanPolicy = provider.OrphanPolicy(strings.ToLower(opts.orphanPolicy))
+	p.RestoreMode = restoreMode
 	return p, nil
 }
 

provider/cocoon/create.go

@@ -167,12 +167,12 @@ func (p *Provider) bringUpVM(ctx context.Context, pod *corev1.Pod, spec meta.VMS
 			return nil, "", fmt.Errorf("annotation %s is incompatible with fork-from %q", meta.AnnotationCloneFromDir, spec.ForkFrom)
 		}
 		v, err := p.Runtime.Clone(ctx, vm.CloneOptions{
-			FromDir:    fromDir,
-			To:         spec.VMName,
-			Network:    spec.Network,
-			Backend:    backend,
-			NoDirectIO: noDirectIO,
-			OnDemand:   useOnDemandClone(spec.OS),
+			FromDir:     fromDir,
+			To:          spec.VMName,
+			Network:     spec.Network,
+			Backend:     backend,
+			NoDirectIO:  noDirectIO,
+			RestoreMode: restoreModeFor(p.RestoreMode, spec.OS),
 		})
 		if err != nil {
 			metrics.CloneFromDirTotal.WithLabelValues("failed").Inc()
@@ -187,12 +187,12 @@ func (p *Provider) bringUpVM(ctx context.Context, pod *corev1.Pod, spec meta.VMS
 			return nil, "", err
 		}
 		v, err := p.Runtime.Clone(ctx, vm.CloneOptions{
-			From:       cloneFrom,
-			To:         spec.VMName,
-			Network:    spec.Network,
-			Backend:    backend,
-			NoDirectIO: noDirectIO,
-			OnDemand:   useOnDemandClone(spec.OS),
+			From:        cloneFrom,
+			To:          spec.VMName,
+			Network:     spec.Network,
+			Backend:     backend,
+			NoDirectIO:  noDirectIO,
+			RestoreMode: restoreModeFor(p.RestoreMode, spec.OS),
 		})
 		if err != nil {
 			return nil, "", fmt.Errorf("clone vm %s from %s: %w", spec.VMName, cloneFrom, err)
@@ -254,13 +254,13 @@ func (p *Provider) bringUpVM(ctx context.Context, pod *corev1.Pod, spec meta.VMS
 		}
 
 		v, err := p.Runtime.Clone(ctx, vm.CloneOptions{
-			From:       local,
-			To:         spec.VMName,
-			Network:    spec.Network,
-			Backend:    backend,
-			NoDirectIO: noDirectIO,
-			Pull:       srcImage != "",
-			OnDemand:   useOnDemandClone(spec.OS),
+			From:        local,
+			To:          spec.VMName,
+			Network:     spec.Network,
+			Backend:     backend,
+			NoDirectIO:  noDirectIO,
+			Pull:        srcImage != "",
+			RestoreMode: restoreModeFor(p.RestoreMode, spec.OS),
 		})
 		if err != nil {
 			return nil, "", fmt.Errorf("clone vm %s from %s: %w", spec.VMName, local, err)
@@ -444,9 +444,12 @@ func parseCloneFromDirAnnotation(pod *corev1.Pod) (string, error) {
 	return raw, nil
 }
 
-// useOnDemandClone is off for Windows: UFFD lazy paging stalls DHCP boot.
-func useOnDemandClone(os string) bool {
-	return os != string(cocoonv1.OSWindows)
+// Windows forces copy: lazy memory restore stalls DHCP boot.
+func restoreModeFor(mode vm.RestoreMode, os string) vm.RestoreMode {
+	if os == string(cocoonv1.OSWindows) {
+		return vm.RestoreCopy
+	}
+	return mode
 }
 
 // isClonedBoot reports whether bringUpVM took a clone path. spec.Mode alone

provider/cocoon/create_test.go

@@ -318,8 +318,8 @@ func TestCreatePodCloneFromDirAnnotationDispatches(t *testing.T) {
 	if rt.cloned.From != "" {
 		t.Errorf("clone From = %q, want empty when FromDir is set", rt.cloned.From)
 	}
-	if !rt.cloned.OnDemand {
-		t.Errorf("OnDemand should be true on clone-from-dir path")
+	if rt.cloned.RestoreMode != vm.RestoreOnDemand {
+		t.Errorf("RestoreMode = %q, want %q on clone-from-dir path", rt.cloned.RestoreMode, vm.RestoreOnDemand)
 	}
 	if rt.snapshotSaveCount != 0 || len(rt.ensuredImages) != 0 {
 		t.Errorf("from-dir path should bypass snapshot/ensure: save=%d images=%v",

provider/cocoon/provider.go

@@ -80,6 +80,7 @@ type Provider struct {
 	NodeName string
 
 	OrphanPolicy provider.OrphanPolicy
+	RestoreMode  vm.RestoreMode
 
 	Clientset   kubernetes.Interface
 	Runtime     vm.Runtime
@@ -133,6 +134,7 @@ func NewProvider() *Provider {
 		lifecycleCtx:     lifecycleCtx,
 		lifecycleStop:    lifecycleStop,
 		OrphanPolicy:     provider.OrphanDestroy,
+		RestoreMode:      vm.RestoreOnDemand,
 		Pinger:           network.NopPinger{},
 		pods:             map[string]*corev1.Pod{},
 		vmsByPod:         map[string]*vm.VM{},

provider/cocoon/update.go

@@ -202,13 +202,13 @@ func (p *Provider) cloneFromHibernate(ctx context.Context, spec meta.VMSpec, sou
 		}
 	}
 	opts := vm.CloneOptions{
-		From:       sourceName,
-		To:         spec.VMName,
-		Network:    spec.Network,
-		Backend:    spec.Backend,
-		NoDirectIO: spec.NoDirectIO,
-		OnDemand:   useOnDemandClone(spec.OS),
-		Pull:       snapshot != nil && snapshot.Image != "",
+		From:        sourceName,
+		To:          spec.VMName,
+		Network:     spec.Network,
+		Backend:     spec.Backend,
+		NoDirectIO:  spec.NoDirectIO,
+		RestoreMode: restoreModeFor(p.RestoreMode, spec.OS),
+		Pull:        snapshot != nil && snapshot.Image != "",
 	}
 	if shouldDropNICBeforeHibernate(spec) {
 		opts.NICs = ptr.To(1)

provider/cocoon/update_test.go

@@ -15,21 +15,22 @@ import (
 	"github.com/cocoonstack/vk-cocoon/vm"
 )
 
-func TestUseOnDemandClone(t *testing.T) {
+func TestRestoreModeFor(t *testing.T) {
 	cases := []struct {
 		name string
 		os   string
-		want bool
+		mode vm.RestoreMode
+		want vm.RestoreMode
 	}{
-		{"linux", string(cocoonv1.OSLinux), true},
-		{"windows off", string(cocoonv1.OSWindows), false},
-		{"android counts as non-windows", string(cocoonv1.OSAndroid), true},
-		{"empty OS defaults to on", "", true},
+		{"linux gets configured mode", string(cocoonv1.OSLinux), vm.RestoreMmap, vm.RestoreMmap},
+		{"windows forced to copy", string(cocoonv1.OSWindows), vm.RestoreMmap, vm.RestoreCopy},
+		{"android counts as non-windows", string(cocoonv1.OSAndroid), vm.RestoreOnDemand, vm.RestoreOnDemand},
+		{"empty OS gets configured mode", "", vm.RestoreOnDemand, vm.RestoreOnDemand},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			if got := useOnDemandClone(tc.os); got != tc.want {
-				t.Errorf("useOnDemandClone(%q) = %v, want %v", tc.os, got, tc.want)
+			if got := restoreModeFor(tc.mode, tc.os); got != tc.want {
+				t.Errorf("restoreModeFor(%q, %q) = %q, want %q", tc.mode, tc.os, got, tc.want)
 			}
 		})
 	}

vm/cocoon_cli.go

@@ -421,10 +421,9 @@ func buildCloneArgs(opts CloneOptions) []string {
 	if opts.Pull || opts.FromDir != "" {
 		args = append(args, "--pull")
 	}
-	if opts.OnDemand && opts.Backend != BackendFirecracker {
-		// UFFD lazy memory restore is CH-only; skipping on FC keeps the
-		// same CloneOptions usable for both backends.
-		args = append(args, "--on-demand")
+	// copy emits no flag, keeping the argv valid on cocoon builds predating --restore-mode.
+	if opts.RestoreMode != "" && opts.RestoreMode != RestoreCopy && opts.Backend != BackendFirecracker {
+		args = append(args, "--restore-mode", string(opts.RestoreMode))
 	}
 	if opts.NICs != nil && opts.Backend != BackendFirecracker {
 		args = append(args, "--nics", strconv.Itoa(*opts.NICs))

vm/cocoon_cli_test.go

@@ -1,6 +1,7 @@
 package vm
 
 import (
+	"cmp"
 	"errors"
 	"reflect"
 	"testing"
@@ -72,8 +73,8 @@ func TestBuildCloneArgs(t *testing.T) {
 			want: []string{"vm", "clone", "--output", "json", "--name", "vm-a", "snap-a"},
 		},
 		{
-			name: "firecracker clone strips on-demand",
-			opts: CloneOptions{From: "snap-a", To: "vm-b", Backend: "firecracker", OnDemand: true},
+			name: "firecracker clone strips restore-mode",
+			opts: CloneOptions{From: "snap-a", To: "vm-b", Backend: "firecracker", RestoreMode: RestoreOnDemand},
 			want: []string{"vm", "clone", "--output", "json", "--name", "vm-b", "snap-a"},
 		},
 		{
@@ -92,23 +93,38 @@ func TestBuildCloneArgs(t *testing.T) {
 			want: []string{"vm", "clone", "--output", "json", "--name", "vm-e", "--pull", "snap-a"},
 		},
 		{
-			name: "on-demand appended on cloud-hypervisor",
-			opts: CloneOptions{From: "snap-a", To: "vm-od", Backend: "cloud-hypervisor", OnDemand: true},
-			want: []string{"vm", "clone", "--output", "json", "--name", "vm-od", "--on-demand", "snap-a"},
+			name: "restore-mode ondemand appended on cloud-hypervisor",
+			opts: CloneOptions{From: "snap-a", To: "vm-od", Backend: "cloud-hypervisor", RestoreMode: RestoreOnDemand},
+			want: []string{"vm", "clone", "--output", "json", "--name", "vm-od", "--restore-mode", "ondemand", "snap-a"},
+		},
+		{
+			name: "restore-mode mmap appended on cloud-hypervisor",
+			opts: CloneOptions{From: "snap-a", To: "vm-mm", Backend: "cloud-hypervisor", RestoreMode: RestoreMmap},
+			want: []string{"vm", "clone", "--output", "json", "--name", "vm-mm", "--restore-mode", "mmap", "snap-a"},
+		},
+		{
+			name: "restore-mode copy emits no flag",
+			opts: CloneOptions{From: "snap-a", To: "vm-cp", Backend: "cloud-hypervisor", RestoreMode: RestoreCopy},
+			want: []string{"vm", "clone", "--output", "json", "--name", "vm-cp", "snap-a"},
+		},
+		{
+			name: "empty backend treated as cloud-hypervisor for restore-mode",
+			opts: CloneOptions{From: "snap-a", To: "vm-eb", RestoreMode: RestoreOnDemand},
+			want: []string{"vm", "clone", "--output", "json", "--name", "vm-eb", "--restore-mode", "ondemand", "snap-a"},
 		},
 		{
 			name: "from-dir replaces positional and forces --pull",
 			opts: CloneOptions{To: "vm-f", FromDir: "/var/lib/cocoon/snaps/foo", Backend: "cloud-hypervisor"},
 			want: []string{"vm", "clone", "--output", "json", "--name", "vm-f", "--pull", "--from-dir", "/var/lib/cocoon/snaps/foo"},
 		},
 		{
-			name: "from-dir on cloud-hypervisor with on-demand",
-			opts: CloneOptions{To: "vm-g", FromDir: "/snaps/bar", Backend: "cloud-hypervisor", OnDemand: true},
-			want: []string{"vm", "clone", "--output", "json", "--name", "vm-g", "--pull", "--on-demand", "--from-dir", "/snaps/bar"},
+			name: "from-dir on cloud-hypervisor with restore-mode",
+			opts: CloneOptions{To: "vm-g", FromDir: "/snaps/bar", Backend: "cloud-hypervisor", RestoreMode: RestoreOnDemand},
+			want: []string{"vm", "clone", "--output", "json", "--name", "vm-g", "--pull", "--restore-mode", "ondemand", "--from-dir", "/snaps/bar"},
 		},
 		{
-			name: "from-dir on firecracker skips on-demand",
-			opts: CloneOptions{To: "vm-h", FromDir: "/snaps/fc", Backend: "firecracker", OnDemand: true},
+			name: "from-dir on firecracker skips restore-mode",
+			opts: CloneOptions{To: "vm-h", FromDir: "/snaps/fc", Backend: "firecracker", RestoreMode: RestoreOnDemand},
 			want: []string{"vm", "clone", "--output", "json", "--name", "vm-h", "--pull", "--from-dir", "/snaps/fc"},
 		},
 		{
@@ -132,9 +148,9 @@ func TestBuildCloneArgs(t *testing.T) {
 			want: []string{"vm", "clone", "--output", "json", "--name", "vm-l", "--nics", "0", "snap-a"},
 		},
 		{
-			name: "nics combines with on-demand on CH",
-			opts: CloneOptions{From: "snap-a", To: "vm-m", Backend: "cloud-hypervisor", OnDemand: true, NICs: ptr.To(2)},
-			want: []string{"vm", "clone", "--output", "json", "--name", "vm-m", "--on-demand", "--nics", "2", "snap-a"},
+			name: "nics combines with restore-mode on CH",
+			opts: CloneOptions{From: "snap-a", To: "vm-m", Backend: "cloud-hypervisor", RestoreMode: RestoreOnDemand, NICs: ptr.To(2)},
+			want: []string{"vm", "clone", "--output", "json", "--name", "vm-m", "--restore-mode", "ondemand", "--nics", "2", "snap-a"},
 		},
 		{
 			name: "firecracker clone strips --nics",
@@ -152,6 +168,41 @@ func TestBuildCloneArgs(t *testing.T) {
 	}
 }
 
+func TestParseRestoreMode(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		in      string
+		want    RestoreMode
+		wantErr bool
+	}{
+		{in: "copy", want: RestoreCopy},
+		{in: "ondemand", want: RestoreOnDemand},
+		{in: "mmap", want: RestoreMmap},
+		{in: "MMAP", want: RestoreMmap},
+		{in: " ondemand ", want: RestoreOnDemand},
+		{in: "", wantErr: true},
+		{in: "on-demand", wantErr: true},
+	}
+	for _, tc := range cases {
+		t.Run(cmp.Or(tc.in, "empty"), func(t *testing.T) {
+			got, err := ParseRestoreMode(tc.in)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatalf("ParseRestoreMode(%q) = %q, want error", tc.in, got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("ParseRestoreMode(%q): %v", tc.in, err)
+			}
+			if got != tc.want {
+				t.Errorf("ParseRestoreMode(%q) = %q, want %q", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
 func TestBuildRunArgs(t *testing.T) {
 	t.Parallel()