feat(hibernate): release the DHCP lease pre-snapshot; renew-nudge stalled wakes in-window

docs/metrics.md

@@ -40,9 +40,10 @@ Prometheus endpoint with vk-cocoon-specific metrics:
 | `cocoon_vk_pod_lifecycle_total{op,result,reason}` | Counter | Pod lifecycle operations (`result=ok\|failed\|skipped`, `reason` sub-classifies) |
 | `cocoon_vk_snapshot_pull_total{result}` / `save_total` / `push_total` | Counter | Snapshot pull/save/push counts |
 | `cocoon_vk_clone_from_dir_total{result}` | Counter | Annotation-driven `--from-dir` clone attempts |
-| `cocoon_vk_hibernate_total{phase,result}` | Counter | Hibernate stage outcomes (`phase=netresize\|snapshot\|push\|remove`) |
+| `cocoon_vk_hibernate_total{phase,result}` | Counter | Hibernate stage outcomes (`phase=dhcp_release\|netresize\|snapshot\|push\|remove`) |
 | `cocoon_vk_wake_total{result}` | Counter | Wake operation outcomes |
 | `cocoon_vk_wake_ip_wait_total{result}` | Counter | Post-clone and wake DHCP-lease-wait outcomes — both the CH+Windows dropNIC wake and every clone's post-clone IP wait (`result=ok\|timeout`) |
+| `cocoon_vk_wake_renew_nudge_total{result}` | Counter | `ipconfig /renew` nudges sent to Windows guests still lease-less mid lease-wait (`result=ok\|failed`; failed means the exec didn't confirm — the in-guest renew may still have taken effect, the lease re-check decides) |
 | `cocoon_vk_postclone_total{kind,result}` | Counter | Post-clone fixup outcomes (`kind=linux_static\|linux_fc\|windows\|sac`) |
 | `cocoon_vk_postclone_retry_attempts{result}` | Histogram | Attempts consumed before post-clone exec succeeded or failed (`result=ok\|failed`) |
 | `cocoon_vk_vm_table_size` | Gauge | Tracked VM count |

main.go

@@ -248,7 +248,7 @@ func buildProvider(ctx context.Context, opts buildOpts) (*cocoon.Provider, error
 	}
 	logger.Infof(ctx, "registry backend: OCI %s", opts.ociRegistry)
 	runtime := vm.NewCocoonCLI(opts.cocoonBin)
-	p := cocoon.NewProvider()
+	p := cocoon.NewProvider(ctx)
 	p.NodeName = opts.nodeName
 	p.Clientset = opts.clientset
 	p.Recorder = opts.recorder

metrics/metrics.go

@@ -190,6 +190,16 @@ var (
 		[]string{"result"}, // result=ok|timeout
 	)
 
+	WakeRenewNudgeTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricNamespace,
+			Subsystem: metricSubsystem,
+			Name:      "wake_renew_nudge_total",
+			Help:      "ipconfig /renew nudges sent to Windows guests still lease-less mid lease-wait.",
+		},
+		[]string{"result"}, // result=ok|failed
+	)
+
 	PostCloneTotal = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: metricNamespace,
@@ -233,6 +243,7 @@ func Register(reg prometheus.Registerer) {
 		HibernateTotal,
 		WakeTotal,
 		WakeIPWaitTotal,
+		WakeRenewNudgeTotal,
 		PostCloneTotal,
 		PostCloneRetryAttempts,
 	)

provider/cocoon/create_test.go

@@ -1607,12 +1607,7 @@ func TestGetPodStatusRefreshesIPFromLease(t *testing.T) {
 	p.Probes = probes.NewManager(t.Context())
 	p.Probes.Set("ns/demo-0", probes.Result{Ready: true})
 
-	leasePath := filepath.Join(t.TempDir(), "leases.json")
-	leases := `[{"mac":"aa:bb:cc:dd:ee:ff","ip":"172.20.0.88","expiry":"2099-01-01T00:00:00Z"}]`
-	if err := os.WriteFile(leasePath, []byte(leases), 0o644); err != nil {
-		t.Fatalf("write leases: %v", err)
-	}
-	p.LeaseParser = network.NewLeaseParser(leasePath)
+	p.LeaseParser = newLeaseParser(t, "aa:bb:cc:dd:ee:ff", "172.20.0.88")
 
 	pod := newPodWithSpec(meta.VMSpec{VMName: "vk-ns-demo-0", Mode: "run"})
 	p.trackPod(pod, &vm.VM{ID: "vmid", Name: "vk-ns-demo-0", MAC: "aa:bb:cc:dd:ee:ff"})
@@ -1678,6 +1673,8 @@ type fakeRuntime struct {
 
 	// onRemove, when set, fires at Remove entry — for ordering / failure tests.
 	onRemove func()
+	// onExec, when set, fires at Exec entry — lets tests block or mutate state mid-exec.
+	onExec func()
 	// removeErr, when set, makes Remove fail with this error.
 	removeErr error
 	// snapshotSaveErr, when set, makes SnapshotSave fail with this error.
@@ -1878,7 +1875,10 @@ type netResizeCall struct {
 	target int
 }
 
-func (f *fakeRuntime) NetResize(_ context.Context, vmID string, target int) error {
+func (f *fakeRuntime) NetResize(ctx context.Context, vmID string, target int) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	f.netResizeCalls = append(f.netResizeCalls, netResizeCall{vmID: vmID, target: target})
 	return f.netResizeErr
 }
@@ -1890,7 +1890,13 @@ type fakeExecCall struct {
 	stdin string
 }
 
-func (f *fakeRuntime) Exec(_ context.Context, vmID string, argv []string, env map[string]string, stdin io.Reader, stdout, stderr io.Writer) error {
+func (f *fakeRuntime) Exec(ctx context.Context, vmID string, argv []string, env map[string]string, stdin io.Reader, stdout, stderr io.Writer) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	if f.onExec != nil {
+		f.onExec()
+	}
 	call := fakeExecCall{vmID: vmID, argv: argv, env: env}
 	if stdin != nil {
 		buf, _ := io.ReadAll(stdin)
@@ -1942,11 +1948,22 @@ func (nopWriteCloser) Close() error                { return nil }
 // race with the next test on a recycled pod heap address.
 func newTestProvider(t *testing.T) *Provider {
 	t.Helper()
-	p := NewProvider()
+	p := NewProvider(t.Context())
 	t.Cleanup(p.Close)
 	return p
 }
 
+// newLeaseParser writes a one-entry leases.json and returns a parser for it.
+func newLeaseParser(t *testing.T, mac, ip string) *network.LeaseParser {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "leases.json")
+	entry := `[{"mac":"` + mac + `","ip":"` + ip + `","expiry":"2099-01-01T00:00:00Z"}]`
+	if err := os.WriteFile(path, []byte(entry), 0o644); err != nil {
+		t.Fatalf("write leases: %v", err)
+	}
+	return network.NewLeaseParser(path)
+}
+
 func newPodWithSpec(spec meta.VMSpec) *corev1.Pod {
 	pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "demo-0", Namespace: "ns"}}
 	spec.Managed = true

provider/cocoon/postclone.go

@@ -123,7 +123,7 @@ func (p *Provider) runPostCloneSetup(ctx context.Context, pod *corev1.Pod, spec
 // ready then would promise an IP resolveVMIP can't yet return. On timeout mark
 // failed, never ready-without-IP.
 func (p *Provider) markReadyAfterIP(ctx context.Context, pod *corev1.Pod, v *vm.VM) {
-	gotIP := p.waitForFreshIP(ctx, pod.Namespace, pod.Name)
+	gotIP := p.waitForFreshIP(ctx, pod, v.ID)
 	if ctx.Err() != nil {
 		return
 	}

provider/cocoon/provider.go

@@ -114,12 +114,14 @@ type Provider struct {
 	// dropNIC wake tunables; defaults live in update.go.
 	wakeFreshIPBudget   time.Duration
 	wakeFreshIPInterval time.Duration
+	wakeRenewNudgeDelay time.Duration
 }
 
-// NewProvider constructs a Provider with empty tables.
-// Default Pinger is NopPinger so tests degrade gracefully.
-func NewProvider() *Provider {
-	lifecycleCtx, lifecycleStop := context.WithCancel(context.Background())
+// NewProvider constructs a Provider with empty tables; background work stops
+// when ctx is canceled or Close is called. Default Pinger is NopPinger so
+// tests degrade gracefully.
+func NewProvider(ctx context.Context) *Provider {
+	lifecycleCtx, lifecycleStop := context.WithCancel(ctx)
 	return &Provider{
 		startTime:        time.Now(),
 		lifecycleCtx:     lifecycleCtx,
@@ -295,20 +297,21 @@ func (p *Provider) vmForPod(namespace, name string) *vm.VM {
 }
 
 // setVMIP updates the tracked VM's IP (copy-on-write for concurrency safety).
-func (p *Provider) setVMIP(namespace, name, ip string) {
+func (p *Provider) setVMIP(namespace, name, vmID, ip string) bool {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	key := meta.PodKey(namespace, name)
 	v, ok := p.vmsByPod[key]
-	if !ok {
-		return
+	if !ok || v.ID != vmID {
+		return false
 	}
 	updated := *v
 	updated.IP = ip
 	p.vmsByPod[key] = &updated
 	if updated.Name != "" {
 		p.vmsByName[updated.Name] = &updated
 	}
+	return true
 }
 
 // resolveVMIP returns the VM's IP, falling back to a cocoon-net lease
@@ -322,7 +325,11 @@ func (p *Provider) resolveVMIP(namespace, name string, v *vm.VM) string {
 	if err != nil {
 		return ""
 	}
-	p.setVMIP(namespace, name, lease.IP)
+	// The lease belongs to v's MAC; if a same-name recreate swapped the tracked
+	// VM during the lookup, the IP must not leak onto the successor.
+	if !p.setVMIP(namespace, name, v.ID, lease.IP) {
+		return ""
+	}
 	return lease.IP
 }
 

provider/cocoon/update.go

@@ -1,10 +1,12 @@
 package cocoon
 
 import (
+	"bytes"
 	"cmp"
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/projecteru2/core/log"
@@ -26,6 +28,15 @@ const (
 	// the budget is generous.
 	defaultWakeFreshIPBudget   = 45 * time.Second
 	defaultWakeFreshIPInterval = 200 * time.Millisecond
+
+	// defaultWakeRenewNudgeDelay leaves natural DHCP the first 30s of the lease
+	// budget; the one-shot renew after it restarts DORA and lands in ~1s.
+	defaultWakeRenewNudgeDelay = 30 * time.Second
+
+	// guestIpconfigTimeout bounds the vsock exec for release/renew so a sick
+	// guest (dead agent, stopped DHCP service) cannot stall hibernate or wake.
+	guestIpconfigTimeout     = 20 * time.Second
+	hibernateRollbackTimeout = 30 * time.Second
 )
 
 // UpdatePod handles hibernate/wake transitions; other spec changes are no-ops
@@ -72,13 +83,10 @@ func (p *Provider) hibernate(ctx context.Context, pod *corev1.Pod, v *vm.VM) err
 	p.markLifecycleState(ctx, pod, meta.LifecycleStateHibernating, "")
 	dropNIC := shouldDropNICBeforeHibernate(spec)
 	if dropNIC {
-		if err := p.Runtime.NetResize(ctx, v.ID, 0); err != nil {
-			metrics.HibernateTotal.WithLabelValues("netresize", "failed").Inc()
-			err = fmt.Errorf("drop NIC pre-hibernate %s: %w", v.Name, err)
+		if err := p.dropNICForHibernate(ctx, v); err != nil {
 			p.failOp(ctx, pod, "HibernateNetResizeFailed", "update", err)
 			return err
 		}
-		metrics.HibernateTotal.WithLabelValues("netresize", "ok").Inc()
 	}
 	saveStart := time.Now()
 	if err := p.Runtime.SnapshotSave(ctx, v.Name, v.ID); err != nil {
@@ -142,13 +150,47 @@ func (p *Provider) hibernate(ctx context.Context, pod *corev1.Pod, v *vm.VM) err
 	return nil
 }
 
+// dropNICForHibernate releases the lease then detaches the NIC (VMware Tools'
+// suspend default): the snapshot carries no cached lease, so restored clones
+// DISCOVER instead of drawing a NAK. Best-effort: a sick guest still hibernates.
+func (p *Provider) dropNICForHibernate(ctx context.Context, v *vm.VM) error {
+	logger := log.WithFunc("Provider.dropNICForHibernate")
+	if err := p.execGuestIpconfig(ctx, v.ID, "release"); err != nil {
+		metrics.HibernateTotal.WithLabelValues("dhcp_release", "failed").Inc()
+		logger.Warnf(ctx, "dhcp release before hibernate %s: %v (proceeding)", v.Name, err)
+	} else {
+		metrics.HibernateTotal.WithLabelValues("dhcp_release", "ok").Inc()
+	}
+	if err := p.Runtime.NetResize(ctx, v.ID, 0); err != nil {
+		metrics.HibernateTotal.WithLabelValues("netresize", "failed").Inc()
+		// Renew regardless, detached from ctx: an exec error does not prove the
+		// guest skipped the release, and the drop may have failed because ctx died.
+		if renewErr := p.execGuestIpconfig(context.WithoutCancel(ctx), v.ID, "renew"); renewErr != nil {
+			logger.Warnf(ctx, "dhcp renew after failed NIC drop %s: %v", v.Name, renewErr)
+		}
+		return fmt.Errorf("drop NIC pre-hibernate %s: %w", v.Name, err)
+	}
+	metrics.HibernateTotal.WithLabelValues("netresize", "ok").Inc()
+	return nil
+}
+
 // rollbackHibernateNIC re-adds the NIC dropped pre-snapshot.
 func (p *Provider) rollbackHibernateNIC(ctx context.Context, v *vm.VM, dropped bool) {
 	if !dropped {
 		return
 	}
+	logger := log.WithFunc("Provider.rollbackHibernateNIC")
+	// Cancel-detached: the failure that triggered the rollback may be ctx dying,
+	// and an online VM must not be left NIC-less.
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), hibernateRollbackTimeout)
+	defer cancel()
 	if err := p.Runtime.NetResize(ctx, v.ID, 1); err != nil {
-		log.WithFunc("Provider.rollbackHibernateNIC").Errorf(ctx, err, "re-add NIC after hibernate failure %s", v.Name)
+		logger.Errorf(ctx, err, "re-add NIC after hibernate failure %s", v.Name)
+		return
+	}
+	// The pre-hibernate release left the guest unbound; nudge it to re-acquire.
+	if err := p.execGuestIpconfig(ctx, v.ID, "renew"); err != nil {
+		logger.Warnf(ctx, "dhcp renew after hibernate rollback %s: %v", v.Name, err)
 	}
 }
 
@@ -229,7 +271,7 @@ func (p *Provider) dispatchHibernateRestore(pod *corev1.Pod, spec meta.VMSpec, v
 
 // finalizeDropNICWake holds Ready until the fresh NIC's lease lands.
 func (p *Provider) finalizeDropNICWake(ctx context.Context, pod *corev1.Pod, v *vm.VM) {
-	gotIP := p.waitForFreshIP(ctx, pod.Namespace, pod.Name)
+	gotIP := p.waitForFreshIP(ctx, pod, v.ID)
 	if ctx.Err() != nil {
 		return
 	}
@@ -281,18 +323,37 @@ func (p *Provider) markLifecycleStateForWake(ctx context.Context, pod *corev1.Po
 // resumes contend, so the first lease can land many seconds after resume;
 // a short budget would misread that as lifecycle=failed and trigger an
 // operator rebuild.
-func (p *Provider) waitForFreshIP(ctx context.Context, namespace, name string) bool {
+func (p *Provider) waitForFreshIP(ctx context.Context, pod *corev1.Pod, vmID string) bool {
 	budget := cmp.Or(p.wakeFreshIPBudget, defaultWakeFreshIPBudget)
 	interval := cmp.Or(p.wakeFreshIPInterval, defaultWakeFreshIPInterval)
 	deadline := time.Now().Add(budget)
+	renewAt := time.Now().Add(cmp.Or(p.wakeRenewNudgeDelay, defaultWakeRenewNudgeDelay))
+	nudged := meta.ParseVMSpec(pod).OS != string(cocoonv1.OSWindows)
 	for {
-		v := p.vmForPod(namespace, name)
-		if v == nil {
+		v := p.vmForPod(pod.Namespace, pod.Name)
+		// A same-name recreate swaps the tracked VM; never touch the successor.
+		if v == nil || v.ID != vmID {
 			return false
 		}
-		if ip := p.resolveVMIP(namespace, name, v); ip != "" {
+		if ip := p.resolveVMIP(pod.Namespace, pod.Name, v); ip != "" {
 			return true
 		}
+		// One-shot renew: win11 can wedge on APIPA after a NAK and never re-DISCOVER.
+		if !nudged && !time.Now().Before(renewAt) {
+			nudged = true
+			// Capped by the wake deadline so a hung agent cannot push the verdict past it.
+			nudgeCtx, cancel := context.WithDeadline(ctx, deadline)
+			err := p.execGuestIpconfig(nudgeCtx, v.ID, "renew")
+			cancel()
+			if err != nil {
+				metrics.WakeRenewNudgeTotal.WithLabelValues("failed").Inc()
+				log.WithFunc("Provider.waitForFreshIP").Warnf(ctx, "renew nudge %s/%s: %v", pod.Namespace, pod.Name, err)
+			} else {
+				metrics.WakeRenewNudgeTotal.WithLabelValues("ok").Inc()
+			}
+			// The lease may have landed during the exec; re-check before the deadline verdict.
+			continue
+		}
 		if !time.Now().Before(deadline) {
 			return false
 		}
@@ -302,6 +363,34 @@ func (p *Provider) waitForFreshIP(ctx context.Context, namespace, name string) b
 	}
 }
 
+// execGuestIpconfig runs `ipconfig /<verb>` in the guest over vsock, so the
+// exec path never depends on the IP it is repairing.
+func (p *Provider) execGuestIpconfig(ctx context.Context, vmID, verb string) error {
+	ctx, cancel := context.WithTimeout(ctx, guestIpconfigTimeout)
+	defer cancel()
+	var out bytes.Buffer
+	if err := p.Runtime.Exec(ctx, vmID, []string{"cmd", "/c", "ipconfig /" + verb}, nil, nil, &out, &out); err != nil {
+		// Surface the guest-side reason ("The RPC server is unavailable", ...).
+		if line := lastNonEmptyLine(out.String()); line != "" {
+			return fmt.Errorf("%w: %s", err, line)
+		}
+		return err
+	}
+	return nil
+}
+
+// lastNonEmptyLine returns the last non-blank line of s, trimmed. ipconfig
+// prints a banner first and the actual error last, so the tail is the signal.
+func lastNonEmptyLine(s string) string {
+	last := ""
+	for line := range strings.SplitSeq(s, "\n") {
+		if trimmed := strings.TrimSpace(line); trimmed != "" {
+			last = trimmed
+		}
+	}
+	return last
+}
+
 // resolveWakeSource returns the clone source name and its snapshot metadata —
 // the local snapshot when present, else pulled from the registry.
 func (p *Provider) resolveWakeSource(ctx context.Context, vmName string) (string, *vm.Snapshot, error) {