diff --git a/docs/metrics.md b/docs/metrics.md index d417513..8940fca 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -40,9 +40,10 @@ Prometheus endpoint with vk-cocoon-specific metrics: | `cocoon_vk_pod_lifecycle_total{op,result,reason}` | Counter | Pod lifecycle operations (`result=ok\|failed\|skipped`, `reason` sub-classifies) | | `cocoon_vk_snapshot_pull_total{result}` / `save_total` / `push_total` | Counter | Snapshot pull/save/push counts | | `cocoon_vk_clone_from_dir_total{result}` | Counter | Annotation-driven `--from-dir` clone attempts | -| `cocoon_vk_hibernate_total{phase,result}` | Counter | Hibernate stage outcomes (`phase=netresize\|snapshot\|push\|remove`) | +| `cocoon_vk_hibernate_total{phase,result}` | Counter | Hibernate stage outcomes (`phase=dhcp_release\|netresize\|snapshot\|push\|remove`) | | `cocoon_vk_wake_total{result}` | Counter | Wake operation outcomes | | `cocoon_vk_wake_ip_wait_total{result}` | Counter | Post-clone and wake DHCP-lease-wait outcomes — both the CH+Windows dropNIC wake and every clone's post-clone IP wait (`result=ok\|timeout`) | +| `cocoon_vk_wake_renew_nudge_total{result}` | Counter | `ipconfig /renew` nudges sent to Windows guests still lease-less mid lease-wait (`result=ok\|failed`; failed means the exec didn't confirm — the in-guest renew may still have taken effect, the lease re-check decides) | | `cocoon_vk_postclone_total{kind,result}` | Counter | Post-clone fixup outcomes (`kind=linux_static\|linux_fc\|windows\|sac`) | | `cocoon_vk_postclone_retry_attempts{result}` | Histogram | Attempts consumed before post-clone exec succeeded or failed (`result=ok\|failed`) | | `cocoon_vk_vm_table_size` | Gauge | Tracked VM count | diff --git a/main.go b/main.go index b93ddbf..17bc2fc 100644 --- a/main.go +++ b/main.go @@ -248,7 +248,7 @@ func buildProvider(ctx context.Context, opts buildOpts) (*cocoon.Provider, error } logger.Infof(ctx, "registry backend: OCI %s", opts.ociRegistry) runtime := vm.NewCocoonCLI(opts.cocoonBin) - p := cocoon.NewProvider() + p := cocoon.NewProvider(ctx) p.NodeName = opts.nodeName p.Clientset = opts.clientset p.Recorder = opts.recorder diff --git a/metrics/metrics.go b/metrics/metrics.go index 5f4116a..50ce6ad 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -190,6 +190,16 @@ var ( []string{"result"}, // result=ok|timeout ) + WakeRenewNudgeTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricNamespace, + Subsystem: metricSubsystem, + Name: "wake_renew_nudge_total", + Help: "ipconfig /renew nudges sent to Windows guests still lease-less mid lease-wait.", + }, + []string{"result"}, // result=ok|failed + ) + PostCloneTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, @@ -233,6 +243,7 @@ func Register(reg prometheus.Registerer) { HibernateTotal, WakeTotal, WakeIPWaitTotal, + WakeRenewNudgeTotal, PostCloneTotal, PostCloneRetryAttempts, ) diff --git a/provider/cocoon/create_test.go b/provider/cocoon/create_test.go index a9c0331..3eb1192 100644 --- a/provider/cocoon/create_test.go +++ b/provider/cocoon/create_test.go @@ -1607,12 +1607,7 @@ func TestGetPodStatusRefreshesIPFromLease(t *testing.T) { p.Probes = probes.NewManager(t.Context()) p.Probes.Set("ns/demo-0", probes.Result{Ready: true}) - leasePath := filepath.Join(t.TempDir(), "leases.json") - leases := `[{"mac":"aa:bb:cc:dd:ee:ff","ip":"172.20.0.88","expiry":"2099-01-01T00:00:00Z"}]` - if err := os.WriteFile(leasePath, []byte(leases), 0o644); err != nil { - t.Fatalf("write leases: %v", err) - } - p.LeaseParser = network.NewLeaseParser(leasePath) + p.LeaseParser = newLeaseParser(t, "aa:bb:cc:dd:ee:ff", "172.20.0.88") pod := newPodWithSpec(meta.VMSpec{VMName: "vk-ns-demo-0", Mode: "run"}) p.trackPod(pod, &vm.VM{ID: "vmid", Name: "vk-ns-demo-0", MAC: "aa:bb:cc:dd:ee:ff"}) @@ -1678,6 +1673,8 @@ type fakeRuntime struct { // onRemove, when set, fires at Remove entry — for ordering / failure tests. onRemove func() + // onExec, when set, fires at Exec entry — lets tests block or mutate state mid-exec. + onExec func() // removeErr, when set, makes Remove fail with this error. removeErr error // snapshotSaveErr, when set, makes SnapshotSave fail with this error. @@ -1878,7 +1875,10 @@ type netResizeCall struct { target int } -func (f *fakeRuntime) NetResize(_ context.Context, vmID string, target int) error { +func (f *fakeRuntime) NetResize(ctx context.Context, vmID string, target int) error { + if err := ctx.Err(); err != nil { + return err + } f.netResizeCalls = append(f.netResizeCalls, netResizeCall{vmID: vmID, target: target}) return f.netResizeErr } @@ -1890,7 +1890,13 @@ type fakeExecCall struct { stdin string } -func (f *fakeRuntime) Exec(_ context.Context, vmID string, argv []string, env map[string]string, stdin io.Reader, stdout, stderr io.Writer) error { +func (f *fakeRuntime) Exec(ctx context.Context, vmID string, argv []string, env map[string]string, stdin io.Reader, stdout, stderr io.Writer) error { + if err := ctx.Err(); err != nil { + return err + } + if f.onExec != nil { + f.onExec() + } call := fakeExecCall{vmID: vmID, argv: argv, env: env} if stdin != nil { buf, _ := io.ReadAll(stdin) @@ -1942,11 +1948,22 @@ func (nopWriteCloser) Close() error { return nil } // race with the next test on a recycled pod heap address. func newTestProvider(t *testing.T) *Provider { t.Helper() - p := NewProvider() + p := NewProvider(t.Context()) t.Cleanup(p.Close) return p } +// newLeaseParser writes a one-entry leases.json and returns a parser for it. +func newLeaseParser(t *testing.T, mac, ip string) *network.LeaseParser { + t.Helper() + path := filepath.Join(t.TempDir(), "leases.json") + entry := `[{"mac":"` + mac + `","ip":"` + ip + `","expiry":"2099-01-01T00:00:00Z"}]` + if err := os.WriteFile(path, []byte(entry), 0o644); err != nil { + t.Fatalf("write leases: %v", err) + } + return network.NewLeaseParser(path) +} + func newPodWithSpec(spec meta.VMSpec) *corev1.Pod { pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "demo-0", Namespace: "ns"}} spec.Managed = true diff --git a/provider/cocoon/postclone.go b/provider/cocoon/postclone.go index 8ee6bce..8a6f33d 100644 --- a/provider/cocoon/postclone.go +++ b/provider/cocoon/postclone.go @@ -123,7 +123,7 @@ func (p *Provider) runPostCloneSetup(ctx context.Context, pod *corev1.Pod, spec // ready then would promise an IP resolveVMIP can't yet return. On timeout mark // failed, never ready-without-IP. func (p *Provider) markReadyAfterIP(ctx context.Context, pod *corev1.Pod, v *vm.VM) { - gotIP := p.waitForFreshIP(ctx, pod.Namespace, pod.Name) + gotIP := p.waitForFreshIP(ctx, pod, v.ID) if ctx.Err() != nil { return } diff --git a/provider/cocoon/provider.go b/provider/cocoon/provider.go index 2df3f8e..ca5e290 100644 --- a/provider/cocoon/provider.go +++ b/provider/cocoon/provider.go @@ -114,12 +114,14 @@ type Provider struct { // dropNIC wake tunables; defaults live in update.go. wakeFreshIPBudget time.Duration wakeFreshIPInterval time.Duration + wakeRenewNudgeDelay time.Duration } -// NewProvider constructs a Provider with empty tables. -// Default Pinger is NopPinger so tests degrade gracefully. -func NewProvider() *Provider { - lifecycleCtx, lifecycleStop := context.WithCancel(context.Background()) +// NewProvider constructs a Provider with empty tables; background work stops +// when ctx is canceled or Close is called. Default Pinger is NopPinger so +// tests degrade gracefully. +func NewProvider(ctx context.Context) *Provider { + lifecycleCtx, lifecycleStop := context.WithCancel(ctx) return &Provider{ startTime: time.Now(), lifecycleCtx: lifecycleCtx, @@ -295,13 +297,13 @@ func (p *Provider) vmForPod(namespace, name string) *vm.VM { } // setVMIP updates the tracked VM's IP (copy-on-write for concurrency safety). -func (p *Provider) setVMIP(namespace, name, ip string) { +func (p *Provider) setVMIP(namespace, name, vmID, ip string) bool { p.mu.Lock() defer p.mu.Unlock() key := meta.PodKey(namespace, name) v, ok := p.vmsByPod[key] - if !ok { - return + if !ok || v.ID != vmID { + return false } updated := *v updated.IP = ip @@ -309,6 +311,7 @@ func (p *Provider) setVMIP(namespace, name, ip string) { if updated.Name != "" { p.vmsByName[updated.Name] = &updated } + return true } // resolveVMIP returns the VM's IP, falling back to a cocoon-net lease @@ -322,7 +325,11 @@ func (p *Provider) resolveVMIP(namespace, name string, v *vm.VM) string { if err != nil { return "" } - p.setVMIP(namespace, name, lease.IP) + // The lease belongs to v's MAC; if a same-name recreate swapped the tracked + // VM during the lookup, the IP must not leak onto the successor. + if !p.setVMIP(namespace, name, v.ID, lease.IP) { + return "" + } return lease.IP } diff --git a/provider/cocoon/update.go b/provider/cocoon/update.go index e731ab7..f3cb3ea 100644 --- a/provider/cocoon/update.go +++ b/provider/cocoon/update.go @@ -1,10 +1,12 @@ package cocoon import ( + "bytes" "cmp" "context" "errors" "fmt" + "strings" "time" "github.com/projecteru2/core/log" @@ -26,6 +28,15 @@ const ( // the budget is generous. defaultWakeFreshIPBudget = 45 * time.Second defaultWakeFreshIPInterval = 200 * time.Millisecond + + // defaultWakeRenewNudgeDelay leaves natural DHCP the first 30s of the lease + // budget; the one-shot renew after it restarts DORA and lands in ~1s. + defaultWakeRenewNudgeDelay = 30 * time.Second + + // guestIpconfigTimeout bounds the vsock exec for release/renew so a sick + // guest (dead agent, stopped DHCP service) cannot stall hibernate or wake. + guestIpconfigTimeout = 20 * time.Second + hibernateRollbackTimeout = 30 * time.Second ) // UpdatePod handles hibernate/wake transitions; other spec changes are no-ops @@ -72,13 +83,10 @@ func (p *Provider) hibernate(ctx context.Context, pod *corev1.Pod, v *vm.VM) err p.markLifecycleState(ctx, pod, meta.LifecycleStateHibernating, "") dropNIC := shouldDropNICBeforeHibernate(spec) if dropNIC { - if err := p.Runtime.NetResize(ctx, v.ID, 0); err != nil { - metrics.HibernateTotal.WithLabelValues("netresize", "failed").Inc() - err = fmt.Errorf("drop NIC pre-hibernate %s: %w", v.Name, err) + if err := p.dropNICForHibernate(ctx, v); err != nil { p.failOp(ctx, pod, "HibernateNetResizeFailed", "update", err) return err } - metrics.HibernateTotal.WithLabelValues("netresize", "ok").Inc() } saveStart := time.Now() if err := p.Runtime.SnapshotSave(ctx, v.Name, v.ID); err != nil { @@ -142,13 +150,47 @@ func (p *Provider) hibernate(ctx context.Context, pod *corev1.Pod, v *vm.VM) err return nil } +// dropNICForHibernate releases the lease then detaches the NIC (VMware Tools' +// suspend default): the snapshot carries no cached lease, so restored clones +// DISCOVER instead of drawing a NAK. Best-effort: a sick guest still hibernates. +func (p *Provider) dropNICForHibernate(ctx context.Context, v *vm.VM) error { + logger := log.WithFunc("Provider.dropNICForHibernate") + if err := p.execGuestIpconfig(ctx, v.ID, "release"); err != nil { + metrics.HibernateTotal.WithLabelValues("dhcp_release", "failed").Inc() + logger.Warnf(ctx, "dhcp release before hibernate %s: %v (proceeding)", v.Name, err) + } else { + metrics.HibernateTotal.WithLabelValues("dhcp_release", "ok").Inc() + } + if err := p.Runtime.NetResize(ctx, v.ID, 0); err != nil { + metrics.HibernateTotal.WithLabelValues("netresize", "failed").Inc() + // Renew regardless, detached from ctx: an exec error does not prove the + // guest skipped the release, and the drop may have failed because ctx died. + if renewErr := p.execGuestIpconfig(context.WithoutCancel(ctx), v.ID, "renew"); renewErr != nil { + logger.Warnf(ctx, "dhcp renew after failed NIC drop %s: %v", v.Name, renewErr) + } + return fmt.Errorf("drop NIC pre-hibernate %s: %w", v.Name, err) + } + metrics.HibernateTotal.WithLabelValues("netresize", "ok").Inc() + return nil +} + // rollbackHibernateNIC re-adds the NIC dropped pre-snapshot. func (p *Provider) rollbackHibernateNIC(ctx context.Context, v *vm.VM, dropped bool) { if !dropped { return } + logger := log.WithFunc("Provider.rollbackHibernateNIC") + // Cancel-detached: the failure that triggered the rollback may be ctx dying, + // and an online VM must not be left NIC-less. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), hibernateRollbackTimeout) + defer cancel() if err := p.Runtime.NetResize(ctx, v.ID, 1); err != nil { - log.WithFunc("Provider.rollbackHibernateNIC").Errorf(ctx, err, "re-add NIC after hibernate failure %s", v.Name) + logger.Errorf(ctx, err, "re-add NIC after hibernate failure %s", v.Name) + return + } + // The pre-hibernate release left the guest unbound; nudge it to re-acquire. + if err := p.execGuestIpconfig(ctx, v.ID, "renew"); err != nil { + logger.Warnf(ctx, "dhcp renew after hibernate rollback %s: %v", v.Name, err) } } @@ -229,7 +271,7 @@ func (p *Provider) dispatchHibernateRestore(pod *corev1.Pod, spec meta.VMSpec, v // finalizeDropNICWake holds Ready until the fresh NIC's lease lands. func (p *Provider) finalizeDropNICWake(ctx context.Context, pod *corev1.Pod, v *vm.VM) { - gotIP := p.waitForFreshIP(ctx, pod.Namespace, pod.Name) + gotIP := p.waitForFreshIP(ctx, pod, v.ID) if ctx.Err() != nil { return } @@ -281,18 +323,37 @@ func (p *Provider) markLifecycleStateForWake(ctx context.Context, pod *corev1.Po // resumes contend, so the first lease can land many seconds after resume; // a short budget would misread that as lifecycle=failed and trigger an // operator rebuild. -func (p *Provider) waitForFreshIP(ctx context.Context, namespace, name string) bool { +func (p *Provider) waitForFreshIP(ctx context.Context, pod *corev1.Pod, vmID string) bool { budget := cmp.Or(p.wakeFreshIPBudget, defaultWakeFreshIPBudget) interval := cmp.Or(p.wakeFreshIPInterval, defaultWakeFreshIPInterval) deadline := time.Now().Add(budget) + renewAt := time.Now().Add(cmp.Or(p.wakeRenewNudgeDelay, defaultWakeRenewNudgeDelay)) + nudged := meta.ParseVMSpec(pod).OS != string(cocoonv1.OSWindows) for { - v := p.vmForPod(namespace, name) - if v == nil { + v := p.vmForPod(pod.Namespace, pod.Name) + // A same-name recreate swaps the tracked VM; never touch the successor. + if v == nil || v.ID != vmID { return false } - if ip := p.resolveVMIP(namespace, name, v); ip != "" { + if ip := p.resolveVMIP(pod.Namespace, pod.Name, v); ip != "" { return true } + // One-shot renew: win11 can wedge on APIPA after a NAK and never re-DISCOVER. + if !nudged && !time.Now().Before(renewAt) { + nudged = true + // Capped by the wake deadline so a hung agent cannot push the verdict past it. + nudgeCtx, cancel := context.WithDeadline(ctx, deadline) + err := p.execGuestIpconfig(nudgeCtx, v.ID, "renew") + cancel() + if err != nil { + metrics.WakeRenewNudgeTotal.WithLabelValues("failed").Inc() + log.WithFunc("Provider.waitForFreshIP").Warnf(ctx, "renew nudge %s/%s: %v", pod.Namespace, pod.Name, err) + } else { + metrics.WakeRenewNudgeTotal.WithLabelValues("ok").Inc() + } + // The lease may have landed during the exec; re-check before the deadline verdict. + continue + } if !time.Now().Before(deadline) { return false } @@ -302,6 +363,34 @@ func (p *Provider) waitForFreshIP(ctx context.Context, namespace, name string) b } } +// execGuestIpconfig runs `ipconfig /` in the guest over vsock, so the +// exec path never depends on the IP it is repairing. +func (p *Provider) execGuestIpconfig(ctx context.Context, vmID, verb string) error { + ctx, cancel := context.WithTimeout(ctx, guestIpconfigTimeout) + defer cancel() + var out bytes.Buffer + if err := p.Runtime.Exec(ctx, vmID, []string{"cmd", "/c", "ipconfig /" + verb}, nil, nil, &out, &out); err != nil { + // Surface the guest-side reason ("The RPC server is unavailable", ...). + if line := lastNonEmptyLine(out.String()); line != "" { + return fmt.Errorf("%w: %s", err, line) + } + return err + } + return nil +} + +// lastNonEmptyLine returns the last non-blank line of s, trimmed. ipconfig +// prints a banner first and the actual error last, so the tail is the signal. +func lastNonEmptyLine(s string) string { + last := "" + for line := range strings.SplitSeq(s, "\n") { + if trimmed := strings.TrimSpace(line); trimmed != "" { + last = trimmed + } + } + return last +} + // resolveWakeSource returns the clone source name and its snapshot metadata — // the local snapshot when present, else pulled from the registry. func (p *Provider) resolveWakeSource(ctx context.Context, vmName string) (string, *vm.Snapshot, error) { diff --git a/provider/cocoon/update_test.go b/provider/cocoon/update_test.go index 083203a..384eb0f 100644 --- a/provider/cocoon/update_test.go +++ b/provider/cocoon/update_test.go @@ -1,13 +1,16 @@ package cocoon import ( + "context" "errors" "maps" "reflect" + "strings" "testing" "time" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/fake" cocoonv1 "github.com/cocoonstack/cocoon-common/apis/v1" "github.com/cocoonstack/cocoon-common/meta" @@ -303,7 +306,7 @@ func TestFinalizeDropNICWakeMarksReadyWhenIPArrives(t *testing.T) { }() time.AfterFunc(10*time.Millisecond, func() { - p.setVMIP("ns", "demo-0", "172.20.1.228") + p.setVMIP("ns", "demo-0", v.ID, "172.20.1.228") }) select { @@ -364,7 +367,7 @@ func TestFinalizeDropNICWakeSkipsLifecycleWhenHibernateRequested(t *testing.T) { time.AfterFunc(10*time.Millisecond, func() { meta.HibernateState(true).Apply(pod) - p.setVMIP("ns", "demo-0", "172.20.1.228") + p.setVMIP("ns", "demo-0", v.ID, "172.20.1.228") }) select { @@ -412,3 +415,286 @@ func newDropNICWakeFixture(t *testing.T, budget, interval time.Duration) (*Provi p.markLifecycleState(t.Context(), pod, meta.LifecycleStateCreating, "") return p, pod, v } + +func execArgvs(rt *fakeRuntime) []string { + var out []string + for _, c := range rt.execCalls { + out = append(out, strings.Join(c.argv, " ")) + } + return out +} + +func TestHibernateReleasesLeaseBeforeNICDrop(t *testing.T) { + rt := &fakeRuntime{} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + v := &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"} + + if err := p.hibernate(t.Context(), pod, v); err != nil { + t.Fatalf("hibernate: %v", err) + } + if got := execArgvs(rt); len(got) != 1 || got[0] != "cmd /c ipconfig /release" { + t.Errorf("exec calls = %v, want exactly [cmd /c ipconfig /release]", got) + } + if len(rt.netResizeCalls) != 1 || rt.netResizeCalls[0].target != 0 { + t.Errorf("NetResize calls = %#v, want one drop-to-0", rt.netResizeCalls) + } +} + +func TestHibernateReleaseFailureDoesNotBlock(t *testing.T) { + rt := &fakeRuntime{execErr: errors.New("agent down")} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + if err := p.hibernate(t.Context(), pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err != nil { + t.Fatalf("hibernate must proceed past a failed release: %v", err) + } + if rt.removedID != "vmid-1" { + t.Errorf("hibernate did not complete: removedID=%q", rt.removedID) + } +} + +func TestHibernateSkipsReleaseOnNonDropNIC(t *testing.T) { + rt := &fakeRuntime{} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSLinux), + }) + if err := p.hibernate(t.Context(), pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err != nil { + t.Fatalf("hibernate: %v", err) + } + if len(rt.execCalls) != 0 { + t.Errorf("linux hibernate must not exec ipconfig, got %v", execArgvs(rt)) + } +} + +func TestHibernateRollbackRenews(t *testing.T) { + rt := &fakeRuntime{snapshotSaveErr: errors.New("save boom")} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + p.Clientset = fake.NewSimpleClientset() + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + if err := p.hibernate(t.Context(), pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err == nil { + t.Fatal("hibernate should fail on snapshot save error") + } + got := execArgvs(rt) + if len(got) != 2 || got[0] != "cmd /c ipconfig /release" || got[1] != "cmd /c ipconfig /renew" { + t.Errorf("exec calls = %v, want [release, renew-on-rollback]", got) + } + if len(rt.netResizeCalls) != 2 || rt.netResizeCalls[1].target != 1 { + t.Errorf("NetResize calls = %#v, want drop then rollback re-add", rt.netResizeCalls) + } +} + +func TestHibernateRenewsWhenNICDropFails(t *testing.T) { + rt := &fakeRuntime{netResizeErr: errors.New("resize boom")} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + p.Clientset = fake.NewSimpleClientset() + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + if err := p.hibernate(t.Context(), pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err == nil { + t.Fatal("hibernate should fail when the NIC drop fails") + } + got := execArgvs(rt) + if len(got) != 2 || got[0] != "cmd /c ipconfig /release" || got[1] != "cmd /c ipconfig /renew" { + t.Errorf("exec calls = %v, want [release, renew] around the failed drop", got) + } + if len(rt.netResizeCalls) != 1 { + t.Errorf("NetResize calls = %#v, want only the failed drop (NIC never dropped, no re-add)", rt.netResizeCalls) + } +} + +func TestHibernateRenewsEvenWhenReleaseVerdictUnknown(t *testing.T) { + rt := &fakeRuntime{execErr: errors.New("vsock timeout"), netResizeErr: errors.New("resize boom")} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + p.Clientset = fake.NewSimpleClientset() + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + if err := p.hibernate(t.Context(), pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err == nil { + t.Fatal("hibernate should fail when the NIC drop fails") + } + got := execArgvs(rt) + if len(got) != 2 || got[1] != "cmd /c ipconfig /renew" { + t.Errorf("exec calls = %v, want a renew attempt even though the release verdict was an error", got) + } +} + +func TestHibernateRenewSurvivesCancelledContext(t *testing.T) { + rt := &fakeRuntime{} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + p.Clientset = fake.NewSimpleClientset() + + ctx, cancel := context.WithCancel(t.Context()) + rt.onExec = func() { cancel() } // the request dies while the release is in flight + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + if err := p.hibernate(ctx, pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err == nil { + t.Fatal("hibernate should fail when the NIC drop fails") + } + got := execArgvs(rt) + if len(got) != 2 || got[1] != "cmd /c ipconfig /renew" { + t.Errorf("exec calls = %v, want the renew to outlive the cancelled request", got) + } +} + +func TestHibernateRollbackSurvivesCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + rt := &fakeRuntime{snapshotSaveErr: errors.New("save boom"), snapshotSaveHook: cancel} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + p.Clientset = fake.NewSimpleClientset() + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSWindows), + }) + if err := p.hibernate(ctx, pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}); err == nil { + t.Fatal("hibernate should fail on snapshot save error") + } + if len(rt.netResizeCalls) != 2 || rt.netResizeCalls[1].target != 1 { + t.Errorf("NetResize calls = %#v, want the NIC re-add to outlive the cancelled request", rt.netResizeCalls) + } + if got := execArgvs(rt); len(got) != 2 || got[1] != "cmd /c ipconfig /renew" { + t.Errorf("exec calls = %v, want the rollback renew to share the detached lifetime", got) + } +} + +func TestResolveVMIPRefusesWriteToSwappedVM(t *testing.T) { + p := newTestProvider(t) + p.LeaseParser = newLeaseParser(t, "aa:bb:cc:dd:ee:ff", "172.20.0.10") + pod := newPodWithSpec(meta.VMSpec{VMName: "vk-ns-demo-0"}) + p.trackPod(pod, &vm.VM{ID: "vmid-successor", Name: "vk-ns-demo-0"}) + + stale := &vm.VM{ID: "vmid-old", Name: "vk-ns-demo-0", MAC: "aa:bb:cc:dd:ee:ff"} + if ip := p.resolveVMIP("ns", "demo-0", stale); ip != "" { + t.Errorf("stale VM's lease resolved to %q, want refused", ip) + } + if got := p.vmForPod("ns", "demo-0").IP; got != "" { + t.Errorf("successor VM polluted with IP %q", got) + } +} + +func TestWaitForFreshIPBailsWhenVMSwapped(t *testing.T) { + p, pod, _ := newDropNICWakeFixture(t, 500*time.Millisecond, 10*time.Millisecond) + rt := p.Runtime.(*fakeRuntime) + p.trackPod(pod, &vm.VM{ID: "vmid-successor", Name: "vk-ns-demo-0", IP: "10.0.0.9"}) + + if p.waitForFreshIP(t.Context(), pod, "vmid-wake") { + t.Fatal("waiter armed for a replaced VM must fail, not adopt the successor") + } + if len(rt.execCalls) != 0 { + t.Errorf("waiter must not exec into the successor VM, got %v", execArgvs(rt)) + } +} + +func TestWaitForFreshIPRenewNudgeWhenLeaseMissing(t *testing.T) { + p, pod, _ := newDropNICWakeFixture(t, 200*time.Millisecond, 10*time.Millisecond) + rt := p.Runtime.(*fakeRuntime) + p.wakeRenewNudgeDelay = 30 * time.Millisecond + + if p.waitForFreshIP(t.Context(), pod, "vmid-wake") { + t.Fatal("no IP should time out") + } + got := execArgvs(rt) + if len(got) != 1 || got[0] != "cmd /c ipconfig /renew" { + t.Errorf("exec calls = %v, want exactly one renew nudge", got) + } +} + +func TestWaitForFreshIPNoRenewWhenIPPresent(t *testing.T) { + p, pod, v := newDropNICWakeFixture(t, 200*time.Millisecond, 10*time.Millisecond) + rt := p.Runtime.(*fakeRuntime) + p.wakeRenewNudgeDelay = time.Nanosecond // even an instant nudge window must not fire + v.IP = "10.0.0.9" + + if !p.waitForFreshIP(t.Context(), pod, "vmid-wake") { + t.Fatal("IP present should return true") + } + if len(rt.execCalls) != 0 { + t.Errorf("guest already holds a lease; renew must not fire, got %v", execArgvs(rt)) + } +} + +func TestWaitForFreshIPNoRenewForLinux(t *testing.T) { + rt := &fakeRuntime{} + p := newTestProvider(t) + p.Runtime = rt + p.Probes = probes.NewManager(t.Context()) + p.wakeFreshIPBudget = 120 * time.Millisecond + p.wakeFreshIPInterval = 10 * time.Millisecond + p.wakeRenewNudgeDelay = 20 * time.Millisecond + + pod := newPodWithSpec(meta.VMSpec{ + VMName: "vk-ns-demo-0", + Backend: string(cocoonv1.BackendCloudHypervisor), + OS: string(cocoonv1.OSLinux), + }) + p.trackPod(pod, &vm.VM{ID: "vmid-1", Name: "vk-ns-demo-0"}) + + if p.waitForFreshIP(t.Context(), pod, "vmid-1") { + t.Fatal("no IP should time out") + } + if len(rt.execCalls) != 0 { + t.Errorf("linux guest must not get ipconfig, got %v", execArgvs(rt)) + } +} + +func TestWaitForFreshIPLeaseLandingDuringNudgeWins(t *testing.T) { + // A lease landing while the renew exec blocks past the deadline must win over timeout. + p, pod, _ := newDropNICWakeFixture(t, 100*time.Millisecond, 10*time.Millisecond) + rt := p.Runtime.(*fakeRuntime) + p.wakeRenewNudgeDelay = 20 * time.Millisecond + + rt.onExec = func() { + time.Sleep(150 * time.Millisecond) // block past the 100ms deadline + p.setVMIP("ns", "demo-0", "vmid-wake", "10.0.0.9") + } + + if !p.waitForFreshIP(t.Context(), pod, "vmid-wake") { + t.Fatal("lease landed during the nudge exec; verdict must be success") + } +}