From 65639c481fc4a72a29a444a75afb44104e30333c Mon Sep 17 00:00:00 2001 From: Vadim Date: Wed, 5 Aug 2026 16:24:31 +0300 Subject: [PATCH 1/3] add dry-run mode --- api/shim/v1/config.go | 18 +++++++ api/shim/v1/config_test.go | 25 ++++++++++ api/shim/v1/shim.pb.go | 13 ++++- api/shim/v1/shim.proto | 1 + config/examples/nginx.yaml | 3 +- docs/configuration/README.md | 17 +++++++ e2e/e2e_test.go | 7 +++ e2e/setup_test.go | 6 +++ manager/event_creator.go | 14 ++++-- manager/event_creator_test.go | 23 +++++++++ shim/checkpoint.go | 4 ++ shim/container.go | 17 +++++++ shim/dryrun.go | 92 +++++++++++++++++++++++++++++++++++ shim/task/service_zeropod.go | 3 ++ 14 files changed, 235 insertions(+), 8 deletions(-) create mode 100644 shim/dryrun.go diff --git a/api/shim/v1/config.go b/api/shim/v1/config.go index 8dd2822f..b5b36aad 100644 --- a/api/shim/v1/config.go +++ b/api/shim/v1/config.go @@ -25,6 +25,7 @@ const ( ContainerNamesAnnotationKey = "zeropod.ctrox.dev/container-names" ScaleDownDurationAnnotationKey = "zeropod.ctrox.dev/scaledown-duration" DisableCheckpoiningAnnotationKey = "zeropod.ctrox.dev/disable-checkpointing" + DryRunAnnotationKey = "zeropod.ctrox.dev/dry-run" PreDumpAnnotationKey = "zeropod.ctrox.dev/pre-dump" MigrateAnnotationKey = "zeropod.ctrox.dev/migrate" LiveMigrateAnnotationKey = "zeropod.ctrox.dev/live-migrate" @@ -60,6 +61,7 @@ var ContainerdAnnotations = []string{ ContainerNamesAnnotationKey, ScaleDownDurationAnnotationKey, DisableCheckpoiningAnnotationKey, + DryRunAnnotationKey, PreDumpAnnotationKey, MigrateAnnotationKey, LiveMigrateAnnotationKey, @@ -76,6 +78,7 @@ type AnnotationConfig struct { Ports []uint16 ScaleDownDuration time.Duration DisableCheckpointing bool + DryRun bool PreDump bool Migrate []string LiveMigrate string @@ -175,6 +178,20 @@ func NewConfig(ctx context.Context, spec *specs.Spec) (*Config, error) { migrate = strings.Split(migrateValue, containersDelim) } + dryRunValue := spec.Annotations[DryRunAnnotationKey] + dryRun := false + if dryRunValue != "" { + dryRun, err = strconv.ParseBool(dryRunValue) + if err != nil { + return nil, err + } + } + + liveMigrateValue := spec.Annotations[LiveMigrateAnnotationKey] + if dryRun && (slices.Contains(migrate, containerName) || (liveMigrateValue != "" && liveMigrateValue == containerName)) { + return nil, fmt.Errorf("dry-run (%s) cannot be combined with migrate/live-migrate for container %q", DryRunAnnotationKey, containerName) + } + ns, ok := namespaces.Namespace(ctx) if !ok { ns = defaultContainerdNS @@ -243,6 +260,7 @@ func NewConfig(ctx context.Context, spec *specs.Spec) (*Config, error) { Ports: containerPorts, ScaleDownDuration: dur, DisableCheckpointing: disableCheckpointing, + DryRun: dryRun, PreDump: preDump, Migrate: migrate, LiveMigrate: spec.Annotations[LiveMigrateAnnotationKey], diff --git a/api/shim/v1/config_test.go b/api/shim/v1/config_test.go index 572232bd..8250cb35 100644 --- a/api/shim/v1/config_test.go +++ b/api/shim/v1/config_test.go @@ -60,6 +60,20 @@ func TestNewConfig(t *testing.T) { assert.False(t, cfg.DisableCheckpointing) }, }, + "dry run": { + annotations: map[string]string{ + DryRunAnnotationKey: "true", + }, + assertCfg: func(t *testing.T, cfg *Config) { + assert.True(t, cfg.DryRun) + }, + }, + "dry run default false": { + annotations: map[string]string{}, + assertCfg: func(t *testing.T, cfg *Config) { + assert.False(t, cfg.DryRun) + }, + }, "predump": { annotations: map[string]string{ PreDumpAnnotationKey: "true", @@ -119,3 +133,14 @@ func TestNewConfig(t *testing.T) { }) } } + +func TestNewConfigDryRunMigrateConflict(t *testing.T) { + _, err := NewConfig(context.Background(), &specs.Spec{ + Annotations: map[string]string{ + CRIContainerNameAnnotation: "app", + DryRunAnnotationKey: "true", + MigrateAnnotationKey: "app", + }, + }) + require.Error(t, err) +} diff --git a/api/shim/v1/shim.pb.go b/api/shim/v1/shim.pb.go index 14e9e8e9..b4078e15 100644 --- a/api/shim/v1/shim.pb.go +++ b/api/shim/v1/shim.pb.go @@ -265,6 +265,7 @@ type ContainerStatus struct { EventTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` EventDuration *durationpb.Duration `protobuf:"bytes,7,opt,name=event_duration,json=eventDuration,proto3" json:"event_duration,omitempty"` EventLog string `protobuf:"bytes,8,opt,name=event_log,json=eventLog,proto3" json:"event_log,omitempty"` + DryRun bool `protobuf:"varint,9,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -355,6 +356,13 @@ func (x *ContainerStatus) GetEventLog() string { return "" } +func (x *ContainerStatus) GetDryRun() bool { + if x != nil { + return x.DryRun + } + return false +} + type ContainerMetrics struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -484,7 +492,7 @@ const file_shim_proto_rawDesc = "" + "\x0fMetricsResponse\x12;\n" + "\ametrics\x18\x01 \x03(\v2!.zeropod.shim.v1.ContainerMetricsR\ametrics\"\"\n" + "\x10ContainerRequest\x12\x0e\n" + - "\x02id\x18\x01 \x01(\tR\x02id\"\xc6\x02\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"\xdf\x02\n" + "\x0fContainerStatus\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x12\x19\n" + @@ -494,7 +502,8 @@ const file_shim_proto_rawDesc = "" + "\n" + "event_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\teventTime\x12@\n" + "\x0eevent_duration\x18\a \x01(\v2\x19.google.protobuf.DurationR\reventDuration\x12\x1b\n" + - "\tevent_log\x18\b \x01(\tR\beventLog\"\xf6\x03\n" + + "\tevent_log\x18\b \x01(\tR\beventLog\x12\x17\n" + + "\adry_run\x18\t \x01(\bR\x06dryRun\"\xf6\x03\n" + "\x10ContainerMetrics\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" + "\bpod_name\x18\x02 \x01(\tR\apodName\x12#\n" + diff --git a/api/shim/v1/shim.proto b/api/shim/v1/shim.proto index 84e08a8a..220a8498 100644 --- a/api/shim/v1/shim.proto +++ b/api/shim/v1/shim.proto @@ -46,6 +46,7 @@ message ContainerStatus { google.protobuf.Timestamp event_time = 6; google.protobuf.Duration event_duration = 7; string event_log = 8; + bool dry_run = 9; } message ContainerMetrics { diff --git a/config/examples/nginx.yaml b/config/examples/nginx.yaml index 21e223fa..df0abd0f 100644 --- a/config/examples/nginx.yaml +++ b/config/examples/nginx.yaml @@ -12,7 +12,8 @@ spec: labels: app: nginx annotations: - zeropod.ctrox.dev/scaledown-duration: 10s + zeropod.ctrox.dev/scaledown-duration: 15s + zeropod.ctrox.dev/dry-run: "true" spec: runtimeClassName: zeropod containers: diff --git a/docs/configuration/README.md b/docs/configuration/README.md index 597973b1..3681cd13 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -160,3 +160,20 @@ been established. Defaults to `5s` if unset. Features that are marked as experimental might change form in the future or could be removed entirely in future releases depending on the stability and need. + +### Dry Run + +```yaml +zeropod.ctrox.dev/dry-run: "true" +``` + +Runs the scale-down timer exactly as normal (same activity tracking, same +scale-down duration logic) but never actually checkpoints or kills the +process and never enables traffic redirection - the container keeps running +and serving traffic uninterrupted. Instead, zeropod logs when it *would have* +scaled down and when it *would have* restored. Use this to evaluate zeropod's +behaviour on a workload risk-free before enabling it for real. Cannot be +combined with `migrate`/`live-migrate`. Note that in-place resource scaling +(`cpu-requests`/`memory-requests`) is also not previewed - since the +container is never actually marked as scaled down, its resource requests are +never touched while dry-run is active. diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 0a74c07e..5065d5fa 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -104,6 +104,13 @@ func TestE2E(t *testing.T) { maxReqDuration: time.Second, expectRunning: true, }, + "pod with dry-run": { + pod: testPod(dryRun(true), defaultScaleDownAfter), + parallelReqs: 1, + sequentialReqs: 1, + maxReqDuration: time.Second, + expectRunning: true, + }, "pod with multiple containers": { pod: testPod(agnContainer("c1", 8080), agnContainer("c2", 8081), defaultScaleDownAfter), svc: testService(8081), diff --git a/e2e/setup_test.go b/e2e/setup_test.go index 0ce2ab86..a1eafacf 100644 --- a/e2e/setup_test.go +++ b/e2e/setup_test.go @@ -387,6 +387,12 @@ func disableCheckpointing(disable bool) podOption { }) } +func dryRun(dryRun bool) podOption { + return annotations(map[string]string{ + shimv1.DryRunAnnotationKey: strconv.FormatBool(dryRun), + }) +} + func scaleDownAfter(dur time.Duration) podOption { return annotations(map[string]string{ shimv1.ScaleDownDurationAnnotationKey: dur.String(), diff --git a/manager/event_creator.go b/manager/event_creator.go index 43c78576..17c022eb 100644 --- a/manager/event_creator.go +++ b/manager/event_creator.go @@ -20,6 +20,7 @@ const ( reasonScaledDown = "Scaled down" reasonCheckpointFailed = "Checkpoint failed" reasonRestoreFailed = "Restore failed" + reasonDryRun = "Dry run" ) type EventCreator struct { @@ -44,24 +45,27 @@ func (ec *EventCreator) Handle(ctx context.Context, status *v1.ContainerStatus, clog.Info("status event") message, reason := "", "" - switch status.Phase { - case v1.ContainerPhase_RUNNING: + switch { + case status.DryRun: + reason = reasonDryRun + message = fmt.Sprintf("Dry-run: container %s %s", status.Name, status.EventLog) + case status.Phase == v1.ContainerPhase_RUNNING: reason = reasonRunning // don't create an event if container was simply started without being restored if status.EventDuration == nil || status.EventDuration.AsDuration() == 0 { return nil } message = fmt.Sprintf("Restored container %s in %s", status.Name, status.EventDuration.AsDuration()) - case v1.ContainerPhase_SCALED_DOWN: + case status.Phase == v1.ContainerPhase_SCALED_DOWN: reason = reasonScaledDown message = fmt.Sprintf("Scaled down container %s", status.Name) if status.EventDuration != nil { message += " in " + status.EventDuration.AsDuration().String() } - case v1.ContainerPhase_CHECKPOINT_FAILED: + case status.Phase == v1.ContainerPhase_CHECKPOINT_FAILED: reason = reasonCheckpointFailed message = fmt.Sprintf("Checkpoint failed for container %s", status.Name) - case v1.ContainerPhase_RESTORE_FAILED: + case status.Phase == v1.ContainerPhase_RESTORE_FAILED: reason = reasonRestoreFailed message = fmt.Sprintf("Restore failed for container %s", status.Name) } diff --git a/manager/event_creator_test.go b/manager/event_creator_test.go index e8b3e9ec..0442d242 100644 --- a/manager/event_creator_test.go +++ b/manager/event_creator_test.go @@ -25,6 +25,8 @@ func TestEventCreator(t *testing.T) { for name, tc := range map[string]struct { statusPhase v1.ContainerPhase statusDuration time.Duration + statusDryRun bool + statusEventLog string containerName string expectedReason string expectedMessage string @@ -58,6 +60,25 @@ func TestEventCreator(t *testing.T) { expectedReason: reasonScaledDown, expectedMessage: "Scaled down container c in 1s", }, + "dry run scaled down": { + containerName: "c", + // dry-run never changes the real phase, unlike a real scale down. + statusPhase: v1.ContainerPhase_RUNNING, + statusDryRun: true, + statusEventLog: "would have scaled down after 10s of inactivity", + expectEventCreated: true, + expectedReason: reasonDryRun, + expectedMessage: "Dry-run: container c would have scaled down after 10s of inactivity", + }, + "dry run restored": { + containerName: "c", + statusPhase: v1.ContainerPhase_RUNNING, + statusDryRun: true, + statusEventLog: "would have restored (got exec)", + expectEventCreated: true, + expectedReason: reasonDryRun, + expectedMessage: "Dry-run: container c would have restored (got exec)", + }, } { t.Run(name, func(t *testing.T) { client := fake.NewClientBuilder().WithScheme(scheme).Build() @@ -71,6 +92,8 @@ func TestEventCreator(t *testing.T) { PodName: pod.Name, PodNamespace: pod.Namespace, Phase: tc.statusPhase, + DryRun: tc.statusDryRun, + EventLog: tc.statusEventLog, } if tc.statusDuration != 0 { status.EventDuration = durationpb.New(tc.statusDuration) diff --git a/shim/checkpoint.go b/shim/checkpoint.go index f2f96fd3..5e6e3afb 100644 --- a/shim/checkpoint.go +++ b/shim/checkpoint.go @@ -20,6 +20,10 @@ import ( ) func (c *Container) scaleDown(ctx context.Context) error { + if c.cfg.DryRun { + return c.dryRunScaleDown(ctx) + } + if c.ScaledDown() { return nil } diff --git a/shim/container.go b/shim/container.go index 3ce79b5a..53fd2d85 100644 --- a/shim/container.go +++ b/shim/container.go @@ -44,6 +44,9 @@ type Container struct { cgroup any logPath string scaledDown bool + dryRunScaledDown bool + dryRunSince time.Time + dryRunPollTimer *time.Timer skipStart bool netNS ns.NetNS scaleDownTimer *time.Timer @@ -234,6 +237,19 @@ func (c *Container) sendFailEvent(phase v1.ContainerPhase, l string) { c.sendEvent(status) } +// sendDryRunEvent sends a status event carrying a simulated dry-run action in +// EventLog, with DryRun set and the real phase left untouched (dry-run never +// changes it), so the manager can create a distinct Kubernetes Event without +// affecting phase-based logic like in-place resource scaling or status +// labels. +func (c *Container) sendDryRunEvent(eventLog string) { + status := c.Status() + status.DryRun = true + status.EventTime = timestamppb.Now() + status.EventLog = eventLog + c.sendEvent(status) +} + func (c *Container) SetSkipStart(skip bool) { c.skipStart = skip } @@ -318,6 +334,7 @@ func (c *Container) DeleteCheckpointedPID(pid int) { func (c *Container) Stop(ctx context.Context) { c.cancelInit() c.CancelScaleDown() + c.cancelDryRunPoll() status := c.Status() status.Phase = v1.ContainerPhase_STOPPING c.sendEvent(status) diff --git a/shim/dryrun.go b/shim/dryrun.go new file mode 100644 index 00000000..eb41f232 --- /dev/null +++ b/shim/dryrun.go @@ -0,0 +1,92 @@ +package shim + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/containerd/log" + "github.com/ctrox/zeropod/activator" +) + +// dryRunPollInterval is how often we recheck activity while in a simulated +// scaled-down state, to detect a would-be restore. +const dryRunPollInterval = time.Second + +// dryRunScaleDown simulates a scale down: it does not checkpoint/kill the +// process and does not enable the eBPF redirect, so the real process keeps +// serving traffic uninterrupted. It logs and starts polling for activity so +// it can log a "would have restored" once traffic resumes. +func (c *Container) dryRunScaleDown(ctx context.Context) error { + if c.dryRunScaledDown { + return nil + } + c.dryRunScaledDown = true + c.dryRunSince = time.Now() + log.G(ctx).Infof("dry-run: would have scaled down container %s after %s of inactivity", c.ID(), c.cfg.ScaleDownDuration) + c.sendDryRunEvent(fmt.Sprintf("would have scaled down after %s of inactivity", c.cfg.ScaleDownDuration)) + c.scheduleDryRunRestoreCheck() + return nil +} + +func (c *Container) scheduleDryRunRestoreCheck() { + if c.dryRunPollTimer == nil { + c.dryRunPollTimer = time.AfterFunc(dryRunPollInterval, c.dryRunRestoreCheck) + return + } + c.dryRunPollTimer.Reset(dryRunPollInterval) +} + +func (c *Container) dryRunRestoreCheck() { + last, err := c.lastActivity() + if err != nil && !errors.Is(err, activator.NoActivityRecordedErr{}) { + log.G(c.context).Warnf("dry-run: unable to get last TCP activity: %s", err) + } + if err == nil && last.After(c.dryRunSince) { + // activity-triggered restore mirrors the real network-triggered + // restore (activator restoreHandler), which reschedules immediately. + c.dryRunRestore(c.context, fmt.Sprintf("last activity %s ago", time.Since(last)), true) + return + } + c.scheduleDryRunRestoreCheck() +} + +// DryRunExec simulates a restore triggered by kubectl exec, mirroring the +// real restore-on-exec behaviour (see task.wrapper.Exec) without touching the +// real process. No-op unless dry-run is currently simulating a scaled down +// state. Like the real exec path, it deliberately does not reschedule the +// scale-down timer itself: that is left to wrapper.Delete, which only +// reschedules once all running execs for the container have ended, so a +// long-running exec session (e.g. an interactive shell) keeps suppressing +// (simulated) scale-down for its entire duration, not just at the moment +// exec was called. +func (c *Container) DryRunExec(ctx context.Context) { + if !c.cfg.DryRun || !c.dryRunScaledDown { + return + } + c.dryRunRestore(ctx, "got exec", false) +} + +// dryRunRestore logs a simulated restore and clears the simulated scaled-down +// state. If reschedule is true, it also resumes the normal scale-down timer +// cycle immediately; otherwise the caller is responsible for rescheduling +// once appropriate. +func (c *Container) dryRunRestore(ctx context.Context, reason string, reschedule bool) { + log.G(ctx).Infof("dry-run: would have restored container %s, %s", c.ID(), reason) + c.sendDryRunEvent(fmt.Sprintf("would have restored (%s)", reason)) + c.dryRunScaledDown = false + c.cancelDryRunPoll() + if reschedule { + c.ScheduleScaleDown() + } +} + +// cancelDryRunPoll stops the dry-run restore-check poller, if running. Must +// be called on container shutdown to avoid leaking the timer goroutine. +func (c *Container) cancelDryRunPoll() { + if c.dryRunPollTimer == nil { + return + } + c.dryRunPollTimer.Stop() +} diff --git a/shim/task/service_zeropod.go b/shim/task/service_zeropod.go index 632e88b1..2660fc5e 100644 --- a/shim/task/service_zeropod.go +++ b/shim/task/service_zeropod.go @@ -275,6 +275,9 @@ func (w *wrapper) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (*emp } } + // no-op unless dry-run is active and currently simulating scaled down + zeropodContainer.DryRunExec(ctx) + return w.service.Exec(ctx, r) } From feaea810c7caa3619b669df38d4b9ae83be4d33b Mon Sep 17 00:00:00 2001 From: Vadim Date: Mon, 10 Aug 2026 15:17:14 +0300 Subject: [PATCH 2/3] add separate pod for dry-run --- config/examples/nginx-dry-run.yaml | 31 ++++++++++++++++++++++++++++++ config/examples/nginx.yaml | 3 +-- 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 config/examples/nginx-dry-run.yaml diff --git a/config/examples/nginx-dry-run.yaml b/config/examples/nginx-dry-run.yaml new file mode 100644 index 00000000..9959d37d --- /dev/null +++ b/config/examples/nginx-dry-run.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-dry-run +spec: + replicas: 1 + selector: + matchLabels: + app: nginx-dry-run + template: + metadata: + labels: + app: nginx-dry-run + annotations: + zeropod.ctrox.dev/scaledown-duration: 15s + zeropod.ctrox.dev/dry-run: "true" + spec: + runtimeClassName: zeropod + containers: + - image: nginx + name: nginx + ports: + - containerPort: 80 + livenessProbe: + periodSeconds: 1 + httpGet: + port: 80 + resources: + requests: + cpu: 100m + memory: 128Mi diff --git a/config/examples/nginx.yaml b/config/examples/nginx.yaml index df0abd0f..21e223fa 100644 --- a/config/examples/nginx.yaml +++ b/config/examples/nginx.yaml @@ -12,8 +12,7 @@ spec: labels: app: nginx annotations: - zeropod.ctrox.dev/scaledown-duration: 15s - zeropod.ctrox.dev/dry-run: "true" + zeropod.ctrox.dev/scaledown-duration: 10s spec: runtimeClassName: zeropod containers: From 8fe20083081cf46cf65d4d518a570106acdd9bc9 Mon Sep 17 00:00:00 2001 From: Vadim Date: Tue, 11 Aug 2026 08:46:49 +0300 Subject: [PATCH 3/3] add metrics --- api/shim/v1/shim.pb.go | 22 +++++++++++++-- api/shim/v1/shim.proto | 2 ++ docs/configuration/README.md | 14 ++++++---- docs/metrics.md | 6 ++++ manager/metrics_collector.go | 54 +++++++++++++++++++++++++----------- shim/container.go | 2 ++ shim/dryrun.go | 2 ++ 7 files changed, 78 insertions(+), 24 deletions(-) diff --git a/api/shim/v1/shim.pb.go b/api/shim/v1/shim.pb.go index b4078e15..3aef04e2 100644 --- a/api/shim/v1/shim.pb.go +++ b/api/shim/v1/shim.pb.go @@ -375,6 +375,8 @@ type ContainerMetrics struct { Running bool `protobuf:"varint,8,opt,name=running,proto3" json:"running,omitempty"` CheckpointErrors int64 `protobuf:"varint,9,opt,name=checkpoint_errors,json=checkpointErrors,proto3" json:"checkpoint_errors,omitempty"` RestoreErrors int64 `protobuf:"varint,10,opt,name=restore_errors,json=restoreErrors,proto3" json:"restore_errors,omitempty"` + DryRunScaleDowns int64 `protobuf:"varint,11,opt,name=dry_run_scale_downs,json=dryRunScaleDowns,proto3" json:"dry_run_scale_downs,omitempty"` + DryRunWouldRestores int64 `protobuf:"varint,12,opt,name=dry_run_would_restores,json=dryRunWouldRestores,proto3" json:"dry_run_would_restores,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -479,6 +481,20 @@ func (x *ContainerMetrics) GetRestoreErrors() int64 { return 0 } +func (x *ContainerMetrics) GetDryRunScaleDowns() int64 { + if x != nil { + return x.DryRunScaleDowns + } + return 0 +} + +func (x *ContainerMetrics) GetDryRunWouldRestores() int64 { + if x != nil { + return x.DryRunWouldRestores + } + return 0 +} + var File_shim_proto protoreflect.FileDescriptor const file_shim_proto_rawDesc = "" + @@ -503,7 +519,7 @@ const file_shim_proto_rawDesc = "" + "event_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\teventTime\x12@\n" + "\x0eevent_duration\x18\a \x01(\v2\x19.google.protobuf.DurationR\reventDuration\x12\x1b\n" + "\tevent_log\x18\b \x01(\tR\beventLog\x12\x17\n" + - "\adry_run\x18\t \x01(\bR\x06dryRun\"\xf6\x03\n" + + "\adry_run\x18\t \x01(\bR\x06dryRun\"\xda\x04\n" + "\x10ContainerMetrics\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" + "\bpod_name\x18\x02 \x01(\tR\apodName\x12#\n" + @@ -515,7 +531,9 @@ const file_shim_proto_rawDesc = "" + "\arunning\x18\b \x01(\bR\arunning\x12+\n" + "\x11checkpoint_errors\x18\t \x01(\x03R\x10checkpointErrors\x12%\n" + "\x0erestore_errors\x18\n" + - " \x01(\x03R\rrestoreErrors*g\n" + + " \x01(\x03R\rrestoreErrors\x12-\n" + + "\x13dry_run_scale_downs\x18\v \x01(\x03R\x10dryRunScaleDowns\x123\n" + + "\x16dry_run_would_restores\x18\f \x01(\x03R\x13dryRunWouldRestores*g\n" + "\x0eContainerPhase\x12\x0f\n" + "\vSCALED_DOWN\x10\x00\x12\v\n" + "\aRUNNING\x10\x01\x12\f\n" + diff --git a/api/shim/v1/shim.proto b/api/shim/v1/shim.proto index 220a8498..61a1b210 100644 --- a/api/shim/v1/shim.proto +++ b/api/shim/v1/shim.proto @@ -60,4 +60,6 @@ message ContainerMetrics { bool running = 8; int64 checkpoint_errors = 9; int64 restore_errors = 10; + int64 dry_run_scale_downs = 11; + int64 dry_run_would_restores = 12; } diff --git a/docs/configuration/README.md b/docs/configuration/README.md index 3681cd13..311535aa 100644 --- a/docs/configuration/README.md +++ b/docs/configuration/README.md @@ -171,9 +171,11 @@ Runs the scale-down timer exactly as normal (same activity tracking, same scale-down duration logic) but never actually checkpoints or kills the process and never enables traffic redirection - the container keeps running and serving traffic uninterrupted. Instead, zeropod logs when it *would have* -scaled down and when it *would have* restored. Use this to evaluate zeropod's -behaviour on a workload risk-free before enabling it for real. Cannot be -combined with `migrate`/`live-migrate`. Note that in-place resource scaling -(`cpu-requests`/`memory-requests`) is also not previewed - since the -container is never actually marked as scaled down, its resource requests are -never touched while dry-run is active. +scaled down and when it *would have* restored (also emitted as a Kubernetes +event on the pod, and counted in the `zeropod_dry_run_scale_downs_total`/ +`zeropod_dry_run_would_restores_total` metrics - see [metrics](../metrics.md)). +Use this to evaluate zeropod's behaviour on a workload risk-free before +enabling it for real. Cannot be combined with `migrate`/`live-migrate`. Note +that in-place resource scaling (`cpu-requests`/`memory-requests`) is also not +previewed - since the container is never actually marked as scaled down, its +resource requests are never touched while dry-run is active. diff --git a/docs/metrics.md b/docs/metrics.md index 29b7d702..368480ac 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -30,4 +30,10 @@ zeropod_checkpoint_errors_total{container="nginx",namespace="default",pod="nginx # HELP zeropod_restore_errors_total Total number of restore errors. # TYPE zeropod_restore_errors_total counter zeropod_restore_errors_total{container="nginx",namespace="default",pod="nginx"} 0 +# HELP zeropod_dry_run_scale_downs_total Total number of simulated dry-run scale downs. +# TYPE zeropod_dry_run_scale_downs_total counter +zeropod_dry_run_scale_downs_total{container="nginx",namespace="default",pod="nginx"} 0 +# HELP zeropod_dry_run_would_restores_total Total number of simulated dry-run restores. +# TYPE zeropod_dry_run_would_restores_total counter +zeropod_dry_run_would_restores_total{container="nginx",namespace="default",pod="nginx"} 0 ``` diff --git a/manager/metrics_collector.go b/manager/metrics_collector.go index ab13e630..5a2eb8c4 100644 --- a/manager/metrics_collector.go +++ b/manager/metrics_collector.go @@ -17,14 +17,16 @@ const ( LabelPodName = "pod" LabelPodNamespace = "namespace" - MetricsNamespace = "zeropod" - MetricCheckpointDuration = "checkpoint_duration_seconds" - MetricRestoreDuration = "restore_duration_seconds" - MetricLastCheckpointTime = "last_checkpoint_time" - MetricLastRestoreTime = "last_restore_time" - MetricRunning = "running" - MetricCheckpointErrorsTotal = "checkpoint_errors_total" - MetricRestoreErrorsTotal = "restore_errors_total" + MetricsNamespace = "zeropod" + MetricCheckpointDuration = "checkpoint_duration_seconds" + MetricRestoreDuration = "restore_duration_seconds" + MetricLastCheckpointTime = "last_checkpoint_time" + MetricLastRestoreTime = "last_restore_time" + MetricRunning = "running" + MetricCheckpointErrorsTotal = "checkpoint_errors_total" + MetricRestoreErrorsTotal = "restore_errors_total" + MetricDryRunScaleDownsTotal = "dry_run_scale_downs_total" + MetricDryRunWouldRestoresTotal = "dry_run_would_restores_total" ) var ( @@ -37,14 +39,16 @@ var ( ) type Collector struct { - log *slog.Logger - checkpointDuration *prometheus.HistogramVec - restoreDuration *prometheus.HistogramVec - lastCheckpointTime *prometheus.GaugeVec - lastRestoreTime *prometheus.GaugeVec - running *prometheus.GaugeVec - checkpointErrors *prometheus.CounterVec - restoreErrors *prometheus.CounterVec + log *slog.Logger + checkpointDuration *prometheus.HistogramVec + restoreDuration *prometheus.HistogramVec + lastCheckpointTime *prometheus.GaugeVec + lastRestoreTime *prometheus.GaugeVec + running *prometheus.GaugeVec + checkpointErrors *prometheus.CounterVec + restoreErrors *prometheus.CounterVec + dryRunScaleDowns *prometheus.CounterVec + dryRunWouldRestores *prometheus.CounterVec } func NewCollector(log *slog.Logger) *Collector { @@ -93,6 +97,18 @@ func NewCollector(log *slog.Logger) *Collector { Name: MetricRestoreErrorsTotal, Help: "Total number of restore errors.", }, commonLabels), + + dryRunScaleDowns: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: MetricDryRunScaleDownsTotal, + Help: "Total number of simulated dry-run scale downs.", + }, commonLabels), + + dryRunWouldRestores: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: MetricDryRunWouldRestoresTotal, + Help: "Total number of simulated dry-run restores.", + }, commonLabels), } } @@ -136,6 +152,8 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { } c.checkpointErrors.With(l).Add(float64(metrics.CheckpointErrors)) c.restoreErrors.With(l).Add(float64(metrics.RestoreErrors)) + c.dryRunScaleDowns.With(l).Add(float64(metrics.DryRunScaleDowns)) + c.dryRunWouldRestores.With(l).Add(float64(metrics.DryRunWouldRestores)) } } c.running.Collect(ch) @@ -145,6 +163,8 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { c.lastRestoreTime.Collect(ch) c.checkpointErrors.Collect(ch) c.restoreErrors.Collect(ch) + c.dryRunScaleDowns.Collect(ch) + c.dryRunWouldRestores.Collect(ch) } func (c *Collector) Describe(ch chan<- *prometheus.Desc) { @@ -186,4 +206,6 @@ func (c *Collector) deleteMetrics(status *v1.ContainerStatus) { c.lastRestoreTime.Delete(l) c.checkpointErrors.Delete(l) c.restoreErrors.Delete(l) + c.dryRunScaleDowns.Delete(l) + c.dryRunWouldRestores.Delete(l) } diff --git a/shim/container.go b/shim/container.go index 53fd2d85..1fbe433b 100644 --- a/shim/container.go +++ b/shim/container.go @@ -531,4 +531,6 @@ func (c *Container) clearMetrics() { c.metrics.LastRestoreDuration = nil c.metrics.CheckpointErrors = 0 c.metrics.RestoreErrors = 0 + c.metrics.DryRunScaleDowns = 0 + c.metrics.DryRunWouldRestores = 0 } diff --git a/shim/dryrun.go b/shim/dryrun.go index eb41f232..50a222cb 100644 --- a/shim/dryrun.go +++ b/shim/dryrun.go @@ -24,6 +24,7 @@ func (c *Container) dryRunScaleDown(ctx context.Context) error { } c.dryRunScaledDown = true c.dryRunSince = time.Now() + c.metrics.DryRunScaleDowns += 1 log.G(ctx).Infof("dry-run: would have scaled down container %s after %s of inactivity", c.ID(), c.cfg.ScaleDownDuration) c.sendDryRunEvent(fmt.Sprintf("would have scaled down after %s of inactivity", c.cfg.ScaleDownDuration)) c.scheduleDryRunRestoreCheck() @@ -73,6 +74,7 @@ func (c *Container) DryRunExec(ctx context.Context) { // cycle immediately; otherwise the caller is responsible for rescheduling // once appropriate. func (c *Container) dryRunRestore(ctx context.Context, reason string, reschedule bool) { + c.metrics.DryRunWouldRestores += 1 log.G(ctx).Infof("dry-run: would have restored container %s, %s", c.ID(), reason) c.sendDryRunEvent(fmt.Sprintf("would have restored (%s)", reason)) c.dryRunScaledDown = false