fix: say so when an upgraded cluster is serving an older CRD

.github/workflows/ci.yaml

@@ -100,6 +100,12 @@ jobs:
       - name: CRD copy in the chart matches config/
         run: diff -u config/crd/bases/cells.kubeswift.io_gpucellpools.yaml charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml
 
+      # The manager embeds the same CRD to detect a cluster serving an older schema.
+      # A stale embed would make it compare against the wrong baseline and report
+      # either nothing or a phantom drift.
+      - name: CRD copy embedded in the manager matches config/
+        run: diff -u config/crd/bases/cells.kubeswift.io_gpucellpools.yaml internal/crdcheck/crd/cells.kubeswift.io_gpucellpools.yaml
+
   image:
     name: Image builds
     runs-on: ubuntu-latest

.github/workflows/release.yaml

@@ -50,6 +50,8 @@ jobs:
         with:
           context: .
           push: true
+          build-args: |
+            VERSION=${{ github.ref_name }}
           platforms: linux/amd64,linux/arm64
           tags: |
             ${{ env.IMAGE }}:${{ github.ref_name }}

Dockerfile

@@ -14,8 +14,14 @@ COPY api/ api/
 COPY internal/ internal/
 
 # CGO off and a static binary, so the image below needs no libc at all.
+# VERSION is stamped into the binary so the CRD-drift report can print a fix
+# command pointing at THIS release's manifest rather than at main. Unset in a local
+# build, which is why the default in crdcheck is "main".
+ARG VERSION=main
 RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \
-    go build -a -trimpath -ldflags="-s -w" -o manager ./cmd/manager
+    go build -a -trimpath \
+      -ldflags="-s -w -X github.com/kubeswift-io/gpucellpool/internal/crdcheck.Version=${VERSION}" \
+      -o manager ./cmd/manager
 
 # Distroless static, non-root. This operator reconciles two API servers and
 # nothing else: no shell, no package manager, no writable filesystem, and — unlike

Makefile

@@ -40,6 +40,11 @@ manifests: controller-gen ## Generate CRDs and RBAC into config/, and sync the c
 	# kubebuilder markers and the chart's hand-written copy drifted independently —
 	# and the chart silently lacked the Cluster API rules it needed.
 	sed -n '/^rules:/,$$p' config/rbac/role.yaml | tail -n +2 > charts/gpucellpool/rules.yaml
+	# And a third copy, embedded INTO the binary: it is what lets the manager notice
+	# at startup that the cluster is serving an older schema than it was built
+	# against — the failure mode is otherwise silent, because the apiserver just
+	# drops the fields it does not know.
+	cp config/crd/bases/*.yaml internal/crdcheck/crd/
 	$(MAKE) dashboards-sync
 
 .PHONY: dashboards-sync

charts/gpucellpool/rules.yaml

@@ -16,6 +16,12 @@
   verbs:
   - create
   - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
 - apiGroups:
   - bootstrap.cluster.x-k8s.io
   resources:

charts/gpucellpool/templates/NOTES.txt

@@ -25,6 +25,18 @@ Watch both layers:
 
 A cell only becomes Ready when the VM runs, its Node registers Ready, AND HAMi
 advertises the GPU. If it sits in AwaitingGPUCapacity, the message names why.
+{{- if .Release.IsUpgrade }}
+
+APPLY THE CRD. `helm upgrade` never updates a chart's crds/, and the apiserver
+then SILENTLY DROPS any field the older schema does not know -- a spec you write
+is accepted and ignored. Run:
+
+  kubectl apply -f https://raw.githubusercontent.com/kubeswift-io/gpucellpool/{{ printf "v%s" .Chart.AppVersion }}/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml
+
+The manager checks this at startup and logs exactly which fields are missing, so
+`kubectl -n {{ .Release.Namespace }} logs deploy/{{ include "gpucellpool.fullname" . }} | head` will tell you
+whether you needed it.
+{{- end }}
 {{- if not .Values.webhook.enabled }}
 
 WARNING: the validating webhook is disabled. The guestTemplate denylist is a

charts/gpucellpool/values.yaml

@@ -69,10 +69,11 @@ monitoring:
     enabled: true
     additionalLabels: {}
 
-# Install the CRDs. Note that `helm upgrade` never updates files in crds/, so a
-# CRD change needs `kubectl apply -f charts/gpucellpool/crds/` — the same caveat
-# KubeSwift has.
-installCRDs: true
+# There is deliberately no installCRDs toggle. The CRD ships in crds/, which Helm
+# installs unconditionally and never updates, so a flag claiming to control it
+# would be a no-op — as one here silently was. Upgrades therefore need the CRD
+# applied by hand; NOTES.txt prints the command, docs/upgrading.md explains why,
+# and the manager logs the specific fields being dropped if it is skipped.
 
 podAnnotations: {}
 podLabels: {}

cmd/manager/main.go

@@ -7,21 +7,26 @@
 package main
 
 import (
+	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
 	"time"
 
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	cellsv1alpha1 "github.com/kubeswift-io/gpucellpool/api/v1alpha1"
 	"github.com/kubeswift-io/gpucellpool/internal/controller"
+	"github.com/kubeswift-io/gpucellpool/internal/crdcheck"
 	webhookv1alpha1 "github.com/kubeswift-io/gpucellpool/internal/webhook/v1alpha1"
 	"github.com/kubeswift-io/gpucellpool/internal/workload"
 )
@@ -99,9 +104,53 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Say so if the cluster is serving an older CRD than this binary was built
+	// against. `helm upgrade` never updates a chart's crds/, and the apiserver then
+	// silently drops every field the old schema lacks — the operator writes them,
+	// they vanish, and nothing reports a problem. Not fatal: the rest of the
+	// operator still works, and refusing to start would be a worse answer than
+	// naming the gap.
+	if err := reportCRDDrift(context.Background(), mgr.GetConfig()); err != nil {
+		setupLog.Info("could not compare the served CRD schema", "error", err.Error())
+	}
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}
 }
+
+func reportCRDDrift(ctx context.Context, cfg *rest.Config) error {
+	cs, err := apiextensionsclient.NewForConfig(cfg)
+	if err != nil {
+		return err
+	}
+	results, err := crdcheck.Verify(ctx, cs.ApiextensionsV1().CustomResourceDefinitions())
+	if err != nil {
+		return err
+	}
+	for _, r := range results {
+		switch {
+		case !r.Checked:
+			// At default verbosity, not V(1): a check that quietly does not run is
+			// the same silent failure it exists to catch. On an upgraded release the
+			// commonest cause is the operator's ClusterRole predating the
+			// apiextensions grant, which reads as Forbidden here.
+			setupLog.Info("CRD schema NOT compared — a stale schema would go unnoticed",
+				"crd", r.Name, "reason", r.Err.Error(), "needs", "apiextensions.k8s.io/customresourcedefinitions: get")
+		case len(r.Missing) > 0:
+			// A real error value, not nil: a nil error logs a stacktrace pointing at
+			// this function, which reads as a crash rather than the configuration
+			// problem it is.
+			setupLog.Error(fmt.Errorf("%d field(s) dropped by a stale CRD", len(r.Missing)),
+				"the cluster is serving an OLDER CRD than this operator was built against; "+
+					"the apiserver will SILENTLY DROP the fields below, so features that depend on them "+
+					"will appear to be accepted and then do nothing. helm upgrade does not update CRDs — apply it yourself",
+				"crd", r.Name, "missingFields", r.Missing, "fix", crdcheck.FixCommand(r.File))
+		default:
+			setupLog.Info("CRD schema matches this build", "crd", r.Name)
+		}
+	}
+	return nil
+}

config/rbac/role.yaml

@@ -22,6 +22,12 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
 - apiGroups:
   - bootstrap.cluster.x-k8s.io
   resources:

docs/README.md

@@ -15,6 +15,7 @@ Start here if you are installing or operating a pool.
 | [security](security.md) | why creating a pool is node-root-equivalent authority, the webhook, the two RBAC scopes |
 | [cell-image](cell-image.md) | building and publishing a cell image with `hack/build-cell-image.sh` |
 | [clusterapi-cells](clusterapi-cells.md) | `provisioner: ClusterAPI` — cells as Cluster API Machines |
+| [upgrading](upgrading.md) | the CRD step `helm upgrade` will not do for you, and what it silently breaks if skipped |
 | [limitations](limitations.md) | what is not implemented, not validated on hardware, or deliberately manual |
 | [observability](observability.md) | the twelve `gpucell_*` metrics, the dashboard, and the alert pack |
 | [runbook](runbook.md) | what to check when a pool is not doing what you expect |

docs/upgrading.md

@@ -0,0 +1,86 @@
+# Upgrading
+
+## The one thing that is not automatic: the CRD
+
+`helm upgrade` does **not** update a chart's `crds/`. Helm installs those files
+once and never touches them again. Nothing warns you, and the upgrade succeeds.
+
+What happens next is the problem. The apiserver validates against the schema it is
+serving — the old one — and **silently drops** every field that schema does not
+know. Not rejects: drops. You apply a pool spec, the apiserver stores a version of
+it with your new fields removed, and reports success.
+
+So after every `helm upgrade`, apply the CRD:
+
+```bash
+kubectl apply -f https://raw.githubusercontent.com/kubeswift-io/gpucellpool/vX.Y.Z/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml
+```
+
+Order does not matter much — before or after the upgrade is fine — but it must
+happen. `helm upgrade --install` does not do it, `--force` does not do it, and
+reinstalling the release does not do it either (the CRD already exists).
+
+### You do not have to remember
+
+The manager compares the schema the cluster is serving with the one it was built
+against, at startup, and names the exact fields being dropped:
+
+```
+ERROR  the cluster is serving an OLDER CRD than this operator was built against;
+       the apiserver will SILENTLY DROP the fields below, so features that depend
+       on them will appear to be accepted and then do nothing. helm upgrade does
+       not update CRDs — apply it yourself
+       {"crd": "gpucellpools.cells.kubeswift.io",
+        "missingFields": ["v1alpha1.spec.updatePolicy"],
+        "fix": "kubectl apply -f https://.../cells.kubeswift.io_gpucellpools.yaml"}
+```
+
+Check it after any upgrade:
+
+```bash
+kubectl -n gpucellpool-system logs deploy/gpucellpool | grep -i 'CRD schema'
+```
+
+`CRD schema matches this build` means there is nothing to do. If the operator's
+credential cannot read CRDs the line says `CRD schema not compared` — the check is
+read-only and optional (`apiextensions.k8s.io/customresourcedefinitions: get`), and
+its absence degrades the check rather than the operator.
+
+### What v0.1.0 → v0.1.1 drops if you skip it
+
+Both fields `updatePolicy` added:
+
+| Field | Consequence of the old schema |
+|---|---|
+| `spec.updatePolicy` | `type: RollingUpdate` is accepted and discarded. The pool reports `Manual` and never replaces a stale cell — the feature looks enabled and does nothing |
+| `status.cells[].templateHash` | the operator's writes vanish, so `Updated` compares against an empty hash. An empty hash counts as current by design, so **drift reports as up-to-date forever** |
+
+## The rest of an upgrade
+
+```bash
+helm upgrade gpucellpool oci://ghcr.io/kubeswift-io/charts/gpucellpool \
+  --version X.Y.Z -n gpucellpool-system \
+  -f <(helm get values gpucellpool -n gpucellpool-system -o yaml)
+```
+
+Two notes on that command:
+
+- Pass the old values through a file rather than using `--reuse-values`, which
+  skips defaulting for value blocks a new chart version added and can leave nil
+  where the templates expect a map.
+- `helm get values -o yaml` output is the values document itself, with no header
+  line to strip.
+
+Running pools are not disturbed by an operator upgrade: cells are separate objects
+in the infrastructure cluster, and the manager rebuilds its view from their live
+state on the first reconcile. There is no state in the manager to migrate.
+
+The webhook's self-signed certificate survives upgrades — the chart reuses the
+existing Secret rather than minting a new one, so the `caBundle` and the serving
+cert cannot drift apart.
+
+## Downgrading
+
+Downgrade the release, and leave the CRD alone. A newer CRD serving an older
+operator is safe: the extra fields are simply unused. Removing fields from a CRD
+that stored objects still carry is not.

go.mod

@@ -5,6 +5,7 @@ go 1.25.0
 require (
 	github.com/prometheus/client_golang v1.23.2
 	k8s.io/api v0.35.4
+	k8s.io/apiextensions-apiserver v0.35.0
 	k8s.io/apimachinery v0.35.4
 	k8s.io/client-go v0.35.4
 	sigs.k8s.io/controller-runtime v0.23.3
@@ -56,7 +57,6 @@ require (
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.35.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
 	k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect

internal/controller/controller.go

@@ -76,6 +76,10 @@ type GPUCellPoolReconciler struct {
 // describe gpucellpool`. The core grant stays for clients that still read there.
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch
+// Read-only, and only its own CRD: the manager compares the served schema with the
+// one it was built against at startup, because helm upgrade never updates a chart's
+// crds/ and the apiserver then drops the new fields without a word (internal/crdcheck).
+// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get
 
 // Reconcile implements the loop in docs/design/gpucellpool-reconciliation.md §4.
 func (r *GPUCellPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {