From 7b367aa87eafd1f2a489ede7f4e3c9771632c29d Mon Sep 17 00:00:00 2001 From: William Rizzo Date: Sat, 8 Aug 2026 16:07:07 +0000 Subject: [PATCH 1/2] fix: five defects found reviewing the repo for a public release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An architecture review of the docs turned up five things that are code or config, not prose. In severity order. **A session-local path in a script we were about to publish.** hack/build-cell-image.sh defaulted OUT to a scratchpad directory on the authoring machine — it would not work for anyone else and should not be in a public repo. Now $PWD/build. The driver comment also now says plainly that the metapackage tracks a BRANCH (570-server resolved to 580.173.02), so "pinned" meant less than it claimed. **The observer RBAC we tell users to apply cannot tear a cell down.** nodes: delete was commented out and labelled a Phase 4 drain feature, but DeleteNode is called from two ALWAYS-ON paths: reaping a Node left by a previous incarnation of a cell, and removing a cell's Node when the cell goes. Anyone following our documented least-privilege path got Forbidden on every teardown, and stale Nodes then accumulate — which is exactly the condition that produces a phantom Ready cell or HAMi advertising a GPU that is gone. Granted, with the real reason. The commented-out pods/eviction block is deleted: this operator waits for a GPU to be released rather than evicting, so it never needed it, and shipping it as a suggestion was misleading. **Stale text was being served to users by the apiserver.** Doc comments in api/ are compiled into the CRD's OpenAPI, so `kubectl explain gpucellpool.spec.autoscaling` told people the feature is not implemented, and the provisioner field documented the MachineDeployment design we explicitly REJECTED. Six such claims, all now corrected and regenerated: autoscaling "a later phase", "Scale-UP only", ScaleDownAuto "Not implemented", Manual "the only supported value", ClusterAPI "the only mode today / sizes a MachineDeployment", and the obsolete ~14-minute startup figure. **bootstrap.provider: KubeadmToken was accepted and unimplemented.** The enum admitted it; nothing implemented it; and the failure was not even clean — with no joinSecretRef it passed admission and then errored about the Opaque provider the user had not selected, and with one it silently behaved as Opaque and minted no token. Removed from the enum rather than left accepted, since a value the apiserver admits and the controller ignores is the silent failure this project refuses. **A shared DRA claim was not held to one cell.** V3's justification says a named ResourceClaim is one claim and a VFIO device backs one VM, so N cells double-book it — but the rule only enforced the XOR between the two references. resourceClaimName with replicas: 2 was admitted. Now rejected, including via autoscaling.maxReplicas, because a pool that can grow past one cell is the same bug deferred until demand arrives. Signed-off-by: William Rizzo --- api/v1alpha1/gpucellpool_types.go | 53 ++-- .../crds/cells.kubeswift.io_gpucellpools.yaml | 30 +- .../cells.kubeswift.io_gpucellpools.yaml | 30 +- config/rbac/workload-cluster-observer.yaml | 23 +- docs/api-reference.md | 277 ++++++++++++++++++ docs/concepts.md | 116 ++++++++ hack/build-cell-image.sh | 6 +- internal/metrics/metrics.go | 5 +- .../webhook/v1alpha1/gpucellpool_validator.go | 29 ++ .../v1alpha1/gpucellpool_validator_test.go | 34 +++ 10 files changed, 534 insertions(+), 69 deletions(-) create mode 100644 docs/api-reference.md create mode 100644 docs/concepts.md diff --git a/api/v1alpha1/gpucellpool_types.go b/api/v1alpha1/gpucellpool_types.go index aee7de7..43fee83 100644 --- a/api/v1alpha1/gpucellpool_types.go +++ b/api/v1alpha1/gpucellpool_types.go @@ -15,9 +15,9 @@ import ( // POLICY. They never mix (design principle 9.3). type GPUCellPoolSpec struct { // Replicas is the desired number of cells. Scaled via the scale subresource, - // so `kubectl scale` and an HPA both work. Autoscaling (min/max, demand - // signals) is a later phase and will arrive as a separate `autoscaling` block - // rather than by changing this field. + // so `kubectl scale` and an HPA both work. When spec.autoscaling is enabled it + // overrides this: the pool then aims for status.desiredReplicas, and replicas + // acts as the default floor. // +kubebuilder:validation:Minimum=0 // +kubebuilder:default=1 Replicas int32 `json:"replicas"` @@ -36,7 +36,8 @@ type GPUCellPoolSpec struct { Capacity CapacitySpec `json:"capacity,omitempty"` // Autoscaling, when enabled, lets unsatisfiable GPU demand in the WORKLOAD - // cluster create cells. Scale-UP only in v1alpha1 (see AutoscalingSpec). + // cluster create cells, and lets idle cells be removed again (see + // AutoscalingSpec). // +optional Autoscaling *AutoscalingSpec `json:"autoscaling,omitempty"` @@ -48,18 +49,20 @@ type GPUCellPoolSpec struct { // Scale-down modes. const ( // ScaleDownManual leaves shrinking to the operator: change spec.replicas (or - // minReplicas) and the drain path runs. The default, and the only supported - // value in v1alpha1. + // minReplicas) and the drain path runs. The default. ScaleDownManual = "Manual" - // ScaleDownAuto lets the pool shrink itself. Not implemented: destroying - // someone's running work on a heuristic is the one unrecoverable mistake in - // this architecture, so it is a separate phase with its own evidence. + // ScaleDownAuto lets the pool shrink itself, and is deliberately conservative + // because destroying someone's running work is the one unrecoverable mistake in + // this architecture: only cells the capacity provider reports as IDLE are + // candidates, demand must have been absent for the whole window, and the drain + // gate re-checks allocations again before the cell is removed. Requires + // MinReplicas to be set. ScaleDownAuto = "Auto" ) // AutoscalingSpec turns unsatisfiable GPU demand into cells. // -// Only scale-UP is implemented. Two filters gate every decision, and they are the +// Both directions are implemented. Two filters gate every decision, and they are the // entire safety of the feature: the demand must be GPU-capacity-constrained, AND a // fresh cell of THIS pool's shape must actually satisfy it. A pod pending on a // missing ConfigMap, a wrong nodeSelector, an impossible GPU model, or a request @@ -83,10 +86,10 @@ type AutoscalingSpec struct { MaxReplicas *int32 `json:"maxReplicas,omitempty"` // StabilizationWindow is how long to wait after a scale-up before scaling up - // again. A cell takes minutes to become Ready (measured: ~14 minutes with - // install-at-boot, less with a baked image), and demand does not clear until - // it does — so without this window one burst of pending pods creates a cell - // per reconcile. + // again. A cell takes minutes to become Ready (measured: 4m45s from a baked + // image, about three minutes of which is cloning the root disk), and demand does + // not clear until it does — so without this window one burst of pending pods + // creates a cell per reconcile. // +kubebuilder:default="10m" // +optional StabilizationWindow *metav1.Duration `json:"stabilizationWindow,omitempty"` @@ -116,10 +119,11 @@ type AutoscalingSpec struct { // CellSpec is the shape of a single cell. type CellSpec struct { // Provisioner creates the outer objects for a cell. - // SwiftGuest: create a KubeSwift SwiftGuest directly (the only mode today). - // ClusterAPI: size a MachineDeployment (later phase; requires a CAPI-managed - // workload cluster, so it is an additional mode, never a - // replacement). + // SwiftGuest: create a KubeSwift SwiftGuest directly. + // ClusterAPI: create one Cluster API Machine + KubeSwiftMachine per cell, so + // the cell is a member of a CAPI-managed cluster. An additional + // mode, never a replacement: CAPI can only add Machines to a + // cluster it already manages. // +kubebuilder:validation:Enum=SwiftGuest;ClusterAPI // +kubebuilder:default=SwiftGuest // +optional @@ -297,17 +301,18 @@ const ( // BootstrapProviderOpaque uses user-supplied cloud-init verbatim (with a // small, closed substitution set). Makes no assumption about the workload // distribution — k0s, kubeadm, RKE2 and k3s all work. + // It is the only provider. A KubeadmToken provider that mints TTL'd join + // tokens per cell was specified and is NOT implemented; it was removed from + // the enum rather than left accepted, because a value the apiserver admits and + // the controller ignores fails silently — and did: it passed admission and + // then errored about the provider the user had not selected. BootstrapProviderOpaque = "Opaque" - // BootstrapProviderKubeadmToken mints a TTL'd bootstrap token in the - // workload cluster per cell. Kubeadm-style clusters only; needs Secret - // write rights in kube-system. - BootstrapProviderKubeadmToken = "KubeadmToken" ) // BootstrapSpec describes how a cell VM joins the workload cluster. type BootstrapSpec struct { - // Provider selects the join-credential mechanism. - // +kubebuilder:validation:Enum=Opaque;KubeadmToken + // Provider selects the join-credential mechanism. Opaque is the only value. + // +kubebuilder:validation:Enum=Opaque // +kubebuilder:default=Opaque // +optional Provider string `json:"provider,omitempty"` diff --git a/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml b/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml index ff11729..a238c87 100644 --- a/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml +++ b/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml @@ -70,7 +70,8 @@ spec: autoscaling: description: |- Autoscaling, when enabled, lets unsatisfiable GPU demand in the WORKLOAD - cluster create cells. Scale-UP only in v1alpha1 (see AutoscalingSpec). + cluster create cells, and lets idle cells be removed again (see + AutoscalingSpec). properties: enabled: description: |- @@ -118,10 +119,10 @@ spec: default: 10m description: |- StabilizationWindow is how long to wait after a scale-up before scaling up - again. A cell takes minutes to become Ready (measured: ~14 minutes with - install-at-boot, less with a baked image), and demand does not clear until - it does — so without this window one burst of pending pods creates a cell - per reconcile. + again. A cell takes minutes to become Ready (measured: 4m45s from a baked + image, about three minutes of which is cloning the root disk), and demand does + not clear until it does — so without this window one burst of pending pods + creates a cell per reconcile. type: string type: object bootstrap: @@ -166,10 +167,10 @@ spec: x-kubernetes-map-type: atomic provider: default: Opaque - description: Provider selects the join-credential mechanism. + description: Provider selects the join-credential mechanism. Opaque + is the only value. enum: - Opaque - - KubeadmToken type: string readyTimeout: default: 15m @@ -383,10 +384,11 @@ spec: default: SwiftGuest description: |- Provisioner creates the outer objects for a cell. - SwiftGuest: create a KubeSwift SwiftGuest directly (the only mode today). - ClusterAPI: size a MachineDeployment (later phase; requires a CAPI-managed - workload cluster, so it is an additional mode, never a - replacement). + SwiftGuest: create a KubeSwift SwiftGuest directly. + ClusterAPI: create one Cluster API Machine + KubeSwiftMachine per cell, so + the cell is a member of a CAPI-managed cluster. An additional + mode, never a replacement: CAPI can only add Machines to a + cluster it already manages. enum: - SwiftGuest - ClusterAPI @@ -428,9 +430,9 @@ spec: default: 1 description: |- Replicas is the desired number of cells. Scaled via the scale subresource, - so `kubectl scale` and an HPA both work. Autoscaling (min/max, demand - signals) is a later phase and will arrive as a separate `autoscaling` block - rather than by changing this field. + so `kubectl scale` and an HPA both work. When spec.autoscaling is enabled it + overrides this: the pool then aims for status.desiredReplicas, and replicas + acts as the default floor. format: int32 minimum: 0 type: integer diff --git a/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml b/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml index ff11729..a238c87 100644 --- a/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml +++ b/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml @@ -70,7 +70,8 @@ spec: autoscaling: description: |- Autoscaling, when enabled, lets unsatisfiable GPU demand in the WORKLOAD - cluster create cells. Scale-UP only in v1alpha1 (see AutoscalingSpec). + cluster create cells, and lets idle cells be removed again (see + AutoscalingSpec). properties: enabled: description: |- @@ -118,10 +119,10 @@ spec: default: 10m description: |- StabilizationWindow is how long to wait after a scale-up before scaling up - again. A cell takes minutes to become Ready (measured: ~14 minutes with - install-at-boot, less with a baked image), and demand does not clear until - it does — so without this window one burst of pending pods creates a cell - per reconcile. + again. A cell takes minutes to become Ready (measured: 4m45s from a baked + image, about three minutes of which is cloning the root disk), and demand does + not clear until it does — so without this window one burst of pending pods + creates a cell per reconcile. type: string type: object bootstrap: @@ -166,10 +167,10 @@ spec: x-kubernetes-map-type: atomic provider: default: Opaque - description: Provider selects the join-credential mechanism. + description: Provider selects the join-credential mechanism. Opaque + is the only value. enum: - Opaque - - KubeadmToken type: string readyTimeout: default: 15m @@ -383,10 +384,11 @@ spec: default: SwiftGuest description: |- Provisioner creates the outer objects for a cell. - SwiftGuest: create a KubeSwift SwiftGuest directly (the only mode today). - ClusterAPI: size a MachineDeployment (later phase; requires a CAPI-managed - workload cluster, so it is an additional mode, never a - replacement). + SwiftGuest: create a KubeSwift SwiftGuest directly. + ClusterAPI: create one Cluster API Machine + KubeSwiftMachine per cell, so + the cell is a member of a CAPI-managed cluster. An additional + mode, never a replacement: CAPI can only add Machines to a + cluster it already manages. enum: - SwiftGuest - ClusterAPI @@ -428,9 +430,9 @@ spec: default: 1 description: |- Replicas is the desired number of cells. Scaled via the scale subresource, - so `kubectl scale` and an HPA both work. Autoscaling (min/max, demand - signals) is a later phase and will arrive as a separate `autoscaling` block - rather than by changing this field. + so `kubectl scale` and an HPA both work. When spec.autoscaling is enabled it + overrides this: the pool then aims for status.desiredReplicas, and replicas + acts as the default floor. format: int32 minimum: 0 type: integer diff --git a/config/rbac/workload-cluster-observer.yaml b/config/rbac/workload-cluster-observer.yaml index 65b889f..be2ed83 100644 --- a/config/rbac/workload-cluster-observer.yaml +++ b/config/rbac/workload-cluster-observer.yaml @@ -19,9 +19,15 @@ metadata: rules: # Cell correlation + readiness, and applying the identity labels/taints the # kubelet may not have set itself. + # + # DELETE is not optional and is not a drain feature. Two ALWAYS-ON paths remove a + # workload Node: reaping a Node left behind by a previous incarnation of a cell + # (without which the pool reports a phantom Ready cell, or HAMi advertises a GPU + # that no longer exists), and removing a cell's Node when the cell goes. Grant it + # or every teardown fails Forbidden and stale Nodes accumulate. - apiGroups: [""] resources: [nodes] - verbs: [get, list, watch, patch, update] + verbs: [get, list, watch, patch, update, delete] # HAMi DevicePlugin-mode accounting: allocations live on pod annotations. - apiGroups: [""] @@ -34,18 +40,9 @@ rules: resources: [resourceslices, resourceclaims] verbs: [get, list, watch] - # Phase 4 — draining a cell before it is removed. - # - apiGroups: [""] - # resources: [pods/eviction] - # verbs: [create] - # - apiGroups: [""] - # resources: [nodes] - # verbs: [delete] - - # bootstrap.provider: KubeadmToken only — minting TTL'd join tokens. - # - apiGroups: [""] - # resources: [secrets] - # verbs: [create, get, update, delete] + # Nothing else is needed. In particular this operator does NOT evict pods: it + # waits for a cell's GPU to be released rather than taking it away, so it needs no + # pods/eviction right. If a cell must be emptied, drain it with kubectl. --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 0000000..e4a43b9 --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,277 @@ +# API reference — `cells.kubeswift.io/v1alpha1` `GPUCellPool` + +> Generated by reading `api/v1alpha1/gpucellpool_types.go` and +> `internal/webhook/v1alpha1/gpucellpool_validator.go` directly — this is the +> ground truth, not the design doc. See `docs/concepts.md` for the ideas behind +> the fields, and `docs/autoscaling.md` for `spec.autoscaling` in depth. + +``` +group cells.kubeswift.io +version v1alpha1 +kind GPUCellPool namespaced +short cellpool +scale .spec.replicas -> .status.replicas +``` + +A pool never crosses namespaces: the workload-cluster kubeconfig Secret, the +join Secret, and (under `provisioner: ClusterAPI`) the CAPI `Cluster` and +bootstrap template must all live in the pool's own namespace. + +## `spec` + +```yaml +spec: + replicas: 2 # int32, default 1, min 0 + + cell: {} # CellSpec — required + bootstrap: {} # BootstrapSpec — required + workloadCluster: {} # WorkloadClusterSpec — required + capacity: {} # CapacitySpec — optional + autoscaling: {} # AutoscalingSpec — optional, see autoscaling.md + deletion: {} # DeletionSpec — optional +``` + +`replicas` is the desired cell count when `autoscaling` is absent or disabled, +and the default floor when it is enabled — the pool then aims for +`status.desiredReplicas` instead. + +### `spec.cell` (`CellSpec`) + +| Field | Type | Default | Notes | +|---|---|---|---| +| `provisioner` | `SwiftGuest \| ClusterAPI` | `SwiftGuest` | which outer objects a cell becomes | +| `guestTemplate` | opaque object (`x-kubernetes-preserve-unknown-fields`) | — | required; see the field contract below | +| `gpu` | `CellGPUSpec` | — | required | +| `spreadPolicy` | `Pack \| Spread` | `Spread` | hostname topology spread of cell VMs, matching `SwiftGuestPool` | +| `nodeIPFrom` | string | primary interface | the `guestTemplate.interfaces[].name` whose address the kubelet should register as `--node-ip` — see `docs/networking.md` | +| `clusterAPI` | `*CellClusterAPISpec` | — | required iff `provisioner: ClusterAPI`; rejected otherwise | + +#### `spec.cell.guestTemplate` field contract + +`guestTemplate` is a verbatim `SwiftGuestSpec` passthrough — this API deliberately +never mirrors KubeSwift's. The validating webhook enforces three field classes, +so a mistake is a rejection, never a silent override: + +| Class | Fields | Behaviour | +|---|---|---| +| **operator-owned** | `seedProfileRef`, `gpuProfileRef`, `gpuResourceClaim`, `nodeName`, `migration`, `runPolicy` | rejected if set by the user; the controller writes them | +| **denied** | `kernelRef`, `cloneFromSnapshot`, `vhostUserDevices`, `filesystems`, `osType: windows` | rejected, with the reason quoted (GPU is disk-boot only; virtiofs/vhost-user are rejected by KubeSwift itself on the GPU path) | +| **required** | `guestClassRef`, `imageRef` | a GPU cell is a disk boot | +| **passthrough** | everything else (`storage`, `interfaces`, `topologySpreadConstraints`, `guestAgent`, …) | copied verbatim into every cell guest | + +Rendering a cell = `guestTemplate` + the operator-owned overlay: + +``` +spec.seedProfileRef = -seed +spec.gpuResourceClaim = from cell.gpu.dra (or spec.gpuProfileRef from cell.gpu.native) +spec.migration.enabled = false # cells are replaced, not migrated +spec.runPolicy = Always +spec.topologySpreadConstraints = a hostname spread over this pool's cells, + unless the template already sets one (spreadPolicy: Spread only) +metadata.name = - +metadata.labels = cells.kubeswift.io/{pool,cell,cell-index} +metadata.annotations = cells.kubeswift.io/template-hash= +``` + +A template change bumps the hash but does **not** roll existing cells: v1alpha1 +has no rolling update for `guestTemplate` (see `docs/limitations.md`). + +Under `provisioner: ClusterAPI` the allowed `guestTemplate` fields shrink to +`imageRef`, `guestClassRef`, `interfaces` — everything else a `KubeSwiftMachine` +cannot express and is rejected at admission rather than dropped. + +### `spec.cell.gpu` (`CellGPUSpec`) + +| Field | Type | Default | Notes | +|---|---|---|---| +| `count` | int32, enum `1` | `1` | only 1 GPU per cell in v1alpha1 | +| `backend` | `DRA \| Native` | `DRA` | which KubeSwift GPU allocation backend | +| `dra` | `*CellGPUDRASpec` | — | required iff `backend: DRA`; forbidden otherwise | +| `native` | `*CellGPUNativeSpec` | — | required iff `backend: Native`; forbidden otherwise | + +`CellGPUDRASpec`: + +| Field | Type | Default | Notes | +|---|---|---|---| +| `resourceClaimTemplateName` | string | — | mints a per-cell `ResourceClaim`; recommended | +| `resourceClaimName` | string | — | one pre-created, shared claim — correct only for a single-cell pool, since a VFIO device backs exactly one running VM | +| `requestName` | string | `gpu` | the device-request name inside the claim | +| `tier` | enum `pcie` | `pcie` | the only supported tier for a cell — `hgx-shared` needs QEMU + a host Fabric Manager, `hgx-full` is rejected by KubeSwift at allocation | +| `hugepages` | `"" \| "1Gi" \| "2Mi"` | `""` | GPU memory hugepage backing | + +Exactly one of `resourceClaimTemplateName` / `resourceClaimName` must be set — +setting both or neither is rejected. A shared `resourceClaimName` is only valid +when the pool's `replicas` (and `autoscaling.maxReplicas`, if set) is 1: a VFIO +device backs one running VM, so N cells sharing one claim double-book the +device. + +`CellGPUNativeSpec`: `gpuProfileRef` (`corev1.LocalObjectReference`, required) — +a `SwiftGPUProfile` in the pool's namespace. + +### `spec.cell.clusterAPI` (`CellClusterAPISpec`) + +Required iff `provisioner: ClusterAPI`. + +| Field | Type | Notes | +|---|---|---| +| `clusterName` | string, required | the CAPI `Cluster` in the pool's namespace that cells join | +| `version` | string | stamped on `Machine.spec.version`; the bootstrap provider uses it to pick the kubelet | +| `bootstrapConfigTemplateRef` | `*ClusterAPIObjectRef` | a bootstrap config **template** (e.g. `KubeadmConfigTemplate`), instantiated once per cell like a `MachineSet` does | + +`ClusterAPIObjectRef`: `apiGroup`, `kind` (must end in `Template`), `name` — no +version field; CAPI resolves the version from the CRD's contract labels. + +When `bootstrapConfigTemplateRef` is set, the workload cluster's own bootstrap +provider supplies the join data and `spec.bootstrap.joinSecretRef` is +**forbidden**. Left unset, the pool's own rendered bootstrap Secret is handed to +the Machine as `bootstrap.dataSecretName` — that works, but you own keeping the +join data valid (token expiry, CA rotation). + +### `spec.bootstrap` (`BootstrapSpec`) + +| Field | Type | Default | Notes | +|---|---|---|---| +| `provider` | enum `Opaque` | `Opaque` | the only provider — see below | +| `joinSecretRef` | `*corev1.LocalObjectReference` | — | required for `Opaque`, unless `cell.clusterAPI.bootstrapConfigTemplateRef` is set (then forbidden) | +| `joinSecretKey` | string | `user-data` | key inside `joinSecretRef` holding the cloud-init template | +| `hostname` | `CellName \| None` | `CellName` | `CellName` makes the guest hostname — and so the workload Node name — equal to the cell name | +| `readyTimeout` | duration | `15m` | bounds `Booting`+`Joining`; past it the cell is marked `Failed` | + +`Opaque` is the only bootstrap provider. A `KubeadmToken` provider that would +mint TTL'd join tokens per cell was designed but is **not implemented** and was +removed from the API enum rather than left accepted-but-ignored — an admitted +value the controller silently does nothing with is exactly the kind of failure +this project refuses to ship. See `docs/limitations.md`. + +### `spec.workloadCluster` (`WorkloadClusterSpec`) + +| Field | Type | Default | Notes | +|---|---|---|---| +| `kubeconfigSecretRef` | `corev1.LocalObjectReference`, required | — | a Secret in the pool's namespace | +| `key` | string | `value` | the Secret key holding the kubeconfig | +| `node` | `*WorkloadNodeSpec` | — | labels/annotations/taints applied to every cell's workload Node | + +`WorkloadNodeSpec.labels` is where HAMi's own scheduling gate (`gpu: "on"`) is +declared — by you, not by the operator; GPUCellPool applies labels, it does not +interpret them. `taints` must be tolerated by HAMi's DaemonSets or the device +plugin never lands on the cell. + +### `spec.capacity` (`CapacitySpec`) + +| Field | Type | Default | Notes | +|---|---|---|---| +| `provider` | enum `HAMi` | `HAMi` | the only capacity provider | +| `hami` | `*HAMiSpec` | — | | +| `readyTimeout` | duration | `5m` | bounds `Joining` -> `Ready`: how long a joined Node may go without advertising the expected GPU before the cell is marked `Failed` | + +`HAMiSpec`: + +| Field | Type | Default | Notes | +|---|---|---|---| +| `mode` | `DevicePlugin \| DRA` | `DevicePlugin` | `DRA` mode is **not implemented** — see `docs/limitations.md` | +| `expectedDevicesPerCell` | int32 | `cell.gpu.count` | devices HAMi must advertise before a cell counts Ready | +| `deviceClassName` | string | — | required in `DRA` mode | + +### `spec.deletion` (`DeletionSpec`) + +| Field | Type | Default | Notes | +|---|---|---|---| +| `policy` | `Drain \| Force` | `Drain` | applies to explicit pool deletion only | +| `drainTimeout` | duration | `10m` | after which explicit deletion proceeds anyway | + +`Force` skips draining entirely and **destroys running workloads** — that is why +it is not the default. `drainTimeout` only bounds *explicit pool deletion*; +implicit scale-down never force-proceeds (see `docs/limitations.md`). + +## `status` + +| Field | Type | Notes | +|---|---|---| +| `observedGeneration` | int64 | | +| `replicas` | int32 | total cells owned (scale subresource statuspath) | +| `readyCells` / `creatingCells` / `drainingCells` / `failedCells` | int32 | | +| `desiredReplicas` | int32 | what the scaling policy asked for; equals `spec.replicas` unless autoscaling is enabled | +| `demand` | `*DemandStatus` | present only when autoscaling is enabled — see `docs/autoscaling.md` | +| `cellDeviceShape` | `*CellDeviceShape` | the remembered shape of one cell's GPU; outlives the cells so a pool scaled to zero can still judge new demand — see `docs/autoscaling.md` | +| `lastScaleUpTime` / `lastScaleDownTime` | `*metav1.Time` | gate the stabilization windows | +| `demandFreeSince` | `*metav1.Time` | when GPU demand last went to zero | +| `physicalCapacity` | `*PhysicalCapacityStatus` | outer — whole GPUs | +| `workloadCapacity` | `*WorkloadCapacityStatus` | inner — HAMi fractions | +| `cells` | `[]CellStatus` | one entry per owned cell, `+listMapKey=name` | +| `conditions` | `[]metav1.Condition` | `+listMapKey=type` | + +`PhysicalCapacityStatus`: `gpus` (held by this pool's cells), `freeGPUsInCluster` +(nil means unknown — never reported as zero), `model`. + +`WorkloadCapacityStatus`: `provider`, `mode`, `gpuDevices`, `homogeneous`, +`gpuMemory` (`MemoryCapacity{total,allocated,available}`, always valid — bytes +are commensurable across models), `gpuCompute` (`ComputeCapacity`, percent, +published **only** while `homogeneous == true`), `byModel` (always populated, +the only compute truth when the pool is mixed-model), `lastObserved`. + +`CellStatus`: `name`, `index`, `phase` (see below), `guestUID` (distinguishes +this incarnation of the cell from a previous one), `hostNode`, `devices` (PCI +BDFs), `nodeName`, `nodeReady`, `capacityDevices`, `message`, `failureCount`, +`readyOnce`, `lastTransitionTime`. + +`CellDeviceShape`: `model`, `memoryMiB`, `corePercent`, `lastObserved`. + +`DemandStatus`: `pendingRequests`, `satisfiableByOneCell`, `lastObserved`. + +### Cell phases + +``` +Pending -> AllocatingGPU -> GuestProvisioning -> Booting -> Joining + -> AwaitingGPUCapacity -> Ready -> Draining -> Deleting + +any state -> Failed (terminal outer failure, or the state's timeout elapsed) +Ready -> AwaitingGPUCapacity (capacity/Node regression — not a failure) +``` + +`Ready` requires all three: the outer VM `Running`, the inner Node `Ready`, and +the capacity provider advertising `>= expectedDevicesPerCell` devices on that +Node. See `docs/runbook.md` for what each phase waits for and how to diagnose a +stuck one. + +### Conditions + +| Type | True when | Notable False reasons | +|---|---|---| +| `Ready` | `readyCells == status.desiredReplicas` (accounts for autoscaling and `ScaledToZero`) | `CellsNotReady`, `WorkloadClusterUnreachable` | +| `Progressing` | a cell is creating/booting/joining/draining | `Idle` (steady state), `Stalled` | +| `WorkloadClusterReachable` | last inner API call succeeded | `Unreachable`, `CredentialInvalid`, `Forbidden` | +| `CapacityProviderReady` | HAMi detected and parseable on every ready cell's Node | `HAMiNotDetected`, `RegistrationUnparseable`, `DRAFeatureGateMissing` | +| `CapacityAvailable` | some ready cell has free memory *and* free compute | `Saturated`, `Unknown`, `Heterogeneous` | +| `PhysicalGPUsAvailable` | outer inventory has >=1 free device, or no cell is waiting for one | `InsufficientPhysicalGPU` | +| `ScalingActive` | autoscaling is enabled and demand was read successfully | see `docs/autoscaling.md` for every reason | +| `CellDrainRequested` | an outer node holding a cell is cordoned/draining | — | + +## Validation rules (webhook) + +Enforced twice — at admission and again at render time in +`internal/provisioner`, so a spec that slipped past a disabled webhook still +cannot misconfigure a cell. + +| Rule | Fires on | Rejects | +|---|---|---| +| pool name is a usable DNS label with room for the cell index | create, update | a name too long to become `-` as a hostname | +| `cell.gpu.count != 1` | create, update | multi-GPU cells | +| `cell.gpu.backend` set with the wrong sub-struct (`native` with `DRA`, `dra` with `Native`, or missing) | create, update | mixed or absent backend config | +| `cell.gpu.dra`: not exactly one of `resourceClaimTemplateName`/`resourceClaimName` | create, update | both or neither set | +| `cell.gpu.dra.tier != pcie` | create, update | `hgx-shared`, `hgx-full` | +| `guestTemplate` sets an operator-owned or denied field | create, update | see the field contract above | +| `guestTemplate.guestClassRef` / `.imageRef` unset | create, update | a GPU cell must be a disk boot | +| `cell.provisioner: ClusterAPI` without `cell.clusterAPI`, or `SwiftGuest` with it set | create, update | mode and config must agree | +| under `ClusterAPI`, `guestTemplate` uses a field a `KubeSwiftMachine` cannot express | create, update | anything beyond `imageRef`, `guestClassRef`, `interfaces` | +| `clusterAPI.bootstrapConfigTemplateRef.kind` does not end in `Template` | create, update | the per-cell object's kind is the template's kind minus that suffix | +| `bootstrap.provider: Opaque` without `joinSecretRef` (unless CAPI owns bootstrap) | create, update | a cell that would boot with no cloud-init | +| `capacity.hami.mode: DRA` without `deviceClassName` | create, update | no way to find the ResourceSlices | +| `cell.nodeIPFrom` names an interface absent from `guestTemplate.interfaces` | create, update | kubelet would bind no address | +| `autoscaling.enabled` without `maxReplicas` | create, update | an unbounded pool that misreads demand can consume every GPU in the cluster | +| `autoscaling.minReplicas > maxReplicas` | create, update | | +| `autoscaling.scaleDown: Auto` without `minReplicas` | create, update | without a floor the pool can shrink to zero and every later request pays a full cell boot | +| `spec.replicas` decreased while `WorkloadClusterReachable != True` | **update only** | cells cannot be drained without access to the workload cluster | +| `spec.workloadCluster.kubeconfigSecretRef.name` empty | create, update | | + +Deletion is never blocked at admission — teardown safety is the reconciler's +drain gate (`spec.deletion`), not a validating-webhook concern. diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..75ae6ef --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,116 @@ +# Concepts + +## The invariant + +``` +OUTER KubeSwift physical GPU -> VM whole device, VFIO +INNER HAMi VM's GPU -> workloads memory + core fractions +BETWEEN GPUCellPool lifecycle + composition no resource translation +``` + +KubeSwift owns the physical-GPU-to-VM boundary. HAMi owns GPU-to-workload +allocation inside that VM. GPUCellPool owns the lifecycle between them and knows +about both — but a HAMi allocation is never handed to VFIO, and a VFIO device is +never handed to HAMi. There is no code path between the two DRA domains: they run +in different clusters, use different drivers, and are never joined. + +Positioning, precisely: GPUCellPool *composes* KubeSwift VM isolation with HAMi +GPU sharing. It is not "HAMi integration for KubeSwift" — neither project is +modified, and neither learns the other exists. + +## A cell + +A **cell** is one KubeSwift VM holding one whole passthrough GPU, running as a +worker node in a separate *workload* Kubernetes cluster, where HAMi shares that +GPU between workloads. A `GPUCellPool` declares N cells; there is no `GPUCell` +CRD — a cell is an owned `SwiftGuest` (or, under `provisioner: ClusterAPI`, a +`Machine` + `KubeSwiftMachine`) named `-`, with per-cell state +recorded in `status.cells[]`. + +``` + cell "inference-0" +┌─────────────────────────────────────────────────────────┐ +│ SwiftGuest inference-0 (infrastructure cluster) │ +│ ┌───────────────────────────────────────────────────┐ │ +│ │ VM: Linux + NVIDIA driver + containerd + kubelet │ │ +│ │ physical GPU ─VFIO──────────────┐ │ │ +│ └────────────────────────────────────┼─────────────────┘ │ +└─────────────────────────────────────────┼─────────────────┘ + │ joins as + ▼ + Node inference-0 (workload cluster) + ┌───────────────┼───────────────┐ + pod A pod B pod C + 4Gi / 30% 8Gi / 50% 2Gi / 20% ← HAMi fractions +``` + +## Two identities, never conflated + +| | outer (infrastructure cluster) | inner (workload cluster) | +|---|---|---| +| object | `SwiftGuest` (or `Machine`) + launcher Pod | `Node` | +| resource | 1 physical GPU (PCI BDF), CPU, RAM, disk | HAMi device + memory/core capacity | +| owned by | KubeSwift | HAMi | +| read by | the pool's outer client | the pool's workload-cluster client | + +Correlation between the two is by **name**, not IP or any other derived value: +cell name = `-` = guest hostname = workload Node name. The Node +additionally carries `cells.kubeswift.io/instance` (the outer guest's UID), so a +stale Node left behind by a *previous* incarnation of the same cell index is +detected and reaped before its replacement is allowed to join — otherwise the +pool would report a phantom Ready cell, or HAMi would advertise capacity for a +GPU that no longer exists. + +## Two capacities, never merged + +`status.physicalCapacity` (outer) and `status.workloadCapacity` (inner) are +computed by separate code paths and are never inferred from one another: + +- **`physicalCapacity`** — whole devices. Read from KubeSwift's and the outer + DRA driver's inventory. `gpus` is how many this pool's cells hold; + `freeGPUsInCluster` is how many more the pool could still claim. +- **`workloadCapacity`** — HAMi's fractional accounting. `gpuMemory` and + `gpuCompute` are total/allocated/available, in HAMi's units (bytes, percent of + one device). + +The physical GPU only returns to the outer pool when the cell's guest is +deleted — draining a cell does not free it, because KubeSwift's own GPU release +(a finalizer in native mode, claim/pod garbage collection in DRA mode) happens +at guest deletion, not at drain. + +A third-party consequence worth knowing: HAMi inflates the `nvidia.com/gpu` +allocatable count by `deviceSplitCount` (default ×10) so the scheduler can place +multiple fractional requests against one physical device. `allocatable` is +**not** a device count — a cell node with one physical GPU reports +`nvidia.com/gpu: 10`. Any other consumer of that Node in the workload cluster +(a cluster autoscaler reading capacity, a quota object, a dashboard) sees the +same inflation. GPUCellPool reads the device count from HAMi's +`hami.io/node-nvidia-register` annotation, never from `allocatable`, for exactly +this reason. + +## Cells are cattle + +A cell is a VFIO passthrough guest. KubeSwift can only move a VFIO guest with an +**offline** migration — a VM restart — never a live one +(`HasVFIODevices` in KubeSwift's migration gate). Applied to a cell that is also +a Kubernetes worker with running HAMi workloads on it, an offline migration is a +silent node reboot under live work. GPUCellPool therefore pins every cell with +`migration.enabled: false` and treats a cell that needs to move as one to be +**replaced**: drained, deleted, and recreated (possibly on another host), never +migrated. If the outer node holding a cell is cordoned or drained, the pool +surfaces `CellDrainRequested` for a human to act on — it does not attempt an +automated sequence in v1alpha1 (see `docs/limitations.md`). + +## Layered isolation + +A cell gives each group of workloads a VM boundary around the physical GPU; +HAMi gives efficiency inside that boundary. What that combination is worth +depends on the GPU hardware's DMA/IOMMU isolation, firmware, driver, and your +threat model — and on the infrastructure cluster, where every KubeSwift +launcher pod is `privileged: true` by design. That makes **the right to create +a `GPUCellPool` node-root-equivalent authority in the infrastructure cluster** +(see `docs/security.md`). + +Say *layered isolation*. Do not claim absolute tenant isolation — a cell is a +stronger boundary between workload groups than GPU-sharing alone provides, not +a hard security perimeter on its own. diff --git a/hack/build-cell-image.sh b/hack/build-cell-image.sh index cf1586c..b546a7f 100755 --- a/hack/build-cell-image.sh +++ b/hack/build-cell-image.sh @@ -9,7 +9,7 @@ # needed at build time — only at cell boot. set -euo pipefail -OUT="${OUT:-/tmp/claude-1000/-home-wrkode-code-vmm-kubeswift-kubeswift/c6001214-37be-44cd-ae40-c65efde23337/scratchpad/cellpoc/build}" +OUT="${OUT:-$PWD/build}" # The DISTRO qemu, deliberately: a Kata build (which may come first in PATH at # /opt/kata/bin) is compiled without user-mode networking, so the guest gets no # egress and the bake dies mid-apt with nothing obviously wrong. @@ -17,7 +17,9 @@ QEMU="${QEMU:-/usr/bin/qemu-system-x86_64}" BIOS="${BIOS:-/usr/share/seabios/bios-256k.bin}" BASE_URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" # Pascal (GTX 1080) needs the PROPRIETARY driver: the open kernel modules are -# Turing+ only. Keep this pinned and recorded — it is part of the image contract. +# Turing+ only. Note this metapackage tracks a BRANCH — 570-server resolved to +# 580.173.02 on the validated build — so the manifest records the version that was +# actually installed. Pin NVIDIA_DRIVER_PKG to an exact package for reproducibility. NVIDIA_DRIVER_PKG="${NVIDIA_DRIVER_PKG:-nvidia-driver-570-server}" DISK_SIZE="${DISK_SIZE:-30G}" MEM="${MEM:-4096}" diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 1f9042d..6dc0d00 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -33,8 +33,9 @@ var ( // // This is the number that decides how autoscaling should be framed: a cell // that takes fifteen minutes is a capacity-planning unit, not something a - // reactive autoscaler can chase. Buckets span "baked image, thin enrollment" - // through "installs a driver at first boot", which measured ~14 minutes. + // reactive autoscaler can chase. Buckets span "baked image, thin enrollment" — + // measured at 4m45s, about three minutes of which is cloning the root disk — + // through "installs a driver at first boot", which measured about 14 minutes. CellStartupSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: prefix + "cell_startup_seconds", Help: "Seconds from cell creation to Ready.", diff --git a/internal/webhook/v1alpha1/gpucellpool_validator.go b/internal/webhook/v1alpha1/gpucellpool_validator.go index 9e0fb5a..1acfa6f 100644 --- a/internal/webhook/v1alpha1/gpucellpool_validator.go +++ b/internal/webhook/v1alpha1/gpucellpool_validator.go @@ -76,6 +76,7 @@ func Validate(pool, old *cellsv1alpha1.GPUCellPool) field.ErrorList { errs = append(errs, validateProvisioner(spec.Child("cell"), pool)...) errs = append(errs, validateGPU(spec.Child("cell", "gpu"), pool.Spec.Cell.GPU)...) + errs = append(errs, validateSharedClaimIsSingleCell(spec, pool)...) errs = append(errs, validateTemplate(spec.Child("cell"), pool)...) errs = append(errs, validateBootstrap(spec.Child("bootstrap"), pool)...) errs = append(errs, validateWorkloadCluster(spec.Child("workloadCluster"), pool.Spec.WorkloadCluster)...) @@ -144,6 +145,34 @@ func validateGPU(p *field.Path, gpu cellsv1alpha1.CellGPUSpec) field.ErrorList { return errs } +// validateSharedClaimIsSingleCell enforces what V3's justification always said but +// the rule never checked: a named ResourceClaim is ONE claim, a VFIO device backs +// exactly one running VM, so N cells sharing it double-book the device. +// +// The XOR between the two references was enforced; the pairing with the replica count +// was not, so `resourceClaimName` with `replicas: 2` was admitted and the second cell +// contended for a device the first one held. Use resourceClaimTemplateName for any +// pool that can grow — the template mints a claim per cell. +func validateSharedClaimIsSingleCell(p *field.Path, pool *cellsv1alpha1.GPUCellPool) field.ErrorList { + dra := pool.Spec.Cell.GPU.DRA + if dra == nil || dra.ResourceClaimName == "" { + return nil + } + const why = "a named resourceClaimName is a single claim and a VFIO device backs one VM, " + + "so more than one cell would double-book it; use resourceClaimTemplateName instead" + + var errs field.ErrorList + if pool.Spec.Replicas > 1 { + errs = append(errs, field.Invalid(p.Child("replicas"), pool.Spec.Replicas, why)) + } + if as := pool.Spec.Autoscaling; as != nil && as.Enabled && + as.MaxReplicas != nil && *as.MaxReplicas > 1 { + errs = append(errs, field.Invalid(p.Child("autoscaling", "maxReplicas"), + *as.MaxReplicas, why)) + } + return errs +} + // capiExpressibleTemplateFields are the guestTemplate fields a KubeSwiftMachine // can carry. Everything else has nowhere to go under the ClusterAPI provisioner. var capiExpressibleTemplateFields = []string{"imageRef", "guestClassRef", "interfaces"} diff --git a/internal/webhook/v1alpha1/gpucellpool_validator_test.go b/internal/webhook/v1alpha1/gpucellpool_validator_test.go index 1d8336c..f2b4646 100644 --- a/internal/webhook/v1alpha1/gpucellpool_validator_test.go +++ b/internal/webhook/v1alpha1/gpucellpool_validator_test.go @@ -251,3 +251,37 @@ func TestValidatorEntryPoints(t *testing.T) { t.Errorf("ValidateDelete blocked a deletion: %v", err) } } + +// A named ResourceClaim is ONE claim and a VFIO device backs one VM, so a pool that +// can hold more than one cell would double-book the device. V3's justification always +// said this; the rule never checked it. +func TestSharedClaimRequiresASingleCell(t *testing.T) { + i32 := func(i int32) *int32 { return &i } + shared := func() *cellsv1alpha1.GPUCellPool { + p := validPool() + p.Spec.Replicas = 1 + p.Spec.Cell.GPU.DRA = &cellsv1alpha1.CellGPUDRASpec{ + ResourceClaimName: "one-gpu", Tier: "pcie", + } + return p + } + + assertValid(t, shared()) + + p := shared() + p.Spec.Replicas = 2 + assertRejected(t, p, "double-book") + + // The ceiling counts too: a pool that can GROW past one cell is the same bug, + // just deferred until demand arrives. + p = shared() + p.Spec.Autoscaling = &cellsv1alpha1.AutoscalingSpec{ + Enabled: true, MinReplicas: i32(1), MaxReplicas: i32(3), + } + assertRejected(t, p, "double-book") + + // A claim TEMPLATE mints one claim per cell, so it has no such limit. + p = validPool() + p.Spec.Replicas = 4 + assertValid(t, p) +} From 3b48e52813624e6599f67479e989ae435eff2e77 Mon Sep 17 00:00:00 2001 From: William Rizzo Date: Sat, 8 Aug 2026 16:30:05 +0000 Subject: [PATCH 2/2] docs: a documentation set a stranger can succeed with MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written for the repo going public. The problem was not missing prose — it was that the first documentation click landed on a pre-implementation design doc whose header said "No hardware proof yet", and that two things a user needs to reach a running cell were never shipped at all. New, user-facing: - quickstart.md — the whole journey, ending on the assertion that is the product: two pods, each limited to a fraction, both naming the SAME GPU UUID. - concepts.md, api-reference.md (written from the types and the validator, not from the design doc), autoscaling.md (the headline feature had no user-facing page at all), networking.md, security.md, limitations.md, cell-image.md, and docs/README.md as an index that separates "using this" from "why it is built this way". - config/samples/cell-join-secret.yaml — k0s and kubeadm cloud-init. There was no join example anywhere, and nobody was going to reconstruct one from prose: it needs the closed substitution set, a quoted token (a bare {{ token }} in a value position is not valid YAML), the node IP derived by subnet, containerd 2.x's v3 config schema, a pinned resolver, and ssh keys or a broken cell is undiagnosable. Plus a NAD sample. Two API fixes fell out of verifying the samples actually apply: - spec.bootstrap is now optional. It is genuinely unused when Cluster API's own bootstrap provider supplies the join data, and requiring it forced an empty `bootstrap: {}` into the manifest to satisfy the schema — a field you must write and nothing reads. The ClusterAPI sample could not be applied at all before this. - A test now runs every shipped GPUCellPool sample through the real webhook rules. A sample the webhook rejects is worse than no sample: it is the first thing a new user applies and it fails looking like their mistake. The CRD schema half is covered by a server-side dry-run. The seven design docs are kept — the D1-D10 decision record is worth publishing — but demoted behind a banner saying they document rationale, not behaviour, and every "Phase N", "later", "not implemented" and "to be measured" that had become false is corrected. gpucellpool-poc.md is renamed to gpucellpool-validation-record.md, because it is a results record and had been read as a plan. Signed-off-by: William Rizzo --- CHANGELOG.md | 240 ++++++++--------- README.md | 24 +- api/v1alpha1/gpucellpool_types.go | 9 +- .../crds/cells.kubeswift.io_gpucellpools.yaml | 11 +- .../cells.kubeswift.io_gpucellpools.yaml | 11 +- config/samples/cell-join-secret.yaml | 233 +++++++++++++++++ ...cells_v1alpha1_gpucellpool_clusterapi.yaml | 5 +- .../network-attachment-definition.yaml | 35 +++ docs/README.md | 40 +++ docs/api-reference.md | 1 + docs/autoscaling.md | 139 ++++++++++ docs/cell-image.md | 118 +++++++++ docs/clusterapi-cells.md | 208 ++++++++------- docs/design/clusterapi-cells-validation.md | 110 ++++++++ docs/design/gpucellpool-api.md | 8 + docs/design/gpucellpool-bootstrap.md | 70 +++-- docs/design/gpucellpool-capacity.md | 27 +- docs/design/gpucellpool-failure-model.md | 13 +- docs/design/gpucellpool-overview.md | 50 ++-- docs/design/gpucellpool-reconciliation.md | 137 ++++++---- ...oc.md => gpucellpool-validation-record.md} | 22 +- docs/limitations.md | 89 +++++++ docs/networking.md | 137 ++++++++++ docs/quickstart.md | 242 ++++++++++++++++++ docs/runbook.md | 75 ++++++ docs/security.md | 89 +++++++ internal/webhook/v1alpha1/samples_test.go | 57 +++++ 27 files changed, 1863 insertions(+), 337 deletions(-) create mode 100644 config/samples/cell-join-secret.yaml create mode 100644 config/samples/network-attachment-definition.yaml create mode 100644 docs/README.md create mode 100644 docs/autoscaling.md create mode 100644 docs/cell-image.md create mode 100644 docs/design/clusterapi-cells-validation.md rename docs/design/{gpucellpool-poc.md => gpucellpool-validation-record.md} (94%) create mode 100644 docs/limitations.md create mode 100644 docs/networking.md create mode 100644 docs/quickstart.md create mode 100644 docs/security.md create mode 100644 internal/webhook/v1alpha1/samples_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 402355a..eb5e0ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,128 +5,136 @@ All notable changes to this project are documented here. The format follows ## [v0.1.0] — 2026-08-08 -### Added +First release. Every capability below has been run on real hardware (one +physical GPU, one KubeSwift VM, two independent workloads sharing it through +HAMi) except where noted in **Known gaps**. See `docs/quickstart.md` to try it +and `docs/design/gpucellpool-validation-record.md` / +`docs/design/clusterapi-cells-validation.md` for the full hardware record, +including the bugs a live cluster found that the test harness could not. + +### Added — core API and controller - `GPUCellPool` v1alpha1 API (`cells.kubeswift.io`): one CRD, namespaced, with a scale subresource on `spec.replicas`. A cell is an owned `SwiftGuest` named `-`, not a second kind. `spec.cell.guestTemplate` is an opaque - `SwiftGuestSpec` passthrough so this API never mirrors KubeSwift's. -- Controller: cell state machine, membership planning with churn control, - physical-inventory pre-flight, per-cell bootstrap rendering, drain-gated - teardown, and status that separates the two layers (`physicalCapacity` = - whole devices, `workloadCapacity` = HAMi fractions). -- HAMi capacity provider (DevicePlugin mode), written against annotations - captured from real hardware rather than upstream documentation. -- Validating webhook enforcing the API's rules, fail-closed. -- Helm chart, container image (distroless, non-root, read-only rootfs), CI. -- Design set under `docs/design/`, including the Phase-1 hardware proof: one - physical GPU, one KubeSwift VM, two workloads sharing it through HAMi, with - neither upstream project modified. - -### Added — demand-driven scale-up (Phase 3) - -- `spec.autoscaling`: `enabled`, `minReplicas`, `maxReplicas`, `stabilizationWindow`, - `scaleDown` (`Manual` only; `Auto` is rejected). `maxReplicas` is required when - enabled, because an unbounded pool that misreads demand can consume every GPU in - the cluster. -- `PendingDemand` for HAMi DevicePlugin mode. Two filters are the whole safety of - the feature: only pods the scheduler could not place count (`PodScheduled=False` - / `Unschedulable`, which naturally excludes a pod stuck on a missing ConfigMap — - that pod was scheduled), and only requests one fresh cell could actually satisfy - count toward a decision. -- Scale-up is one cell at a time, behind a stabilization window, and never on - unread demand, an unknown cell shape, a saturated cluster, or a ceiling already - reached. `status.demand`, `status.desiredReplicas`, `status.lastScaleUpTime` and - the `ScalingActive` condition report every decision and its reason. - -### Added — automatic scale-down (Phase 4) - -- `spec.autoscaling.scaleDown: Auto` now works, gated on `minReplicas` being set: - without a floor the pool could shrink to zero and every later request would pay a - full cell boot. -- Only cells the capacity provider reports as **idle** are removable, and a cell - whose allocations cannot be read is never treated as idle — "empty" and "unknown" - are different answers, and only the first may lead to a deletion. -- Demand must have been absent for the whole `scaleDownStabilizationWindow` - (default 30m, deliberately longer than scale-up), tracked by - `status.demandFreeSince`. Absent *right now* is not the same thing: a pool that - shrinks between two bursts is worse than one that waits. -- Membership removes the autoscaler's named idle cells rather than the highest - index; highest-index-first remains the rule for an operator-driven shrink, where - the intent is "make it smaller" rather than "remove that one". - -### Added — metrics - -- `gpucell_*` Prometheus metrics, with the two layers deliberately kept apart: - `physical_gpus{held,free}` counts whole devices from the infrastructure cluster, - `capacity_gpu_*` reports fractional capacity from the workload cluster. An alert - can therefore tell "no GPU left in the cluster" from "the shared GPU is full". -- `cell_startup_seconds` measures creation to first Ready — the number that decides - whether autoscaling can be reactive at all. -- `cell_transitions_total` makes an oscillating cell visible even though its - instantaneous phase looks healthy; `capacity_scrape_errors_total` says the gauges - went stale, since a failed read retains the previous values rather than zeroing; - `scale_decisions_total` records refusals as well as scale-ups. - -### Fixed (from the first live run) + `SwiftGuestSpec` passthrough so this API never mirrors KubeSwift's — see + `docs/api-reference.md`. +- Cell state machine (`Pending` → `AllocatingGPU` → `GuestProvisioning` → + `Booting` → `Joining` → `AwaitingGPUCapacity` → `Ready`), membership planning + with churn control, physical-inventory pre-flight, per-cell bootstrap + rendering, drain-gated teardown, and status that separates the two layers + (`physicalCapacity` = whole devices, `workloadCapacity` = HAMi fractions). +- Validating webhook enforcing the API's rules, fail-closed — see + `docs/security.md`. + +### Added — GPU allocation + +- Two backends behind `spec.cell.gpu.backend`: `DRA` + (`gpuResourceClaim`/`ResourceClaimTemplate`, scheduler-time) and `Native` + (`gpuProfileRef`, controller-time). One whole `pcie`-tier GPU per cell. + +### Added — bootstrap + +- `spec.bootstrap.provider: Opaque` — user-supplied cloud-init with a closed + substitution set (`{{ cellName }}`, `{{ poolName }}`, `{{ nodeLabels }}`, + `{{ nodeIPInterface }}`, `{{ expectedGPUs }}`), rendered per cell into an + operator-owned Secret. The join credential is only ever referenced, never + written into a CR. +- `hack/build-cell-image.sh`: a reference cell image build (Ubuntu Noble + + NVIDIA driver + `nvidia-container-toolkit` + k0s worker), documented in + `docs/cell-image.md`. + +### Added — capacity and autoscaling + +- HAMi capacity provider (`DevicePlugin` mode), written against annotations + captured from real hardware rather than upstream documentation — see + `docs/design/gpucellpool-capacity.md`. +- `spec.autoscaling`, both directions — see `docs/autoscaling.md`: + - **Scale-up**: `enabled`, `minReplicas`, `maxReplicas` (required when + enabled), `stabilizationWindow` (default 10m). Demand is read from pods + the scheduler could not place (`PodScheduled=False`/`Unschedulable`) and + filtered against whether a fresh cell of this pool's shape would actually + satisfy it — the only two gates that make scale-up safe. + - **Scale-down**: `scaleDown: Auto` (requires `minReplicas`), removing only + cells the capacity provider reports **idle**, after demand has been + absent for the whole `scaleDownStabilizationWindow` (default 30m, longer + than scale-up on purpose). + - `status.cellDeviceShape`: the remembered GPU shape, kept after the last + cell is removed, so `minReplicas: 0` is recoverable instead of a one-way + door. + +### Added — Cluster API cells + +- `spec.cell.provisioner: ClusterAPI` + `spec.cell.clusterAPI` + (`clusterName`, `version`, `bootstrapConfigTemplateRef`): each cell becomes + a `Machine` + `KubeSwiftMachine`, one per cell (never a `MachineDeployment` + — see `docs/clusterapi-cells.md` for why that would have broken cell + identity). When `bootstrapConfigTemplateRef` is set, the workload cluster's + own bootstrap provider supplies join data instead of `spec.bootstrap`. +- `KubeSwiftMachine` can express only `imageRef`, `guestClassRef`, + `interfaces` — any other `guestTemplate` field is rejected at admission + under this provisioner rather than silently dropped. + +### Added — observability + +- `gpucell_*` Prometheus metrics (twelve series, `{pool, namespace}` labels + throughout): cell counts by phase, `cell_startup_seconds`, + `cell_transitions_total` (catches an oscillating cell even when its current + phase looks healthy), physical vs. workload capacity kept as separate + metric families, `capacity_scrape_errors_total` (a failed read retains the + previous value rather than reporting zero — the errors counter is how you'd + know), `scale_decisions_total` (including refusals), `reconcile_errors_total`. + +### Added — packaging and security + +- Helm chart, distroless/non-root/read-only-rootfs container image, CI. +- `config/rbac/workload-cluster-observer.yaml`: the minimal + `ServiceAccount`+`ClusterRole` a workload-cluster credential needs — no + `cluster-admin`, no `pods/eviction`, `nodes: delete` granted as an + always-on right (stale-Node reaping and cell teardown both need it + unconditionally, not only under automatic scale-down). + +### Fixed (found during hardware validation) - `cell.nodeIPFrom` is an observation field, not a readiness gate. KubeSwift - v0.13.4 reports a secondary NAD interface's MAC but not its IP, and the operator - does not control what the kubelet registers anyway (cloud-init derives the node - IP in-guest), so gating on it parked every bridge-NAD cell in `Booting`. -- The sample and design doc showed a `networkRef.kind` field that does not exist; - KubeSwift's strict decoding rejects it, so the sample could not have been applied. - -### Added — Cluster API cells (Phase 5) - -- `cell.provisioner: ClusterAPI` plus `cell.clusterAPI` (`clusterName`, `version`, - `bootstrapConfigTemplateRef`). Each cell becomes a `Machine` and a - `KubeSwiftMachine`, so a GPU cell added to a CAPI-managed cluster is a member of - it — with a `providerID`, visible to the cluster's own controllers — rather than a - node attached out of band. -- One Machine per cell, named after the cell, not a MachineDeployment sized to the - replica count. A MachineDeployment generates Machine names, and capi-kubeswift - derives the guest hostname (and so the Node name) from the Machine name, which - would break the cell-name==Node-name identity and leave no way to drain one cell. -- Bootstrap comes from the workload cluster's own provider when - `bootstrapConfigTemplateRef` is set: the operator instantiates the template once - per cell, as a MachineSet does, so tokens and CA hashes are the cluster's rather - than a secret somebody maintains. `spec.bootstrap.joinSecretRef` is then rejected - instead of ignored. Without a template ref, the pool's rendered Secret is handed - over as `dataSecretName`. -- A `KubeSwiftMachine` can express only image, class, two networks and GPU, so any - other `guestTemplate` field is rejected at admission rather than silently dropped. -- Validated end to end on hardware: pool to Ready in 6m29s, two workloads sharing the - cell's GTX 1080, teardown returning the GPU claim in under a minute. - See `docs/clusterapi-cells.md`, which also records what the workload cluster needs. - -### Fixed (from the Cluster API validation) - -- The pool claimed the **controller** owner reference on its Machines. Kubernetes - allows one per object and Cluster API needs it, so CAPI failed every reconcile with - "already owned by another GPUCellPool controller" — no bootstrap data, no VM, ever. - Cells are co-owned now, which is all garbage collection requires. -- Pool teardown listed SwiftGuests, so a ClusterAPI pool saw no cells, drained - nothing and dropped its own finalizer — orphaning each Machine with the drain - finalizer still on it, unclearable, GPU claim leaked. Teardown goes through the - provisioner now. -- Scale-to-zero was a one-way door. The satisfiability reference device came from - live capacity only, so an emptied pool had nothing to judge a request against, - counted nothing satisfiable, and never grew back — while blaming the request. - `status.cellDeviceShape` outlives the cells; a pool that never advertised a device - reports `CellShapeUnknown` and says what to do about it. -- `Ready` was measured against `spec.replicas`, which stops being the target once - autoscaling is on: a healthy pool holding two autoscaled cells reported "2 of 1 - cells are Ready" and False. A pool that deliberately holds none also reported - itself broken; both now say `ScaledToZero`. + v0.13.4 reports a secondary NAD interface's MAC but not its IP, and the + operator does not control what the kubelet registers anyway (cloud-init + derives the node IP in-guest), so gating readiness on it parked every + bridge-NAD cell in `Booting` forever. +- `networkRef` takes `{name, namespace}` only; an earlier sample and design + doc showed a `kind` field that does not exist and that KubeSwift's strict + decoding rejects — the sample as first written could not have been applied. +- Scale-to-zero was a one-way door: with the satisfiability reference device + coming from live capacity only, an emptied pool had nothing to judge new + demand against, counted nothing satisfiable, and never grew back — while + blaming the request. Fixed by `status.cellDeviceShape` (above). +- `Ready` was measured against `spec.replicas`, which stops being the target + once autoscaling is on: a healthy pool holding two autoscaled cells reported + "2 of 1 cells are Ready" and `False`. A pool deliberately holding zero cells + also reported itself broken. Both now report `ScaledToZero` correctly + against `status.desiredReplicas`. +- The pool claimed the **controller** owner reference on ClusterAPI Machines. + Kubernetes allows one per object and Cluster API needs it, so every + reconcile failed with "already owned by another GPUCellPool controller" — + no bootstrap data, no VM, ever. Cells are co-owned now. +- ClusterAPI pool teardown listed SwiftGuests, so it saw no cells, drained + nothing, and dropped its own finalizer — orphaning each Machine with the + drain finalizer still on it, GPU claim leaked. Teardown goes through the + provisioner's own lister now. ### Known gaps -- `hami.mode: DRA` reports `ErrUnsupported`; only DevicePlugin mode is implemented. -- Pools of two or more cells are covered by the two-apiserver harness, not by - hardware — the lab has one GPU. So are `deletion.policy: Force` and cell - replacement backoff. -- Cell startup measures 4m45s, about three minutes of which is cloning a 30 GiB root - disk. A smaller disk or a copy-on-write clone is where the next minute is. -- A join template must derive the cell's routable address by subnet: `nodeIPFrom` - names a KubeSwift interface, and cloud-init cannot map that to a guest device. +- `hami.mode: DRA` reports `ErrUnsupported`; only `DevicePlugin` mode is + implemented. +- Pools of two or more cells are covered by the two-apiserver test harness, + not by hardware — the reference lab has one GPU. So are + `deletion.policy: Force` and cell replacement backoff. +- Cell startup measures 4m45s to Ready, about three minutes of which is + cloning a 30 GiB root disk. A smaller disk or a copy-on-write clone + strategy is where the next minute would come from. +- A join template must derive the cell's routable address by subnet: + `nodeIPFrom` names a KubeSwift interface, and cloud-init cannot map that to + a guest device. +- No rolling update on `guestTemplate` change, no automated outer-drain + sequencing, no bootstrap token minting. See `docs/limitations.md` for the + full list and the operational workarounds. diff --git a/README.md b/README.md index 90f238a..8a54501 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ boundary; HAMi owns GPU → workload allocation; this operator owns the lifecycl between them. A HAMi fraction is never handed to VFIO — the two layers nest, they do not translate. +This is **layered isolation**, not tenant isolation: a cell is a VM boundary +around the GPU, not a hard security perimeter, and every KubeSwift launcher pod +is privileged in the infrastructure cluster — see `docs/security.md`. + ## Status **v0.1.0 — alpha.** Every capability has been run on real hardware: one physical GPU @@ -65,6 +69,7 @@ spec: ```bash helm install gpucellpool oci://ghcr.io/kubeswift-io/charts/gpucellpool \ + --version 0.1.0 \ --namespace gpucellpool-system --create-namespace ``` @@ -85,22 +90,19 @@ privileged in the infrastructure cluster. | | | |---|---| -| infrastructure cluster | KubeSwift ≥ v0.13.4, a GPU node (`kubeswift.io/gpu-node=true`), a `DeviceClass` for VFIO GPUs | +| infrastructure cluster | KubeSwift ≥ v0.13.4, a GPU node (`kubeswift.io/gpu-node=true`), a `DeviceClass` + `ResourceClaimTemplate` for VFIO GPUs, Multus + a NAD carrying a routable address, a `SwiftGuestClass` for the cell VM | | workload cluster | HAMi installed, reachable from the operator, and reachable **both ways** for kubelet (cells need a routable interface, not just egress) | | cell image | a `SwiftImage` with the NVIDIA driver, containerd + CDI, and your distribution's node binaries | +Budget **~5 minutes** for a first cell to go `Pending` → `Ready` — most of it is +cloning the root disk, not booting. See `docs/quickstart.md`. + ## Documentation -| Doc | Contents | -|---|---| -| [overview](docs/design/gpucellpool-overview.md) | architecture, decisions, scope, phases | -| [api](docs/design/gpucellpool-api.md) | the v1alpha1 CRD and its validation rules | -| [reconciliation](docs/design/gpucellpool-reconciliation.md) | cell state machine, identity, RBAC, deletion | -| [bootstrap](docs/design/gpucellpool-bootstrap.md) | cell image strategy, join credentials | -| [capacity](docs/design/gpucellpool-capacity.md) | how HAMi capacity is read | -| [failure-model](docs/design/gpucellpool-failure-model.md) | what breaks and what the operator does about it | -| [poc](docs/design/gpucellpool-poc.md) | hardware proof and test strategy | -| [runbook](docs/runbook.md) | what to check when a pool misbehaves | +Start at [`docs/README.md`](docs/README.md) — it separates operator-facing +docs (quickstart, concepts, API reference, networking, security, autoscaling, +runbook) from the design record (`docs/design/`, decisions and rationale, kept +for history rather than as the current spec). ## Licence diff --git a/api/v1alpha1/gpucellpool_types.go b/api/v1alpha1/gpucellpool_types.go index 43fee83..8da8023 100644 --- a/api/v1alpha1/gpucellpool_types.go +++ b/api/v1alpha1/gpucellpool_types.go @@ -26,7 +26,14 @@ type GPUCellPoolSpec struct { Cell CellSpec `json:"cell"` // Bootstrap describes how a cell becomes a worker of the workload cluster. - Bootstrap BootstrapSpec `json:"bootstrap"` + // + // Optional because it is genuinely unused when the workload cluster's own Cluster + // API bootstrap provider supplies the join data (cell.clusterAPI. + // bootstrapConfigTemplateRef). Requiring it there forced an empty `bootstrap: {}` + // into the manifest to satisfy the schema — a field you must write and nothing + // reads. The webhook still requires joinSecretRef whenever it IS the join path. + // +optional + Bootstrap BootstrapSpec `json:"bootstrap,omitempty"` // WorkloadCluster is the cluster the cells join and where HAMi runs. WorkloadCluster WorkloadClusterSpec `json:"workloadCluster"` diff --git a/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml b/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml index a238c87..73f5dbe 100644 --- a/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml +++ b/charts/gpucellpool/crds/cells.kubeswift.io_gpucellpools.yaml @@ -126,8 +126,14 @@ spec: type: string type: object bootstrap: - description: Bootstrap describes how a cell becomes a worker of the - workload cluster. + description: |- + Bootstrap describes how a cell becomes a worker of the workload cluster. + + Optional because it is genuinely unused when the workload cluster's own Cluster + API bootstrap provider supplies the join data (cell.clusterAPI. + bootstrapConfigTemplateRef). Requiring it there forced an empty `bootstrap: {}` + into the manifest to satisfy the schema — a field you must write and nothing + reads. The webhook still requires joinSecretRef whenever it IS the join path. properties: hostname: default: CellName @@ -517,7 +523,6 @@ spec: - kubeconfigSecretRef type: object required: - - bootstrap - cell - replicas - workloadCluster diff --git a/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml b/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml index a238c87..73f5dbe 100644 --- a/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml +++ b/config/crd/bases/cells.kubeswift.io_gpucellpools.yaml @@ -126,8 +126,14 @@ spec: type: string type: object bootstrap: - description: Bootstrap describes how a cell becomes a worker of the - workload cluster. + description: |- + Bootstrap describes how a cell becomes a worker of the workload cluster. + + Optional because it is genuinely unused when the workload cluster's own Cluster + API bootstrap provider supplies the join data (cell.clusterAPI. + bootstrapConfigTemplateRef). Requiring it there forced an empty `bootstrap: {}` + into the manifest to satisfy the schema — a field you must write and nothing + reads. The webhook still requires joinSecretRef whenever it IS the join path. properties: hostname: default: CellName @@ -517,7 +523,6 @@ spec: - kubeconfigSecretRef type: object required: - - bootstrap - cell - replicas - workloadCluster diff --git a/config/samples/cell-join-secret.yaml b/config/samples/cell-join-secret.yaml new file mode 100644 index 0000000..3263f29 --- /dev/null +++ b/config/samples/cell-join-secret.yaml @@ -0,0 +1,233 @@ +# The cloud-init a cell uses to join YOUR workload cluster. +# +# GPUCellPool ships no default here on purpose (docs/design/gpucellpool-bootstrap.md +# §4): the join blob is distribution-specific and carries your cluster's credential. +# Pick ONE of the two Secrets below (k0s or kubeadm), fill in the placeholders that +# are NOT `{{ }}` tokens (search this file for "FILL IN"), and apply it. Reference +# whichever name you keep from spec.bootstrap.joinSecretRef. +# +# ── The substitution set is CLOSED (internal/bootstrap/render.go) ────────────── +# +# {{ cellName }} -, e.g. inference-0 — becomes the guest +# hostname AND the workload Node name +# {{ poolName }} the GPUCellPool's name +# {{ nodeLabels }} a ready-to-use `k=v,k=v` string: your +# spec.workloadCluster.node.labels PLUS the identity +# labels (cells.kubeswift.io/{pool,cell,cell-index}) — +# pass this straight to --node-labels +# {{ nodeIPInterface }} the guestTemplate interface name from cell.nodeIPFrom +# (e.g. "node") — for logging only. It is a KubeSwift +# interface NAME, not a guest device (not "ens4"), and +# cloud-init cannot map one to the other. Derive the +# actual address by SUBNET instead — see below. +# {{ expectedGPUs }} cell.gpu.count, for the preflight log line +# +# An unknown token (a typo, or one from an older draft of this file) is a +# RENDER ERROR, not a silent pass-through — the operator refuses to create a +# cell from a template it cannot fully substitute. +# +# ── Quote every token in YAML value position ─────────────────────────────────── +# +# hostname: {{ cellName }} is INVALID YAML — {{...}} parses as a nested +# flow mapping and the file cannot be validated +# before substitution. +# hostname: "{{ cellName }}" is valid before AND after substitution. +# +# Every token below is already quoted for this reason. Keep it that way if you +# add more. +# +# ── The node IP must be derived by subnet, with a wait loop ─────────────────── +# +# The controller does not know the cell's routable address at render time (DHCP +# on the networkRef NAD assigns it after the guest exists), and KubeSwift +# v0.13.4 does not report a secondary NAD interface's IP at all — only its MAC +# (docs/networking.md). So cloud-init must read it from the guest's own +# interfaces, by subnet, and wait for it to appear: +# +# for i in $(seq 1 30); do +# NODE_IP=$(ip -4 -o addr show | awk '/10\.77\.0\./ {split($4,a,"/"); print a[1]; exit}') +# [ -n "$NODE_IP" ] && break +# sleep 2 +# done +# [ -n "$NODE_IP" ] || { echo "no routable address after 60s" >&2; exit 1; } +# +# Replace `10\.77\.0\.` with YOUR NAD's subnet (config/samples/ +# network-attachment-definition.yaml uses 10.77.0.0/24). This hardcodes your +# cell subnet into the template — that is the least-bad option available today. +# +# ── containerd 2.x needs the v3 config schema ────────────────────────────────── +# +# k0s >= 1.34 ships containerd 2.x. A drop-in written in the OLDER v1 CRI form +# (`version = 2` with `[plugins."io.containerd.grpc.v1.cri"]`) is REJECTED at +# k0s pre-flight: +# +# Rejected: unsupported configuration version: expected 3, got 2 +# property=/etc/k0s/containerd.d/nvidia.toml +# Error: pre-flight checks failed +# +# and the worker service never starts — SILENTLY, from the outside: the VM +# boots, sshd answers, both addresses come up, cloud-init reports success and +# even logs `EXIT=0`, because the JOIN SCRIPT finished; it was the SERVICE it +# installed that refused to start. This cost two cell rebuilds to find. Use the +# v3 form, `[plugins."io.containerd.cri.v1.runtime"]`, as below. +# +# ── DNS breaks once the inner CNI comes up ───────────────────────────────────── +# +# systemd-resolved's stub forwards to the pod-netns dnsmasq (192.168.99.1) by +# default. Once the workload cluster's CNI programs the node, that path stops +# answering while raw egress keeps working — so `apt`/image pulls fail right +# when the join is finishing. Pin an upstream resolver (below) rather than +# trust the default. +# +# ── Without an SSH key, a broken cell is undiagnosable ───────────────────────── +# +# The baked image resets machine-id and host keys, and KubeSwift's Cloud +# Hypervisor drops the serial console entirely when no client is attached — so +# cloud-init's own output is not captured anywhere you can read after the fact. +# `ssh_authorized_keys` is not optional if you want to debug a cell that joins +# wrong. +--- +# ═══════════════════════════════════════════════════════════════════════════ +# Variant 1: k0s worker join +# ═══════════════════════════════════════════════════════════════════════════ +apiVersion: v1 +kind: Secret +metadata: + name: inference-cluster-join-k0s + namespace: gpu-cells +stringData: + user-data: | + #cloud-config + hostname: "{{ cellName }}" + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... FILL IN your public key + + write_files: + # Pin an upstream resolver before the inner CNI can break the default one. + - path: /etc/systemd/resolved.conf.d/10-upstream.conf + content: | + [Resolve] + DNS=1.1.1.1 8.8.8.8 + + # containerd 2.x / v3 schema — the v1 CRI form is rejected by k0s >= 1.34. + - path: /etc/k0s/containerd.d/nvidia.toml + content: | + [plugins."io.containerd.cri.v1.runtime"] + enable_cdi = true + cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] + [plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "nvidia" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + + # The join script itself. Written to a file, not inlined into runcmd as a + # flow-sequence list item — cloud-init's runcmd accepts either a plain + # string or a [cmd, arg, ...] list, and a list item cannot itself hold a + # multi-line block scalar; a YAML parser rejects it outright. + - path: /usr/local/bin/gpu-cell-join.sh + permissions: "0755" + content: | + #!/bin/bash + set -euo pipefail + + # FILL IN: your NAD's subnet — see network-attachment-definition.yaml + for i in $(seq 1 30); do + NODE_IP=$(ip -4 -o addr show | awk '/10\.77\.0\./ {split($4,a,"/"); print a[1]; exit}') + [ -n "$NODE_IP" ] && break + sleep 2 + done + [ -n "$NODE_IP" ] || { echo "no routable address after 60s" >&2; exit 1; } + echo "gpu-cell: node IP $NODE_IP (interface role: {{ nodeIPInterface }}, expecting {{ expectedGPUs }} GPU(s))" + + # k0s installs one role per cell — mutually exclusive with a + # controller install, and k0s reset does not clean up a second + # installed unit, so guard against a stale one before installing. + if ! systemctl is-enabled k0sworker.service >/dev/null 2>&1; then + # FILL IN: your k0s join token (a k0s token, NOT a kubeadm bootstrap + # token — mint it with `k0s token create --role=worker` on the + # control plane, then paste the resulting blob here as one line). + echo "FILL_IN_K0S_JOIN_TOKEN_BASE64" | base64 -d > /etc/k0s-join-token + + k0s install worker \ + --token-file=/etc/k0s-join-token \ + --labels="{{ nodeLabels }}" \ + --kubelet-extra-args="--node-ip=${NODE_IP}" + systemctl enable --now k0sworker.service + fi + + runcmd: + - systemctl restart systemd-resolved + - /usr/local/bin/gpu-cell-join.sh +--- +# ═══════════════════════════════════════════════════════════════════════════ +# Variant 2: kubeadm join +# ═══════════════════════════════════════════════════════════════════════════ +apiVersion: v1 +kind: Secret +metadata: + name: inference-cluster-join-kubeadm + namespace: gpu-cells +stringData: + user-data: | + #cloud-config + hostname: "{{ cellName }}" + ssh_authorized_keys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA... FILL IN your public key + + write_files: + - path: /etc/systemd/resolved.conf.d/10-upstream.conf + content: | + [Resolve] + DNS=1.1.1.1 8.8.8.8 + + # containerd 2.x / v3 schema. If your kubeadm distribution ships + # containerd 1.7, use the v1 CRI form instead + # ([plugins."io.containerd.grpc.v1.cri"]) — check `containerd --version` + # on a control-plane node before assuming this one applies. + - path: /etc/containerd/conf.d/nvidia.toml + content: | + [plugins."io.containerd.cri.v1.runtime"] + enable_cdi = true + cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"] + [plugins."io.containerd.cri.v1.runtime".containerd] + default_runtime_name = "nvidia" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia.options] + BinaryName = "/usr/bin/nvidia-container-runtime" + SystemdCgroup = true + + # The join script itself. Written to a file, not inlined into runcmd as a + # flow-sequence list item — cloud-init's runcmd accepts either a plain + # string or a [cmd, arg, ...] list, and a list item cannot itself hold a + # multi-line block scalar; a YAML parser rejects it outright. + - path: /usr/local/bin/gpu-cell-join.sh + permissions: "0755" + content: | + #!/bin/bash + set -euo pipefail + + # FILL IN: your NAD's subnet. + for i in $(seq 1 30); do + NODE_IP=$(ip -4 -o addr show | awk '/10\.77\.0\./ {split($4,a,"/"); print a[1]; exit}') + [ -n "$NODE_IP" ] && break + sleep 2 + done + [ -n "$NODE_IP" ] || { echo "no routable address after 60s" >&2; exit 1; } + echo "gpu-cell: node IP $NODE_IP (interface role: {{ nodeIPInterface }}, expecting {{ expectedGPUs }} GPU(s))" + + # FILL IN: your control-plane endpoint, token and CA hash — + # kubeadm token create --print-join-command + # on a control-plane node prints all three. + kubeadm join FILL_IN_ENDPOINT:6443 \ + --token FILL_IN_TOKEN \ + --discovery-token-ca-cert-hash sha256:FILL_IN_CA_HASH \ + --node-name "{{ cellName }}" \ + --kubelet-extra-args "--node-ip=${NODE_IP} --node-labels={{ nodeLabels }}" + + runcmd: + - systemctl restart systemd-resolved containerd + - /usr/local/bin/gpu-cell-join.sh diff --git a/config/samples/cells_v1alpha1_gpucellpool_clusterapi.yaml b/config/samples/cells_v1alpha1_gpucellpool_clusterapi.yaml index e239439..8ae9d32 100644 --- a/config/samples/cells_v1alpha1_gpucellpool_clusterapi.yaml +++ b/config/samples/cells_v1alpha1_gpucellpool_clusterapi.yaml @@ -12,8 +12,9 @@ # # Extra prerequisites in the INFRASTRUCTURE cluster (which is also the CAPI # management cluster here): -# - Cluster API >= v1.11 (contract v1beta2) and cluster-api-provider-kubeswift -# >= the release carrying KubeSwiftMachine spec.backend.swiftGuest.gpu +# - Cluster API >= v1.13.4 (the validated floor; see docs/clusterapi-cells.md) +# and cluster-api-provider-kubeswift >= v0.2.0, which ships the GPU surface +# (KubeSwiftMachine spec.backend.swiftGuest.gpu) and nodeName placement # - a CAPI Cluster named below, already provisioned, in this namespace # - the bootstrap template below, if you use one apiVersion: cells.kubeswift.io/v1alpha1 diff --git a/config/samples/network-attachment-definition.yaml b/config/samples/network-attachment-definition.yaml new file mode 100644 index 0000000..c8fd3c2 --- /dev/null +++ b/config/samples/network-attachment-definition.yaml @@ -0,0 +1,35 @@ +# A minimal Multus NAD carrying a ROUTABLE address for cell traffic. +# +# Every GPU cell needs a routable interface, not just KubeSwift's node-local nat +# egress: the workload apiserver must be able to dial the cell's kubelet for +# logs, exec, port-forward and metrics (docs/networking.md). Reference this NAD +# from cell.guestTemplate.interfaces[].networkRef, and point cell.nodeIPFrom at +# that interface's name. +# +# This bridge+host-local shape is enough while every VM that needs to reach it +# (including a workload-cluster control plane, if it is itself a KubeSwift +# guest) is scheduled on the SAME infrastructure node — KubeSwift's br0 lives +# inside each launcher pod's own network namespace, so there is no way for two +# guests on different nodes to reach each other over it. Cross-node reachability +# needs KubeSwift's routable secondary-NAD shape (see the KubeSwift networking +# docs) — apply the equivalent of that NAD instead if your cells span nodes. +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: cell-net + namespace: gpu-cells +spec: + config: | + { + "cniVersion": "0.4.0", + "name": "cell-net", + "type": "bridge", + "bridge": "cellbr0", + "isGateway": true, + "ipam": { + "type": "host-local", + "subnet": "10.77.0.0/24", + "rangeStart": "10.77.0.10", + "rangeEnd": "10.77.0.250" + } + } diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..4d1a9c0 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,40 @@ +# Documentation index + +## Using gpucellpool + +Start here if you are installing or operating a pool. + +| Doc | Read it for | +|---|---| +| [quickstart](quickstart.md) | the shortest path from nothing to two workloads sharing one GPU in a cell | +| [concepts](concepts.md) | the two-layer model — cells, the two identities, the two capacities, layered isolation | +| [api-reference](api-reference.md) | the full `GPUCellPool` v1alpha1 spec/status, generated from the Go types and the validating webhook | +| [autoscaling](autoscaling.md) | `spec.autoscaling` — both directions, the safety gates, the remembered cell shape | +| [networking](networking.md) | the routable-interface requirement, `nodeIPFrom`, NADs, DNS, `port-forward` | +| [security](security.md) | why creating a pool is node-root-equivalent authority, the webhook, the two RBAC scopes | +| [cell-image](cell-image.md) | building and publishing a cell image with `hack/build-cell-image.sh` | +| [clusterapi-cells](clusterapi-cells.md) | `provisioner: ClusterAPI` — cells as Cluster API Machines | +| [limitations](limitations.md) | what is not implemented, not validated on hardware, or deliberately manual | +| [runbook](runbook.md) | what to check when a pool is not doing what you expect | + +`config/samples/` has ready-to-apply manifests: a `SwiftGuest`-provisioned pool, +a `ClusterAPI`-provisioned pool, a join-Secret template (k0s + kubeadm), and a +NAD. + +## Design and rationale (historical) + +`docs/design/*.md` is the design record: why the API and controller look the +way they do. It predates and partially postdates implementation — each file +carries a banner saying so. Read it when you want the reasoning behind a +decision, not the current behaviour; for current behaviour, use the docs above. + +| Doc | Contents | +|---|---| +| [overview](design/gpucellpool-overview.md) | the architectural invariant, decisions D1–D10, scope, phases | +| [api](design/gpucellpool-api.md) | the original API design (superseded by `api-reference.md` for current fields) | +| [reconciliation](design/gpucellpool-reconciliation.md) | the cell state machine, identity model, RBAC, deletion sequencing | +| [bootstrap](design/gpucellpool-bootstrap.md) | why prebaked-image + thin cloud-init, the measured startup budget | +| [capacity](design/gpucellpool-capacity.md) | how HAMi capacity is read, both accounting modes | +| [failure-model](design/gpucellpool-failure-model.md) | the failure scenario matrix and churn control | +| [validation-record](design/gpucellpool-validation-record.md) | the hardware proof: Phase 1–2 results, what was measured, what broke | +| [clusterapi-cells (validation)](design/clusterapi-cells-validation.md) | the CAPI provisioner post-mortem: two bugs only a live cluster found, the dev recipe | diff --git a/docs/api-reference.md b/docs/api-reference.md index e4a43b9..0fa5ca3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -258,6 +258,7 @@ cannot misconfigure a cell. | `cell.gpu.count != 1` | create, update | multi-GPU cells | | `cell.gpu.backend` set with the wrong sub-struct (`native` with `DRA`, `dra` with `Native`, or missing) | create, update | mixed or absent backend config | | `cell.gpu.dra`: not exactly one of `resourceClaimTemplateName`/`resourceClaimName` | create, update | both or neither set | +| `cell.gpu.dra.resourceClaimName` set with `spec.replicas > 1`, or with `autoscaling.enabled` and `autoscaling.maxReplicas > 1` | create, update | a shared claim backing more than one cell would double-book the device — use `resourceClaimTemplateName` for a pool that can grow | | `cell.gpu.dra.tier != pcie` | create, update | `hgx-shared`, `hgx-full` | | `guestTemplate` sets an operator-owned or denied field | create, update | see the field contract above | | `guestTemplate.guestClassRef` / `.imageRef` unset | create, update | a GPU cell must be a disk boot | diff --git a/docs/autoscaling.md b/docs/autoscaling.md new file mode 100644 index 0000000..4a348fc --- /dev/null +++ b/docs/autoscaling.md @@ -0,0 +1,139 @@ +# Autoscaling + +`spec.autoscaling` lets unsatisfiable GPU demand in the *workload* cluster grow +a pool, and lets it shrink again once that demand is gone. Both directions are +shipped. This page is the operator-facing view; the mechanism is designed in +`docs/design/gpucellpool-capacity.md` §8 and implemented in +`internal/controller/scaling.go`. + +```yaml +spec: + autoscaling: + enabled: true + minReplicas: 0 # optional; defaults to spec.replicas when omitted + maxReplicas: 4 # required whenever enabled: true + stabilizationWindow: 10m # default + scaleDown: Auto # Manual (default) | Auto + scaleDownStabilizationWindow: 30m # default +``` + +## Why two filters gate every scale-up + +A pod pending for a GPU is not, by itself, a reason to create a cell. Two gates +are the entire safety of the feature — get either wrong and the pool burns a +physical GPU and several minutes of boot for nothing: + +1. **The demand must be GPU-capacity-constrained.** Only pods with + `PodScheduled=False` and reason `Unschedulable` count. A pod pending on a + missing ConfigMap or an unpullable image has already been *scheduled* + (`PodScheduled=True`) — it can never look like GPU demand, because the + discriminator excludes it by construction, not by a heuristic that might + miss a case. +2. **A fresh cell of *this pool's* shape must actually satisfy the request.** + A request for two devices, or for more memory or compute than one device + has, is counted as pending but explicitly **not** satisfiable — adding a + cell would change nothing. Only `status.demand.satisfiableByOneCell` drives + scale-up decisions, never the raw pending count. + +HAMi itself helps here: a request larger than a single device is refused at +**scheduling** time (`NodeUnfitPod`), so an over-large request reliably shows up +as an unschedulable pod rather than silently binding somewhere wrong — which is +exactly the signal the first gate needs. + +## Scale-up + +One cell at a time, gated in order: + +| Gate | `ScalingActive` reason if it blocks | +|---|---| +| demand could not be read from the workload cluster | `Unknown` | +| the pool has never advertised a device (no shape to compare against) | `CellShapeUnknown` | +| something is pending, but none of it fits one cell of this pool's shape | `DemandUnsatisfiable` | +| already at `maxReplicas` | `AtMaxReplicas` | +| demand is satisfiable, but no free GPU remains in the infrastructure cluster | `InsufficientPhysicalGPU` | +| a cell was created inside the `stabilizationWindow` | `Stabilizing` | +| — none of the above — | scales up by exactly one cell, reason `ScaledUp` | + +`stabilizationWindow` defaults to **10 minutes**. A cell measures ~4m45s to +Ready (see `docs/design/gpucellpool-bootstrap.md` §6), and demand does not clear +until the new cell is actually Ready — without this window, one burst of +pending pods would create a cell per reconcile. + +## Scale-down + +Only reached once demand is entirely satisfied (`satisfiableByOneCell == 0`). +Every gate here is a reason **not** to shrink, because the failure mode is +asymmetric: holding an idle GPU costs money, removing a wanted one costs a full +cell boot and the workload that was about to run on it. + +| Gate | `ScalingActive` reason if it blocks | +|---|---| +| `scaleDown != Auto` | `Idle` (nothing to do — use `spec.replicas`/`minReplicas` to shrink manually) | +| already at `minReplicas` | `AtMinReplicas` | +| every Ready cell still holds workloads | `NoIdleCell` | +| demand has not been absent for the *whole* `scaleDownStabilizationWindow` yet, or a scale action happened inside it | `Stabilizing` | +| — none of the above — | removes exactly one **idle** cell, reason `ScaledDown` | + +`scaleDownStabilizationWindow` defaults to **30 minutes** — deliberately longer +than the scale-up window. Removing a cell that is about to be wanted again costs +a full boot; a pool that shrinks between two demand bursts is worse than one +that waits. "Demand absent" is tracked from `status.demandFreeSince`: absence +*right now* is not enough, it must have held for the entire window. + +Only cells the capacity provider reports as **idle** (zero HAMi allocations) are +candidates, and a cell whose allocations cannot be read is never treated as +idle — "empty" and "unknown" are different answers, and only the first may lead +to a deletion. The drain gate re-checks allocations again immediately before the +cell is actually removed, so a workload that lands on a cell between the +autoscaler's decision and the deletion is not destroyed. + +`scaleDown: Auto` **requires `minReplicas` to be set explicitly** — without a +floor the pool could shrink to zero and every later request would pay a full +cell boot. This is enforced by the validating webhook. + +## `minReplicas: 0` and the remembered cell shape + +`minReplicas: 0` is allowed and safe to use. The mechanism that makes it +recoverable is `status.cellDeviceShape`: whenever a live cell advertises a +device, the pool remembers its model, `memoryMiB` and `corePercent`. That +memory **outlives the cells** — it is not cleared when the pool scales to zero. + +Without it, an emptied pool would have nothing to judge a request against: with +no live cell to read a shape from, every request would compare against nothing, +read as "does not fit", and the pool would refuse to grow back — forever, +while reporting a misleading `DemandUnsatisfiable`. Resolution order is +live-then-remembered: live capacity is preferred because it is current: the +remembered shape is consulted only in exactly the scaled-to-zero case. + +**Caveat, stated plainly: nothing invalidates the remembered shape.** If a pool +scales to zero and its `guestTemplate.imageRef` or `cell.gpu` claim template is +then repointed at a **different GPU model**, `status.cellDeviceShape` still +holds the old model's numbers. A fresh cell judged against the stale shape can +be created (or refused) based on a device the pool no longer actually brings. +There is no drift detection for this in v1alpha1 — if you change a scaled-to- +zero pool's GPU shape, scale it to at least 1 once so the shape re-learns, +rather than trusting the memory. + +When a pool has *never* advertised a device (a brand-new pool with +`minReplicas: 0`), it reports `CellShapeUnknown` rather than +`DemandUnsatisfiable` — a distinct reason, because the fix differs: set +`replicas` or `minReplicas` to 1 once so the pool learns its shape, rather than +concluding demand does not fit. + +## Watching a decision happen + +```bash +kubectl get cellpool -n -o jsonpath='{.status.conditions[?(@.type=="ScalingActive")]}' +kubectl get cellpool -n -o jsonpath='{.status.demand}' +kubectl get cellpool -n -o jsonpath='{.status.cellDeviceShape}' +``` + +`status.demand.pendingRequests` is the raw pending count; +`status.demand.satisfiableByOneCell` is what actually drives the decision — the +two can legitimately differ (e.g. two pods pending, one of them asking for more +memory than a device has). + +## Known gap + +`hami.mode: DRA` capacity reads `PendingDemand` as `ErrUnsupported` — demand +signal is implemented for `DevicePlugin` mode only. See `docs/limitations.md`. diff --git a/docs/cell-image.md b/docs/cell-image.md new file mode 100644 index 0000000..01615bd --- /dev/null +++ b/docs/cell-image.md @@ -0,0 +1,118 @@ +# The cell image + +A cell boots from a prebaked disk image — a GPU-capable Kubernetes worker with +the driver, container runtime and node binaries already installed, so joining +is thin cloud-init rather than a 5-15 minute install-at-boot. This page covers +building one and what it must contain. + +## What the image must satisfy + +| Requirement | Why | +|---|---| +| NVIDIA driver | HAMi-core needs >= 440; match it to the CUDA version your workload images expect | +| `nvidia-container-toolkit` + CDI enabled | HAMi's interposition library needs the nvidia container runtime; CDI generation must happen on the cell itself, at first boot — the GPU is not present until then | +| your distribution's node binaries | kubelet + the container runtime for k0s/kubeadm/RKE2/k3s, whichever the workload cluster expects | +| a pinned upstream DNS resolver | once the inner CNI programs the node, the pod-netns resolver stops answering — see `docs/networking.md` | +| an SSH key in the join template | the bake resets `machine-id` and host keys, and KubeSwift's Cloud Hypervisor drops the serial console when nothing is attached — without SSH a broken cell is undiagnosable | + +Nothing above is HAMi- or KubeSwift-specific configuration baked into the +image beyond what any GPU Kubernetes worker needs. HAMi's own DaemonSets +(device plugin / scheduler extension) run in the *workload* cluster, not on +the cell. + +## `hack/build-cell-image.sh` — the reference build + +The script is a first-class, documented path — not a one-off used for the +validation record. It bakes Ubuntu Noble 24.04 + the NVIDIA proprietary driver ++ `nvidia-container-toolkit` + a k0s worker binary into a raw disk, under plain +QEMU with user-mode networking (no root, no libguestfs). + +```bash +NVIDIA_DRIVER_PKG=nvidia-driver-570-server \ +OUT=./build \ + hack/build-cell-image.sh +``` + +| Variable | Default | Notes | +|---|---|---| +| `OUT` | `./build` | output directory; produces `$OUT/gpu-worker-noble.raw` | +| `QEMU` | `/usr/bin/qemu-system-x86_64` | must be the **distribution** QEMU — a Kata-bundled build (sometimes first on `PATH` at `/opt/kata/bin`) is compiled without user-mode networking, so the guest gets no egress and the bake dies mid-`apt` with nothing obviously wrong | +| `NVIDIA_DRIVER_PKG` | `nvidia-driver-570-server` | see "driver selection" below | +| `DISK_SIZE` | `30G` | the validated size; sparse output is ~5.5 GiB | +| `MEM` / `CPUS` | `4096` / `4` | build-VM resources, not the cell's | + +Output: a 30 GiB raw disk (~5.5 GiB sparse on disk), plus an in-guest +`/etc/gpucell-image-manifest` recording exactly what was installed (kernel, +driver version, k0s version) — read that file rather than assuming the script +defaults match what actually landed, since some of the pinned inputs (below) +resolve to a range, not an exact version. + +**No GPU is needed at build time.** The NVIDIA driver builds via DKMS against +the image's own kernel; it only needs the physical GPU present at *cell boot*, +when DKMS's kernel module actually loads. + +## Driver selection + +Pascal (GTX 1080, the reference-lab GPU) needs the **proprietary** driver — +NVIDIA's open kernel modules are Turing-and-later only. `nvidia-driver- +-server` is correct for Pascal; if your GPU is Turing or newer, an +`-open` variant is also valid, but the script defaults to `-server` because +it is the one validated on hardware. + +The metapackage tracks a **branch, not a pin**: `nvidia-driver-570-server` +resolved to `580.173.02` on the validated build. For a reproducible image, set +`NVIDIA_DRIVER_PKG` to an exact package version; otherwise treat +`/etc/gpucell-image-manifest` inside the built image as the record of what you +actually shipped, not the variable you set. + +## Publishing + +The image is imported as a `SwiftImage` via an OCI artifact, using KubeSwift's +`swiftctl`: + +```bash +swiftctl image publish ./build/gpu-worker-noble.raw \ + --to ghcr.io/your-org/gpu-worker-noble \ + --tag noble-570 \ + --chunk-size-mib 256 +``` + +**Set `--chunk-size-mib 256`.** The 64 MiB default issues far more requests +for a disk this size than GitHub Container Registry's per-repository +secondary rate limit tolerates, and the push fails partway through with a 429 +that looks unrelated to chunk size unless you already know to look for it. + +Then reference it from a `SwiftImage`: + +```yaml +apiVersion: image.kubeswift.io/v1alpha1 +kind: SwiftImage +metadata: + name: gpu-worker-noble-570 +spec: + format: raw # this is the INPUT format — the raw disk above, not qcow2 + source: + oci: + ref: ghcr.io/your-org/gpu-worker-noble:noble-570 +``` + +Getting `spec.format` wrong is a trap independent of this script: if you ever +import a **qcow2** cloud image (e.g. Ubuntu's stock cloud image) instead of +this script's raw output, `spec.format` must say `qcow2`. Declaring `raw` for +a qcow2 source skips conversion and hands Cloud Hypervisor a file it reads as +raw, failing fast during import ("Failed to get refcount"). + +## Alternatives to a prebaked image + +Two other strategies exist and are documented in +`docs/design/gpucellpool-bootstrap.md` §2 for completeness, but neither is +implemented or recommended over the baked image: + +- **install-at-boot** — a generic cloud image, driver installed by cloud-init + on every boot. Measured ~14 minutes to Ready versus ~4m45s baked; not worth + it unless you cannot maintain a custom image at all. +- **driver from the workload cluster** (a GPU-operator-style driver container, + no driver baked into the cell) — the better long-term answer once + driver-version churn across several workload clusters costs more than the + extra boot minutes, but it needs no API change: it is just a different + `imageRef`. Nothing in GPUCellPool prevents building this yourself. diff --git a/docs/clusterapi-cells.md b/docs/clusterapi-cells.md index 06c3dc0..931cd6b 100644 --- a/docs/clusterapi-cells.md +++ b/docs/clusterapi-cells.md @@ -1,105 +1,119 @@ -# ClusterAPI cells — validated recipe and what it cost +# Cells as Cluster API Machines -> `provisioner: ClusterAPI` was validated end to end on real hardware on -> **2026-08-08** (dev/boba, GTX 1080, CAPI v1.13.4, capi-kubeswift @ main, -> Kubernetes v1.33.3, HAMi 2.9.0). Everything below is measured, including the -> mistakes. +`spec.cell.provisioner: ClusterAPI` makes each cell a Cluster API `Machine` + +`KubeSwiftMachine`, instead of a bare `SwiftGuest`. Use it when the workload +cluster your cells join is **already Cluster-API-managed** and you want the +GPU cells to be first-class members of it — visible as Machines, carrying a +`providerID`, participating in the cluster's own node lifecycle — rather than +nodes that joined out of band. -## What was proven +It is an **additional** provisioner, never a replacement for the default +(`SwiftGuest`): Cluster API can only add Machines to a cluster it already +manages, and the common case for this operator is bring-your-own workload +cluster, which is not CAPI-managed at all. If your workload cluster is not +CAPI-managed, use the default `SwiftGuest` provisioner instead — see +`docs/quickstart.md`. -A `GPUCellPool` added the only worker of a CAPI-managed workload cluster: +Validated end to end on hardware (dev/boba, GTX 1080): pool created to cell +`Ready` in 6m29s, two workloads sharing the cell's GPU through HAMi, clean +teardown. The post-mortem — two bugs a live cluster found that the test +harness could not, plus a step-by-step recipe for standing up a CAPI-managed +workload cluster on a lab with no cross-node L2 — is in +`docs/design/clusterapi-cells-validation.md`; this page is the how-to. + +## Prerequisites + +In addition to everything in `docs/quickstart.md`'s prerequisite table: | | | |---|---| -| pool created → cell `Ready` | **6 min 29 s** | -| Cluster API's view | `Machine cells-0` `Running`, `providerID kubeswift://capicell/cells-0`, `nodeRef cells-0` | -| workload cluster's view | Node `cells-0` `Ready`, same providerID | -| HAMi's view | `GPU-e71afe85…` GTX 1080, `devmem 8192`, `devcore 100`, healthy | -| the pool | `physicalCapacity{gpus 1}` + `workloadCapacity{1 device, 8Gi}`, reported separately | -| two workloads at 3000 MiB / 30 % | both `Running` on `cells-0`, **same GPU UUID**, pool at `6000Mi allocated / 2192Mi available`, compute `60/40` | -| teardown | pool deleted → Machine and KubeSwiftMachine gone in ~16 s, pool finalizer released at 47 s, GPU claim returned, Node reaped | - -The FSM walked `AllocatingGPU → Joining → AwaitingGPUCapacity → Ready`, holding in -`AwaitingGPUCapacity` exactly while HAMi's device plugin started. The three-way Ready -gate behaves the same through Cluster API as it does without it. - -## Two bugs only a live cluster could find - -Both were invisible to the envtest harness, for the same reason: **a CRD stub accepts -anything, because the thing that objects is the controller that is not running there.** - -1. **The pool claimed the controller owner reference.** Kubernetes allows one per - object and Cluster API needs it (the Cluster on the Machine, the Machine on the - infrastructure object). CAPI failed every reconcile with *"Object cells-0 is already - owned by another GPUCellPool controller"* — a hard stop: no bootstrap data, no VM, - ever. Cells are now **co-owned** (plain owner reference), which is all that garbage - collection needs. - -2. **Pool teardown listed SwiftGuests.** A ClusterAPI pool owns Machines, so deletion - saw no cells, drained nothing, and dropped the pool finalizer — orphaning each cell - Machine with `cells.kubeswift.io/cell-drain` still on it. Nothing was left to clear - it, so the Machine could never be deleted and its GPU claim never came back. - Observed exactly that, and cleared it only by patching the finalizer out by hand. - Deletion now goes through `prov.List`, and the dead lister is gone. - -## What the workload cluster needs (dev recipe) - -The pool is the easy half. Getting a CAPI-managed cluster onto KubeSwift on a lab with -no cross-node L2 took four things worth writing down. - -**A way to place the control plane.** The CP VM and the GPU cell must share the -node-local bridge NAD, which means sharing a host — and the cell's host is fixed by -where its GPU is. Cluster API cannot place an individual Machine, so -`KubeSwiftMachine.spec.backend.swiftGuest.nodeName` was added -(capi-kubeswift [#20](https://github.com/kubeswift-io/cluster-api-provider-kubeswift/pull/20)). -It rejects being combined with `gpu`: pinning bypasses the scheduler, and a DRA claim -is allocated *by* the scheduler, so the pair yields a VM with an unallocated claim and -no device. - -**The control-plane endpoint hairpins back into the control plane VM.** With -`endpoint.mode: Service` the endpoint is a ClusterIP in the *management* cluster whose -backend is the CP's own launcher pod. `kubeadm init`'s `wait-control-plane` phase health -checks through it, so the request goes VM → in-pod MASQUERADE → node → Service → the -same pod → back to the VM, and conntrack cannot match the reply: - -``` -error execution phase wait-control-plane: kube-apiserver check failed at -https://192.168.99.10:6443/livez: Get "https://10.96.212.189:6443/livez?timeout=10s": -context deadline exceeded +| Cluster API | `>= v1.13.4` — the validated floor. (An earlier draft of this project's sample referenced `>= v1.11`; `v1.13.4` is the version this was actually proven against and is the one to target.) | +| `cluster-api-provider-kubeswift` | `>= v0.2.0` — the release that ships `KubeSwiftMachine.spec.backend.swiftGuest.gpu` and `nodeName` placement. Earlier versions have no GPU surface at all | +| a CAPI `Cluster` | already provisioned, in the pool's own namespace (a pool never crosses namespaces) | +| a bootstrap config template (optional) | e.g. a `KubeadmConfigTemplate`, if you want the workload cluster's own bootstrap provider to mint join credentials per cell instead of maintaining `spec.bootstrap` yourself | + +## Why one Machine per cell, not a MachineDeployment + +A cell's identity model (`docs/concepts.md`) depends on the cell name, the +guest hostname, and the workload Node name being the *same string*. A +`MachineDeployment` generates its own Machine names, and `capi-kubeswift` +derives the guest hostname — and therefore the Node name — from the Machine +name. Random names would break that identity and leave no way to drain or +replace one specific cell. So GPUCellPool creates exactly one `Machine` + +`KubeSwiftMachine` per cell, named `-` like every other cell +object, and reconciles them the way it reconciles `SwiftGuest`s under the +default provisioner. + +## What a `KubeSwiftMachine` can express + +A `KubeSwiftMachine` exposes a curated subset of `SwiftGuestSpec`: image, +guest class, up to two network interfaces, and GPU. Under +`provisioner: ClusterAPI`, `guestTemplate` is therefore restricted to +`imageRef`, `guestClassRef`, `interfaces` — anything else (data disks, +storage class, topology spread constraints) is **rejected at admission** +rather than silently dropped, because a cell that boots with less than you +asked for is worse than a pool that refuses to be created. See +`docs/api-reference.md` for the exact validation rule. + +## Apply + +```yaml +apiVersion: cells.kubeswift.io/v1alpha1 +kind: GPUCellPool +metadata: + name: inference + namespace: gpu-cells +spec: + replicas: 2 + cell: + provisioner: ClusterAPI + clusterAPI: + clusterName: inference # the CAPI Cluster in this namespace + version: v1.33.3 + bootstrapConfigTemplateRef: # optional — omit to use spec.bootstrap instead + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: gpu-workers + guestTemplate: + guestClassRef: {name: gpu-worker-32c-128g} + imageRef: {name: gpu-worker-noble-580} + interfaces: + - {name: mgmt, primary: true} + - {name: node, networkRef: {name: cell-udn}} + nodeIPFrom: node + gpu: + backend: DRA + dra: {resourceClaimTemplateName: single-vfio-gpu} + workloadCluster: + kubeconfigSecretRef: {name: inference-kubeconfig} + node: + labels: {gpu: "on"} ``` -The failure is quiet in the worst way: every control-plane static pod is `Running`, so -the cluster looks alive, while `kubeadm-config`, `kubelet-config`, `cluster-info`, the -bootstrap tokens, the control-plane role label, kube-proxy and CoreDNS are all absent. -Fix: alias the endpoint address on `lo` in the guest, so in-guest clients reach the -apiserver locally. **Control plane only** — a worker must reach the real Service. - -**That alias must come after the node-IP derivation.** It is a scope-global address, so -deriving "the global IPv4 that is not on the default route" afterwards picks up the -alias and the node registers with the endpoint ClusterIP as its `InternalIP`. Measured, -and it silently breaks apiserver→kubelet. Exclude `lo` as well. - -**Then the documented single-CP hairpin still applies** *inside* the workload cluster: -`masqueradeAll: true` in the kube-proxy ConfigMap (capi-kubeswift -`docs/operations/single-control-plane-hairpin.md`, fix 3), or the CNI on the lone -control-plane node never starts. And the CNI must be pinned to the datapath interface — -flannel picks the default-route interface, which here is KubeSwift's node-local nat -primary, the wrong side (`--iface-regex=10\.79\.0\.\d+`). - -Two smaller ones: a CP-only cluster needs its control-plane taint removed or nothing -schedules, and `SwiftImage.spec.format` is the **input** format — declaring `raw` for -the Ubuntu qcow2 cloud image skips conversion and hands Cloud Hypervisor a qcow2 it -reads as raw (`Failed to get refcount`, in under a minute of "importing"). - -## Capacity arithmetic, since it bites - -boba has 8 cores. A 4-vCPU control plane plus a 2-vCPU cell does not fit alongside the -node's existing load, and with `nodeName` the kubelet says so immediately — -`OutOfcpu: requested 4000, used 7790, capacity 8000` — rather than leaving the pod -Pending. The validated shape is a **2-vCPU** control plane (`capicell-cp`) and a 2-vCPU -cell. - -Do not delete Machines mid-rollout to force a change: KubeadmControlPlane holds a -pre-terminate hook on the last control plane and will not release it until a -replacement joins, so you deadlock (`stage: WaitingForPreTerminateHook`). Delete the -`Cluster` and rebuild instead. +The full, heavily-commented sample is at +`config/samples/cells_v1alpha1_gpucellpool_clusterapi.yaml`. + +When `bootstrapConfigTemplateRef` is set, do **not** also set +`spec.bootstrap.joinSecretRef` — the workload cluster's own bootstrap provider +supplies the join data, instantiated once per cell the way a `MachineSet` +does, and the webhook rejects the redundant Secret reference rather than +silently ignoring it. Omit `bootstrapConfigTemplateRef` and the pool's own +rendered Secret is handed to the Machine as `bootstrap.dataSecretName` instead +— that works, but the join data (token validity, CA rotation) is then yours +to keep working, same as under the `SwiftGuest` provisioner. + +## Watching and diagnosing + +The FSM and `status.cells[]` behave identically to the `SwiftGuest` +provisioner — see `docs/api-reference.md` and `docs/runbook.md`. Two +CAPI-specific entries worth knowing: + +- Cells are discovered by listing **Machines** labelled + `cells.kubeswift.io/pool=`, not SwiftGuests. If the pool reports no + cells despite Machines existing, the label did not survive — see the + runbook. +- A cell only leaves `AllocatingGPU` once `capi-kubeswift` has provisioned the + backing `SwiftGuest` and stamped a `providerID` on the `KubeSwiftMachine` — + the GPU is found by following that reference. An empty `providerID` with the + `Machine` still `Provisioning` is normal; a `Provisioned` `Machine` with no + `providerID` is a `capi-kubeswift` problem, not a pool problem. diff --git a/docs/design/clusterapi-cells-validation.md b/docs/design/clusterapi-cells-validation.md new file mode 100644 index 0000000..3a6b331 --- /dev/null +++ b/docs/design/clusterapi-cells-validation.md @@ -0,0 +1,110 @@ +> Design record — validation and post-mortem. Written after +> `provisioner: ClusterAPI` was implemented and validated on hardware +> (2026-08-08). It documents what broke and why, not how to use the feature — +> see `docs/clusterapi-cells.md` for that. + +# ClusterAPI cells — validated recipe and what it cost + +`provisioner: ClusterAPI` was validated end to end on real hardware on +**2026-08-08** (dev/boba, GTX 1080, CAPI v1.13.4, capi-kubeswift @ main, +Kubernetes v1.33.3, HAMi 2.9.0). Everything below is measured, including the +mistakes. + +## What was proven + +A `GPUCellPool` added the only worker of a CAPI-managed workload cluster: + +| | | +|---|---| +| pool created → cell `Ready` | **6 min 29 s** | +| Cluster API's view | `Machine cells-0` `Running`, `providerID kubeswift://capicell/cells-0`, `nodeRef cells-0` | +| workload cluster's view | Node `cells-0` `Ready`, same providerID | +| HAMi's view | `GPU-e71afe85…` GTX 1080, `devmem 8192`, `devcore 100`, healthy | +| the pool | `physicalCapacity{gpus 1}` + `workloadCapacity{1 device, 8Gi}`, reported separately | +| two workloads at 3000 MiB / 30 % | both `Running` on `cells-0`, **same GPU UUID**, pool at `6000Mi allocated / 2192Mi available`, compute `60/40` | +| teardown | pool deleted → Machine and KubeSwiftMachine gone in ~16 s, pool finalizer released at 47 s, GPU claim returned, Node reaped | + +The FSM walked `AllocatingGPU → Joining → AwaitingGPUCapacity → Ready`, holding in +`AwaitingGPUCapacity` exactly while HAMi's device plugin started. The three-way Ready +gate behaves the same through Cluster API as it does without it. + +## Two bugs only a live cluster could find + +Both were invisible to the envtest harness, for the same reason: **a CRD stub accepts +anything, because the thing that objects is the controller that is not running there.** + +1. **The pool claimed the controller owner reference.** Kubernetes allows one per + object and Cluster API needs it (the Cluster on the Machine, the Machine on the + infrastructure object). CAPI failed every reconcile with *"Object cells-0 is already + owned by another GPUCellPool controller"* — a hard stop: no bootstrap data, no VM, + ever. Cells are now **co-owned** (plain owner reference), which is all that garbage + collection needs. + +2. **Pool teardown listed SwiftGuests.** A ClusterAPI pool owns Machines, so deletion + saw no cells, drained nothing, and dropped the pool finalizer — orphaning each cell + Machine with `cells.kubeswift.io/cell-drain` still on it. Nothing was left to clear + it, so the Machine could never be deleted and its GPU claim never came back. + Observed exactly that, and cleared it only by patching the finalizer out by hand. + Deletion now goes through `prov.List`, and the dead lister is gone. + +## What the workload cluster needs (dev recipe) + +The pool is the easy half. Getting a CAPI-managed cluster onto KubeSwift on a lab with +no cross-node L2 took four things worth writing down. + +**A way to place the control plane.** The CP VM and the GPU cell must share the +node-local bridge NAD, which means sharing a host — and the cell's host is fixed by +where its GPU is. Cluster API cannot place an individual Machine, so +`KubeSwiftMachine.spec.backend.swiftGuest.nodeName` was added +(capi-kubeswift [#20](https://github.com/kubeswift-io/cluster-api-provider-kubeswift/pull/20)). +It rejects being combined with `gpu`: pinning bypasses the scheduler, and a DRA claim +is allocated *by* the scheduler, so the pair yields a VM with an unallocated claim and +no device. + +**The control-plane endpoint hairpins back into the control plane VM.** With +`endpoint.mode: Service` the endpoint is a ClusterIP in the *management* cluster whose +backend is the CP's own launcher pod. `kubeadm init`'s `wait-control-plane` phase health +checks through it, so the request goes VM → in-pod MASQUERADE → node → Service → the +same pod → back to the VM, and conntrack cannot match the reply: + +``` +error execution phase wait-control-plane: kube-apiserver check failed at +https://192.168.99.10:6443/livez: Get "https://10.96.212.189:6443/livez?timeout=10s": +context deadline exceeded +``` + +The failure is quiet in the worst way: every control-plane static pod is `Running`, so +the cluster looks alive, while `kubeadm-config`, `kubelet-config`, `cluster-info`, the +bootstrap tokens, the control-plane role label, kube-proxy and CoreDNS are all absent. +Fix: alias the endpoint address on `lo` in the guest, so in-guest clients reach the +apiserver locally. **Control plane only** — a worker must reach the real Service. + +**That alias must come after the node-IP derivation.** It is a scope-global address, so +deriving "the global IPv4 that is not on the default route" afterwards picks up the +alias and the node registers with the endpoint ClusterIP as its `InternalIP`. Measured, +and it silently breaks apiserver→kubelet. Exclude `lo` as well. + +**Then the documented single-CP hairpin still applies** *inside* the workload cluster: +`masqueradeAll: true` in the kube-proxy ConfigMap (capi-kubeswift +`docs/operations/single-control-plane-hairpin.md`, fix 3), or the CNI on the lone +control-plane node never starts. And the CNI must be pinned to the datapath interface — +flannel picks the default-route interface, which here is KubeSwift's node-local nat +primary, the wrong side (`--iface-regex=10\.79\.0\.\d+`). + +Two smaller ones: a CP-only cluster needs its control-plane taint removed or nothing +schedules, and `SwiftImage.spec.format` is the **input** format — declaring `raw` for +the Ubuntu qcow2 cloud image skips conversion and hands Cloud Hypervisor a qcow2 it +reads as raw (`Failed to get refcount`, in under a minute of "importing"). + +## Capacity arithmetic, since it bites + +boba has 8 cores. A 4-vCPU control plane plus a 2-vCPU cell does not fit alongside the +node's existing load, and with `nodeName` the kubelet says so immediately — +`OutOfcpu: requested 4000, used 7790, capacity 8000` — rather than leaving the pod +Pending. The validated shape is a **2-vCPU** control plane (`capicell-cp`) and a 2-vCPU +cell. + +Do not delete Machines mid-rollout to force a change: KubeadmControlPlane holds a +pre-terminate hook on the last control plane and will not release it until a +replacement joins, so you deadlock (`stage: WaitingForPreTerminateHook`). Delete the +`Cluster` and rebuild instead. diff --git a/docs/design/gpucellpool-api.md b/docs/design/gpucellpool-api.md index 1076bc0..9f6d066 100644 --- a/docs/design/gpucellpool-api.md +++ b/docs/design/gpucellpool-api.md @@ -1,5 +1,13 @@ # GPUCellPool — v1alpha1 API +> Design record, written before implementation (2026-07-30) and not updated +> since — `spec.autoscaling` and `cell.clusterAPI` are absent below because +> they were designed after this file was written, and the shipped status +> fields (`status.desiredReplicas`, `status.demand`, `status.cellDeviceShape`) +> are likewise missing. **For the current API, read `docs/api-reference.md` +> instead** — it is generated from the Go types and the validating webhook. +> This file remains for the field-contract *reasoning* (§4, §8). + > One CRD. Cells are owned `SwiftGuest`s, not a second kind (D1). The cell's VM > shape is an **opaque `SwiftGuestSpec` passthrough** so this API never chases > KubeSwift's (D4); GPU, identity, bootstrap and node enrollment are first-class diff --git a/docs/design/gpucellpool-bootstrap.md b/docs/design/gpucellpool-bootstrap.md index c964361..7cda24b 100644 --- a/docs/design/gpucellpool-bootstrap.md +++ b/docs/design/gpucellpool-bootstrap.md @@ -1,5 +1,10 @@ # GPUCellPool — cell bootstrap +> Design record, written before implementation (2026-07-30) and partially +> updated. It documents rationale, not current behaviour — see `docs/` for +> that, in particular `docs/cell-image.md`, `docs/networking.md` and +> `config/samples/cell-join-secret.yaml`. + > A cell must become a GPU-capable Kubernetes worker in the *workload* cluster. > Decision: **prebaked image + thin cloud-init enrollment**, with the join > credential delivered by Secret reference and never written into a CR. @@ -23,7 +28,7 @@ Notes that bite: - HAMi's `LD_PRELOAD` core also documents **glibc ≥ 2.17 and < 2.30** — that is a constraint on the *workload container image*, not the node, but it must be - verified in Phase 1 or every PoC workload silently runs unlimited (`-poc.md` R6). + verified in Phase 1 or every PoC workload silently runs unlimited (`-validation-record.md` R6). - A cell needs **no** Fabric Manager, no IOMMU config in the guest, and no hugepages unless `cell.gpu.dra.hugepages` is set: it is a flat single-GPU `pcie` guest. @@ -90,10 +95,15 @@ Two deliberate choices: kubelet must bind the latter — the same rule `capi-kubeswift` follows. - **Preflight is non-fatal.** A node that joins *without* a GPU is diagnosable — the pool parks it in `AwaitingGPUCapacity` with a message and a timeout. A node - that refuses to join is opaque. Loud, not silent: the preflight result is logged - to the console and, in Phase 2+, written as a node annotation - `cells.kubeswift.io/gpu-preflight: "0/1"` which the controller surfaces verbatim - in `status.cells[].message`. + that refuses to join is opaque. **As shipped, preflight is log-only, not the + node-annotation surface this paragraph originally planned**: the reference + cell image (`hack/build-cell-image.sh`) writes the result to + `/run/gpu-cell-preflight` inside the guest, and the diagnostic path is + `swiftctl ssh -- cat /run/gpu-cell-preflight` (see `docs/runbook.md`). + The Go constant `cells.kubeswift.io/gpu-preflight` + (`AnnotationGPUPreflight`, `api/v1alpha1/conditions.go`) exists for a future + node-annotation surface but nothing writes it or reads it yet — do not + document it as available. Kubelet self-labelling with a third-party prefix is permitted under the `NodeRestriction` admission plugin (it only constrains `kubernetes.io`/`k8s.io` @@ -232,30 +242,44 @@ default and the one that must work. --- -## 5. Why not delegate bootstrap to Cluster API in the MVP +## 5. Why not delegate bootstrap to Cluster API in the MVP (and what shipped instead) + +**This section is entirely historical.** It was written to argue against +building a CAPI provisioner for the MVP, before one existed. `provisioner: +ClusterAPI` has since shipped and is hardware-validated — see +`docs/clusterapi-cells.md`. Two things below were wrong even as a plan, not +just overtaken by events: it describes the mechanism as sizing a +**`MachineDeployment`**, but the shipped provisioner creates **one `Machine` + +`KubeSwiftMachine` per cell**, named after the cell — a `MachineDeployment` +generates its own Machine names, which would have broken the +cell-name-equals-Node-name identity the whole design rests on (see D8 in +`-overview.md` and `docs/clusterapi-cells.md`). Kept for the reasoning that is +still valid: why CAPI was, and remains, a *second* provisioner rather than a +replacement for `SwiftGuest`. CAPI solves exactly this problem — a bootstrap provider produces the cloud-init Secret, and `capi-kubeswift` already consumes it verbatim into a `SwiftSeedProfile`. -So the honest evaluation: +So the honest evaluation, as it stood before implementation: -**Blocking reason:** a `MachineDeployment` can only add Machines to a +**Blocking reason (still true):** Cluster API can only add Machines to a **CAPI-managed** cluster. The target use case is *bring your own workload cluster* (HAMi already installed, possibly not CAPI-managed at all). A CAPI-only design -cannot serve it, so CAPI must be a **second provisioner**, never the only one (D8). - -**Secondary reason:** `capi-kubeswift` has no GPU surface today (verified: zero -`gpu` hits across `api/`, `internal/`, `templates/`). Phase 5 needs -`KubeSwiftMachine.spec.backend.swiftGuest.gpu` (a `gpuResourceClaim`/`gpuProfileRef` -passthrough) added there first — a small, well-scoped change in a repo the same team -owns, but a cross-repo dependency the MVP should not be blocked on. - -**What CAPI buys when the workload cluster *is* CAPI-managed** — and why Phase 5 is -worth doing: bootstrap providers + token rotation for free, `Machine`↔`Node` -correlation for free, node drain on Machine deletion for free, rolling updates for -free. Under `provisioner: ClusterAPI` the pool becomes a thin controller that sizes -a `MachineDeployment` and reads HAMi capacity — most of `-failure-model.md` §2 and -§5 becomes someone else's tested code. The `CellProvisioner` seam -(`-reconciliation.md` §1) exists so that transition is additive. +cannot serve it, so CAPI is a **second provisioner**, never the only one (D8). + +**Secondary reason (resolved):** at the time of writing, `capi-kubeswift` had +no GPU surface (verified then: zero `gpu` hits across `api/`, `internal/`, +`templates/`). That blocker is gone: `capi-kubeswift` v0.2.0 ships +`KubeSwiftMachine.spec.backend.swiftGuest.gpu` and `nodeName` placement. + +**What CAPI buys when the workload cluster *is* CAPI-managed** — realized, not +just planned: `Machine`↔`Node` correlation via `providerID`, and the cluster's +own bootstrap provider supplying join credentials per cell +(`bootstrapConfigTemplateRef`, instantiated once per cell the way a +`MachineSet` does — see `docs/clusterapi-cells.md`). **Not** realized: node +drain on Machine deletion and rolling updates are not automatic — see +`docs/limitations.md`, which applies identically to both provisioners. The +`CellProvisioner` seam (`-reconciliation.md` §1) is what made adding the +provisioner additive rather than a rewrite. --- diff --git a/docs/design/gpucellpool-capacity.md b/docs/design/gpucellpool-capacity.md index 28075e1..713a566 100644 --- a/docs/design/gpucellpool-capacity.md +++ b/docs/design/gpucellpool-capacity.md @@ -1,5 +1,9 @@ # GPUCellPool — capacity discovery +> Design record, written before implementation (2026-07-30) and partially +> updated. It documents rationale, not current behaviour — see `docs/` for +> that, in particular `docs/concepts.md` and `docs/autoscaling.md`. + > Two capacities, never merged: **physical** (outer, KubeSwift/DRA — whole devices) > and **workload** (inner, HAMi — memory + core fractions). One internal > `CapacityProvider` interface isolates HAMi so its churn cannot reach the CRD (D5). @@ -297,20 +301,23 @@ Measured note that shaped this: HAMi refuses an over-large request at SCHEDULING time ("0/1 nodes are available: 1 NodeUnfitPod"), so over-large requests do show up as unschedulable pods and the second gate is what stops them driving a scale-up. -DRA mode still returns `ErrUnsupported`. +DRA mode still returns `ErrUnsupported` — that half of the design intent below +remains just intent. The DevicePlugin half shipped, described above. -The original design intent, kept for the DRA implementation: +The original design intent, still outstanding for the DRA implementation: -- **DRA mode is the good signal**: a `ResourceClaim` in `WaitingForFirstConsumer`/ - unallocated state whose `deviceClassName` is the pool's HAMi class, with - `capacity.requests` that **would fit** on a fresh cell but fits nowhere now. That - is unambiguous GPU-capacity demand. -- **DevicePlugin mode is the weak signal**: pending Pods requesting +- **DRA mode would be the good signal**: a `ResourceClaim` in + `WaitingForFirstConsumer`/unallocated state whose `deviceClassName` is the + pool's HAMi class, with `capacity.requests` that **would fit** on a fresh + cell but fits nowhere now. That is unambiguous GPU-capacity demand — and + unimplemented (`docs/limitations.md`). +- **DevicePlugin mode is the weak signal — SHIPPED**: pending Pods requesting `nvidia.com/gpu`+`gpumem` whose `PodScheduled=False` reason is - `Unschedulable`/`FailedScheduling` mentioning the HAMi resources. + `Unschedulable`. This is what §8 above describes. - Both must pass two filters before any cell is created: `demand is GPU-capacity-constrained` **AND** `a fresh cell of this pool's shape would satisfy it`. A Pod pending on a missing ConfigMap, a wrong nodeSelector, an impossible model, or a request larger than one cell's whole GPU must **never** - create a cell. That check is the entire safety of Phase 3, which is why it is - designed here and shipped later. + create a cell. That check is the entire safety of the feature, which is why it + was designed here before either mode shipped. See `docs/autoscaling.md` for + the user-facing view of the shipped (DevicePlugin) behaviour. diff --git a/docs/design/gpucellpool-failure-model.md b/docs/design/gpucellpool-failure-model.md index 0583f80..7f4ca30 100644 --- a/docs/design/gpucellpool-failure-model.md +++ b/docs/design/gpucellpool-failure-model.md @@ -1,5 +1,9 @@ # GPUCellPool — failure model +> Design record, written before implementation (2026-07-30) and partially +> updated. It documents rationale, not current behaviour — see `docs/` for +> that, in particular `docs/limitations.md` and `docs/runbook.md`. + > A two-layer system fails in two layers, and the dangerous failures are the > asymmetric ones: outer success + inner failure (a cell that looks provisioned and > is useless), and inner doubt + outer action (deleting infrastructure because a @@ -151,7 +155,7 @@ boolean: | 13 | Workload API unavailable? | Freeze all destructive paths, retain capacity, `WorkloadClusterReachable=False` (rule 2) | | 14 | Operations during degraded HAMi? | Create/read/report yes; Ready/count/scale-down/drain-completion no (§5) | | 15 | How does CAPI change things? | Adds `cell.provisioner: ClusterAPI`; needs GPU fields in `capi-kubeswift`; only possible for CAPI-managed workload clusters — hence a second provisioner, never a replacement (D8, bootstrap §5) | -| 16 | What is postponed? | Automatic scale-down, demand-driven scale-up, MIG, multi-GPU cells, overcommit, multi-cluster pools, HAMi install, workload integrations (overview §5) | +| 16 | What is postponed? | **Automatic scale-down and demand-driven scale-up are SHIPPED, not postponed** (`docs/autoscaling.md`) — this row is stale. Still postponed/not implemented: HAMi DRA capacity mode, MIG, multi-GPU cells, overcommit, multi-cluster pools, HAMi install, workload integrations, automated outer-drain sequencing (overview §5, `docs/limitations.md`) | --- @@ -175,8 +179,11 @@ boolean: - U2 **HAMi annotation format stability** — currently an internal protocol. Should the provider pin a supported HAMi version range and refuse newer ones loudly? Leaning yes, with an override annotation. -- U3 **HAMi DRA maturity** — `v0.1.0`, k8s ≥ 1.34. `mode: DRA` should probably ship - behind an explicit `experimental` acknowledgement until validated. +- U3 **HAMi DRA maturity** — `v0.1.0`, k8s ≥ 1.34. **Resolved, more strictly + than this question proposed**: `mode: DRA` did not ship behind an + experimental flag; it does not ship at all. `internal/capacity`'s DRA path + returns `ErrUnsupported` for every operation. `DevicePlugin` is the only + implemented mode (`docs/limitations.md`). - U4 **Preflight reporting from inside the guest** (bootstrap §7.3). - U5 **Cell replacement policy on GPU-absent failures** (scenario 6): replace-once is a guess; the real number comes from Phase 1/2 experience. diff --git a/docs/design/gpucellpool-overview.md b/docs/design/gpucellpool-overview.md index bad4dfe..f702b42 100644 --- a/docs/design/gpucellpool-overview.md +++ b/docs/design/gpucellpool-overview.md @@ -1,16 +1,23 @@ # GPUCellPool — overview and architecture +> Design record, written before implementation (2026-07-30) and partially +> updated. It documents rationale, not current behaviour — see `docs/` for +> that. + > A **composition operator**: KubeSwift provides VM-isolated, whole-GPU-passthrough > Kubernetes workers; HAMi fractionally shares the GPU inside them. `GPUCellPool` > owns the lifecycle between the two layers and knows about both — while neither > KubeSwift nor HAMi is modified, and neither learns about the other. > -> Status: **DESIGN (first pass)** — grounded in source inspection of -> `kubeswift-io/kubeswift` @ v0.13.4 and `capi-kubeswift`, plus HAMi upstream -> (`Project-HAMi/HAMi`, `Project-HAMi/k8s-dra-driver`, `Project-HAMi/HAMi-DRA`). -> No hardware proof yet — Phase 1 is the hardware gate. +> Status: **SHIPPED as v0.1.0 (2026-08-08), hardware-validated** — see §5 and +> §8. Originally written as a first-pass design grounded in source inspection +> of `kubeswift-io/kubeswift` @ v0.13.4 and `capi-kubeswift`, plus HAMi +> upstream (`Project-HAMi/HAMi`, `Project-HAMi/k8s-dra-driver`, +> `Project-HAMi/HAMi-DRA`), before any hardware proof existed. Left largely +> as-written for the decision record; do not trust phase/status language +> below over §5 and §8. > Companions: `gpucellpool-api.md`, `-reconciliation.md`, `-bootstrap.md`, -> `-capacity.md`, `-failure-model.md`, `-poc.md`. Date: 2026-07-30. +> `-capacity.md`, `-failure-model.md`, `-validation-record.md`. Date: 2026-07-30. --- @@ -88,8 +95,12 @@ Kubernetes worker. Reusable findings: - The validated multi-node worker shape is **dual-NIC**: nat `mgmt` primary + `networkRef` `node` interface carrying the routable IP, with kubelet `--node-ip` set to the secondary. -- **It has no GPU support at all** (`grep -i gpu api/ internal/ templates/` → 0 - hits). Any CAPI-based cell provisioner needs a GPU field added there first. → D8. +- **At the time this was written it had no GPU support at all** + (`grep -i gpu api/ internal/ templates/` → 0 hits). Any CAPI-based cell + provisioner needed a GPU field added there first (→ D8). **Resolved**: + `capi-kubeswift` v0.2.0 ships `KubeSwiftMachine.spec.backend.swiftGuest.gpu` + and `nodeName` placement; `provisioner: ClusterAPI` is shipped and + hardware-validated against it (§5, `docs/clusterapi-cells.md`). ### HAMi (inner) — two accounting models, both readable @@ -104,7 +115,7 @@ Kubernetes worker. Reusable findings: Extra HAMi facts that shape the PoC: HAMi-core needs NVIDIA driver ≥ 440 and — notably — **glibc ≥ 2.17 and < 2.30 in the workload container image** for its `LD_PRELOAD` interposition. That is a workload-image constraint, not a node -constraint, and it is a Phase-1 verification item (`-poc.md` R6). +constraint, and it is a Phase-1 verification item (`-validation-record.md` R6). --- @@ -113,7 +124,7 @@ constraint, and it is a Phase-1 verification item (`-poc.md` R6). | # | Question | Decision | Why | |---|---|---|---| | D1 | per-Cell CRD or `SwiftGuest` directly? | **No `GPUCell` CRD.** One CRD (`GPUCellPool`); a cell *is* an owned `SwiftGuest` named `-`, with per-cell state in `status.cells[]` and the drain finalizer on the guest | `SwiftGuestPool` proves counters-plus-owned-objects is enough; and if CAPI lands (D8) the *`Machine`* becomes the per-cell object — inventing `GPUCell` now guarantees a third, doomed object | -| D2 | `replicas` or `min/max` in v1alpha1? | **`spec.replicas: int32` + scale subresource.** `min/max` arrives later inside `spec.autoscaling` | HPA-ready day one (same seam as `SwiftGuestPool`/`SwiftSandboxPool`); a scaling block is additive, whereas demoting `replicas` later is not | +| D2 | `replicas` or `min/max` in v1alpha1? | **`spec.replicas: int32` + scale subresource, PLUS `spec.autoscaling.{minReplicas,maxReplicas}` — shipped, not deferred.** | HPA-ready day one (same seam as `SwiftGuestPool`/`SwiftSandboxPool`); a scaling block is additive, whereas demoting `replicas` later would not have been | | D3 | cell identity across clusters | **Name-derived, not IP-derived**: cell name = `-` = guest name = guest hostname = **Node name**; verified by node labels `gpu-cell.kubeswift.io/{pool,cell}` and an *instance* label carrying the guest UID | deterministic, reconstructible after operator restart, and detects a stale Node left by a previous incarnation of the same index | | D4 | cell shape in the API | **Opaque `SwiftGuestSpec` passthrough** (`cell.guestTemplate`, preserve-unknown-fields) + first-class `cell.gpu`, with an operator-owned/denied field contract enforced by a webhook | never chases KubeSwift's API surface (mirrors `SwiftGuestPool.spec.template.spec`); GPU stays first-class because the operator must own the tier/backend semantics | | D5 | HAMi coupling | **`CapacityProvider` interface, one implementation (`HAMi`), two modes** (`DevicePlugin`, `DRA`). No HAMi Go dependency; annotations/ResourceSlices are read as data | HAMi DRA is `v0.1.0` — isolate it so its churn cannot reach the CRD | @@ -132,11 +143,11 @@ constraint, and it is a Phase-1 verification item (`-poc.md` R6). ┌───────────────────────────────────────────────────────────────┐ │ GPUCellPool controller │ │ ├── CellManager index/name allocation, per-cell FSM │ - │ ├── CellProvisioner SwiftGuest (v1) | ClusterAPI (later) │ + │ ├── CellProvisioner SwiftGuest | ClusterAPI (both shipped)│ │ ├── PhysicalInventory outer ResourceSlices/Claims → free GPUs│ │ ├── WorkloadClient one cached client per kubeconfig │ │ ├── CapacityProvider HAMi{DevicePlugin|DRA} (inner reads) │ - │ └── ScalingPolicy static (v1) → demand-driven (Ph. 3) │ + │ └── ScalingPolicy static, or demand-driven (both ways) │ │ │ owns │ │ ▼ │ │ SwiftGuest -0 … -N (+ SwiftSeedProfile each) │ @@ -202,9 +213,12 @@ migration — a VM restart. Applied to a cell, that silently reboots a Kubernete worker with running HAMi workloads on it. v1alpha1 therefore stamps `migration.enabled: false` on every cell guest (pinned) and surfaces a `CellDrainRequested` condition when the outer node is cordoned or KubeSwift sets -`kubeswift.io/drain-requested`. The operator (human) then scales/replaces. Phase 4 -automates the correct order: cordon inner Node → drain inner workloads → delete -cell → recreate elsewhere. Never the reverse. +`kubeswift.io/drain-requested`. The operator (human) then scales/replaces. **This +remains true in the shipped v0.1.0**: no phase automates the correct order +(cordon inner Node → drain inner workloads → delete cell → recreate elsewhere). +It is a manual runbook step today — see `docs/limitations.md` and +`docs/runbook.md`. If automated forever, delete this sentence; if it lands, cite +the PR instead. **(b) The GPU is released by deleting the cell, not by draining it.** Outer capacity only returns to the pool when the `SwiftGuest` is gone (native backend: @@ -233,13 +247,13 @@ GPU-sharing alone provides. | 0 | this design set | — | | 1 | hardware proof: GPU → cell VM → nvidia driver → HAMi → 2 fractional workloads, done by hand | **boba/GTX 1080; blocks everything** | | 2 | static `GPUCellPool` (MVP, §5) | Phase 1 PASS | -| 3 | demand-driven scale-**up** (pending-pod signal, guarded) — **DONE** | Phase 2 stable | -| 4 | safe scale-**down** + automated outer-drain sequencing | Phase 3 stable | -| 5 | `provisioner: ClusterAPI` — the `capi-kubeswift` GPU field landed as PR #19; the provisioner itself is next | Phase 2; independent of 3/4 | +| 3 | demand-driven scale-**up** (pending-pod signal, guarded) — **SHIPPED** | Phase 2 stable | +| 4 | safe scale-**down** — **SHIPPED**; automated outer-drain sequencing (cordon inner Node → drain → delete → recreate) — **NOT implemented**, see §6(a) | Phase 3 stable | +| 5 | `provisioner: ClusterAPI` — **SHIPPED and hardware-validated** (2026-08-08); the `capi-kubeswift` GPU field landed as its PR #19/v0.2.0 | Phase 2; independent of 3/4 | The lab has exactly **one** GPU (boba, GTX 1080). Phase 1 is fully doable; `replicas ≥ 2` is hardware-gated and must be validated against a faked capacity -provider + faked inner Nodes (`-poc.md` §6), the same way KubeSwift validates HGX. +provider + faked inner Nodes (`-validation-record.md` §6), the same way KubeSwift validates HGX. --- diff --git a/docs/design/gpucellpool-reconciliation.md b/docs/design/gpucellpool-reconciliation.md index fff9f66..32514b8 100644 --- a/docs/design/gpucellpool-reconciliation.md +++ b/docs/design/gpucellpool-reconciliation.md @@ -1,5 +1,11 @@ # GPUCellPool — reconciliation, identity, RBAC, deletion +> Design record, written before implementation (2026-07-30) and partially +> updated. It documents rationale, not current behaviour — see `docs/` for +> that, in particular `docs/api-reference.md` and `docs/autoscaling.md`. +> Several "v1"/"later phase" markers below are now stale — scale-down and +> demand-driven scale-up are both shipped; see the inline corrections. + > One controller, two clusters, one cached client per pool. Every state is derived > from cluster objects, never from controller memory — the reconciler must survive > a restart with no bookkeeping. @@ -19,7 +25,7 @@ | `PhysicalInventory` | free/held GPUs from outer `ResourceSlice` + `ResourceClaim` | allocate anything | | `WorkloadClusterClient` | one cached, scoped client + informers per kubeconfig; Node get/label/cordon/drain | interpret GPU capacity | | `CapacityProvider` (iface) | HAMi device/memory/core capacity from inner objects | write to the inner cluster | -| `ScalingPolicy` | `desiredCells` (v1: `= spec.replicas`) | delete cells directly | +| `ScalingPolicy` | `desiredCells` — **shipped as `DecideScale`**, static (`= spec.replicas`) or demand-driven (`spec.autoscaling`, both directions); see `docs/autoscaling.md` | delete cells directly | `CellProvisioner` mirrors KubeSwift's own `gpualloc.Backend` seam — two phases, one struct as the contract: @@ -149,12 +155,15 @@ cells.kubeswift.io/instance = unreachable → set WorkloadClusterReachable=False, SUSPEND all destructive decisions, requeue, return 4 list owned cells (outer objects by label) → reconstruct status.cells[] -5 desired = ScalingPolicy.Desired(pool) // v1: spec.replicas +5 desired = ScalingPolicy.Desired(pool) // DecideScale: spec.replicas, + // or the autoscaler's decision 6 reconcile membership create while len(cells) < desired and inFlight < maxCreating(=2) and PhysicalGPUsAvailable replace Failed cells, per-index exponential backoff (30s→30m, capped) - drain while len(cells) > desired: pick highest index → Draining [manual only in v1] + drain while len(cells) > desired: pick highest index (operator-driven + shrink) OR the autoscaler's named idle cells (scaleDown: Auto) + → Draining 7 advance each cell's FSM (§2) using outer + inner reads 8 reap stale inner Nodes (identity mismatch, or cell gone) → §3 9 ensure inner Node labels/annotations/taints match spec.workloadCluster.node @@ -209,6 +218,20 @@ Invariants: ## 6. RBAC +The two role manifests below are the design draft. **The shipped roles are +`config/rbac/role.yaml`** (outer) **and `config/rbac/workload-cluster-observer.yaml`** +(inner) — read those for ground truth; this section is kept for the reasoning. +Notable drift from the draft: the shipped outer role additionally grants +`cluster.x-k8s.io/clusters`, `cluster.x-k8s.io/machines` and +`infrastructure.cluster.x-k8s.io/kubeswiftmachines` (all +create/delete/get/list/patch/update/watch except `clusters`, which is read-only) +plus `bootstrap.cluster.x-k8s.io/*` — needed once `provisioner: ClusterAPI` +shipped (§1, `docs/clusterapi-cells.md`) and absent from the draft below because +it predates that provisioner. The shipped inner role grants `nodes: delete` as +an **always-on** right, not a Phase-4/drain-only one (see the correction after +the inner block) and has no `KubeadmToken`-only block, because that bootstrap +provider was never implemented (`docs/limitations.md`). + ### Outer (management) cluster — the operator's own ServiceAccount ```yaml @@ -234,50 +257,67 @@ rules: - apiGroups: [coordination.k8s.io] resources: [leases] verbs: [get, create, update] # leader election + # ClusterAPI provisioner only (shipped after this draft was written — see + # config/rbac/role.yaml for the exact, generated rules): + - apiGroups: [cluster.x-k8s.io] + resources: [clusters] + verbs: [get, list, watch] + - apiGroups: [cluster.x-k8s.io] + resources: [machines] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [infrastructure.cluster.x-k8s.io] + resources: [kubeswiftmachines] + verbs: [get, list, watch, create, update, patch, delete] + - apiGroups: [bootstrap.cluster.x-k8s.io] + resources: ["*"] + verbs: [get, list, watch, create, update, patch, delete] ``` -Note what is **absent**: no `pods`, no `nodes`, no `swiftgpuprofiles` write, no -`resourceclaims` write. The operator never allocates a GPU — KubeSwift and the -scheduler do. +Note what is still **absent**: no `pods`, no `nodes`, no `swiftgpuprofiles` +write, no `resourceclaims` write. The operator never allocates a GPU — +KubeSwift and the scheduler do. ### Inner (workload) cluster — the credential in `kubeconfigSecretRef` -Minimum for the MVP; every verb has a named consumer: +Every verb has a named consumer: ```yaml # ClusterRole gpu-cell-pool-observer - apiGroups: [""] resources: [nodes] - verbs: [get, list, watch] # correlation, readiness -- apiGroups: [""] - resources: [nodes] - verbs: [patch, update] # identity labels, taints, cordon + verbs: [get, list, watch, patch, update, delete] + # DELETE IS ALWAYS-ON, NOT A DRAIN/PHASE-4 FEATURE — the draft below marked + # it Phase-4-only; that was wrong even before scale-down shipped. Two paths + # unconditionally need it: reaping a stale Node left by a replaced cell + # (§3, §8), and removing a cell's Node when the cell itself is deleted. + # Without it every teardown fails Forbidden and stale Nodes accumulate. - apiGroups: [""] resources: [pods] verbs: [get, list, watch] # HAMi DevicePlugin accounting - apiGroups: [resource.k8s.io] resources: [resourceslices, resourceclaims] verbs: [get, list, watch] # HAMi DRA accounting -# Phase 4 only (drain): -- apiGroups: [""] - resources: [pods/eviction] - verbs: [create] -- apiGroups: [""] - resources: [nodes] - verbs: [delete] # remove the Node of a deleted cell -# KubeadmToken bootstrap mode only, scoped by resourceNames prefix if the -# distribution permits it: -- apiGroups: [""] - resources: [secrets] - verbs: [create, get, update, delete] # kube-system bootstrap tokens ``` -`cluster-admin` is never required. The three privilege escalations to be conscious -of are `nodes/patch` (can cordon/label any node), `pods/eviction` (Phase 4), and -bootstrap-token creation (Phase-dependent, and the reason `Opaque` is the default -provider — see `-bootstrap.md` §5). Ship a ready-made -`ClusterRole`+`ServiceAccount`+token manifest so operators do not hand over an -admin kubeconfig out of convenience. +Deliberately absent, both permanently: `pods/eviction` (the operator waits for +a cell's GPU to be released rather than evicting workloads to force it empty — +drain with `kubectl` if you need a cell emptied faster) and any `secrets` +write in the workload cluster (the `KubeadmToken` bootstrap provider that +would have needed it was never implemented — see `docs/limitations.md`). + +`cluster-admin` is never required. `nodes/patch`+`delete` (can cordon/label/ +remove any node in the workload cluster) is the one privilege escalation to be +conscious of, since it is the broadest grant here — it is why the shipped +`config/rbac/workload-cluster-observer.yaml` is a separate, minimal manifest +rather than something operators are expected to hand-write, and why the +default `Opaque` bootstrap provider (`-bootstrap.md` §5) needs no +bootstrap-token-minting rights at all (`pods/eviction` and bootstrap-token +creation, both once planned as later-phase grants here, were never needed: +the former because the operator never evicts, the latter because +`KubeadmToken` bootstrap was never implemented — `docs/limitations.md`). Ship +a ready-made `ClusterRole`+`ServiceAccount`+token manifest so operators do not +hand over an admin kubeconfig out of convenience — `config/rbac/ +workload-cluster-observer.yaml` is exactly that. --- @@ -337,23 +377,30 @@ Nothing lives in memory. On startup, for each pool: ## 9. Metrics -Prefix `gpucell_`, one namespace for both levels, `pool` label everywhere (the -brief's `gpucellpool_*`/`gpucell_*` split is noise): +Prefix `gpucell_`, one namespace for both levels. **Every metric actually +shipped carries `{pool, namespace}`, not just `{pool}`** as drafted below — a +pool name is only unique within a namespace, and the draft's label sets were +written before that was caught. See `internal/metrics/metrics.go` and +`docs/runbook.md` for the ground truth (twelve metrics, one more than drafted +here: `scale_decisions_total` was added with autoscaling): ``` -gpucell_cells_desired{pool} gauge -gpucell_cells{pool,phase} gauge # Ready|Booting|Joining|Awaiting…|Draining|Failed -gpucell_cell_startup_seconds{pool} histogram # Pending→Ready -gpucell_cell_transitions_total{pool,from,to} counter -gpucell_physical_gpus{pool,state} gauge # held|free-in-cluster -gpucell_capacity_gpu_devices{pool} gauge -gpucell_capacity_gpu_memory_bytes{pool,state} gauge # total|allocated|available -gpucell_capacity_gpu_compute_percent{pool,state} gauge -gpucell_capacity_scrape_errors_total{pool,reason} counter -gpucell_workload_cluster_reachable{pool} gauge # 1|0 -gpucell_reconcile_errors_total{pool,reason} counter +gpucell_cells_desired{pool,namespace} gauge +gpucell_cells{pool,namespace,phase} gauge # Ready|Booting|Joining|Awaiting…|Draining|Failed +gpucell_cell_startup_seconds{pool,namespace} histogram # creation→first Ready +gpucell_cell_transitions_total{pool,namespace,from,to} counter +gpucell_physical_gpus{pool,namespace,state} gauge # held|free-in-cluster +gpucell_capacity_gpu_devices{pool,namespace} gauge +gpucell_capacity_gpu_memory_bytes{pool,namespace,state} gauge # total|allocated|available +gpucell_capacity_gpu_compute_percent{pool,namespace,state} gauge +gpucell_capacity_scrape_errors_total{pool,namespace,reason} counter +gpucell_workload_cluster_reachable{pool,namespace} gauge # 1|0 +gpucell_scale_decisions_total{pool,namespace,reason,scaled_up} counter +gpucell_reconcile_errors_total{pool,namespace} counter ``` -`gpucell_cell_startup_seconds` is the number that decides whether Phase 3 -autoscaling is worth building: if a cell takes 12 minutes to become Ready, -demand-driven scale-up is a capacity planner, not an autoscaler. +`gpucell_cell_startup_seconds` is the number that decides whether +demand-driven autoscaling is worth using reactively: at the measured ~4m45s +(`-bootstrap.md` §6) it is workable with the default 10-minute stabilization +window; a cell in the range this draft worried about (12+ minutes) would make +scale-up a capacity planner rather than an autoscaler. diff --git a/docs/design/gpucellpool-poc.md b/docs/design/gpucellpool-validation-record.md similarity index 94% rename from docs/design/gpucellpool-poc.md rename to docs/design/gpucellpool-validation-record.md index 2b03956..daa42ae 100644 --- a/docs/design/gpucellpool-poc.md +++ b/docs/design/gpucellpool-validation-record.md @@ -1,4 +1,11 @@ -# GPUCellPool — proof of concept and test strategy +# GPUCellPool — validation record + +> Design record — a **results record, not a plan** (renamed from `-poc.md`, +> which it was originally written as before the results in §8a/§8b/§9 landed). +> Written before implementation (2026-07-30) and partially updated. It +> documents rationale and measurements, not current behaviour — see `docs/` +> for that, in particular `docs/quickstart.md` and +> `docs/design/gpucellpool-bootstrap.md` §6 for the current startup numbers. > Phase 1 is a **hardware proof done by hand**: physical GPU → KubeSwift VM → > in-guest NVIDIA driver → HAMi → two fractionally-limited workloads on the same @@ -6,7 +13,7 @@ > > Lab: dev k0s cluster (frida CP, miles/boba workers), **one GTX 1080 on boba**, > CH v53.0, KubeSwift v0.13.4. Point `KUBECONFIG` at the infrastructure cluster. -> Date: 2026-07-30. +> Date: 2026-07-30. Results recorded 2026-08-07/08 in §8a/§8b/§9 below — **PASS**. --- @@ -97,7 +104,10 @@ Install HAMi on the inner cluster (prerequisite, D6), label the node `gpu=on`, t ```bash kubectl get node -o jsonpath='{.metadata.annotations.hami\.io/node-nvidia-register}' | jq . -# [{"id":"GPU-…","count":10,"devmem":8192,"devcore":100,"type":"NVIDIA-GeForce-GTX-1080","numa":0,"health":true}] +# The real captured shape (§8a): the model string has SPACES, not hyphens, and +# there is no "numa" field — older documentation showing "numa" is wrong; do +# not assume it will be present. +# [{"id":"GPU-…","count":10,"devmem":8192,"devcore":100,"type":"NVIDIA GeForce GTX 1080","health":true}] kubectl get node -o jsonpath='{.status.allocatable}' | jq '."nvidia.com/gpu"' # 10, NOT 1 ``` @@ -242,8 +252,10 @@ Findings that change the design or the image recipe: `build-cell-image.sh` produced a 30 GiB raw disk, **5.5 GiB sparse**, `BAKE_EXIT=0`, containing driver 580.173.02 (kernel 6.8.0-136-generic), the NVIDIA container toolkit, `k0s v1.36.3+k0s.0`, the k0s containerd nvidia drop-in, the CDI -generate unit, and a reset cloud-init/machine-id/host-keys. Not yet published; -`swiftctl image publish --to ghcr.io/… --tag …` is the remaining step. +generate unit, and a reset cloud-init/machine-id/host-keys. At the time this +paragraph was written it was not yet published — it was, moments later, by +`swiftctl image publish`; see §8b below, where it boots under Cloud Hypervisor, +and `docs/cell-image.md` for the current publishing recipe. Notes for the next bake: the local build needs the **distro** QEMU (`/usr/bin/qemu-system-x86_64`) — Kata's bundled build has no user-mode networking diff --git a/docs/limitations.md b/docs/limitations.md new file mode 100644 index 0000000..d142497 --- /dev/null +++ b/docs/limitations.md @@ -0,0 +1,89 @@ +# Limitations + +Everything here is a real, current gap — not a hedge. Where there is a +practical workaround, it is stated; where the answer is "a human has to act", +that is stated too. + +## HAMi DRA mode is unimplemented + +`spec.capacity.hami.mode: DRA` is accepted by the API but +`internal/capacity`'s DRA path returns `ErrUnsupported` — capacity reads, +demand signal (`docs/autoscaling.md`), and readiness all fail loudly rather +than silently reading zero. **`DevicePlugin` mode is the only implementation.** +Use it (the default) even if your HAMi install also has DRA mode available. + +## No rolling update on `guestTemplate` change + +Changing `spec.cell.guestTemplate` (a new `imageRef`, a driver bump, a +different `guestClassRef`) bumps the per-cell template-hash annotation but +**does not roll existing cells**. `status.conditions` will not tell you a +template drifted either — there is no `Updated` condition in v1alpha1. + +This is the most likely real operational task you will hit: **updating the +NVIDIA driver (or anything else) in the cell image means manually recreating +every cell**, one at a time, and doing the *inner* drain yourself first — +cordon and drain the cell's workload Node before deleting the cell's +`SwiftGuest`/`Machine`, or you destroy running HAMi workloads. See +`docs/runbook.md` for the sequence. + +## No automated outer-drain sequencing + +A cell is a VFIO guest, so KubeSwift can only move it with an *offline* +migration — a VM restart (`docs/concepts.md` — "Cells are cattle"). Every +cell is therefore pinned with `migration.enabled: false`. If the +infrastructure node holding a cell is cordoned or drained, the pool surfaces +`CellDrainRequested` and **does nothing further** — it does not cordon the +inner Node, drain inner workloads, delete the cell, or recreate it elsewhere. +**A human has to act**: drain the cell's workload Node, delete the cell, and +let the pool recreate it (on a different infrastructure node, if the original +one stays cordoned). + +## Bootstrap token minting is not implemented + +`spec.bootstrap.provider` accepts only `Opaque` — user-supplied cloud-init with +a closed substitution set. A `KubeadmToken` provider that would mint TTL'd join +tokens per cell (removing the token-expiry footgun for kubeadm-style clusters) +was designed but never implemented, and the value was **removed from the API +enum** rather than left accepted-and-ignored — an earlier iteration did exactly +that, and it failed silently: the object passed admission, then the controller +errored about a provider the user never actually selected. If your bootstrap +token expires, new cells fail at `Joining` with `JoinTimeout` +(`docs/runbook.md` has the diagnostic). + +## Pools of two or more cells: harness-only + +The reference lab has exactly one GPU. Everything about a *single* cell — +provisioning, boot, join, HAMi accounting, autoscale-to-one, teardown — is +hardware-validated. Behaviour specific to **multiple concurrent cells** — +membership planning with more than one cell in flight, per-index replacement +backoff at scale, cross-cell scheduling spread under real contention — is +covered by the two-envtest test harness (a real apiserver as the outer +cluster, a second as the workload cluster) but has not been run against real +hardware with two or more physical GPUs. + +The same caveat applies to `spec.deletion.policy: Force` (skips drain +entirely — validated in the harness, not against a hardware cell with live +HAMi workloads on it) and to cell replacement backoff under repeated real +failures. + +## One GPU per cell + +`spec.cell.gpu.count` accepts only `1`. Multi-GPU cells (NVLink-connected +GPUs behind one VM) need PCIe/NUMA topology work this project has not done; +there is no roadmap item committed for it. + +## `resourceClaimName` is single-cell-only + +`spec.cell.gpu.dra.resourceClaimName` references one pre-created, shared +`ResourceClaim`. A VFIO device backs exactly one running VM, so pointing more +than one cell at the same claim double-books the device. This is only ever +correct for a pool whose `replicas` (and `autoscaling.maxReplicas`, if set) +is 1 — use `resourceClaimTemplateName` for anything larger. + +## What is *not* a limitation, stated for clarity + +- Scale-up and scale-down are both implemented (`docs/autoscaling.md`) — the + earlier design draft that called scale-down "postponed" is stale; ignore + any doc under `docs/design/` that still says so (they carry a banner). +- `provisioner: ClusterAPI` is shipped and hardware-validated, not "later" — + see `docs/clusterapi-cells.md`. diff --git a/docs/networking.md b/docs/networking.md new file mode 100644 index 0000000..63dbce9 --- /dev/null +++ b/docs/networking.md @@ -0,0 +1,137 @@ +# Networking + +This is the single authority on cell networking. Read it before applying a +pool — this is where deployments get stuck most often, and every consequence +below was measured on real hardware, not inferred. + +## A cell needs a routable interface, not just egress + +A Kubernetes node must be **dialable by its own apiserver** — the apiserver +initiates connections to a node's kubelet for logs, `exec`, `port-forward`, and +metrics. Egress alone (a cell that can reach *out* to the workload cluster) +is not enough: the cell also needs an address the workload apiserver can reach +*in* on. + +KubeSwift's default nat primary interface gives a cell node-local egress only +(`192.168.99.x`, private to that launcher pod's network namespace — see below). +A cell on that interface alone will join the workload cluster and then +half-work: it registers Ready, but `kubectl logs`/`exec` against pods on it +fail, and (with `capacity.hami.mode: DevicePlugin`) HAMi's own accounting reads +depend on the same path. + +**Therefore every cell needs a second, routable interface**, and +`cell.nodeIPFrom` must name it: + +```yaml +cell: + guestTemplate: + interfaces: + - name: mgmt + primary: true # KubeSwift's node-local nat egress + - name: node + networkRef: + name: cell-net # a Multus NetworkAttachmentDefinition, routable + nodeIPFrom: node # kubelet registers THIS interface's address +``` + +`networkRef` takes `{name, namespace}` only — KubeSwift's strict decoding +rejects a `kind` field on it (an earlier draft of this project's samples had +one; it does not exist). + +## `br0` is per launcher pod, not per node — there is no same-node shortcut + +Measured: KubeSwift's `br0` bridge lives inside **each launcher pod's own +network namespace**. Two guests scheduled on the *same physical host* both +receive `192.168.99.10` in separate namespaces and **cannot reach each other** +over that interface at all. Co-locating a workload-cluster control-plane VM and +a GPU cell on one host does not give them a shared L2 for free. + +Anything a cell needs to talk to beyond its own pod's egress — an apiserver +running in another VM, a kubelet dialed by that apiserver — needs a +`networkRef` NAD. There is no shortcut for VMs that happen to land on the same +node. + +## A minimal NAD + +A node-local bridge NAD is enough while every VM involved is on one host (the +common single-GPU-node case); cross-node reachability needs KubeSwift's +secondary-NAD-with-routable-IP shape (see the KubeSwift networking docs) — +GPUCellPool does not add anything beyond what a cell's `networkRef` already +gives it. + +```yaml +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: cell-net + namespace: gpu-cells +spec: + config: | + { + "cniVersion": "0.4.0", + "name": "cell-net", + "type": "bridge", + "bridge": "cellbr0", + "isGateway": true, + "ipam": { + "type": "host-local", + "subnet": "10.77.0.0/24", + "rangeStart": "10.77.0.10", + "rangeEnd": "10.77.0.250" + } + } +``` + +A ready-to-apply copy is at `config/samples/network-attachment-definition.yaml`. +Every VM that must reach, or be reached by, another VM on this network — the +workload cluster's control plane included, if it is also a KubeSwift guest — +needs an `interfaces[].networkRef` entry pointing at the same NAD. + +## `nodeIPFrom` is observation-only, not a readiness gate + +`cell.nodeIPFrom` names a **KubeSwift interface**, not a guest device — the +operator has no way to map `node` to `ens4` inside the guest, and the address +is not known at render time anyway (DHCP/IPAM assign it after the guest +exists). Two consequences: + +- **The join template must derive the node IP itself, by subnet, with a wait + loop** — see `config/samples/cell-join-secret.yaml` for the pattern. There is + no `{{ nodeIP }}` substitution token, because the controller does not know + the value at render time either. +- **KubeSwift v0.13.4 reports a secondary NAD interface's MAC in + `status.network.interfaces[]`, but not its IP** — measured; the guest + genuinely has the address, KubeSwift just does not surface it. GPUCellPool + therefore treats `nodeIPFrom` as an *observation* field: it reports the + routable address once KubeSwift exposes one and falls back to the primary + interface for the cell's own readiness gate, verifying which address the + Node actually registered against the *workload cluster*, where the ground + truth lives. Gating cell readiness on KubeSwift reporting the routable + address would park every NAD-attached cell in `Booting` forever — this was a + real bug, fixed before v0.1.0 (see `CHANGELOG.md`). + +## `kubectl port-forward` cannot reach a nat-exposed VM + +`kubectl port-forward` dials `localhost` **inside the pod's network +namespace**, while KubeSwift's nat primary DNAT maps `podIP -> VM`. The two do +not compose: port-forwarding to a pod fronting a nat-exposed guest does not +reach the guest. To reach a VM-hosted service (an apiserver, for debugging) from +outside its own cluster: + +- expose it as a Kubernetes `Service` and reach that, or +- put the client on the same `networkRef` NAD as the VM. + +This bit the operator's own development: the operator pod that drives a pool +must itself join the workload cluster's NAD (via +`podAnnotations: k8s.v1.cni.cncf.io/networks` plus a matching `nodeSelector`) +to reach a workload apiserver that is itself a KubeSwift guest with no other +path in. + +## DNS inside the cell + +A measured failure worth knowing before you bake an image: once the inner CNI +(k0s's kube-router, or whatever your distribution ships) programs the cell's +node, `systemd-resolved`'s stub stops resolving through the launcher pod's +dnsmasq at `192.168.99.1`, while raw egress keeps working. Image pulls and +package installs then fail at the worst possible moment — mid-join. **Pin an +upstream resolver in the cell image** (`/etc/systemd/resolved.conf.d/`) rather +than relying on the pod-netns default. See `docs/cell-image.md`. diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..1f1d3d5 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,242 @@ +# Quickstart + +The shortest path from nothing to two workloads sharing one GPU inside a cell. +Budget **~5 minutes** for the first cell to boot and join (measured: 4m45s +guest-created to Ready, most of it cloning the root disk — see +`docs/design/gpucellpool-bootstrap.md` §6) plus a couple of minutes to install +the chart and stage prerequisites. + +Two clusters are involved throughout: the **infrastructure** cluster (runs +KubeSwift and this operator) and the **workload** cluster (runs HAMi and +receives cells as worker nodes). They can be the same cluster in a lab, but +keep the distinction straight — every command below says which one it targets. + +## Prerequisites + +You must supply all of these before applying a pool. None of them is created +by GPUCellPool. + +**In the infrastructure cluster:** + +| | | +|---|---| +| KubeSwift | `>= v0.13.4`, installed and healthy | +| a GPU node | labelled `kubeswift.io/gpu-node=true`, with a physical GPU KubeSwift can pass through | +| a `DeviceClass` + `ResourceClaimTemplate` for the VFIO GPU | do not re-derive these — apply KubeSwift's own samples at `config/samples/dra-gpu/` in the KubeSwift repo (`resourceclaimtemplate-single-gpu.yaml` creates the `single-vfio-gpu` template this quickstart references) | +| Multus + a `NetworkAttachmentDefinition` carrying a **routable** address | not optional — see `docs/networking.md`. A minimal sample is at `config/samples/network-attachment-definition.yaml` | +| a `SwiftGuestClass` | CPU/memory/disk shape for the cell VM | +| a cell `SwiftImage` | a disk image with the NVIDIA driver, containerd + CDI, and your distribution's node binaries baked in — see `docs/cell-image.md` to build one | + +**In the workload cluster:** + +| | | +|---|---| +| a Kubernetes cluster | any distribution (k0s, kubeadm, RKE2, k3s all work — `spec.bootstrap.provider: Opaque` makes no assumption) | +| HAMi | already installed, with its DaemonSets able to tolerate whatever taints you plan to put on cell nodes | +| network path | the workload apiserver must be able to reach a cell's kubelet — this is the same routable-NAD requirement above, from the other direction | + +**Two Secrets**, staged before you apply the pool (both created below). + +## 1. Install the chart + +In the **infrastructure** cluster: + +```bash +helm install gpucellpool oci://ghcr.io/kubeswift-io/charts/gpucellpool \ + --version 0.1.0 \ + --namespace gpucellpool-system --create-namespace +``` + +The validating webhook installs on by default — leave it on (see +`docs/security.md`). + +## 2. Give the operator a credential for the workload cluster + +Apply the RBAC bundle **in the workload cluster** — it creates a +`ServiceAccount` scoped to exactly what the operator needs, no +`cluster-admin`: + +```bash +kubectl --context workload apply -f config/rbac/workload-cluster-observer.yaml +``` + +Build a kubeconfig from that ServiceAccount's token and store it as a Secret +**in the infrastructure cluster**, in the namespace where you will create the +pool: + +```bash +TOKEN=$(kubectl --context workload -n kube-system create token gpu-cell-pool-observer --duration=8760h) +SERVER=$(kubectl --context workload config view --minify -o jsonpath='{.clusters[0].cluster.server}') +CA=$(kubectl --context workload config view --raw --minify -o jsonpath='{.clusters[0].cluster.certificate-authority-data}') + +cat > workload-kubeconfig.yaml < +kubectl --context workload drain --ignore-daemonsets --delete-emptydir-data + +# 2. Confirm the cell is actually idle before deleting it. +kubectl -n get cellpool \ + -o jsonpath='{range .status.cells[*]}{.name} {.capacityDevices}{"\n"}{end}' + +# 3. Delete the cell's outer object. The pool recreates it from the CURRENT +# guestTemplate (SwiftGuest) or Machine (ClusterAPI) — pick the matching +# command. +kubectl -n delete swiftguest # provisioner: SwiftGuest +kubectl -n delete machine # provisioner: ClusterAPI + +# 4. Watch it come back on the new template. +kubectl -n get cellpool -w +``` + +The cell finalizer (`cells.kubeswift.io/cell-drain`) blocks the delete from +actually completing until HAMi allocations clear — if step 1 was skipped, the +delete in step 3 hangs rather than destroying running work, which is +correct, but slower than doing the drain first. + +--- + +## Metrics + +Twelve `gpucell_*` Prometheus series, every one labelled `{pool, namespace}` +(`internal/metrics/metrics.go`) plus whatever the metric itself is about. +Both layers are kept deliberately apart, so an alert can distinguish "no GPU +left in the infrastructure cluster" from "the shared GPU is full": + +| Metric | Type | Extra labels | What it tells you | +|---|---|---|---| +| `gpucell_cells_desired` | gauge | — | what the scaling policy asked for | +| `gpucell_cells` | gauge | `phase` | cells per phase — every phase is written every pass, so an emptied phase reads `0`, not stale | +| `gpucell_cell_startup_seconds` | histogram | — | creation → first Ready; the number that decides whether reactive autoscaling makes sense | +| `gpucell_cell_transitions_total` | counter | `from`, `to` | catches an oscillating cell (`Ready → AwaitingGPUCapacity → Ready`) even when its current phase looks healthy | +| `gpucell_physical_gpus` | gauge | `state` (`held`\|`free`) | outer, whole devices | +| `gpucell_capacity_gpu_devices` | gauge | — | inner, devices HAMi advertises across this pool's cells | +| `gpucell_capacity_gpu_memory_bytes` | gauge | `state` (`total`\|`allocated`\|`available`) | inner, always valid (bytes are commensurable across GPU models) | +| `gpucell_capacity_gpu_compute_percent` | gauge | `state` | inner, homogeneous pools only | +| `gpucell_workload_cluster_reachable` | gauge | — | `1`/`0` — while `0`, every destructive path is frozen; alert on this to explain why a pool stopped changing | +| `gpucell_capacity_scrape_errors_total` | counter | `reason` | a failed capacity read is reported `Unknown` and the last value retained — this counter is how you'd notice the data went stale, since the gauges alone will not tell you | +| `gpucell_scale_decisions_total` | counter | `reason`, `scaled_up` | every autoscaling decision, including refusals — "did not scale, and here is why" is the interesting case | +| `gpucell_reconcile_errors_total` | counter | — | reconciles that returned an error | + +A pool deleted from the cluster stops reporting — the controller drops its +label series on teardown rather than leaving a torn-down pool's last state +looking current forever. + --- ## Networking: the thing that bites +See `docs/networking.md` for the full treatment; this is the short version. + A Kubernetes node must be **dialable by its own apiserver** (logs, exec, port-forward, metrics). Two measured consequences: diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..d58ee13 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,89 @@ +# Security + +## The launcher pod is a node-level trust boundary + +Every KubeSwift launcher pod runs `privileged: true` **by design** — it is +documented KubeSwift behaviour, not an oversight, and not something this +project works around (KubeSwift tried a capability-scoped launcher and reverted +it; see KubeSwift's own security-audit doc). A cell's launcher pod is no +exception. + +The consequence for this operator: **the right to create a `GPUCellPool` in a +namespace is node-root-equivalent authority in the infrastructure cluster.** +Anything a `GPUCellPool` spec can make a cell's launcher pod mount, request, or +run is effectively node-level access on whichever KubeSwift node schedules it. +This is why `spec.cell.guestTemplate` is not a free-form passthrough — see the +webhook, below — and why standard Kubernetes namespace isolation is **not** +sufficient to isolate `GPUCellPool` creators from each other or from the rest +of the infrastructure cluster. Restrict who may create `GPUCellPool` objects +the same way you would restrict who may create a privileged Pod directly (an +admission policy, a separate namespace with tighter RBAC, or simply: only +infrastructure administrators). + +## Layered isolation, not tenant isolation + +State the isolation claim precisely (see `docs/concepts.md`): a cell gives a +group of workloads a VM boundary around a physical GPU, and HAMi gives sharing +efficiency inside that boundary. That is a real, useful property — a stronger +boundary between groups of workloads than GPU-sharing alone provides — but it +is not absolute tenant isolation, and it does not change who is trusted to +*create* cells in the first place. Do not describe GPUCellPool as providing +multi-tenant security; describe it as providing layered isolation between +workload groups that already trust the same infrastructure operators. + +## The validating webhook is a security control + +The webhook (`internal/webhook/v1alpha1`, on by default, +`failurePolicy: fail`) enforces the `guestTemplate` field contract (see +`docs/api-reference.md`): a set of fields a user is not permitted to set +because the operator owns them, and a set that is denied outright because it +is incompatible with a GPU cell. This denylist is not a convenience check — it +is what stops a `GPUCellPool` spec from asking the launcher pod for something +that should be refused. **That is why the webhook fails closed +(`failurePolicy: fail`)**: if the webhook cannot be reached, admission is +rejected rather than silently allowed through. + +Running with the webhook disabled (`webhook.enabled: false` in the Helm +values) removes this control. The chart's `NOTES.txt` warns about this +explicitly on install. Do not disable it in a cluster where `GPUCellPool` +creation is not already restricted to fully-trusted operators. + +The same rules are enforced a second time at render time +(`internal/provisioner.ValidateTemplate`/`ValidateGPU`), so a spec that slipped +past a disabled or unreachable webhook — an object created before the webhook +was installed, for instance — still cannot misconfigure a cell when the +controller renders it. + +## Two RBAC scopes + +GPUCellPool touches two clusters with two different credentials, and neither +needs broad access: + +**Outer (infrastructure) cluster** — the operator's own ServiceAccount +(`config/rbac/role.yaml`). Notably **absent**: `pods`, `nodes`, and any write +verb on `resourceclaims`/`swiftgpuprofiles`. The operator never allocates a +GPU itself — KubeSwift and the Kubernetes scheduler do that. It creates and +watches `SwiftGuest`/`SwiftSeedProfile` (and, when `provisioner: ClusterAPI`, +`Machine`/`KubeSwiftMachine`), and reads GPU inventory (`resourceslices`, +`resourceclaims`, `resourceclaimtemplates`, `deviceclasses`) read-only. + +**Inner (workload) cluster** — the credential in +`spec.workloadCluster.kubeconfigSecretRef`, scoped by +`config/rbac/workload-cluster-observer.yaml`. `cluster-admin` is never +required or expected; hand over the `ServiceAccount` token this manifest +creates, not an admin kubeconfig. The grants: + +| Resource | Verbs | Consumer | +|---|---|---| +| `nodes` | `get, list, watch, patch, update, delete` | correlation, readiness, applying identity labels/taints, cordoning, and — `delete` is **not optional and not a drain-only feature** — reaping a stale Node left by a replaced cell, and removing a cell's Node when the cell itself is deleted. Both are always-on paths; without `delete`, every teardown fails `Forbidden` and stale Nodes accumulate | +| `pods` | `get, list, watch` | HAMi `DevicePlugin`-mode accounting (allocations live on Pod annotations) | +| `resourceslices`, `resourceclaims` | `get, list, watch` | HAMi `DRA`-mode accounting | + +Nothing else is granted. In particular the operator does **not** hold +`pods/eviction`: it never evicts a workload to force a cell empty — it waits +for the capacity provider to report the cell idle, and if you need a cell +emptied faster, drain it yourself with `kubectl drain`. + +If a pool reports `Forbidden` on the `WorkloadClusterReachable` condition, the +credential Secret is missing one of the grants above — see the runbook entry +in `docs/runbook.md`. diff --git a/internal/webhook/v1alpha1/samples_test.go b/internal/webhook/v1alpha1/samples_test.go new file mode 100644 index 0000000..c6433f5 --- /dev/null +++ b/internal/webhook/v1alpha1/samples_test.go @@ -0,0 +1,57 @@ +package v1alpha1 + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/util/yaml" + + cellsv1alpha1 "github.com/kubeswift-io/gpucellpool/api/v1alpha1" +) + +// TestShippedSamplesPassValidation runs every GPUCellPool in config/samples through +// the real admission rules. +// +// A sample the webhook rejects is worse than no sample: it is the first thing a new +// user applies, and it fails in a way that looks like their mistake. The CRD schema is +// checked by a server dry-run in CI; this covers the half of the contract that lives in +// the webhook and cannot be expressed in OpenAPI. +func TestShippedSamplesPassValidation(t *testing.T) { + dir := filepath.Join("..", "..", "..", "config", "samples") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read samples dir: %v", err) + } + + checked := 0 + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".yaml") { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatalf("read %s: %v", e.Name(), err) + } + for i, doc := range strings.Split(string(raw), "\n---") { + if !strings.Contains(doc, "kind: GPUCellPool") { + continue + } + var pool cellsv1alpha1.GPUCellPool + if err := yaml.NewYAMLOrJSONDecoder(strings.NewReader(doc), 4096).Decode(&pool); err != nil { + t.Errorf("%s doc %d: not decodable as a GPUCellPool: %v", e.Name(), i, err) + continue + } + checked++ + if errs := Validate(&pool, nil); len(errs) > 0 { + t.Errorf("%s (%s) would be REJECTED by our own webhook: %v", + e.Name(), pool.Name, errs.ToAggregate()) + } + } + } + if checked == 0 { + t.Fatal("no GPUCellPool samples found — this test would silently pass forever") + } + t.Logf("validated %d shipped GPUCellPool sample(s)", checked) +}