From e67b536234a6a83e0e72d1e6ce8bd2dec8297034 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 13 Jul 2026 15:54:17 +0100 Subject: [PATCH 01/52] docs: add three-tier StorageNodeSet/StorageNode/StorageNodeOps design document --- .../design-storagenodeset-storagenode.md | 650 ++++++++++++++++++ 1 file changed, 650 insertions(+) create mode 100644 operator/docs/designs/design-storagenodeset-storagenode.md diff --git a/operator/docs/designs/design-storagenodeset-storagenode.md b/operator/docs/designs/design-storagenodeset-storagenode.md new file mode 100644 index 00000000..9f65cab8 --- /dev/null +++ b/operator/docs/designs/design-storagenodeset-storagenode.md @@ -0,0 +1,650 @@ +# Design Document: StorageNodeSet / StorageNode / StorageNodeOps Three-Tier Model + +**Status:** Draft +**Author:** Israel Geoffrey +**Date:** 2026-07-13 + +--- + +## 1. Background + +The current `StorageNodeSet` CRD conflates three distinct concerns: + +| Concern | Current location | Problem | +|---|---|---| +| Fleet management (DaemonSet, RBAC, provisioning) | `StorageNodeSet.spec` | Mixed with per-node config | +| Per-node configuration and status | `StorageNodeSet.status.nodes[]` | No individual CR to target | +| Imperative operations (shutdown, restart, drain) | `StorageNodeSet.spec.action` | One action at a time on a fleet object | + +--- + +## 2. Goals + +- **`StorageNodeSet`** — fleet template: defaults, DaemonSet, RBAC, services. Creates and owns `StorageNode` CRs. +- **`StorageNode`** — individual node: configuration overrides, observed status, health. One CR per (worker, socket). +- **`StorageNodeOps`** — imperative operations: a one-shot CR that targets a `StorageNode` and drives an action (shutdown, restart, suspend, resume, remove/drain) to completion, then records the result. + +## 3. Non-Goals + +- Changing the backend provisioning protocol. +- Replacing DaemonSet/RBAC/Service management. +- Automatic migration of existing `StorageNodeSet`-only deployments (migration strategy is a follow-up). + +--- + +## 4. Architecture Overview + +``` +StorageNodeSet (fleet / template) + │ owns + ├─► StorageNode (worker-A, socket 0) ←── StorageNodeOps (action: remove) + ├─► StorageNode (worker-A, socket 1) + ├─► StorageNode (worker-B, socket 0) ←── StorageNodeOps (action: restart) + └─► StorageNode (worker-C, socket 0) + +VolumeMigration CRs (owned by StorageNodeOps during drain) +``` + +`StorageNodeOps` is scoped to a single `StorageNode`. Multiple operations can run concurrently on different nodes. Only one `StorageNodeOps` can be active per `StorageNode` at a time (enforced by a validating webhook or controller). + +--- + +## 5. API Design + +### 5.1 StorageNodeSet (revised — fleet only) + +Removes all per-node action and status fields. Adds a summary status. + +```go +type StorageNodeSetSpec struct { + ClusterName string `json:"clusterName"` + ClusterImage string `json:"clusterImage,omitempty"` + SpdkImage string `json:"spdkImage,omitempty"` + SpdkProxyImage string `json:"spdkProxyImage,omitempty"` + MgmtIfname string `json:"mgmtIfname"` + DataIfname []string `json:"dataIfname,omitempty"` + WorkerNodes []string `json:"workerNodes"` + + // Fleet-wide defaults — apply to every StorageNode unless overridden by NodeConfigs. + MaxLogicalVolumeCount *int32 `json:"maxLogicalVolumeCount,omitempty"` + Partitions *int32 `json:"partitions,omitempty"` + JournalManagerSpec *JournalManagerSpec `json:"journalManager,omitempty"` + CorePercentage *int32 `json:"corePercentage,omitempty"` + SpdkSystemMemory string `json:"spdkSystemMemory,omitempty"` + MaxSize string `json:"maxSize,omitempty"` + NodesPerSocket *int32 `json:"nodesPerSocket,omitempty"` + SocketsToUse []string `json:"socketsToUse,omitempty"` + // ... tolerations, resources, image pull policy, OpenShift fields, etc. + + // NodeConfigs allows per-worker-node configuration overrides keyed by the + // Kubernetes worker node name. Entries here are propagated to the corresponding + // StorageNode.spec.overrides by the reconciler on every reconcile — the + // StorageNodeSet is the single source of truth for all per-node config, + // including failure domain assignment via nodeConfigs[worker].failureDomain. + // + // Example: + // nodeConfigs: + // vm02.simplyblock3.localdomain: + // maxLogicalVolumeCount: 50 + // spdkSystemMemory: "8G" + // failureDomain: 1 + // vm04.simplyblock3.localdomain: + // maxLogicalVolumeCount: 10 + // failureDomain: 2 + // +optional + NodeConfigs map[string]StorageNodeOverrides `json:"nodeConfigs,omitempty"` + + // REMOVED: action, nodeUUID, force, reattachVolume, workerNode + // REMOVED: systemVolumeFilterRegex (moves to StorageNodeOps) + // REMOVED: nodeFailureDomains (consolidated into NodeConfigs[].failureDomain) +} + +type StorageNodeSetStatus struct { + // Summary counts derived from owned StorageNode statuses. + TotalNodes int `json:"totalNodes,omitempty"` + OnlineNodes int `json:"onlineNodes,omitempty"` + OfflineNodes int `json:"offlineNodes,omitempty"` + + // PendingNodeAdds is the provisioning guard against duplicate POSTs. + PendingNodeAdds map[string]metav1.Time `json:"pendingNodeAdds,omitempty"` + SchedulingFailedWorkers map[string]bool `json:"schedulingFailedWorkers,omitempty"` + DrainCoordination []NodeDrainState `json:"drainCoordination,omitempty"` + + // REMOVED: nodes[], actionStatus (both live on StorageNode) +} +``` + +### 5.2 StorageNode (new CRD) + +One CR per backend storage node instance. Owned by `StorageNodeSet`. + +```go +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Worker",type=string,JSONPath=".spec.workerNode" +// +kubebuilder:printcolumn:name="UUID",type=string,JSONPath=".status.uuid" +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Health",type=boolean,JSONPath=".status.health" +type StorageNode struct { ... } + +type StorageNodeSpec struct { + // StorageNodeSetRef is the owning StorageNodeSet name. Immutable. + // +k8s:immutable + StorageNodeSetRef string `json:"storageNodeSetRef"` + + // WorkerNode is the Kubernetes node hostname. Immutable. + // +k8s:immutable + WorkerNode string `json:"workerNode"` + + // SocketIndex is the NUMA socket index (0-based). Immutable. + // +k8s:immutable + // +optional + SocketIndex *int32 `json:"socketIndex,omitempty"` + + // Overrides allow per-node configuration overrides of the parent StorageNodeSet defaults. + // Immutable fields from the parent (clusterName, mgmtIfname, partitions, etc.) cannot + // be overridden here. + // +optional + Overrides *StorageNodeOverrides `json:"overrides,omitempty"` +} + +type StorageNodeOverrides struct { + MaxLogicalVolumeCount *int32 `json:"maxLogicalVolumeCount,omitempty"` + SpdkSystemMemory string `json:"spdkSystemMemory,omitempty"` + + // FailureDomain is the failure-domain group index (≥ 1) for this node. + // Required when the parent StorageCluster has enableFailureDomains=true. + // Set via StorageNodeSet.spec.nodeConfigs[workerNode].failureDomain. + // +kubebuilder:validation:Minimum=1 + // +optional + FailureDomain *int32 `json:"failureDomain,omitempty"` + + // ... other mutable per-node tuning knobs +} + +type StorageNodeStatus struct { + // UUID is the backend storage node UUID. Set once after node-add completes. + UUID string `json:"uuid,omitempty"` + Status string `json:"status,omitempty"` + Health bool `json:"health,omitempty"` + CPU *int32 `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + Volumes *int32 `json:"volumes,omitempty"` + Devices string `json:"devices,omitempty"` + MgmtIp string `json:"mgmtIp,omitempty"` + Hostname string `json:"hostname,omitempty"` + Uptime string `json:"uptime,omitempty"` + RpcPort *int32 `json:"rpcPort,omitempty"` + LvolPort *int32 `json:"lvolPort,omitempty"` + NvmfPort *int32 `json:"nvmfPort,omitempty"` + + // PostedAt is when the node-add POST was sent (provisioning guard). + PostedAt *metav1.Time `json:"postedAt,omitempty"` + + // ActiveOpsRef is the name of the currently active StorageNodeOps on this node. + // Empty when no operation is in progress. + ActiveOpsRef string `json:"activeOpsRef,omitempty"` +} +``` + +### 5.3 StorageNodeOps (new CRD) + +A one-shot operational CR targeting a single `StorageNode`. Analogous to a Kubernetes `Job`. + +```go +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=".spec.storageNodeRef" +// +kubebuilder:printcolumn:name="Action",type=string,JSONPath=".spec.action" +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp" +type StorageNodeOps struct { ... } + +type StorageNodeOpsSpec struct { + // StorageNodeRef is the name of the target StorageNode. Immutable. + // +k8s:immutable + StorageNodeRef string `json:"storageNodeRef"` + + // Action is the operation to perform. Immutable. + // +kubebuilder:validation:Enum=shutdown;restart;suspend;resume;remove + // +k8s:immutable + Action string `json:"action"` + + // Force enables forced execution where the backend supports it. + // +optional + Force *bool `json:"force,omitempty"` + + // ReattachVolume reattaches volumes during restart. + // +optional + ReattachVolume *bool `json:"reattachVolume,omitempty"` + + // Drain configures the drain workflow for action=remove. + // +optional + Drain *DrainSpec `json:"drain,omitempty"` +} + +type DrainSpec struct { + // SystemVolumeFilterRegex filters system/benchmark volumes from drain migration. + // Defaults to "^sb-fio-baseline-.*". + // +optional + SystemVolumeFilterRegex *string `json:"systemVolumeFilterRegex,omitempty"` +} + +// StorageNodeOpsPhase is the lifecycle phase of a StorageNodeOps. +// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed +type StorageNodeOpsPhase string + +const ( + StorageNodeOpsPhasePending StorageNodeOpsPhase = "Pending" + StorageNodeOpsPhaseRunning StorageNodeOpsPhase = "Running" + StorageNodeOpsPhaseSucceeded StorageNodeOpsPhase = "Succeeded" + StorageNodeOpsPhaseFailed StorageNodeOpsPhase = "Failed" +) + +type StorageNodeOpsStatus struct { + // Phase is the high-level lifecycle phase. + Phase StorageNodeOpsPhase `json:"phase,omitempty"` + + // SubPhase tracks the active drain step (Validating|Suspending|Migrating|Verifying|Removing). + // +kubebuilder:validation:Enum=Validating;Suspending;Migrating;Verifying;Removing + // +optional + SubPhase string `json:"subPhase,omitempty"` + + // Message is a human-readable description of the current state or failure reason. + Message string `json:"message,omitempty"` + + // VolumesMigrated is the count of volumes migrated (drain only). + VolumesMigrated int `json:"volumesMigrated,omitempty"` + // VolumesPending is the count of volumes awaiting migration (drain only). + VolumesPending int `json:"volumesPending,omitempty"` + + // StartedAt is when the operation began. + StartedAt *metav1.Time `json:"startedAt,omitempty"` + // CompletedAt is when the operation finished (successfully or not). + CompletedAt *metav1.Time `json:"completedAt,omitempty"` +} +``` + +--- + +## 6. Controller Changes + +### 6.1 StorageNodeSetReconciler + +Narrowed to **fleet management only**: + +1. Ensure DaemonSet, RBAC, Services, EndpointSlices — unchanged. +2. **Reconcile StorageNode CRs** — for each `(workerNode, socket)` in `spec.workerNodes × spec.socketsToUse`: + - Create a `StorageNode` CR with owner reference if it does not exist. + - On every reconcile, sync `StorageNode.spec.overrides` from `StorageNodeSet.spec.nodeConfigs[workerNode]`. The `StorageNodeSet` is the single source of truth — users edit `spec.nodeConfigs` on the `StorageNodeSet`, never the `StorageNode` directly. + - Sync `StorageNode.spec.overrides.failureDomain` from `StorageNodeSet.spec.nodeConfigs[workerNode].failureDomain`. If the referenced `StorageCluster` has `enableFailureDomains=true` and no `failureDomain` is set for this worker, emit a Warning event and block node-add until it is populated. + - Do NOT manage the backend node lifecycle here. +3. **Remove stale StorageNode CRs** — if a worker is removed from `spec.workerNodes`, delete the owned `StorageNode` CRs (deletion triggers drain via `StorageNodeOps` if the node is online). +4. **Aggregate status** — count online/offline from `StorageNode` statuses. + +### 6.2 StorageNodeReconciler (new) + +Owns the per-node provisioning loop: + +``` +StorageNode.status.uuid == "" → postStorageNode() → pollNodeOnline() +StorageNode.status.uuid != "" → syncStatus() every 30s +StorageNode deletion → ensure a StorageNodeOps(action=remove) exists if node is online +``` + +Watches: +- `StorageNode` (primary) +- `StorageNodeSet` (to read effective config) + +### 6.3 StorageNodeOpsReconciler (new) + +Drives all imperative operations. Replaces the existing action handling in `StorageNodeSetReconciler`. + +``` +StorageNodeOps created → + 1. Check no other active ops on the target StorageNode (set StorageNode.status.activeOpsRef) + 2. Set Phase=Running + 3. Dispatch to handler based on spec.action: + - shutdown/restart/suspend/resume → existing waitForActionCompletion pattern + - remove → existing drain state machine (Validating→Suspending→Migrating→Verifying→Removing) + 4. On completion → set Phase=Succeeded/Failed, clear StorageNode.status.activeOpsRef +``` + +Owns `VolumeMigration` CRs during drain (`.Owns(&VolumeMigration{})`). + +**Mutual exclusion:** The reconciler checks `StorageNode.status.activeOpsRef` before starting. If set, it requeues. A validating webhook may additionally reject a new `StorageNodeOps` if one is already active on the same node. + +--- + +## 7. Migration Strategy + +### Phase 1 — Introduce new CRDs (non-breaking) + +- Add `StorageNode` and `StorageNodeOps` CRDs. +- `StorageNodeSetReconciler` creates `StorageNode` CRs from its existing `status.nodes[]`. +- Existing action fields on `StorageNodeSet` continue to work via a shim that creates a `StorageNodeOps` internally. +- No behavioural change for existing deployments. + +### Phase 2 — Move lifecycle to new controllers + +- `StorageNodeReconciler` takes over node provisioning. +- `StorageNodeOpsReconciler` takes over action handling and drain. +- Feature flag: `StorageNodeSet.spec.nodeManagement: legacy | managed` (default: `legacy`). + +### Phase 3 — Deprecate legacy fields + +- Deprecate `spec.action`, `spec.nodeUUID`, `status.nodes`, `status.actionStatus`, + `status.drainCoordination` on `StorageNodeSet`. +- Remove in the next major version. + +--- + +## 8. Open Questions + +**Q1: StorageNodeOps retention policy** +How long should completed `StorageNodeOps` CRs be retained? Similar to `ttlSecondsAfterFinished` +on Jobs, or retained indefinitely for audit? + +**Q2: Concurrent ops on different nodes of the same set** +Multiple `StorageNodeOps` targeting different `StorageNode` CRs in the same `StorageNodeSet` +should be allowed. How many concurrent drains are permitted given FTT constraints? + +**Q3: StorageNode naming** +Name pattern: `{storagenodeset-name}-{sanitised-worker}-{socket}`. Needs to be deterministic +and DNS-label safe. + +**Q4: Adoption of pre-existing backend nodes** +If a node already exists in the backend (from a prior deployment), the `StorageNode` CR should +be created with `status.uuid` pre-populated rather than triggering a new node-add. + +**Q5: Who triggers the drain StorageNodeOps on scale-down?** +When `StorageNodeSet.spec.workerNodes` shrinks, the controller must decide whether to trigger +`action=remove` or simply delete the `StorageNode` CR. Explicit `StorageNodeOps(action=remove)` +is safer. + +--- + +## 9. Architecture Diagrams + +**Ownership Diagram** — Shows the ownership hierarchy from StorageCluster through StorageNodeSet down to individual StorageNode and StorageNodeOps resources, including how VolumeMigration CRs are created during drain operations. + +``` +StorageCluster + spec.enableFailureDomains = true + │ + │ (referenced by clusterName) + ▼ +StorageNodeSet (fleet / template) + spec.nodeFailureDomains: + worker-A: 1 + worker-B: 2 + worker-C: 1 + spec.nodeConfigs: + worker-A: { maxLogicalVolumeCount: 50 } + spec.workerNodes: [worker-A, worker-B, worker-C] + │ + │ owns (OwnerReference) + ├──────────────────────────────────────────────────────┐ + ▼ ▼ +StorageNode StorageNode + name: sns-worker-a-0 name: sns-worker-b-0 + spec.workerNode: worker-A spec.workerNode: worker-B + spec.socketIndex: 0 spec.socketIndex: 0 + spec.failureDomain: 1 ◄── propagated spec.failureDomain: 2 ◄── propagated + spec.overrides: status.uuid: + maxLogicalVolumeCount: 50 status.activeOpsRef: "ops-restart-b" + status.uuid: │ + status.activeOpsRef: "ops-remove-a" │ targeted by + │ ▼ + │ targeted by StorageNodeOps + ▼ name: ops-restart-b +StorageNodeOps spec.storageNodeRef: sns-worker-b-0 + name: ops-remove-a spec.action: restart + spec.storageNodeRef: sns-worker-a-0 status.phase: Running + spec.action: remove status.subPhase: "" + spec.drain: (no VolumeMigration CRs) + systemVolumeFilterRegex: "^sb-fio-.*" + status.phase: Running + status.subPhase: Migrating + status.volumesMigrated: 3 + status.volumesPending: 2 + │ + │ owns (during drain only) + ├──────────────────────────┐ + ▼ ▼ +VolumeMigration VolumeMigration + name: vm-lvol-aaa name: vm-lvol-bbb + status.phase: Completed status.phase: Running + (deleted on Verifying) (in-flight) + + ┌─────────────────────────────────────────────────────────┐ + │ StorageNode (worker-C, socket 0) │ + │ spec.failureDomain: 1 │ + │ status.activeOpsRef: "" (no active operation) │ + └─────────────────────────────────────────────────────────┘ +``` + +--- + +**Reconciler Flow** — Illustrates the three controllers and what each reconciler watches, manages, and produces. + +``` + ┌───────────────────────────────────────────────────────────────────────────────────┐ + │ Kubernetes Controller Manager │ + └───────────────────────────────────────────────────────────────────────────────────┘ + │ │ │ + ▼ ▼ ▼ + ┌─────────────────────┐ ┌──────────────────────┐ ┌──────────────────────────┐ + │ StorageNodeSet │ │ StorageNode │ │ StorageNodeOps │ + │ Reconciler │ │ Reconciler (new) │ │ Reconciler (new) │ + ├─────────────────────┤ ├──────────────────────┤ ├──────────────────────────┤ + │ WATCHES │ │ WATCHES │ │ WATCHES │ + │ • StorageNodeSet │ │ • StorageNode │ │ • StorageNodeOps │ + │ • Pod (SPDK ready) │ │ • StorageNodeSet │ │ • StorageNode │ + │ • Node (labels) │ │ (read config) │ │ • VolumeMigration │ + ├─────────────────────┤ ├──────────────────────┤ ├──────────────────────────┤ + │ MANAGES │ │ MANAGES │ │ MANAGES │ + │ • DaemonSet │ │ • backend node-add │ │ • backend actions: │ + │ • ServiceAccount │ │ POST (if uuid=="") │ │ shutdown / restart │ + │ • ClusterRole / │ │ • status sync (30s) │ │ suspend / resume │ + │ ClusterRoleBinding│ │ • pollNodeOnline │ │ remove/drain │ + │ • Service │ │ • creates │ │ • VolumeMigration CRs │ + │ • EndpointSlices │ │ StorageNodeOps │ │ (drain only) │ + │ • TLS Certificates │ │ (action=remove) │ │ • StorageNode.status │ + │ • StorageNode CRs │ │ on deletion │ │ .activeOpsRef │ + │ (create/delete) │ │ • StorageNode.status │ │ │ + │ • status aggregate │ │ (uuid, health...) │ │ MUTUAL EXCLUSION │ + │ (totalNodes, │ │ │ │ checks activeOpsRef │ + │ online/offline) │ │ READS EFFECTIVE CFG │ │ before starting; │ + │ • nodeConfigs sync │ │ StorageNodeSet.spec │ │ requeues if set │ + │ → StorageNode │ │ defaults merged with │ │ │ + │ .spec.overrides │ │ StorageNode.spec │ │ ON COMPLETION │ + │ • failureDomain │ │ .overrides │ │ Phase=Succeeded/Failed │ + │ sync │ │ │ │ clears activeOpsRef │ + └─────────────────────┘ └──────────────────────┘ └──────────────────────────┘ + │ creates/deletes │ │ owns + ▼ ▼ ▼ + StorageNode CRs backend API calls VolumeMigration CRs + (one per worker/socket) /api/v2/clusters/... (drain sub-phase only) +``` + +--- + +**Drain State Machine** — The five sequential sub-phases of the remove/drain operation, including the cluster pause guard, migration retry loop, and cancellation path. + +``` + StorageNodeOps(action=remove) created + │ + ▼ + ╔══════════════════════════════════════════════════════╗ + ║ ENTRY: SubPhase=Validating, Triggered=false ║ + ╚══════════════════════════════════════════════════════╝ + │ + ▼ + ┌───────────────────────────────────────────────────────────────┐ + │ [1] VALIDATING │ + │ • Fetch all volumes on the node from backend API │ + │ • Identify pinned / unmanaged volumes (no matching PVC) │ + │ ─────────────────────────────────────────────────────────────│ + │ EXIT → SUSPENDING : no pinned or unmanaged volumes │ + │ BLOCK (requeue 60s): pinned or unmanaged volumes present │ + └───────────────────────────────────────────────────────────────┘ + │ + ┌─────────────────────────▼──────────────────────────────────────────┐ + │ CLUSTER PAUSE GUARD (all phases after Validating) │ + │ if cluster.status != "active" OR rebalancing == true │ + │ → emit DrainPaused event, requeue 60s │ + │ → auto-resumes when cluster is active and not rebalancing │ + └─────────────────────────┬──────────────────────────────────────────┘ + │ + ▼ + ┌───────────────────────────────────────────────────────────────┐ + │ [2] SUSPENDING │ + │ • GET node status; if already suspended → skip POST │ + │ • POST /storage-nodes/{uuid}/suspend, set Triggered=true │ + │ • Poll GET every 10s │ + │ ─────────────────────────────────────────────────────────────│ + │ EXIT → MIGRATING : node.status == "suspended" │ + │ RETRY (10s) : node not yet suspended │ + └───────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌───────────────────────────────────────────────────────────────────────────┐ + │ [3] MIGRATING │ + │ • createMissingVolumeMigrations: one CR per PV-managed volume │ + │ with round-robin target node assignment │ + │ • Track VolumesMigrated / VolumesPending │ + │ ┌────────────────────────────────────────────────────────────┐ │ + │ │ FAILURE RETRY LOOP │ │ + │ │ VolumeMigration.phase == Failed/Aborted: │ │ + │ │ cluster not ready → delete CR, pause (DrainPaused) │ │ + │ │ cluster ready → delete CR, recreate with new target │ │ + │ └────────────────────────────────────────────────────────────┘ │ + │ ─────────────────────────────────────────────────────────────────────────│ + │ EXIT → VERIFYING : all migrations complete, no in-flight │ + │ RETRY (15s) : migrations still running │ + └───────────────────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌───────────────────────────────────────────────────────────────┐ + │ [4] VERIFYING │ + │ • GET all volumes remaining on node │ + │ • Delete system volumes (match systemVolumeFilterRegex) │ + │ • Poll 30s until none remain │ + │ ─────────────────────────────────────────────────────────────│ + │ EXIT → REMOVING : no non-system volumes remain │ + │ RETRY (30s) : volumes still present │ + └───────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌───────────────────────────────────────────────────────────────┐ + │ [5] REMOVING │ + │ • DELETE /api/v2/clusters/{uuid}/storage-nodes/{nodeUUID} │ + │ ─────────────────────────────────────────────────────────────│ + │ 200/204/404 → SUCCEEDED │ + │ error → FAILED (resume node best-effort) │ + └───────────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ + ╔══════════════════════╗ ╔═══════════════════════════════════╗ + ║ SUCCEEDED ║ ║ FAILED ║ + ║ Phase=Succeeded ║ ║ POST /resume (best-effort) ║ + ║ activeOpsRef="" ║ ║ Phase=Failed, activeOpsRef="" ║ + ╚══════════════════════╝ ╚═══════════════════════════════════╝ + + CANCELLATION (any phase after Suspending): + spec.action cleared → POST /resume if suspended + → delete all owned VolumeMigration CRs + → ActionStatus = nil +``` + +--- + +**Failure Domain Propagation** — How `enableFailureDomains` and `nodeFailureDomains` flow from StorageCluster and StorageNodeSet down to individual StorageNode specs and the backend API, including the blocking guard for missing entries. + +``` + ┌─────────────────────────────────────────────────┐ + │ StorageCluster │ + │ spec.enableFailureDomains: true │ + └─────────────────────────────────────────────────┘ + │ referenced by StorageNodeSet.spec.clusterName + ▼ + ┌─────────────────────────────────────────────────┐ + │ StorageNodeSet │ + │ spec.nodeFailureDomains: │ + │ worker-A: 1 ← rack-1 / AZ-1 │ + │ worker-B: 2 ← rack-2 / AZ-2 │ + │ worker-C: 1 ← rack-1 / AZ-1 │ + │ worker-D: ? ← MISSING │ + │ │ + │ CEL rule: all values >= 1 │ + └──────────┬──────────────┬──────────────┬────────┘ + │ sync │ sync │ MISSING + ▼ ▼ ▼ + ┌──────────────┐ ┌──────────────┐ ┌──────────────────────────┐ + │ StorageNode │ │ StorageNode │ │ StorageNode (worker-D) │ + │ worker-A, s0 │ │ worker-B, s0 │ │ │ + │ .failureDomain│ │ .failureDomain│ │ enableFailureDomains=T │ + │ = 1 │ │ = 2 │ │ no entry in map │ + └──────┬───────┘ └──────┬───────┘ │ → Warning event emitted │ + │ │ │ → node-add BLOCKED │ + ▼ ▼ │ → requeue until populated │ + ┌────────────────────────────────┐ └──────────────────────────┘ + │ Backend API POST │ + │ { "failure_domain": 1 } │ + │ { "failure_domain": 2 } │ + └────────────────────────────────┘ +``` + +--- + +**StorageNodeOps Lifecycle** — Full lifecycle from creation through mutual exclusion gate, action dispatch, sub-phases for `action=remove`, and terminal states with `activeOpsRef` cleanup. + +``` + kubectl apply -f ops.yaml (or auto-created on StorageNode deletion) + │ + ▼ + ┌─────────────────────────────────────────────────────────────┐ + │ PENDING — reconciler checks StorageNode.status.activeOpsRef│ + │ │ + │ activeOpsRef == "" activeOpsRef != "" │ + │ → set activeOpsRef = self → requeue (back-off) │ + │ → phase = Running (another op is running) │ + └──────────────────────┬──────────────────────────────────────┘ + │ + ▼ + ╔═════════════════════════════════════════════════════╗ + ║ status.phase: Running ║ + ╚═════════════════════════════════════════════════════╝ + │ + ┌──────────────┴───────────────────────────┐ + │ action dispatch │ + ▼ ▼ + ┌────────────────────────────┐ ┌──────────────────────────────────────┐ + │ shutdown / restart / │ │ remove │ + │ suspend / resume │ │ │ + │ │ │ Validating → Suspending │ + │ POST action to backend │ │ → Migrating │ + │ poll until terminal state │ │ (VolumeMigration CRs owned) │ + │ suspend → "suspended" │ │ → Verifying │ + │ resume → "online" │ │ → Removing │ + │ restart → "online" │ │ │ + │ shutdown → "offline" │ │ VolumesMigrated / VolumesPending │ + └─────────────┬──────────────┘ │ tracked in status │ + │ └──────────────────┬───────────────────┘ + │ │ + └──────────────────┬──────────────────┘ + │ + ┌──────────────┴──────────────┐ + ▼ ▼ + ╔═══════════════════════════╗ ╔═══════════════════════════════╗ + ║ Succeeded ║ ║ Failed ║ + ║ completedAt: ║ ║ message: ║ + ╚═══════════════════════════╝ ║ completedAt: ║ + │ ╚═══════════════════════════════╝ + └──────────────────┬──────────────────────────┘ + │ both terminal states: + ▼ + StorageNode.status.activeOpsRef = "" + (next StorageNodeOps may now acquire the lock) +``` From 8206aa26c0b9295072fcddec8dca23c44e74382c Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 11:38:21 +0100 Subject: [PATCH 02/52] operator: introduce StorageNode and StorageNodeOps three-tier model (Phase 1) --- operator/api/v1alpha1/storagenode_types.go | 229 +++++ operator/api/v1alpha1/storagenodeops_types.go | 152 +++ operator/api/v1alpha1/storagenodeset_types.go | 42 +- .../api/v1alpha1/zz_generated.deepcopy.go | 352 ++++++- operator/cmd/main.go | 18 + ...storage.simplyblock.io_storagenodeops.yaml | 163 +++ .../storage.simplyblock.io_storagenodes.yaml | 259 +++++ ...torage.simplyblock.io_storagenodesets.yaml | 201 ++-- operator/config/rbac/role.yaml | 6 + .../design-storagenodeset-storagenode.md | 42 +- .../simplyblockstoragenodeset_controller.go | 482 +-------- ...lockstoragenodeset_controller_unit_test.go | 944 ------------------ .../simplyblockstoragenodeset_drain.go | 925 +---------------- ...mplyblockstoragenodeset_drain_unit_test.go | 850 +--------------- .../simplyblockstoragenodeset_storagenode.go | 269 +++++ .../controller/storagenode_controller.go | 480 +++++++++ .../storagenode_controller_unit_test.go | 300 ++++++ .../controller/storagenodeops_controller.go | 931 +++++++++++++++++ .../storagenodeops_controller_unit_test.go | 310 ++++++ 19 files changed, 3639 insertions(+), 3316 deletions(-) create mode 100644 operator/api/v1alpha1/storagenode_types.go create mode 100644 operator/api/v1alpha1/storagenodeops_types.go create mode 100644 operator/config/crd/bases/storage.simplyblock.io_storagenodeops.yaml create mode 100644 operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml create mode 100644 operator/internal/controller/simplyblockstoragenodeset_storagenode.go create mode 100644 operator/internal/controller/storagenode_controller.go create mode 100644 operator/internal/controller/storagenode_controller_unit_test.go create mode 100644 operator/internal/controller/storagenodeops_controller.go create mode 100644 operator/internal/controller/storagenodeops_controller_unit_test.go diff --git a/operator/api/v1alpha1/storagenode_types.go b/operator/api/v1alpha1/storagenode_types.go new file mode 100644 index 00000000..ffd8376b --- /dev/null +++ b/operator/api/v1alpha1/storagenode_types.go @@ -0,0 +1,229 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// StorageNodeOverrides holds per-node configuration that overrides the parent +// StorageNodeSet fleet defaults for a specific worker node. Populated by the +// StorageNodeSetReconciler from StorageNodeSet.spec.nodeConfigs[workerNode] on +// every reconcile. The StorageNodeSet is the single source of truth — users +// should not edit this struct directly on the StorageNode. +// +// Fields here mirror the configurable (non-immutable, non-infrastructure) fields +// of StorageNodeSetSpec. When a field is set here it takes precedence over the +// fleet default; when omitted the fleet default applies. +type StorageNodeOverrides struct { + // MaxLogicalVolumeCount overrides the maximum number of logical volumes for this node. + // +optional + MaxLogicalVolumeCount *int32 `json:"maxLogicalVolumeCount,omitempty"` + + // MaxSize overrides the maximum allocatable size of huge pages for this node. + // +optional + MaxSize string `json:"maxSize,omitempty"` + + // SpdkImage overrides the SPDK image for this node (e.g. for phased rollouts). + // +optional + SpdkImage string `json:"spdkImage,omitempty"` + + // SpdkProxyImage overrides the SPDK proxy image for this node. + // +optional + SpdkProxyImage string `json:"spdkProxyImage,omitempty"` + + // CorePercentage overrides the percentage of cores allocated to SPDK for this node (0-99). + // +optional + CorePercentage *int32 `json:"corePercentage,omitempty"` + + // SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + // (e.g. "4G", "512M"). + // +kubebuilder:validation:Pattern=`^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$` + // +optional + SpdkSystemMemory string `json:"spdkSystemMemory,omitempty"` + + // JournalManagerSpec overrides journal manager tuning for this node. + // +optional + JournalManagerSpec *JournalManagerSpec `json:"journalManager,omitempty"` + + // PcieAllowList overrides the list of PCI addresses allowed for use on this node. + // +optional + PcieAllowList []string `json:"pcieAllowList,omitempty"` + + // PcieDenyList overrides the list of PCI addresses excluded from use on this node. + // +optional + PcieDenyList []string `json:"pcieDenyList,omitempty"` + + // PcieModel overrides the PCI model filter for this node. + // +optional + PcieModel string `json:"pcieModel,omitempty"` + + // DriveSizeRange overrides the drive size range filter for this node. + // +optional + DriveSizeRange string `json:"driveSizeRange,omitempty"` + + // DeviceNames explicitly defines the NVMe namespace names to use on this node + // (e.g. ["nvme0n1","nvme1n1"]). + // +optional + DeviceNames []string `json:"deviceNames,omitempty"` + + // EnableCpuTopology overrides topology-aware CPU handling for this node. + // +optional + EnableCpuTopology *bool `json:"enableCpuTopology,omitempty"` + + // ReservedSystemCPU overrides the CPUs reserved for system workloads on this node. + // +optional + ReservedSystemCPU string `json:"reservedSystemCPU,omitempty"` + + // UbuntuHost overrides the Ubuntu host OS flag for this node. + // +optional + UbuntuHost *bool `json:"ubuntuHost,omitempty"` + + // SkipKubeletConfiguration overrides whether kubelet configuration changes are + // skipped for this node. + // +optional + SkipKubeletConfiguration *bool `json:"skipKubeletConfiguration,omitempty"` + + // FailureDomain is the failure-domain group index (≥ 1) for this node. + // Required when the parent StorageCluster has enableFailureDomains=true. + // Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + // +kubebuilder:validation:Minimum=1 + // +optional + FailureDomain *int32 `json:"failureDomain,omitempty"` +} + +// StorageNodeSpec defines the desired state of a StorageNode. +type StorageNodeSpec struct { + // StorageNodeSetRef is the name of the owning StorageNodeSet. Immutable. + // +kubebuilder:validation:Required + // +k8s:immutable + StorageNodeSetRef string `json:"storageNodeSetRef"` + + // WorkerNode is the Kubernetes node hostname this StorageNode runs on. Immutable. + // +kubebuilder:validation:Required + // +k8s:immutable + WorkerNode string `json:"workerNode"` + + // SocketIndex is the NUMA socket index (0-based). Immutable. + // +k8s:immutable + // +optional + SocketIndex *int32 `json:"socketIndex,omitempty"` + + // Overrides holds per-node configuration propagated from + // StorageNodeSet.spec.nodeConfigs[workerNode] on every reconcile. + // +optional + Overrides *StorageNodeOverrides `json:"overrides,omitempty"` +} + +// StorageNodeStatus holds the observed state of a StorageNode. +type StorageNodeStatus struct { + // UUID is the backend storage node UUID. Set once after node-add completes. + // +optional + UUID string `json:"uuid,omitempty"` + + // Status is the backend-reported node status (e.g. online, suspended, offline). + // +optional + Status string `json:"status,omitempty"` + + // Health is the backend-reported node health flag. + // +optional + Health bool `json:"health,omitempty"` + + // CPU is the number of CPU cores allocated to the node. + // +optional + CPU *int32 `json:"cpu,omitempty"` + + // Memory is the memory allocation reported by the backend. + // +optional + Memory string `json:"memory,omitempty"` + + // Volumes is the number of logical volumes on this node. + // +optional + Volumes *int32 `json:"volumes,omitempty"` + + // Devices is the device list reported by the backend. + // +optional + Devices string `json:"devices,omitempty"` + + // MgmtIp is the management IP address of the node. + // +optional + MgmtIp string `json:"mgmtIp,omitempty"` + + // Hostname is the node hostname as reported by the backend. + // +optional + Hostname string `json:"hostname,omitempty"` + + // Uptime is the node uptime as reported by the backend. + // +optional + Uptime string `json:"uptime,omitempty"` + + // RpcPort is the RPC port of the node. + // +optional + RpcPort *int32 `json:"rpcPort,omitempty"` + + // LvolPort is the lvol port of the node. + // +optional + LvolPort *int32 `json:"lvolPort,omitempty"` + + // NvmfPort is the NVMe-oF port of the node. + // +optional + NvmfPort *int32 `json:"nvmfPort,omitempty"` + + // PostedAt is the timestamp when the node-add POST was sent. + // Used as a provisioning guard against duplicate POSTs. + // +optional + PostedAt *metav1.Time `json:"postedAt,omitempty"` + + // ActiveOpsRef is the name of the currently active StorageNodeOps CR targeting + // this node. Empty when no operation is in progress. Used for mutual exclusion. + // +optional + ActiveOpsRef string `json:"activeOpsRef,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=sn +// +kubebuilder:printcolumn:name="Worker",type=string,JSONPath=".spec.workerNode" +// +kubebuilder:printcolumn:name="Socket",type=integer,JSONPath=".spec.socketIndex" +// +kubebuilder:printcolumn:name="UUID",type=string,JSONPath=".status.uuid" +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=".status.status" +// +kubebuilder:printcolumn:name="Health",type=boolean,JSONPath=".status.health" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp" + +// StorageNode is the Schema for a single backend storage node instance. +// One StorageNode CR exists per (workerNode, socketIndex) pair and is owned +// by the parent StorageNodeSet. +type StorageNode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec StorageNodeSpec `json:"spec,omitempty"` + Status StorageNodeStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// StorageNodeList contains a list of StorageNode. +type StorageNodeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StorageNode `json:"items"` +} + +func init() { + SchemeBuilder.Register(&StorageNode{}, &StorageNodeList{}) +} diff --git a/operator/api/v1alpha1/storagenodeops_types.go b/operator/api/v1alpha1/storagenodeops_types.go new file mode 100644 index 00000000..a5f47c44 --- /dev/null +++ b/operator/api/v1alpha1/storagenodeops_types.go @@ -0,0 +1,152 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// StorageNodeOpsPhase is the lifecycle phase of a StorageNodeOps. +// +kubebuilder:validation:Enum=Pending;Running;Succeeded;Failed +type StorageNodeOpsPhase string + +const ( + StorageNodeOpsPhasePending StorageNodeOpsPhase = "Pending" + StorageNodeOpsPhaseRunning StorageNodeOpsPhase = "Running" + StorageNodeOpsPhaseSucceeded StorageNodeOpsPhase = "Succeeded" + StorageNodeOpsPhaseFailed StorageNodeOpsPhase = "Failed" +) + +// StorageNodeOpsSubPhase is the active drain sub-phase when action=remove. +// +kubebuilder:validation:Enum=Validating;Suspending;Migrating;Verifying;Removing +type StorageNodeOpsSubPhase string + +const ( + StorageNodeOpsSubPhaseValidating StorageNodeOpsSubPhase = "Validating" + StorageNodeOpsSubPhaseSuspending StorageNodeOpsSubPhase = "Suspending" + StorageNodeOpsSubPhaseMigrating StorageNodeOpsSubPhase = "Migrating" + StorageNodeOpsSubPhaseVerifying StorageNodeOpsSubPhase = "Verifying" + StorageNodeOpsSubPhaseRemoving StorageNodeOpsSubPhase = "Removing" +) + +// DrainOpsSpec configures the drain workflow for action=remove. +type DrainOpsSpec struct { + // SystemVolumeFilterRegex is a Go regular expression matched against backend + // volume names. Matching volumes are treated as system volumes: excluded from + // drain migration and deleted inline during the Verifying phase. + // Defaults to "^sb-fio-baseline-.*". + // +optional + SystemVolumeFilterRegex *string `json:"systemVolumeFilterRegex,omitempty"` +} + +// StorageNodeOpsSpec defines the desired state of a StorageNodeOps. +type StorageNodeOpsSpec struct { + // StorageNodeRef is the name of the target StorageNode. Immutable. + // +kubebuilder:validation:Required + // +k8s:immutable + StorageNodeRef string `json:"storageNodeRef"` + + // Action is the operation to perform. Immutable. + // +kubebuilder:validation:Enum=shutdown;restart;suspend;resume;remove + // +kubebuilder:validation:Required + // +k8s:immutable + Action string `json:"action"` + + // Force enables forced execution where the backend supports it. + // +optional + Force *bool `json:"force,omitempty"` + + // ReattachVolume reattaches volumes during restart. + // Only applicable when action=restart. + // +optional + ReattachVolume *bool `json:"reattachVolume,omitempty"` + + // Drain configures the drain workflow. Only applicable when action=remove. + // +optional + Drain *DrainOpsSpec `json:"drain,omitempty"` +} + +// StorageNodeOpsStatus holds the observed state of a StorageNodeOps. +type StorageNodeOpsStatus struct { + // Phase is the high-level lifecycle phase. + // +optional + Phase StorageNodeOpsPhase `json:"phase,omitempty"` + + // SubPhase tracks the active drain step when action=remove and phase=Running. + // +optional + SubPhase StorageNodeOpsSubPhase `json:"subPhase,omitempty"` + + // Message is a human-readable description of the current state or failure reason. + // +optional + Message string `json:"message,omitempty"` + + // VolumesMigrated is the count of volumes successfully migrated (drain only). + // +optional + VolumesMigrated int `json:"volumesMigrated,omitempty"` + + // VolumesPending is the count of volumes awaiting migration (drain only). + // +optional + VolumesPending int `json:"volumesPending,omitempty"` + + // Triggered indicates the backend action POST has been sent (used during + // Suspending to avoid duplicate POSTs across reconcile iterations). + // +optional + Triggered bool `json:"triggered,omitempty"` + + // StartedAt is when the operation began. + // +optional + StartedAt *metav1.Time `json:"startedAt,omitempty"` + + // CompletedAt is when the operation finished (successfully or not). + // +optional + CompletedAt *metav1.Time `json:"completedAt,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,shortName=snops +// +kubebuilder:printcolumn:name="Node",type=string,JSONPath=".spec.storageNodeRef" +// +kubebuilder:printcolumn:name="Action",type=string,JSONPath=".spec.action" +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="SubPhase",type=string,JSONPath=".status.subPhase" +// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=".status.message" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp" + +// StorageNodeOps is a one-shot operational CR targeting a single StorageNode. +// Analogous to a Kubernetes Job — it drives an action (shutdown, restart, suspend, +// resume, remove/drain) to completion and records the result. Only one +// StorageNodeOps can be active per StorageNode at a time. +type StorageNodeOps struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec StorageNodeOpsSpec `json:"spec,omitempty"` + Status StorageNodeOpsStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// StorageNodeOpsList contains a list of StorageNodeOps. +type StorageNodeOpsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StorageNodeOps `json:"items"` +} + +func init() { + SchemeBuilder.Register(&StorageNodeOps{}, &StorageNodeOpsList{}) +} diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index 1228a8a1..17d06e1e 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -43,13 +43,6 @@ type StorageNodeSetSpec struct { // Must reference one of the trusted registries (quay.io/simplyblock-io, docker.io/simplyblock, public.ecr.aws/simply-block); digest pinning (@sha256:...) is recommended. // +kubebuilder:validation:Pattern=`^($|(quay\.io/simplyblock-io|docker\.io/simplyblock|public\.ecr\.aws/simply-block)/[a-z0-9][a-z0-9._-]*:[a-zA-Z0-9][a-zA-Z0-9._-]*(@sha256:[a-f0-9]{64})?)$` ClusterImage string `json:"clusterImage,omitempty"` - // +kubebuilder:validation:Enum=shutdown;restart;suspend;resume;remove - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action" - // Action triggers an imperative node operation. - Action string `json:"action,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Node UUID" - // NodeUUID is required when action is specified - NodeUUID string `json:"nodeUUID,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Max Logical Volume Count" // MaxLogicalVolumeCount is the maximum number of logical volumes per node. @@ -106,12 +99,6 @@ type StorageNodeSetSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Worker Nodes" // WorkerNodes is the set of Kubernetes worker nodes to manage. WorkerNodes []string `json:"workerNodes,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Worker Node" - // WorkerNode is a single worker node used by action flows. - WorkerNode string `json:"workerNode,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Reattach Volume" - // ReattachVolume reattaches volumes during restart where supported by the backend. - ReattachVolume *bool `json:"reattachVolume,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OpenShift Cluster" // OpenShiftCluster indicates OpenShift-specific behavior should be enabled. OpenShiftCluster *bool `json:"openShiftCluster,omitempty"` @@ -168,15 +155,6 @@ type StorageNodeSetSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Image Pull Policy" // ImagePullPolicy controls when the container image is pulled. Defaults to IfNotPresent. ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Force" - // Force enables forced action execution where supported. - Force *bool `json:"force,omitempty"` - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="System Volume Filter Regex" - // SystemVolumeFilterRegex is a Go regular expression matched against backend - // volume names. Matching volumes are excluded from drain migration and from - // the final verification check. Defaults to "^sb-fio-baseline-.*". - // +optional - SystemVolumeFilterRegex *string `json:"systemVolumeFilterRegex,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Node Failure Domains" // NodeFailureDomains assigns each worker node to a failure-domain group (integer ≥ 1). @@ -187,6 +165,14 @@ type StorageNodeSetSpec struct { // independent fault groups. // +optional NodeFailureDomains map[string]int32 `json:"nodeFailureDomains,omitempty"` + + // NodeConfigs allows per-worker-node configuration overrides keyed by the + // Kubernetes worker node name. Entries are propagated to the corresponding + // StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. + // The StorageNodeSet is the single source of truth for all per-node config, + // including failure domain assignment via nodeConfigs[worker].failureDomain. + // +optional + NodeConfigs map[string]StorageNodeOverrides `json:"nodeConfigs,omitempty"` } // Drain coordination phases for a worker node undergoing a rolling upgrade drain. @@ -242,11 +228,19 @@ type NodeLatencyMetrics struct { // StorageNodeSetStatus defines the observed state of StorageNodeSet. type StorageNodeSetStatus struct { + // TotalNodes is the total number of owned StorageNode CRs. + // +optional + TotalNodes int `json:"totalNodes,omitempty"` + // OnlineNodes is the count of StorageNode CRs with status "online". + // +optional + OnlineNodes int `json:"onlineNodes,omitempty"` + // OfflineNodes is the count of StorageNode CRs with status "offline" or "suspended". + // +optional + OfflineNodes int `json:"offlineNodes,omitempty"` + // +operator-sdk:csv:customresourcedefinitions:type=status,displayName="Nodes" // Nodes is the observed state of each managed storage node. Nodes []NodeStatus `json:"nodes,omitempty"` - // ActionStatus tracks the latest action execution status. - ActionStatus *ActionStatus `json:"actionStatus,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=status,displayName="Drain Coordination" // DrainCoordination tracks the upgrade-drain state per worker node. DrainCoordination []NodeDrainState `json:"drainCoordination,omitempty"` diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index d5a6e672..2ef09617 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -531,6 +531,26 @@ func (in *ControlPlaneStatus) DeepCopy() *ControlPlaneStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DrainOpsSpec) DeepCopyInto(out *DrainOpsSpec) { + *out = *in + if in.SystemVolumeFilterRegex != nil { + in, out := &in.SystemVolumeFilterRegex, &out.SystemVolumeFilterRegex + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DrainOpsSpec. +func (in *DrainOpsSpec) DeepCopy() *DrainOpsSpec { + if in == nil { + return nil + } + out := new(DrainOpsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HashicorpVaultSettings) DeepCopyInto(out *HashicorpVaultSettings) { *out = *in @@ -1557,6 +1577,242 @@ func (in *StorageClusterStatus) DeepCopy() *StorageClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNode) DeepCopyInto(out *StorageNode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNode. +func (in *StorageNode) DeepCopy() *StorageNode { + if in == nil { + return nil + } + out := new(StorageNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StorageNode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeList) DeepCopyInto(out *StorageNodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StorageNode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeList. +func (in *StorageNodeList) DeepCopy() *StorageNodeList { + if in == nil { + return nil + } + out := new(StorageNodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StorageNodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeOps) DeepCopyInto(out *StorageNodeOps) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeOps. +func (in *StorageNodeOps) DeepCopy() *StorageNodeOps { + if in == nil { + return nil + } + out := new(StorageNodeOps) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StorageNodeOps) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeOpsList) DeepCopyInto(out *StorageNodeOpsList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StorageNodeOps, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeOpsList. +func (in *StorageNodeOpsList) DeepCopy() *StorageNodeOpsList { + if in == nil { + return nil + } + out := new(StorageNodeOpsList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StorageNodeOpsList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeOpsSpec) DeepCopyInto(out *StorageNodeOpsSpec) { + *out = *in + if in.Force != nil { + in, out := &in.Force, &out.Force + *out = new(bool) + **out = **in + } + if in.ReattachVolume != nil { + in, out := &in.ReattachVolume, &out.ReattachVolume + *out = new(bool) + **out = **in + } + if in.Drain != nil { + in, out := &in.Drain, &out.Drain + *out = new(DrainOpsSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeOpsSpec. +func (in *StorageNodeOpsSpec) DeepCopy() *StorageNodeOpsSpec { + if in == nil { + return nil + } + out := new(StorageNodeOpsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeOpsStatus) DeepCopyInto(out *StorageNodeOpsStatus) { + *out = *in + if in.StartedAt != nil { + in, out := &in.StartedAt, &out.StartedAt + *out = (*in).DeepCopy() + } + if in.CompletedAt != nil { + in, out := &in.CompletedAt, &out.CompletedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeOpsStatus. +func (in *StorageNodeOpsStatus) DeepCopy() *StorageNodeOpsStatus { + if in == nil { + return nil + } + out := new(StorageNodeOpsStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeOverrides) DeepCopyInto(out *StorageNodeOverrides) { + *out = *in + if in.MaxLogicalVolumeCount != nil { + in, out := &in.MaxLogicalVolumeCount, &out.MaxLogicalVolumeCount + *out = new(int32) + **out = **in + } + if in.CorePercentage != nil { + in, out := &in.CorePercentage, &out.CorePercentage + *out = new(int32) + **out = **in + } + if in.JournalManagerSpec != nil { + in, out := &in.JournalManagerSpec, &out.JournalManagerSpec + *out = new(JournalManagerSpec) + (*in).DeepCopyInto(*out) + } + if in.PcieAllowList != nil { + in, out := &in.PcieAllowList, &out.PcieAllowList + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PcieDenyList != nil { + in, out := &in.PcieDenyList, &out.PcieDenyList + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DeviceNames != nil { + in, out := &in.DeviceNames, &out.DeviceNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.EnableCpuTopology != nil { + in, out := &in.EnableCpuTopology, &out.EnableCpuTopology + *out = new(bool) + **out = **in + } + if in.UbuntuHost != nil { + in, out := &in.UbuntuHost, &out.UbuntuHost + *out = new(bool) + **out = **in + } + if in.SkipKubeletConfiguration != nil { + in, out := &in.SkipKubeletConfiguration, &out.SkipKubeletConfiguration + *out = new(bool) + **out = **in + } + if in.FailureDomain != nil { + in, out := &in.FailureDomain, &out.FailureDomain + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeOverrides. +func (in *StorageNodeOverrides) DeepCopy() *StorageNodeOverrides { + if in == nil { + return nil + } + out := new(StorageNodeOverrides) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageNodeSet) DeepCopyInto(out *StorageNodeSet) { *out = *in @@ -1669,11 +1925,6 @@ func (in *StorageNodeSetSpec) DeepCopyInto(out *StorageNodeSetSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.ReattachVolume != nil { - in, out := &in.ReattachVolume, &out.ReattachVolume - *out = new(bool) - **out = **in - } if in.OpenShiftCluster != nil { in, out := &in.OpenShiftCluster, &out.OpenShiftCluster *out = new(bool) @@ -1718,16 +1969,6 @@ func (in *StorageNodeSetSpec) DeepCopyInto(out *StorageNodeSetSpec) { } in.ContainerResources.DeepCopyInto(&out.ContainerResources) in.InitContainerResources.DeepCopyInto(&out.InitContainerResources) - if in.Force != nil { - in, out := &in.Force, &out.Force - *out = new(bool) - **out = **in - } - if in.SystemVolumeFilterRegex != nil { - in, out := &in.SystemVolumeFilterRegex, &out.SystemVolumeFilterRegex - *out = new(string) - **out = **in - } if in.NodeFailureDomains != nil { in, out := &in.NodeFailureDomains, &out.NodeFailureDomains *out = make(map[string]int32, len(*in)) @@ -1735,6 +1976,13 @@ func (in *StorageNodeSetSpec) DeepCopyInto(out *StorageNodeSetSpec) { (*out)[key] = val } } + if in.NodeConfigs != nil { + in, out := &in.NodeConfigs, &out.NodeConfigs + *out = make(map[string]StorageNodeOverrides, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeSetSpec. @@ -1757,11 +2005,6 @@ func (in *StorageNodeSetStatus) DeepCopyInto(out *StorageNodeSetStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.ActionStatus != nil { - in, out := &in.ActionStatus, &out.ActionStatus - *out = new(ActionStatus) - (*in).DeepCopyInto(*out) - } if in.DrainCoordination != nil { in, out := &in.DrainCoordination, &out.DrainCoordination *out = make([]NodeDrainState, len(*in)) @@ -1802,6 +2045,75 @@ func (in *StorageNodeSetStatus) DeepCopy() *StorageNodeSetStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeSpec) DeepCopyInto(out *StorageNodeSpec) { + *out = *in + if in.SocketIndex != nil { + in, out := &in.SocketIndex, &out.SocketIndex + *out = new(int32) + **out = **in + } + if in.Overrides != nil { + in, out := &in.Overrides, &out.Overrides + *out = new(StorageNodeOverrides) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeSpec. +func (in *StorageNodeSpec) DeepCopy() *StorageNodeSpec { + if in == nil { + return nil + } + out := new(StorageNodeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeStatus) DeepCopyInto(out *StorageNodeStatus) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(int32) + **out = **in + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = new(int32) + **out = **in + } + if in.RpcPort != nil { + in, out := &in.RpcPort, &out.RpcPort + *out = new(int32) + **out = **in + } + if in.LvolPort != nil { + in, out := &in.LvolPort, &out.LvolPort + *out = new(int32) + **out = **in + } + if in.NvmfPort != nil { + in, out := &in.NvmfPort, &out.NvmfPort + *out = new(int32) + **out = **in + } + if in.PostedAt != nil { + in, out := &in.PostedAt, &out.PostedAt + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeStatus. +func (in *StorageNodeStatus) DeepCopy() *StorageNodeStatus { + if in == nil { + return nil + } + out := new(StorageNodeStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StripeSpec) DeepCopyInto(out *StripeSpec) { *out = *in diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 9771d47d..3349e692 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -362,6 +362,24 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "VolumeMigration") os.Exit(1) } + if err := (&controller.StorageNodeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("storagenode-controller"), + TLSEnabled: tlsEnabled, + TLSMutualEnabled: tlsMutualEnabled, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "StorageNode") + os.Exit(1) + } + if err := (&controller.StorageNodeOpsReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("storagenodeops-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "StorageNodeOps") + os.Exit(1) + } if err := (&controller.VolumeRebalancerReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodeops.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodeops.yaml new file mode 100644 index 00000000..94bf3181 --- /dev/null +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodeops.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: storagenodeops.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: StorageNodeOps + listKind: StorageNodeOpsList + plural: storagenodeops + shortNames: + - snops + singular: storagenodeops + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.storageNodeRef + name: Node + type: string + - jsonPath: .spec.action + name: Action + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.subPhase + name: SubPhase + type: string + - jsonPath: .status.message + name: Message + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StorageNodeOps is a one-shot operational CR targeting a single StorageNode. + Analogous to a Kubernetes Job — it drives an action (shutdown, restart, suspend, + resume, remove/drain) to completion and records the result. Only one + StorageNodeOps can be active per StorageNode at a time. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageNodeOpsSpec defines the desired state of a StorageNodeOps. + properties: + action: + description: Action is the operation to perform. Immutable. + enum: + - shutdown + - restart + - suspend + - resume + - remove + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + drain: + description: Drain configures the drain workflow. Only applicable + when action=remove. + properties: + systemVolumeFilterRegex: + description: |- + SystemVolumeFilterRegex is a Go regular expression matched against backend + volume names. Matching volumes are treated as system volumes: excluded from + drain migration and deleted inline during the Verifying phase. + Defaults to "^sb-fio-baseline-.*". + type: string + type: object + force: + description: Force enables forced execution where the backend supports + it. + type: boolean + reattachVolume: + description: |- + ReattachVolume reattaches volumes during restart. + Only applicable when action=restart. + type: boolean + storageNodeRef: + description: StorageNodeRef is the name of the target StorageNode. + Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - action + - storageNodeRef + type: object + status: + description: StorageNodeOpsStatus holds the observed state of a StorageNodeOps. + properties: + completedAt: + description: CompletedAt is when the operation finished (successfully + or not). + format: date-time + type: string + message: + description: Message is a human-readable description of the current + state or failure reason. + type: string + phase: + description: Phase is the high-level lifecycle phase. + enum: + - Pending + - Running + - Succeeded + - Failed + type: string + startedAt: + description: StartedAt is when the operation began. + format: date-time + type: string + subPhase: + description: SubPhase tracks the active drain step when action=remove + and phase=Running. + enum: + - Validating + - Suspending + - Migrating + - Verifying + - Removing + type: string + triggered: + description: |- + Triggered indicates the backend action POST has been sent (used during + Suspending to avoid duplicate POSTs across reconcile iterations). + type: boolean + volumesMigrated: + description: VolumesMigrated is the count of volumes successfully + migrated (drain only). + type: integer + volumesPending: + description: VolumesPending is the count of volumes awaiting migration + (drain only). + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml new file mode 100644 index 00000000..3029de08 --- /dev/null +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml @@ -0,0 +1,259 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: storagenodes.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: StorageNode + listKind: StorageNodeList + plural: storagenodes + shortNames: + - sn + singular: storagenode + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.workerNode + name: Worker + type: string + - jsonPath: .spec.socketIndex + name: Socket + type: integer + - jsonPath: .status.uuid + name: UUID + type: string + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.health + name: Health + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StorageNode is the Schema for a single backend storage node instance. + One StorageNode CR exists per (workerNode, socketIndex) pair and is owned + by the parent StorageNodeSet. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageNodeSpec defines the desired state of a StorageNode. + properties: + overrides: + description: |- + Overrides holds per-node configuration propagated from + StorageNodeSet.spec.nodeConfigs[workerNode] on every reconcile. + properties: + corePercentage: + description: CorePercentage overrides the percentage of cores + allocated to SPDK for this node (0-99). + format: int32 + type: integer + deviceNames: + description: |- + DeviceNames explicitly defines the NVMe namespace names to use on this node + (e.g. ["nvme0n1","nvme1n1"]). + items: + type: string + type: array + driveSizeRange: + description: DriveSizeRange overrides the drive size range filter + for this node. + type: string + enableCpuTopology: + description: EnableCpuTopology overrides topology-aware CPU handling + for this node. + type: boolean + failureDomain: + description: |- + FailureDomain is the failure-domain group index (≥ 1) for this node. + Required when the parent StorageCluster has enableFailureDomains=true. + Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + format: int32 + minimum: 1 + type: integer + journalManager: + description: JournalManagerSpec overrides journal manager tuning + for this node. + properties: + count: + description: Count is the number of journal managers to configure. + format: int32 + type: integer + percentPerDevice: + description: PercentPerDevice is the journal manager capacity + percentage per device. + format: int32 + type: integer + type: object + maxLogicalVolumeCount: + description: MaxLogicalVolumeCount overrides the maximum number + of logical volumes for this node. + format: int32 + type: integer + maxSize: + description: MaxSize overrides the maximum allocatable size of + huge pages for this node. + type: string + pcieAllowList: + description: PcieAllowList overrides the list of PCI addresses + allowed for use on this node. + items: + type: string + type: array + pcieDenyList: + description: PcieDenyList overrides the list of PCI addresses + excluded from use on this node. + items: + type: string + type: array + pcieModel: + description: PcieModel overrides the PCI model filter for this + node. + type: string + reservedSystemCPU: + description: ReservedSystemCPU overrides the CPUs reserved for + system workloads on this node. + type: string + skipKubeletConfiguration: + description: |- + SkipKubeletConfiguration overrides whether kubelet configuration changes are + skipped for this node. + type: boolean + spdkImage: + description: SpdkImage overrides the SPDK image for this node + (e.g. for phased rollouts). + type: string + spdkProxyImage: + description: SpdkProxyImage overrides the SPDK proxy image for + this node. + type: string + spdkSystemMemory: + description: |- + SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + (e.g. "4G", "512M"). + pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ + type: string + ubuntuHost: + description: UbuntuHost overrides the Ubuntu host OS flag for + this node. + type: boolean + type: object + socketIndex: + description: SocketIndex is the NUMA socket index (0-based). Immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + storageNodeSetRef: + description: StorageNodeSetRef is the name of the owning StorageNodeSet. + Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + workerNode: + description: WorkerNode is the Kubernetes node hostname this StorageNode + runs on. Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - storageNodeSetRef + - workerNode + type: object + x-kubernetes-validations: + - message: field socketIndex is immutable once set + rule: '!has(oldSelf.socketIndex) || has(self.socketIndex)' + status: + description: StorageNodeStatus holds the observed state of a StorageNode. + properties: + activeOpsRef: + description: |- + ActiveOpsRef is the name of the currently active StorageNodeOps CR targeting + this node. Empty when no operation is in progress. Used for mutual exclusion. + type: string + cpu: + description: CPU is the number of CPU cores allocated to the node. + format: int32 + type: integer + devices: + description: Devices is the device list reported by the backend. + type: string + health: + description: Health is the backend-reported node health flag. + type: boolean + hostname: + description: Hostname is the node hostname as reported by the backend. + type: string + lvolPort: + description: LvolPort is the lvol port of the node. + format: int32 + type: integer + memory: + description: Memory is the memory allocation reported by the backend. + type: string + mgmtIp: + description: MgmtIp is the management IP address of the node. + type: string + nvmfPort: + description: NvmfPort is the NVMe-oF port of the node. + format: int32 + type: integer + postedAt: + description: |- + PostedAt is the timestamp when the node-add POST was sent. + Used as a provisioning guard against duplicate POSTs. + format: date-time + type: string + rpcPort: + description: RpcPort is the RPC port of the node. + format: int32 + type: integer + status: + description: Status is the backend-reported node status (e.g. online, + suspended, offline). + type: string + uptime: + description: Uptime is the node uptime as reported by the backend. + type: string + uuid: + description: UUID is the backend storage node UUID. Set once after + node-add completes. + type: string + volumes: + description: Volumes is the number of logical volumes on this node. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index 5779a5eb..89f4adfa 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -39,15 +39,6 @@ spec: spec: description: spec defines the desired state of StorageNodeSet properties: - action: - description: Action triggers an imperative node operation. - enum: - - shutdown - - restart - - suspend - - resume - - remove - type: string clusterImage: description: |- ClusterImage is the container image used for storage-node workloads. @@ -143,9 +134,6 @@ spec: enableCpuTopology: description: EnableCpuTopology enables topology-aware CPU handling. type: boolean - force: - description: Force enables forced action execution where supported. - type: boolean forceFormat4K: description: ForceFormat4K forces 4K blocksize formatting of the NVMe device where supported. @@ -259,6 +247,122 @@ spec: x-kubernetes-validations: - message: field is immutable rule: self == oldSelf + nodeConfigs: + additionalProperties: + description: |- + StorageNodeOverrides holds per-node configuration that overrides the parent + StorageNodeSet fleet defaults for a specific worker node. Populated by the + StorageNodeSetReconciler from StorageNodeSet.spec.nodeConfigs[workerNode] on + every reconcile. The StorageNodeSet is the single source of truth — users + should not edit this struct directly on the StorageNode. + + Fields here mirror the configurable (non-immutable, non-infrastructure) fields + of StorageNodeSetSpec. When a field is set here it takes precedence over the + fleet default; when omitted the fleet default applies. + properties: + corePercentage: + description: CorePercentage overrides the percentage of cores + allocated to SPDK for this node (0-99). + format: int32 + type: integer + deviceNames: + description: |- + DeviceNames explicitly defines the NVMe namespace names to use on this node + (e.g. ["nvme0n1","nvme1n1"]). + items: + type: string + type: array + driveSizeRange: + description: DriveSizeRange overrides the drive size range filter + for this node. + type: string + enableCpuTopology: + description: EnableCpuTopology overrides topology-aware CPU + handling for this node. + type: boolean + failureDomain: + description: |- + FailureDomain is the failure-domain group index (≥ 1) for this node. + Required when the parent StorageCluster has enableFailureDomains=true. + Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + format: int32 + minimum: 1 + type: integer + journalManager: + description: JournalManagerSpec overrides journal manager tuning + for this node. + properties: + count: + description: Count is the number of journal managers to + configure. + format: int32 + type: integer + percentPerDevice: + description: PercentPerDevice is the journal manager capacity + percentage per device. + format: int32 + type: integer + type: object + maxLogicalVolumeCount: + description: MaxLogicalVolumeCount overrides the maximum number + of logical volumes for this node. + format: int32 + type: integer + maxSize: + description: MaxSize overrides the maximum allocatable size + of huge pages for this node. + type: string + pcieAllowList: + description: PcieAllowList overrides the list of PCI addresses + allowed for use on this node. + items: + type: string + type: array + pcieDenyList: + description: PcieDenyList overrides the list of PCI addresses + excluded from use on this node. + items: + type: string + type: array + pcieModel: + description: PcieModel overrides the PCI model filter for this + node. + type: string + reservedSystemCPU: + description: ReservedSystemCPU overrides the CPUs reserved for + system workloads on this node. + type: string + skipKubeletConfiguration: + description: |- + SkipKubeletConfiguration overrides whether kubelet configuration changes are + skipped for this node. + type: boolean + spdkImage: + description: SpdkImage overrides the SPDK image for this node + (e.g. for phased rollouts). + type: string + spdkProxyImage: + description: SpdkProxyImage overrides the SPDK proxy image for + this node. + type: string + spdkSystemMemory: + description: |- + SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + (e.g. "4G", "512M"). + pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ + type: string + ubuntuHost: + description: UbuntuHost overrides the Ubuntu host OS flag for + this node. + type: boolean + type: object + description: |- + NodeConfigs allows per-worker-node configuration overrides keyed by the + Kubernetes worker node name. Entries are propagated to the corresponding + StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. + The StorageNodeSet is the single source of truth for all per-node config, + including failure domain assignment via nodeConfigs[worker].failureDomain. + type: object nodeFailureDomains: additionalProperties: format: int32 @@ -271,9 +375,6 @@ spec: the same group index so the control plane can spread erasure-coding chunks across independent fault groups. type: object - nodeUUID: - description: NodeUUID is required when action is specified - type: string nodesPerSocket: description: NodesPerSocket defines how many storage nodes are created per NUMA socket. @@ -315,10 +416,6 @@ spec: pcieModel: description: PcieModel filters devices by PCI model. type: string - reattachVolume: - description: ReattachVolume reattaches volumes during restart where - supported by the backend. - type: boolean reservedSystemCPU: description: ReservedSystemCPU defines CPUs reserved for system workloads. type: string @@ -349,12 +446,6 @@ spec: When omitted the backend default is used. pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ type: string - systemVolumeFilterRegex: - description: |- - SystemVolumeFilterRegex is a Go regular expression matched against backend - volume names. Matching volumes are excluded from drain migration and from - the final verification check. Defaults to "^sb-fio-baseline-.*". - type: string tolerations: description: Tolerations configures pod tolerations for storage-node pods. @@ -398,9 +489,6 @@ spec: ubuntuHost: description: UbuntuHost indicates the node host OS is Ubuntu. type: boolean - workerNode: - description: WorkerNode is a single worker node used by action flows. - type: string workerNodes: description: WorkerNodes is the set of Kubernetes worker nodes to manage. @@ -422,52 +510,6 @@ spec: status: description: status defines the observed state of StorageNodeSet properties: - actionStatus: - description: ActionStatus tracks the latest action execution status. - properties: - action: - description: Action is the requested action name. - type: string - message: - description: Message is a human-readable action result or error. - type: string - nodeUUID: - description: NodeUUID is the target node UUID for the action. - type: string - observedGeneration: - description: ObservedGeneration is the resource generation observed - by this status. - format: int64 - type: integer - state: - type: string - subPhase: - description: SubPhase tracks the active drain step within the - remove action. - enum: - - Validating - - Suspending - - Migrating - - Verifying - - Removing - type: string - triggered: - description: Triggered indicates whether the underlying backend - action has been fired. - type: boolean - updatedAt: - description: UpdatedAt is the timestamp of the last status transition. - format: date-time - type: string - volumesMigrated: - description: VolumesMigrated is the count of volumes successfully - migrated so far. - type: integer - volumesPending: - description: VolumesPending is the count of volumes still awaiting - migration. - type: integer - type: object drainCoordination: description: DrainCoordination tracks the upgrade-drain state per worker node. @@ -598,6 +640,14 @@ spec: type: integer type: object type: array + offlineNodes: + description: OfflineNodes is the count of StorageNode CRs with status + "offline" or "suspended". + type: integer + onlineNodes: + description: OnlineNodes is the count of StorageNode CRs with status + "online". + type: integer pendingNodeAdds: additionalProperties: format: date-time @@ -617,6 +667,9 @@ spec: a FailedScheduling event during node add. Used to emit a recovery event when the node subsequently comes online. type: object + totalNodes: + description: TotalNodes is the total number of owned StorageNode CRs. + type: integer type: object required: - spec diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index bd69f86c..7a4f0ae4 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -187,6 +187,8 @@ rules: - snapshotreplications - storagebackups - storageclusters + - storagenodeops + - storagenodes - storagenodesets - tasks - volumemigrations @@ -208,6 +210,8 @@ rules: - snapshotreplications/finalizers - storagebackups/finalizers - storageclusters/finalizers + - storagenodeops/finalizers + - storagenodes/finalizers - storagenodesets/finalizers - tasks/finalizers - volumemigrations/finalizers @@ -224,6 +228,8 @@ rules: - snapshotreplications/status - storagebackups/status - storageclusters/status + - storagenodeops/status + - storagenodes/status - storagenodesets/status - tasks/status - volumemigrations/status diff --git a/operator/docs/designs/design-storagenodeset-storagenode.md b/operator/docs/designs/design-storagenodeset-storagenode.md index 9f65cab8..eb1bbbf1 100644 --- a/operator/docs/designs/design-storagenodeset-storagenode.md +++ b/operator/docs/designs/design-storagenodeset-storagenode.md @@ -1,8 +1,8 @@ # Design Document: StorageNodeSet / StorageNode / StorageNodeOps Three-Tier Model -**Status:** Draft +**Status:** Phase 1 Complete **Author:** Israel Geoffrey -**Date:** 2026-07-13 +**Date:** 2026-07-14 --- @@ -318,24 +318,36 @@ Owns `VolumeMigration` CRs during drain (`.Owns(&VolumeMigration{})`). ## 7. Migration Strategy -### Phase 1 — Introduce new CRDs (non-breaking) +### Phase 1 — Complete ✓ -- Add `StorageNode` and `StorageNodeOps` CRDs. -- `StorageNodeSetReconciler` creates `StorageNode` CRs from its existing `status.nodes[]`. -- Existing action fields on `StorageNodeSet` continue to work via a shim that creates a `StorageNodeOps` internally. -- No behavioural change for existing deployments. +Implemented on 2026-07-14. Deviations from the original plan: -### Phase 2 — Move lifecycle to new controllers +- `StorageNodeSet.spec.action`, `spec.nodeUUID`, `spec.workerNode`, `spec.force`, + `spec.reattachVolume`, `spec.systemVolumeFilterRegex` and `status.actionStatus` were + **removed outright** rather than shimmed. All imperative operations must now use a + `StorageNodeOps` CR. This is a breaking API change but results in a significantly + cleaner codebase. +- `StorageNodeSetReconciler` creates `StorageNode` CRs for every `(workerNode, socket)` + pair in `spec.workerNodes × spec.socketsToUse` and aggregates their status into + `StorageNodeSetStatus.TotalNodes / OnlineNodes / OfflineNodes`. +- `StorageNodeOpsReconciler` watches `StorageNode` changes so pending ops acquire the + lock immediately when `activeOpsRef` is cleared, rather than waiting for the poll timer. +- CRD YAMLs generated and deployed to the helm chart. +- 28 unit tests added covering both new reconcilers. -- `StorageNodeReconciler` takes over node provisioning. -- `StorageNodeOpsReconciler` takes over action handling and drain. -- Feature flag: `StorageNodeSet.spec.nodeManagement: legacy | managed` (default: `legacy`). +### Phase 2 — Move provisioning to StorageNodeReconciler -### Phase 3 — Deprecate legacy fields +- `StorageNodeReconciler` takes over node-add POST and status sync from + `StorageNodeSetReconciler.reconcileWorkerNodes`. The `provisionNode` and `syncStatus` + stubs are already in place; they need to consume the existing `postStorageNodeSet` and + `pollNodeOnline` logic refactored to read from `StorageNode` instead of `StorageNodeSet`. -- Deprecate `spec.action`, `spec.nodeUUID`, `status.nodes`, `status.actionStatus`, - `status.drainCoordination` on `StorageNodeSet`. -- Remove in the next major version. +### Phase 3 — Remove legacy provisioning fields + +- Remove `status.nodes[]`, `status.pendingNodeAdds`, `status.schedulingFailedWorkers` + and `status.drainCoordination` from `StorageNodeSet` once `StorageNodeReconciler` + drives provisioning end-to-end. +- Remove `reconcileWorkerNodes` and related helpers from `StorageNodeSetReconciler`. --- diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 59574e09..2764c654 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -22,10 +22,10 @@ import ( "fmt" "net/http" "reflect" - "slices" + "strconv" "strings" - "sync" + "time" appsv1 "k8s.io/api/apps/v1" @@ -61,10 +61,6 @@ type StorageNodeSetReconciler struct { TLSProvider string TLSMutualEnabled bool Recorder record.EventRecorder - // systemVolumeFilterCache caches compiled system-volume filter regexes keyed - // by the pattern string. Compilation errors are surfaced on first use and the - // CR is rejected, so subsequent reconciles reuse the valid compiled regex. - systemVolumeFilterCache sync.Map } type SNODEAPIResponse struct { @@ -93,13 +89,6 @@ var ( waitForNodeOnlineActivationDelay = 120 * time.Second waitForNodeOnlineSleepFn = time.Sleep - performNodeActionPostTriggerDelay = 5 * time.Second - performNodeActionSleepFn = time.Sleep - - waitForActionCompletionRetries = 50 - waitForActionCompletionWaitInterval = 5 * time.Second - waitForActionCompletionSleepFn = time.Sleep - syncNodeStatusInterval = 30 * time.Second spdkPodEventDelay = 20 * time.Second @@ -167,33 +156,6 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque apiClient := webapi.NewClient() - if snCR.Spec.Action != "" { - // spdk-proxy EndpointSlice maintenance is pod-driven and must keep - // running during actions. The restart/migration flow blocks on the - // target worker's spdk-proxy DNS name - // (.simplyblock-spdk-proxy..svc) resolving once the control - // plane (re)creates its SPDK pod there. That name is only published by - // reconcileSpdkProxyEndpointSlices, which the early return below would - // otherwise skip for the entire duration of the action — so the pod - // comes up but its DNS entry never appears and the migration deadlocks. - // Reconcile the slices here so the entry is (re)built as soon as the new - // spdk-proxy pod becomes Ready; pod events and the action's own requeues - // re-drive this path until it converges. - if err := r.reconcileSpdkProxyEndpointSlices(ctx, snCR); err != nil { - log.Error(err, "failed to reconcile spdk-proxy EndpointSlices during action") - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - return r.reconcileAction(ctx, snCR, clusterUUID) - } - - // If the user cleared the remove action while a drain was in progress, resume the node. - if snCR.Status.ActionStatus != nil && - snCR.Status.ActionStatus.Action == utils.NodeActionRemove && - snCR.Status.ActionStatus.SubPhase != "" && - snCR.Status.ActionStatus.State == utils.ActionStateRunning { - return r.drainHandleCancellation(ctx, webapi.NewClient(), clusterUUID, snCR) - } - if err := r.labelWorkerNodes(ctx, snCR); err != nil { return ctrl.Result{}, err } @@ -231,6 +193,13 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque expectedPerHost := utils.ExpectedNodesPerHost(snCR) + // Phase-1 bridge: create/sync/delete owned StorageNode CRs to match + // spec.workerNodes × spec.socketsToUse. The StorageNodeReconciler owns + // the per-node provisioning; this only manages CR lifecycle. + if _, err := r.reconcileStorageNodeCRs(ctx, snCR); err != nil { + log.Error(err, "failed to reconcile StorageNode CRs") + } + if res, err := r.reconcileWorkerNodes(ctx, req, snCR, clusterUUID, apiClient, expectedPerHost); err != nil || res.RequeueAfter > 0 { return res, err } @@ -488,7 +457,6 @@ func (r *StorageNodeSetReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.controlPlaneToStorageNodeSetRequests), builder.WithPredicates(predicate.NewPredicateFuncs(isSimplyblockControlPlane)), ). - Owns(&simplyblockv1alpha1.VolumeMigration{}). Complete(r) } @@ -622,27 +590,6 @@ func (r *StorageNodeSetReconciler) labelWorkerNodes(ctx context.Context, sn *sim return nil } -func (r *StorageNodeSetReconciler) labelWorkerNode(ctx context.Context, sn *simplyblockv1alpha1.StorageNodeSet) error { - var node corev1.Node - if err := r.Get(ctx, client.ObjectKey{Name: sn.Spec.WorkerNode}, &node); err != nil { - return err - } - - if node.Labels == nil { - node.Labels = map[string]string{} - } - - key := "io.simplyblock.node-type" - value := "simplyblock-storage-plane-" + sn.Spec.ClusterName - - node.Labels[key] = value - if err := r.Update(ctx, &node); err != nil { - return err - } - - return nil -} - func (r *StorageNodeSetReconciler) reconcileDaemonSet( ctx context.Context, snCR *simplyblockv1alpha1.StorageNodeSet, @@ -1732,330 +1679,6 @@ func journalManagerCount( return utils.IntPtrOrDefault(snCR.Spec.JournalManagerSpec.Count, 3) } -func (r *StorageNodeSetReconciler) reconcileAction( - ctx context.Context, - snCR *simplyblockv1alpha1.StorageNodeSet, - clusterUUID string, -) (ctrl.Result, error) { - apiClient := webapi.NewClient() - - if snCR.Spec.Action == utils.NodeActionRemove { - return r.performDrainAndRemove(ctx, apiClient, clusterUUID, snCR) - } - - if err := r.handleNodeAction( - ctx, - apiClient, - snCR, - clusterUUID, - ); err != nil { - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - - return ctrl.Result{}, nil -} - -func (r *StorageNodeSetReconciler) handleNodeAction( - ctx context.Context, - apiClient *webapi.Client, - snCR *simplyblockv1alpha1.StorageNodeSet, - clusterUUID string, -) error { - log := logf.FromContext(ctx) - - // Skip if this exact request already succeeded. The generation gate is - // essential: without it a re-request of the same action+node (clear - // spec.action, then set it again) would be skipped forever because the prior - // success is still recorded. A new request bumps the generation, so a stale - // success no longer matches and the action runs again. - if snCR.Status.ActionStatus != nil && - snCR.Status.ActionStatus.Action == snCR.Spec.Action && - snCR.Status.ActionStatus.NodeUUID == snCR.Spec.NodeUUID && - snCR.Status.ActionStatus.State == utils.ActionStateSuccess && - snCR.Status.ActionStatus.ObservedGeneration == snCR.Generation { - log.Info("Action already completed successfully, skipping", - "action", snCR.Spec.Action, - "nodeUUID", snCR.Spec.NodeUUID, - ) - return nil - } - - // Carry the Triggered flag forward for the same action and spec generation - // so a long-running backend action already fired on an earlier reconcile is - // not re-fired on requeue (which would reset the node and spawn a duplicate - // task). A new request bumps the generation and starts untriggered. - triggered := false - if prev := snCR.Status.ActionStatus; prev != nil && - prev.Action == snCR.Spec.Action && - prev.NodeUUID == snCR.Spec.NodeUUID && - prev.ObservedGeneration == snCR.Generation { - triggered = prev.Triggered - } - - snCR.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: snCR.Spec.Action, - NodeUUID: snCR.Spec.NodeUUID, - State: utils.ActionStateRunning, - UpdatedAt: metav1.Now(), - ObservedGeneration: snCR.Generation, - Triggered: triggered, - } - if err := r.Status().Update(ctx, snCR); err != nil { - log.Error(err, "Failed to set action status to running") - return err - } - - if err := r.performNodeAction(ctx, apiClient, clusterUUID, snCR); err != nil { - log.Error(err, "Action failed", "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID) - snCR.Status.ActionStatus.State = utils.ActionStateFailed - snCR.Status.ActionStatus.Message = err.Error() - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - _ = r.Status().Update(ctx, snCR) - return err - } - - snCR.Status.ActionStatus.State = utils.ActionStateSuccess - snCR.Status.ActionStatus.Message = "Action executed successfully" - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Update(ctx, snCR); err != nil { - log.Error(err, "Failed to update action status") - return err - } - - log.Info("Action completed successfully", "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID) - return nil -} - -func (r *StorageNodeSetReconciler) performNodeAction( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) error { - - log := logf.FromContext(ctx) - - // Idempotency guard (prevents the restart/migration retry storm): the - // backend actions are long-running and stateful. Re-issuing one while it is - // already executing resets the node to its in-progress state and spawns a - // duplicate task that the task runner aborts ("node is restarting, stopping - // task"), livelocking the operation. Consult the backend's current node - // status first and skip the trigger when the action is already underway or - // already complete; a subsequent requeue just polls for completion. - if cur, err := getNodeStatus(ctx, apiClient, clusterUUID, snCR.Spec.NodeUUID); err != nil { - log.Info("Could not read backend node status before action; proceeding to trigger", - "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID, "error", err.Error()) - } else if inProgress := actionInProgressStatus(snCR.Spec.Action); inProgress != "" && cur == inProgress { - log.Info("Backend action already in progress; waiting for completion instead of re-triggering", - "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID, "status", cur) - return r.waitForActionCompletion(ctx, apiClient, clusterUUID, snCR.Spec.NodeUUID, snCR.Spec.Action) - } else if target, ok := actionTargetStatus(snCR.Spec.Action); ok && cur == target && actionAlreadyTriggered(snCR) { - log.Info("Backend action already completed; nothing to do", - "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID, "status", cur) - return nil - } - - var ( - endpoint string - method = http.MethodPost - body any - ) - - switch snCR.Spec.Action { - - case utils.NodeActionRemove: - return fmt.Errorf("remove action must be handled by performDrainAndRemove, not performNodeAction") - - case utils.NodeActionRestart: - payload := map[string]any{ - "force": nodeActionForce(snCR, true), - "reattach_volume": utils.BoolPtrOrFalse(snCR.Spec.ReattachVolume), - } - - if snCR.Spec.WorkerNode != "" { - if err := r.labelWorkerNode(ctx, snCR); err != nil { - return fmt.Errorf("failed to label worker node %s: %w", snCR.Spec.WorkerNode, err) - } - - // The action reconcile path skips reconcileEndpointSlice, so the - // headless-service DNS entry for the target worker would be missing. - // Ensure the EndpointSlice is updated before any reachability check. - if err := r.ensureWorkerInEndpointSlice(ctx, snCR, snCR.Spec.WorkerNode); err != nil { - return fmt.Errorf("failed to ensure endpoint for worker %s: %w", snCR.Spec.WorkerNode, err) - } - - if err := waitForNodeInfoReachable(ctx, snCR.Spec.WorkerNode, snCR.Namespace, r.TLSEnabled, r.TLSMutualEnabled); err != nil { - log.Error(err, "node never became reachable") - return err - } - - body = map[string]any{ - "force": nodeActionForce(snCR, true), - "reattach_volume": utils.BoolPtrOrFalse(snCR.Spec.ReattachVolume), - "node_address": utils.StorageNodeSetAPIAddress(snCR.Spec.WorkerNode, snCR.Namespace), - } - } else { - body = payload - } - - endpoint = fmt.Sprintf( - "/api/v2/clusters/%s/storage-nodes/%s/restart", - clusterUUID, - snCR.Spec.NodeUUID, - ) - - default: - body = nil - endpoint = fmt.Sprintf( - "/api/v2/clusters/%s/storage-nodes/%s/%s", - clusterUUID, - snCR.Spec.NodeUUID, - snCR.Spec.Action, - ) - } - - respBody, status, err := apiClient.Do(ctx, method, endpoint, body) - if err != nil || status >= 300 { - if err == nil { - err = fmt.Errorf("unexpected status %d", status) - } - log.Error(err, "Node action API call failed", "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID, "status", status, "response", string(respBody)) - return fmt.Errorf("action API failed: status=%d err=%v", status, err) - } - - // Record that the backend action has been fired so subsequent reconciles - // for this same spec generation poll for completion instead of re-firing. - // Persist immediately (best-effort): if the operator crashes after the POST - // succeeds but before handleNodeAction writes the final status, a restart of - // the operator could otherwise observe the node already back at its target - // state with Triggered still false and re-issue the action — a spurious, - // disruptive second restart of a healthy node. A failed persist is - // non-fatal: the in-progress status guard still prevents re-triggering while - // the action is underway. - if snCR.Status.ActionStatus != nil { - snCR.Status.ActionStatus.Triggered = true - if err := r.Status().Update(ctx, snCR); err != nil { - log.Error(err, "failed to persist Triggered flag after firing action; continuing", - "action", snCR.Spec.Action, "nodeUUID", snCR.Spec.NodeUUID) - } - } - - log.Info( - "Node action triggered", - "nodeUUID", snCR.Spec.NodeUUID, - "action", snCR.Spec.Action, - "response", string(respBody), - ) - - performNodeActionSleepFn(performNodeActionPostTriggerDelay) - - if err := r.waitForActionCompletion( - ctx, - apiClient, - clusterUUID, - snCR.Spec.NodeUUID, - snCR.Spec.Action, - ); err != nil { - return fmt.Errorf( - "node did not reach expected state after action %s: %w", - snCR.Spec.Action, - err, - ) - } - - log.Info( - "Node reached expected state", - "nodeUUID", snCR.Spec.NodeUUID, - "action", snCR.Spec.Action, - ) - - return nil -} - -func nodeActionForce(snCR *simplyblockv1alpha1.StorageNodeSet, defaultValue bool) bool { - if snCR.Spec.Force == nil { - return defaultValue - } - return *snCR.Spec.Force -} - -// ensureWorkerInEndpointSlice adds the target worker to the storage-node-api -// EndpointSlice when it is absent. spec.workerNode holds the migration target -// but is never part of spec.workerNodes, so reconcileEndpointSlice would never -// add a DNS hostname entry for it, causing headless-service lookups to fail. -func (r *StorageNodeSetReconciler) ensureWorkerInEndpointSlice( - ctx context.Context, - snCR *simplyblockv1alpha1.StorageNodeSet, - workerNode string, -) error { - if slices.Contains(snCR.Spec.WorkerNodes, workerNode) { - return nil // already covered by the regular EndpointSlice reconciliation - } - - ip, err := getNodeInternalIP(ctx, r.Client, workerNode) - if err != nil { - return fmt.Errorf("failed to get IP for worker %s: %w", workerNode, err) - } - - log := logf.FromContext(ctx) - nodeIPs := make(map[string]string, len(snCR.Spec.WorkerNodes)+1) - for _, nodeName := range snCR.Spec.WorkerNodes { - nodeIP, err := getNodeInternalIP(ctx, r.Client, nodeName) - if err != nil { - log.Error(err, "failed to get internal IP for EndpointSlice, skipping node", "node", nodeName) - continue - } - nodeIPs[nodeName] = nodeIP - } - nodeIPs[workerNode] = ip - - return r.applyStorageNodeSetEndpointSlice(ctx, snCR, nodeIPs) -} - -// actionTargetStatus returns the backend node status that signals the given -// action completed successfully, and whether the action is recognized. -func actionTargetStatus(action string) (string, bool) { - switch action { - case utils.NodeActionSuspend: - return "suspended", true - case utils.NodeActionResume, utils.NodeActionRestart: - return utils.NodeStatusOnline, true - case utils.NodeActionShutdown: - return utils.NodeStatusOffline, true - case utils.NodeActionRemove: - return "removed", true - default: - return "", false - } -} - -// actionInProgressStatus returns the transient backend status a node reports -// while the given action is executing, or "" if the action has no such state. -// Observing this status means the backend is already working on the action and -// it must not be re-triggered. -func actionInProgressStatus(action string) string { - switch action { - case utils.NodeActionRestart: - return utils.NodeStatusInRestart - case utils.NodeActionShutdown: - return utils.NodeStatusInShutdown - default: - return "" - } -} - -// actionAlreadyTriggered reports whether the backend action for the current -// spec has already been fired, based on the persisted ActionStatus. It is -// scoped to the exact action, node and spec generation so a fresh request -// (new generation) is never mistaken for an in-flight one. -func actionAlreadyTriggered(snCR *simplyblockv1alpha1.StorageNodeSet) bool { - as := snCR.Status.ActionStatus - return as != nil && as.Triggered && - as.Action == snCR.Spec.Action && - as.NodeUUID == snCR.Spec.NodeUUID && - as.ObservedGeneration == snCR.Generation -} - // getNodeStatus reads the current backend status string for a storage node. func getNodeStatus( ctx context.Context, @@ -2078,90 +1701,3 @@ func getNodeStatus( return resp.Status, nil } -func (r *StorageNodeSetReconciler) waitForActionCompletion( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - nodeUUID string, - action string, -) error { - - log := logf.FromContext(ctx) - - expectedStatus := map[string]string{ - utils.NodeActionSuspend: "suspended", - utils.NodeActionResume: "online", - utils.NodeActionShutdown: "offline", - utils.NodeActionRestart: "online", - utils.NodeActionRemove: "removed", - } - - targetStatus, ok := expectedStatus[action] - if !ok { - return fmt.Errorf("unknown action: %s", action) - } - - endpoint := fmt.Sprintf( - "/api/v2/clusters/%s/storage-nodes/%s", - clusterUUID, - nodeUUID, - ) - - for i := 0; i < waitForActionCompletionRetries; i++ { - body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) - - class := webapi.ClassifyError(err, status) - - if class.ContextSpecific { - // 404/409 — interpret per action. - if status == http.StatusNotFound { - // Node no longer exists — treat as completed for shutdown/remove-like actions. - log.Info("Node not found during status poll, treating as complete", - "nodeUUID", nodeUUID, "action", action) - return nil - } - // 409 — unexpected during a status poll; retry. - waitForActionCompletionSleepFn(waitForActionCompletionWaitInterval) - continue - } - - if !class.Retryable && (err != nil || status >= 300) { - // Permanent error — do not retry. - if err == nil { - err = fmt.Errorf("unexpected status %d", status) - } - log.Error(err, "Permanent error polling node status — aborting", - "nodeUUID", nodeUUID, "status", status) - return err - } - - if class.Retryable { - log.Error(err, "Transient error polling node status, retrying", - "nodeUUID", nodeUUID, "status", status) - waitForActionCompletionSleepFn(waitForActionCompletionWaitInterval) - continue - } - - var resp utils.NodeStatusResponse - if err := json.Unmarshal(body, &resp); err != nil { - log.Error(err, "Failed to parse node status response", "body", string(body)) - waitForActionCompletionSleepFn(waitForActionCompletionWaitInterval) - continue - } - - if resp.Status == targetStatus { - log.Info("Node reached expected status", - "nodeUUID", nodeUUID, "status", resp.Status) - return nil - } - - waitForActionCompletionSleepFn(waitForActionCompletionWaitInterval) - } - - return fmt.Errorf( - "node %s did not reach expected status %q after action %q", - nodeUUID, - targetStatus, - action, - ) -} diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go index 772234d2..2de97d7e 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go @@ -2,14 +2,11 @@ package controller import ( "context" - "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "strings" - "sync" - "sync/atomic" "testing" "time" @@ -62,227 +59,6 @@ func TestEnsureNodeStatus(t *testing.T) { } } -func TestWaitForActionCompletionUnknownAction(t *testing.T) { - r := &StorageNodeSetReconciler{} - c := webapi.NewClient("http://127.0.0.1:1") - err := r.waitForActionCompletion(context.Background(), c, "cluster", "node", "invalid-action") - if err == nil { - t.Fatalf("expected error for unknown action") - } -} - -func TestWaitForActionCompletionValidTransitions(t *testing.T) { - tests := []struct { - name string - action string - statusCode int - respStatus string - }{ - { - name: "suspend reaches suspended", - action: "suspend", - statusCode: http.StatusOK, - respStatus: "suspended", - }, - { - name: "resume reaches online", - action: "resume", - statusCode: http.StatusOK, - respStatus: statusOnline, - }, - { - name: "shutdown reaches offline", - action: "shutdown", - statusCode: http.StatusOK, - respStatus: "offline", - }, - { - name: "restart reaches online", - action: "restart", - statusCode: http.StatusOK, - respStatus: statusOnline, - }, - { - name: "remove reaches deleted via 404", - action: "remove", - statusCode: http.StatusNotFound, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if tc.statusCode == http.StatusNotFound { - w.WriteHeader(http.StatusNotFound) - return - } - - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(utils.NodeStatusResponse{ - Status: tc.respStatus, - }) - })) - defer srv.Close() - - r := &StorageNodeSetReconciler{} - c := webapi.NewClient(srv.URL) - err := r.waitForActionCompletion(context.Background(), c, "cluster", "node", tc.action) - if err != nil { - t.Fatalf("waitForActionCompletion returned error: %v", err) - } - }) - } -} - -func TestHandleNodeActionTransitions(t *testing.T) { - t.Run("does not re-enter terminal success for same action and node", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-a", - Namespace: "default", - Generation: 5, - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: "node-1", - }, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - // Success recorded for the current generation must suppress a - // re-run of the same request. - Action: "restart", - NodeUUID: "node-1", - State: utils.ActionStateSuccess, - ObservedGeneration: 5, - }, - }, - } - - r := newStorageNodeSetStateTestReconciler(t, sn) - err := r.handleNodeAction(context.Background(), webapi.NewClient("http://127.0.0.1:1"), sn, "cluster") - if err != nil { - t.Fatalf("handleNodeAction returned error: %v", err) - } - if sn.Status.ActionStatus.State != utils.ActionStateSuccess { - t.Fatalf("expected success to remain stable, got %q", sn.Status.ActionStatus.State) - } - }) - - t.Run("re-executes when a new generation re-requests a previously successful action", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-regen", - Namespace: "default", - Generation: 2, - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: "node-1", - }, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - // Success recorded for an earlier generation must not suppress - // a fresh request at a newer generation. - Action: "restart", - NodeUUID: "node-1", - State: utils.ActionStateSuccess, - ObservedGeneration: 1, - }, - }, - } - - r := newStorageNodeSetStateTestReconciler(t, sn) - // The API is unreachable, so a re-executed action fails; a wrongly-skipped - // action would return nil and leave the state at success. - err := r.handleNodeAction(context.Background(), webapi.NewClient("http://127.0.0.1:1"), sn, "cluster") - if err == nil { - t.Fatalf("expected re-requested action to execute (and fail against the unreachable API), but it was skipped") - } - - current := &simplyblockv1alpha1.StorageNodeSet{} - if getErr := r.Get(context.Background(), client.ObjectKeyFromObject(sn), current); getErr != nil { - t.Fatalf("failed to fetch storagenodeset: %v", getErr) - } - if current.Status.ActionStatus.State != utils.ActionStateFailed { - t.Fatalf("expected re-executed action to transition to failed, got %q", current.Status.ActionStatus.State) - } - }) - - t.Run("transitions running to failed when action call fails", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-b", - Namespace: "default", - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: "node-2", - }, - } - - r := newStorageNodeSetStateTestReconciler(t, sn) - err := r.handleNodeAction(context.Background(), webapi.NewClient("http://127.0.0.1:1"), sn, "cluster") - if err == nil { - t.Fatalf("expected action failure") - } - - current := &simplyblockv1alpha1.StorageNodeSet{} - if getErr := r.Get(context.Background(), client.ObjectKeyFromObject(sn), current); getErr != nil { - t.Fatalf("failed to fetch storagenodeset: %v", getErr) - } - if current.Status.ActionStatus == nil { - t.Fatalf("expected action status") - } - if current.Status.ActionStatus.State != utils.ActionStateFailed { - t.Fatalf("expected failed state, got %q", current.Status.ActionStatus.State) - } - if strings.TrimSpace(current.Status.ActionStatus.Message) == "" { - t.Fatalf("expected failure message to be set") - } - }) -} - -func TestHandleNodeActionRejectsIllegalSuccessIdentity(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-illegal-success", - Namespace: "default", - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: "node-expected", - }, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - // Illegal success for another identity should not be accepted. - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - Action: "restart", - NodeUUID: "node-other", - State: utils.ActionStateSuccess, - }, - }, - } - - r := newStorageNodeSetStateTestReconciler(t, sn) - err := r.handleNodeAction(context.Background(), webapi.NewClient("http://127.0.0.1:1"), sn, "cluster") - if err == nil { - t.Fatalf("expected failure after rejecting illegal success identity") - } - - current := &simplyblockv1alpha1.StorageNodeSet{} - if getErr := r.Get(context.Background(), client.ObjectKeyFromObject(sn), current); getErr != nil { - t.Fatalf("failed to fetch storagenodeset: %v", getErr) - } - if current.Status.ActionStatus == nil { - t.Fatalf("expected action status") - } - if current.Status.ActionStatus.State != utils.ActionStateFailed { - t.Fatalf("expected stale/illegal success to be rejected and transitioned to failed, got %q", current.Status.ActionStatus.State) - } - if strings.TrimSpace(current.Status.ActionStatus.Message) == "" { - t.Fatalf("expected failure message to be set") - } -} - func TestStorageNodeSetFinalizerLifecycleHelpers(t *testing.T) { now := metav1.NewTime(time.Now()) @@ -364,32 +140,6 @@ func TestStorageNodeSetLabelingHelpers(t *testing.T) { } }) - t.Run("labelWorkerNode labels single worker node", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-label-one", - Namespace: "default", - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: "cluster-b", - WorkerNode: "node-one", - }, - } - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-one"}} - r := newStorageNodeSetStateTestReconciler(t, sn, node) - - if err := r.labelWorkerNode(context.Background(), sn); err != nil { - t.Fatalf("labelWorkerNode returned error: %v", err) - } - - var out corev1.Node - if err := r.Get(context.Background(), client.ObjectKey{Name: "node-one"}, &out); err != nil { - t.Fatalf("failed to fetch node: %v", err) - } - if out.Labels["io.simplyblock.node-type"] != "simplyblock-storage-plane-cluster-b" { - t.Fatalf("expected worker node label to be set") - } - }) } func TestStorageNodeSetDaemonSetReconcileCreatesWhenMissing(t *testing.T) { @@ -679,53 +429,6 @@ func TestGetNodeInternalIPNoAddress(t *testing.T) { } } -func TestStorageNodeSetReconcileActionFastPaths(t *testing.T) { - t.Run("reconcileAction returns no requeue when action already successful", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-ra-ok", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: "node-1", - }, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - Action: "restart", - NodeUUID: "node-1", - State: utils.ActionStateSuccess, - }, - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - - res, err := r.reconcileAction(context.Background(), sn, "cluster") - if err != nil { - t.Fatalf("reconcileAction returned error: %v", err) - } - if res.RequeueAfter != 0 { - t.Fatalf("expected no delayed requeue for successful action, got %+v", res) - } - }) - - t.Run("reconcileAction requeues on action failure", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-ra-fail", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: "node-2", - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - - res, err := r.reconcileAction(context.Background(), sn, "cluster") - if err != nil { - t.Fatalf("reconcileAction returned unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Fatalf("expected delayed requeue after failed action, got %+v", res) - } - }) -} - func TestStorageNodeSetHandleDeletionNoopWithoutDeletionTimestamp(t *testing.T) { sn := &simplyblockv1alpha1.StorageNodeSet{ ObjectMeta: metav1.ObjectMeta{ @@ -901,46 +604,6 @@ func TestStorageNodeSetReconcileAddsFinalizer(t *testing.T) { } } -func TestStorageNodeSetReconcileActionPath(t *testing.T) { - const namespace = "default" - const clusterName = "cluster-action" - const clusterUUID = "cluster-uuid-action" - - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: namespace}, - Spec: simplyblockv1alpha1.StorageClusterSpec{}, - Status: simplyblockv1alpha1.StorageClusterStatus{UUID: clusterUUID}, - } - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-action-flow", - Namespace: namespace, - Finalizers: []string{utils.FinalizerStorageNodeSet}, - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: clusterName, - Action: "restart", - NodeUUID: "node-action-1", - }, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - Action: "restart", - NodeUUID: "node-action-1", - State: utils.ActionStateSuccess, - }, - }, - } - - r := newStorageNodeSetStateTestReconciler(t, sn, cluster) - res, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(sn)}) - if err != nil { - t.Fatalf("reconcile returned error: %v", err) - } - if res.RequeueAfter != 0 { - t.Fatalf("expected action fast-path to avoid delayed requeue, got %+v", res) - } -} - func TestStorageNodeSetReconcileLabelWorkerNodesFailure(t *testing.T) { const namespace = "default" const clusterName = "cluster-label-fail" @@ -1615,531 +1278,6 @@ func TestPollNodeOnlineErrorAndTimeoutPaths(t *testing.T) { }) } -// TestPerformNodeActionIdempotentTrigger guards the fix for the restart/ -// migration retry storm (Defect B): performNodeAction must not re-issue a -// backend action that is already in progress or already complete. Re-issuing a -// restart while the node is in_restart resets it and spawns a duplicate task -// the task runner aborts, livelocking the migration. -func TestPerformNodeActionIdempotentTrigger(t *testing.T) { - origWait := waitForActionCompletionWaitInterval - origSleepFn := waitForActionCompletionSleepFn - origPostTriggerSleepFn := performNodeActionSleepFn - t.Cleanup(func() { - waitForActionCompletionWaitInterval = origWait - waitForActionCompletionSleepFn = origSleepFn - performNodeActionSleepFn = origPostTriggerSleepFn - }) - waitForActionCompletionWaitInterval = 0 - waitForActionCompletionSleepFn = func(time.Duration) {} - performNodeActionSleepFn = func(time.Duration) {} - - newServer := func(statuses ...string) (*httptest.Server, *int32) { - var restartPosts int32 - var mu sync.Mutex - idx := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/restart") { - atomic.AddInt32(&restartPosts, 1) - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{}`)) - return - } - // GET node status: walk through the provided statuses, holding on - // the last one. - mu.Lock() - s := statuses[idx] - if idx < len(statuses)-1 { - idx++ - } - mu.Unlock() - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(utils.NodeStatusResponse{UUID: "node-1", Status: s}) - })) - return srv, &restartPosts - } - - t.Run("skips trigger while backend already in_restart", func(t *testing.T) { - srv, restartPosts := newServer(utils.NodeStatusInRestart, utils.NodeStatusOnline) - defer srv.Close() - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-inflight", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{Action: "restart", NodeUUID: "node-1"}, - } - - r := &StorageNodeSetReconciler{} - if err := r.performNodeAction(context.Background(), webapi.NewClient(srv.URL), "cluster-a", sn); err != nil { - t.Fatalf("performNodeAction returned error: %v", err) - } - if got := atomic.LoadInt32(restartPosts); got != 0 { - t.Fatalf("expected no restart to be POSTed while node in_restart, got %d", got) - } - }) - - t.Run("skips trigger when already triggered and node online", func(t *testing.T) { - srv, restartPosts := newServer(utils.NodeStatusOnline) - defer srv.Close() - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-done", Namespace: "default", Generation: 7}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{Action: "restart", NodeUUID: "node-1"}, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - Action: "restart", NodeUUID: "node-1", Triggered: true, ObservedGeneration: 7, - }, - }, - } - - r := &StorageNodeSetReconciler{} - if err := r.performNodeAction(context.Background(), webapi.NewClient(srv.URL), "cluster-a", sn); err != nil { - t.Fatalf("performNodeAction returned error: %v", err) - } - if got := atomic.LoadInt32(restartPosts); got != 0 { - t.Fatalf("expected no restart to be POSTed for a completed action, got %d", got) - } - }) - - t.Run("re-triggers when Triggered is from a stale generation", func(t *testing.T) { - // Same node/action, but the Triggered flag was recorded against an older - // generation. A new request (bumped generation) must not be mistaken for - // the already-fired one, so the action is triggered exactly once. This is - // what makes the ObservedGeneration match load-bearing rather than a - // no-op self-comparison. - srv, restartPosts := newServer(utils.NodeStatusOnline) - defer srv.Close() - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-stale-gen", Namespace: "default", Generation: 8}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{Action: "restart", NodeUUID: "node-1"}, - Status: simplyblockv1alpha1.StorageNodeSetStatus{ - ActionStatus: &simplyblockv1alpha1.ActionStatus{ - Action: "restart", NodeUUID: "node-1", Triggered: true, ObservedGeneration: 7, - }, - }, - } - - // A client-backed reconciler so the best-effort Triggered persist after - // the POST has somewhere to write. - r := newStorageNodeSetStateTestReconciler(t, sn) - if err := r.performNodeAction(context.Background(), webapi.NewClient(srv.URL), "cluster-a", sn); err != nil { - t.Fatalf("performNodeAction returned error: %v", err) - } - if got := atomic.LoadInt32(restartPosts); got != 1 { - t.Fatalf("expected exactly one restart POST for a new generation, got %d", got) - } - }) -} - -func TestWaitForActionCompletionRetryBehavior(t *testing.T) { - origRetries := waitForActionCompletionRetries - origWait := waitForActionCompletionWaitInterval - origSleepFn := waitForActionCompletionSleepFn - t.Cleanup(func() { - waitForActionCompletionRetries = origRetries - waitForActionCompletionWaitInterval = origWait - waitForActionCompletionSleepFn = origSleepFn - }) - - waitForActionCompletionRetries = 4 - waitForActionCompletionWaitInterval = 0 - waitForActionCompletionSleepFn = func(time.Duration) {} - - t.Run("returns terminal error when target status never reached", func(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(utils.NodeStatusResponse{ - Status: "creating", - }) - })) - defer srv.Close() - - r := &StorageNodeSetReconciler{} - err := r.waitForActionCompletion( - context.Background(), - webapi.NewClient(srv.URL), - "cluster-a", - "node-a", - "restart", - ) - if err == nil { - t.Fatalf("expected terminal status error") - } - if !strings.Contains(err.Error(), "did not reach expected status") { - t.Fatalf("unexpected error: %v", err) - } - }) - - t.Run("recovers from transient API and payload errors before success", func(t *testing.T) { - attempts := 0 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - attempts++ - switch attempts { - case 1: - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error":"temporary"}`)) - case 2: - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{`)) - default: - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(utils.NodeStatusResponse{ - Status: statusOnline, - }) - } - })) - defer srv.Close() - - r := &StorageNodeSetReconciler{} - err := r.waitForActionCompletion( - context.Background(), - webapi.NewClient(srv.URL), - "cluster-b", - "node-b", - "restart", - ) - if err != nil { - t.Fatalf("expected eventual success, got error: %v", err) - } - if attempts != 3 { - t.Fatalf("expected 3 attempts, got %d", attempts) - } - }) -} - -func TestPerformNodeActionRemoveReturnsError(t *testing.T) { - const clusterUUID = "cluster-uuid-remove" - const nodeUUID = "node-uuid-remove" - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-remove", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: utils.NodeActionRemove, - NodeUUID: nodeUUID, - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - - err := r.performNodeAction(context.Background(), webapi.NewClient("http://127.0.0.1:1"), clusterUUID, sn) - if err == nil { - t.Fatal("expected performNodeAction(remove) to return an error") - } - if !strings.Contains(err.Error(), "performDrainAndRemove") { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestPerformNodeActionRestartWorkerNodeLabelFailure(t *testing.T) { - const clusterUUID = "cluster-uuid-restart-label-fail" - const nodeUUID = "node-uuid-restart-label-fail" - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-restart-label-fail", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: nodeUUID, - WorkerNode: "missing-node", - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - - err := r.performNodeAction( - context.Background(), - webapi.NewClient("http://127.0.0.1:1"), - clusterUUID, - sn, - ) - if err == nil { - t.Fatalf("expected restart action to fail when worker node lookup fails") - } - if !strings.Contains(err.Error(), "failed to label worker node") { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestPerformNodeActionRestartWorkerNodeReachabilityCanceled(t *testing.T) { - const clusterUUID = "cluster-uuid-restart-cancel" - const nodeUUID = "node-uuid-restart-cancel" - const workerNode = "node-cancel" - - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: workerNode}, - Status: corev1.NodeStatus{ - Addresses: []corev1.NodeAddress{ - {Type: corev1.NodeInternalIP, Address: "10.0.0.99"}, - }, - }, - } - sn := &simplyblockv1alpha1.StorageNodeSet{ - // UID is required for SetControllerReference inside ensureWorkerInEndpointSlice. - ObjectMeta: metav1.ObjectMeta{Name: "sn-restart-cancel", Namespace: "default", UID: "uid-restart-cancel"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: nodeUUID, - WorkerNode: workerNode, - ClusterName: "cluster-a", - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn, node) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - err := r.performNodeAction( - ctx, - webapi.NewClient("http://127.0.0.1:1"), - clusterUUID, - sn, - ) - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected context cancellation error from restart reachability path, got %v", err) - } -} - -func TestPerformNodeActionRestartWorkerNodeEndpointSliceEnsured(t *testing.T) { - const clusterUUID = "cluster-uuid-restart-eps" - const nodeUUID = "node-uuid-restart-eps" - const workerNode = "node-eps" - - origCheckFn := waitForNodeInfoReachableCheckFn - origRetries := waitForNodeInfoReachableMaxRetries - origDelay := waitForNodeInfoReachableRetryDelay - t.Cleanup(func() { - waitForNodeInfoReachableCheckFn = origCheckFn - waitForNodeInfoReachableMaxRetries = origRetries - waitForNodeInfoReachableRetryDelay = origDelay - }) - waitForNodeInfoReachableMaxRetries = 1 - waitForNodeInfoReachableRetryDelay = time.Millisecond - waitForNodeInfoReachableCheckFn = func(context.Context, string, string, bool, bool) error { - return errors.New("unreachable") - } - - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: workerNode}, - Status: corev1.NodeStatus{ - Addresses: []corev1.NodeAddress{ - {Type: corev1.NodeInternalIP, Address: "10.0.2.2"}, - }, - }, - } - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-eps", Namespace: "default", UID: "uid-eps"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: nodeUUID, - WorkerNode: workerNode, - ClusterName: "cluster-eps", - // workerNode is intentionally absent from WorkerNodes. - WorkerNodes: []string{"existing-worker"}, - }, - } - // No existing EndpointSlice — ensureWorkerInEndpointSlice must create one. - r := newStorageNodeSetStateTestReconciler(t, sn, node) - - // The action will fail at waitForNodeInfoReachable (stubbed to fail), but - // ensureWorkerInEndpointSlice runs before that — the slice must be present. - _ = r.performNodeAction( - context.Background(), - webapi.NewClient("http://127.0.0.1:1"), - clusterUUID, - sn, - ) - - var eps discoveryv1.EndpointSlice - if err := r.Get( - context.Background(), - client.ObjectKey{Namespace: "default", Name: "simplyblock-storage-node-api-endpoints"}, - &eps, - ); err != nil { - t.Fatalf("expected EndpointSlice to be created by ensureWorkerInEndpointSlice: %v", err) - } - - found := false - for _, ep := range eps.Endpoints { - if ep.Hostname != nil && *ep.Hostname == workerNode { - found = true - break - } - } - if !found { - t.Fatalf("target worker %q not found in EndpointSlice endpoints: %#v", workerNode, eps.Endpoints) - } -} - -func TestPerformNodeActionAPIFailure(t *testing.T) { - t.Run("restart returns API failure on non-2xx", func(t *testing.T) { - const clusterUUID = "cluster-uuid-restart-api-fail" - const nodeUUID = "node-uuid-restart-api-fail" - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - mock.Register( - http.MethodPost, - "/api/v2/clusters/"+clusterUUID+"/storage-nodes/"+nodeUUID+"/restart", - webapimock.RouteResponse{ - Status: http.StatusInternalServerError, - Body: `{"error":"boom"}`, - Headers: map[string]string{ - "Content-Type": "application/json", - }, - }, - ) - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-restart-api-fail", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "restart", - NodeUUID: nodeUUID, - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - err := r.performNodeAction(context.Background(), webapi.NewClient(mock.URL()), clusterUUID, sn) - if err == nil { - t.Fatalf("expected restart API failure") - } - if !strings.Contains(err.Error(), "action API failed") { - t.Fatalf("unexpected error: %v", err) - } - }) - - t.Run("default action endpoint returns API failure on non-2xx", func(t *testing.T) { - const clusterUUID = "cluster-uuid-default-api-fail" - const nodeUUID = "node-uuid-default-api-fail" - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - mock.Register( - http.MethodPost, - "/api/v2/clusters/"+clusterUUID+"/storage-nodes/"+nodeUUID+"/suspend?force=true", - webapimock.RouteResponse{ - Status: http.StatusBadGateway, - Body: `{"error":"upstream failed"}`, - Headers: map[string]string{ - "Content-Type": "application/json", - }, - }, - ) - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-default-api-fail", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "suspend", - NodeUUID: nodeUUID, - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - err := r.performNodeAction(context.Background(), webapi.NewClient(mock.URL()), clusterUUID, sn) - if err == nil { - t.Fatalf("expected default-action API failure") - } - if !strings.Contains(err.Error(), "action API failed") { - t.Fatalf("unexpected error: %v", err) - } - }) -} - -func TestPerformNodeActionDefaultActionSuccess(t *testing.T) { - const clusterUUID = "cluster-uuid-default-success" - const nodeUUID = "node-uuid-default-success" - - origDelay := performNodeActionPostTriggerDelay - origSleep := performNodeActionSleepFn - t.Cleanup(func() { - performNodeActionPostTriggerDelay = origDelay - performNodeActionSleepFn = origSleep - }) - performNodeActionPostTriggerDelay = 0 - performNodeActionSleepFn = func(time.Duration) {} - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - mock.Register( - http.MethodPost, - "/api/v2/clusters/"+clusterUUID+"/storage-nodes/"+nodeUUID+"/suspend", - webapimock.RouteResponse{ - Status: http.StatusOK, - Body: `{}`, - Headers: map[string]string{ - "Content-Type": "application/json", - }, - }, - ) - mock.Register( - http.MethodGet, - "/api/v2/clusters/"+clusterUUID+"/storage-nodes/"+nodeUUID, - webapimock.RouteResponse{ - Status: http.StatusOK, - Body: `{"status":"suspended"}`, - Headers: map[string]string{ - "Content-Type": "application/json", - }, - }, - ) - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-default-success", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: "suspend", - NodeUUID: nodeUUID, - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - if err := r.performNodeAction(context.Background(), webapi.NewClient(mock.URL()), clusterUUID, sn); err != nil { - t.Fatalf("performNodeAction(default suspend) returned error: %v", err) - } -} - -func TestHandleNodeActionTransitionsToSuccess(t *testing.T) { - const clusterUUID = "cluster-uuid-action-success" - const nodeUUID = "node-uuid-action-success" - - origDelay := performNodeActionPostTriggerDelay - origSleep := performNodeActionSleepFn - t.Cleanup(func() { - performNodeActionPostTriggerDelay = origDelay - performNodeActionSleepFn = origSleep - }) - performNodeActionPostTriggerDelay = 0 - performNodeActionSleepFn = func(time.Duration) {} - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - mock.Register( - http.MethodPost, - "/api/v2/clusters/"+clusterUUID+"/storage-nodes/"+nodeUUID+"/suspend", - webapimock.RouteResponse{Status: http.StatusOK, Body: `{}`}, - ) - mock.Register( - http.MethodGet, - "/api/v2/clusters/"+clusterUUID+"/storage-nodes/"+nodeUUID, - webapimock.RouteResponse{Status: http.StatusOK, Body: `{"status":"suspended"}`}, - ) - - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-action-success", Namespace: "default"}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - Action: utils.NodeActionSuspend, - NodeUUID: nodeUUID, - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - - if err := r.handleNodeAction(context.Background(), webapi.NewClient(mock.URL()), sn, clusterUUID); err != nil { - t.Fatalf("handleNodeAction returned error: %v", err) - } - - current := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), current); err != nil { - t.Fatalf("failed to fetch storagenodeset: %v", err) - } - if current.Status.ActionStatus == nil { - t.Fatalf("expected action status to be set") - } - if current.Status.ActionStatus.State != utils.ActionStateSuccess { - t.Fatalf("expected success action state, got %q", current.Status.ActionStatus.State) - } -} - // testOperatorNamespace is the namespace the test reconciler pretends to run in. // It must match the namespace of the seeded singleton ControlPlane CR below. const testOperatorNamespace = "default" @@ -2622,88 +1760,6 @@ func TestReconcileSpdkProxyEndpointSlices(t *testing.T) { } } -// TestReconcilePublishesSpdkProxyEndpointSliceDuringAction guards the fix for -// the node-migration deadlock: while spec.action is set, Reconcile dispatches to -// the action flow and returns early, so the pod-driven spdk-proxy EndpointSlice -// reconciliation must still run in that path. Otherwise the target worker's -// spdk-proxy DNS name never resolves and the control-plane restart hangs on -// NameResolutionError. Reconcile must publish the slice even with an action set. -func TestReconcilePublishesSpdkProxyEndpointSliceDuringAction(t *testing.T) { - const namespace = "default" - const clusterName = "cluster-action" - - // Point the web API at an unreachable address so the action's own API call - // fails fast (the assertion below only cares that the slice was published - // before/independently of the action attempt). - t.Setenv("SIMPLYBLOCK_WEBAPI_BASE_URL", "http://127.0.0.1:1") - - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: namespace}, - Status: simplyblockv1alpha1.StorageClusterStatus{UUID: "cluster-uuid-action"}, - } - // Finalizer pre-set so ensureFinalizer does not short-circuit before the - // action dispatch. - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-action", - Namespace: namespace, - Finalizers: []string{utils.FinalizerStorageNodeSet}, - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: clusterName, - Action: "restart", - NodeUUID: "node-uuid-1", - }, - } - // A ready spdk-proxy pod that the control plane created on the target worker. - proxyPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "snode-spdk-pod-4422-cid", - Namespace: namespace, - Labels: map[string]string{"role": utils.LabelSpdkProxyRole}, - }, - Spec: corev1.PodSpec{ - NodeName: "worker-4", - Containers: []corev1.Container{ - { - Name: "spdk-proxy-container", - Env: []corev1.EnvVar{{Name: "RPC_PORT", Value: "4422"}}, - }, - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - PodIP: "10.0.0.4", - ContainerStatuses: []corev1.ContainerStatus{ - {Name: "spdk-proxy-container", Ready: true}, - }, - }, - } - - r := newStorageNodeSetStateTestReconciler(t, sn, cluster, proxyPod) - - ctx := context.Background() - if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(sn)}); err != nil { - t.Fatalf("reconcile returned unexpected error: %v", err) - } - - var slices discoveryv1.EndpointSliceList - if err := r.List(ctx, &slices, - client.InNamespace(namespace), - client.MatchingLabels{"kubernetes.io/service-name": "simplyblock-spdk-proxy"}, - ); err != nil { - t.Fatalf("list slices: %v", err) - } - if len(slices.Items) != 1 || slices.Items[0].Name != "spdk-proxy-endpoints-4422" { - t.Fatalf("expected spdk-proxy-endpoints-4422 to be published during action, got %v", sliceNames(slices.Items)) - } - if len(slices.Items[0].Endpoints) != 1 || - slices.Items[0].Endpoints[0].Hostname == nil || - *slices.Items[0].Endpoints[0].Hostname != "worker-4" { - t.Fatalf("expected a worker-4 endpoint, got %#v", slices.Items[0].Endpoints) - } -} - func TestReconcileSpdkProxyEndpointSlices_DuplicateFirstSegment(t *testing.T) { sn := &simplyblockv1alpha1.StorageNodeSet{ ObjectMeta: metav1.ObjectMeta{Name: "sn", Namespace: "ns", UID: "sn-uid"}, diff --git a/operator/internal/controller/simplyblockstoragenodeset_drain.go b/operator/internal/controller/simplyblockstoragenodeset_drain.go index 8b857be2..808a2ce0 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_drain.go +++ b/operator/internal/controller/simplyblockstoragenodeset_drain.go @@ -26,11 +26,8 @@ import ( "time" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" @@ -43,38 +40,6 @@ import ( // is malformed — intentional fast-fail for a hardcoded value. var defaultSystemVolumeFilter = regexp.MustCompile(simplyblockv1alpha1.DefaultSystemVolumeFilterRegex) -// resolveSystemVolumeFilter returns the compiled regex for the CR's system -// volume filter. The default is a package-level constant; custom patterns are -// compiled on first use and cached in the reconciler. If a custom pattern -// fails to compile, an error is returned so the reconciler can surface it to -// the user before touching any volumes. -func resolveSystemVolumeFilter( - snCR *simplyblockv1alpha1.StorageNodeSet, - r *StorageNodeSetReconciler, -) (*regexp.Regexp, error) { - if snCR.Spec.SystemVolumeFilterRegex == nil || *snCR.Spec.SystemVolumeFilterRegex == "" { - return defaultSystemVolumeFilter, nil - } - pattern := *snCR.Spec.SystemVolumeFilterRegex - if cached, ok := r.systemVolumeFilterCache.Load(pattern); ok { - return cached.(*regexp.Regexp), nil - } - re, err := regexp.Compile(pattern) - if err != nil { - return nil, fmt.Errorf("invalid systemVolumeFilterRegex %q: %w", pattern, err) - } - r.systemVolumeFilterCache.Store(pattern, re) - return re, nil -} - -// Drain sub-phase names. -const ( - drainSubPhaseValidating = "Validating" - drainSubPhaseSuspending = "Suspending" - drainSubPhaseMigrating = "Migrating" - drainSubPhaseVerifying = "Verifying" - drainSubPhaseRemoving = "Removing" -) // Requeue intervals used by the drain state machine. const ( @@ -87,890 +52,6 @@ const ( drainRequeueValidate = 30 * time.Second ) -// performDrainAndRemove is the entry point for the drain-remove state machine. -// It is called from reconcileAction when action == "remove" and a drain flow is -// required (i.e. when the caller opts in by leaving the existing remove path and -// routing here). It dispatches to the appropriate sub-phase handler based on the -// current SubPhase stored in ActionStatus. -func (r *StorageNodeSetReconciler) performDrainAndRemove( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - // Skip immediately if this drain already reached a terminal state. - // Without this guard, stale reconciles read SubPhase="" after success and - // re-enter the initialization branch, restarting the drain on a node that - // was already removed. - if snCR.Status.ActionStatus != nil && - snCR.Status.ActionStatus.NodeUUID == snCR.Spec.NodeUUID && - (snCR.Status.ActionStatus.State == utils.ActionStateSuccess || - snCR.Status.ActionStatus.State == utils.ActionStateFailed) { - return ctrl.Result{}, nil - } - - // Determine current sub-phase. - subPhase := "" - if snCR.Status.ActionStatus != nil { - subPhase = snCR.Status.ActionStatus.SubPhase - } - - // If SubPhase is empty or "Validating", ensure ActionStatus is initialised - // then delegate to drainValidate. - if subPhase == "" || subPhase == drainSubPhaseValidating { - if snCR.Status.ActionStatus == nil { - snCR.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{} - } - as := snCR.Status.ActionStatus - if as.State == "" || as.SubPhase == "" { - as.Action = snCR.Spec.Action - as.NodeUUID = snCR.Spec.NodeUUID - as.State = utils.ActionStateRunning - as.SubPhase = drainSubPhaseValidating - as.UpdatedAt = metav1.Now() - if err := r.Status().Update(ctx, snCR); err != nil { - return ctrl.Result{RequeueAfter: drainRequeueValidate}, nil - } - } - return r.drainValidate(ctx, apiClient, clusterUUID, snCR) - } - - // For all active sub-phases (Suspending onward), pause drain if the cluster - // is no longer active or is currently rebalancing. Resume automatically - // on the next reconcile once the cluster is active and not rebalancing. - if paused, reason := r.drainClusterPauseCheck(ctx, snCR); paused { - log := logf.FromContext(ctx) - log.Info("drain: pausing — cluster not ready", - "subPhase", subPhase, "reason", reason) - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Message = "drain paused: " + reason - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch paused message") - } - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "DrainPaused", - "drain paused at %s: %s", subPhase, reason) - return ctrl.Result{RequeueAfter: drainRequeueBlocking}, nil - } - - switch subPhase { - case drainSubPhaseSuspending: - return r.drainSuspend(ctx, apiClient, clusterUUID, snCR) - case drainSubPhaseMigrating: - return r.drainMigrate(ctx, apiClient, clusterUUID, snCR) - case drainSubPhaseVerifying: - return r.drainVerify(ctx, apiClient, clusterUUID, snCR) - case drainSubPhaseRemoving: - return r.drainRemove(ctx, apiClient, clusterUUID, snCR) - default: - return ctrl.Result{}, fmt.Errorf("drain: unknown SubPhase %q", subPhase) - } -} - -// drainHandleCancellation is called when the action field is cleared while a -// drain is in progress. It resumes the node so it returns to online, then -// clears ActionStatus. -func (r *StorageNodeSetReconciler) drainHandleCancellation( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - - if snCR.Status.ActionStatus == nil { - return ctrl.Result{}, nil - } - - // Re-fetch from the API server to guard against stale informer cache reads. - // A stale reconcile may see Spec.Action="" while a concurrent one is actively - // draining — a live read prevents a spurious resume from racing with suspend. - fresh := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(ctx, client.ObjectKeyFromObject(snCR), fresh); err != nil { - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - if fresh.Spec.Action == utils.NodeActionRemove { - // Drain is still active in the live CR — the cache was stale; skip cancellation. - return ctrl.Result{}, nil - } - - nodeUUID := snCR.Status.ActionStatus.NodeUUID - subPhase := snCR.Status.ActionStatus.SubPhase - - // Only attempt resume if we reached or passed the Suspending phase AND the - // node is actually suspended — skip if it's already online. - if subPhase == drainSubPhaseSuspending || - subPhase == drainSubPhaseMigrating || - subPhase == drainSubPhaseVerifying || - subPhase == drainSubPhaseRemoving { - currentStatus, err := getNodeBackendStatus(ctx, apiClient, clusterUUID, nodeUUID) - if err != nil { - log.Error(err, "drain: cancellation could not read node status, skipping resume", "nodeUUID", nodeUUID) - } else if currentStatus == utils.NodeStatusSuspended { - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s/resume", clusterUUID, nodeUUID) - if _, _, err := apiClient.Do(ctx, http.MethodPost, endpoint, nil); err != nil { - log.Error(err, "drain: cancellation resume failed (best effort)", "nodeUUID", nodeUUID) - } else { - r.Recorder.Eventf(snCR, corev1.EventTypeNormal, "NodeResumed", - "drain cancelled: resumed node %s", nodeUUID) - } - } else { - log.Info("drain: cancellation skipping resume, node already online", "nodeUUID", nodeUUID, "status", currentStatus) - } - } - - // Delete all owned VolumeMigration CRs so a subsequent drain starts fresh and - // emits MigrationCreated events. Without this, reusing in-flight CRs silently - // completes the migration with no user-visible indication the drain restarted. - var vmigList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(ctx, &vmigList, - client.InNamespace(snCR.Namespace), - client.MatchingLabels{"storage.simplyblock.io/drain-node": nodeUUID}, - ); err != nil { - log.Error(err, "drain: cancellation failed to list VolumeMigration CRs", "nodeUUID", nodeUUID) - } else { - for i := range vmigList.Items { - vm := &vmigList.Items[i] - if err := r.Delete(ctx, vm); err != nil { - log.Error(err, "drain: cancellation failed to delete VolumeMigration", "name", vm.Name) - } - } - if len(vmigList.Items) > 0 { - log.Info("drain: cancelled — deleted in-flight VolumeMigration CRs", - "nodeUUID", nodeUUID, "count", len(vmigList.Items)) - } - } - - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus = nil - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - return ctrl.Result{}, nil -} - -// drainClusterPauseCheck returns (true, reason) when the cluster is not ready -// for drain progression — either because it is not active or because it is -// currently rebalancing. The drain resumes automatically on the next poll once -// both conditions are cleared. Returns (false, "") when the cluster is ready. -func (r *StorageNodeSetReconciler) drainClusterPauseCheck( - ctx context.Context, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (paused bool, reason string) { - clusterCR, err := utils.ResolveClusterCR(ctx, r.Client, snCR.Namespace, snCR.Spec.ClusterName) - if err != nil { - // Can't determine cluster status — don't block, let the sub-phase handle it. - return false, "" - } - - if clusterCR.Status.Status != "" && clusterCR.Status.Status != utils.ClusterStatusActive { - return true, fmt.Sprintf("cluster status is %q (not active)", clusterCR.Status.Status) - } - - if clusterCR.Status.Rebalancing != nil && *clusterCR.Status.Rebalancing { - return true, "cluster is rebalancing" - } - - return false, "" -} - -// handleFailedVolumeMigrations checks whether any VolumeMigration CR has reached -// a terminal failure state. If the cluster is not ready it treats the failure as -// a transient pause; otherwise it calls resumeAndFail. Returns (result, true) -// when it has handled the situation and the caller should return immediately. -func (r *StorageNodeSetReconciler) handleFailedVolumeMigrations( - ctx context.Context, - snCR *simplyblockv1alpha1.StorageNodeSet, - items []simplyblockv1alpha1.VolumeMigration, -) (ctrl.Result, bool) { - log := logf.FromContext(ctx) - - for i := range items { - vm := &items[i] - if vm.Status.Phase != simplyblockv1alpha1.VolumeMigrationPhaseFailed && - vm.Status.Phase != simplyblockv1alpha1.VolumeMigrationPhaseAborted { - continue - } - - // A suspended/degraded cluster causes ContinueMigration to return 400, - // making the VolumeMigrationReconciler mark the CR as Failed. Treat this - // as a transient pause so the drain resumes once the cluster is active. - if paused, reason := r.drainClusterPauseCheck(ctx, snCR); paused { - log.Info("drain: VolumeMigration failed but cluster is not ready — pausing and deleting failed CR so it is recreated on resume", - "vm", vm.Name, "vmPhase", vm.Status.Phase, "clusterReason", reason) - // Delete ALL Failed/Aborted VMs now so that when the cluster - // recovers drainMigrate will recreate them with fresh target - // assignments. Leaving failed CRs in place would cause - // resumeAndFail to be called as soon as the cluster is active. - for i := range items { - if items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed || - items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseAborted { - if err := r.Delete(ctx, &items[i]); err != nil { - log.Error(err, "drain: failed to delete failed VolumeMigration during pause", - "vm", items[i].Name) - } - } - } - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Message = "drain paused (migration failed due to cluster state): " + reason - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch paused message") - } - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "DrainPaused", - "drain paused at Migrating: VolumeMigration %s failed because cluster is not ready (%s); failed CRs deleted — will recreate when cluster is active", - vm.Name, reason) - return ctrl.Result{RequeueAfter: drainRequeueBlocking}, true - } - - // Delete the Failed VM and let createMissingVolumeMigrations recreate it - // with a fresh roundRobinTargetNodes assignment. The node remains suspended - // while we retry — resumeAndFail is not called here so the drain continues. - // If no target is available the DrainNoMigrationTarget event surfaces the stall. - reason := vm.Status.ErrorMessage - if reason == "" { - reason = fmt.Sprintf("VolumeMigration %s reached phase %s", vm.Name, vm.Status.Phase) - } - log.Info("drain: VolumeMigration failed — deleting and retrying with fresh target assignment", - "vm", vm.Name, "reason", reason) - for i := range items { - if items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed || - items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseAborted { - if err := r.Delete(ctx, &items[i]); err != nil { - log.Error(err, "drain: failed to delete Failed VolumeMigration for retry", "vm", items[i].Name) - } - } - } - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Message = "migration failed, retrying with new target: " + reason - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch retry message") - } - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "MigrationRetry", - "VolumeMigration %s failed (%s); deleted — will recreate with a different target node", - vm.Name, reason) - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, true - } - return ctrl.Result{}, false -} - -// drainValidate classifies volumes on the node into system, PV-managed, pinned, -// and unmanaged buckets and blocks the drain until all blocking volumes are gone. -func (r *StorageNodeSetReconciler) drainValidate( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - // Ensure ActionStatus is in running/Validating state. - if snCR.Status.ActionStatus == nil { - snCR.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: snCR.Spec.Action, - NodeUUID: nodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - UpdatedAt: metav1.Now(), - } - } - - // Block drain if the cluster is currently rebalancing — concurrent volume - // movement would compete with the drain's VolumeMigration CRs. - clusterCR, err := utils.ResolveClusterCR(ctx, r.Client, snCR.Namespace, snCR.Spec.ClusterName) - if err != nil { - log.Error(err, "drain: failed to resolve StorageCluster for rebalancing check") - return ctrl.Result{RequeueAfter: drainRequeueValidate}, nil - } - if clusterCR.Status.Rebalancing != nil && *clusterCR.Status.Rebalancing { - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "ClusterRebalancing", - "drain blocked: cluster is rebalancing, will retry when complete") - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Message = "drain blocked: cluster is currently rebalancing" - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch status (rebalancing blocked)") - } - return ctrl.Result{RequeueAfter: drainRequeueBlocking}, nil - } - - volumes, err := listNodeVolumes(ctx, apiClient, clusterUUID, nodeUUID) - if err != nil { - log.Error(err, "drain: failed to list node volumes", "nodeUUID", nodeUUID) - return ctrl.Result{RequeueAfter: drainRequeueValidate}, nil - } - - sysFilter, err := resolveSystemVolumeFilter(snCR, r) - if err != nil { - log.Error(err, "drain: invalid systemVolumeFilterRegex") - return ctrl.Result{RequeueAfter: drainRequeueValidate}, nil - } - - _, pinned, unmanaged, _, _, err := matchVolumesToPVs(ctx, r, volumes, sysFilter) - if err != nil { - log.Error(err, "drain: matchVolumesToPVs failed") - return ctrl.Result{RequeueAfter: drainRequeueValidate}, nil - } - - // Evaluate ALL blocking conditions before returning so the user sees every - // issue at once — not one per reconcile iteration. - var msgParts []string - if len(pinned) > 0 { - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "PinnedVolumeBlocking", - "drain blocked: pinned volumes %s must be unpinned before drain", - strings.Join(pinned, ", ")) - msgParts = append(msgParts, fmt.Sprintf("%d pinned volume(s): %s", len(pinned), strings.Join(pinned, ", "))) - } - if len(unmanaged) > 0 { - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "UnmanagedVolumeBlocking", - "drain blocked: unmanaged volumes %s must be removed before drain", - strings.Join(unmanaged, ", ")) - msgParts = append(msgParts, fmt.Sprintf("%d unmanaged volume(s): %s", len(unmanaged), strings.Join(unmanaged, ", "))) - } - if len(msgParts) > 0 { - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Message = "drain blocked by " + strings.Join(msgParts, "; ") - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch status (blocking)") - } - return ctrl.Result{RequeueAfter: drainRequeueBlocking}, nil - } - - // Advance to Suspending. - if err := advanceDrainSubPhase(ctx, r, snCR, drainSubPhaseSuspending); err != nil { - log.Error(err, "drain: failed to advance to Suspending") - return ctrl.Result{RequeueAfter: drainRequeueValidate}, nil - } - return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil -} - -// drainSuspend POSTs to the suspend endpoint and polls until the node reports -// status "suspended", then advances to the Migrating sub-phase. -func (r *StorageNodeSetReconciler) drainSuspend( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - if !snCR.Status.ActionStatus.Triggered { - // Check actual node status before POSTing — skip if already suspended. - currentStatus, err := getNodeBackendStatus(ctx, apiClient, clusterUUID, nodeUUID) - if err != nil { - log.Error(err, "drain: could not read node status before suspend, retrying", "nodeUUID", nodeUUID) - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - if currentStatus == utils.NodeStatusSuspended { - log.Info("drain: node already suspended, advancing without POST", "nodeUUID", nodeUUID) - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Triggered = true - snCR.Status.ActionStatus.Message = "node already suspended" - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch Triggered=true (already suspended)") - } - return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil - } - - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s/suspend", clusterUUID, nodeUUID) - _, status, err := apiClient.Do(ctx, http.MethodPost, endpoint, nil) - if err != nil || status >= 300 { - if err == nil { - err = fmt.Errorf("suspend API returned status %d", status) - } - log.Error(err, "drain: suspend POST failed", "nodeUUID", nodeUUID) - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Triggered = true - snCR.Status.ActionStatus.Message = "suspend request sent, waiting for node to suspend" - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch Triggered=true") - } - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - - // Poll node status. - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s", clusterUUID, nodeUUID) - body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) - if err != nil || status >= 300 { - if err == nil { - err = fmt.Errorf("node status GET returned status %d", status) - } - log.Error(err, "drain: failed to GET node status during suspend poll", "nodeUUID", nodeUUID) - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - - var nodeResp utils.NodeStatusResponse - if err := json.Unmarshal(body, &nodeResp); err != nil { - log.Error(err, "drain: failed to unmarshal node status response") - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - - if nodeResp.Status != utils.NodeStatusSuspended { - log.Info("drain: node not yet suspended, requeuing", - "nodeUUID", nodeUUID, "currentStatus", nodeResp.Status) - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "DrainSuspendPending", - "waiting for node %s to suspend (current status: %s)", nodeUUID, nodeResp.Status) - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - - // Node is suspended — advance to Migrating. - if err := advanceDrainSubPhase(ctx, r, snCR, drainSubPhaseMigrating); err != nil { - log.Error(err, "drain: failed to advance to Migrating") - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil -} - -// drainMigrate creates VolumeMigration CRs for all PV-managed volumes on the -// node and waits until they all complete. -func (r *StorageNodeSetReconciler) drainMigrate( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - // List all VolumeMigration CRs owned by this drain. - var vmigList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(ctx, &vmigList, - client.InNamespace(snCR.Namespace), - client.MatchingLabels{"storage.simplyblock.io/drain-node": nodeUUID}, - ); err != nil { - log.Error(err, "drain: failed to list VolumeMigration CRs") - return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil - } - - // If any migration failed, handle it — pausing if the cluster is the cause. - if res, handled := r.handleFailedVolumeMigrations(ctx, snCR, vmigList.Items); handled { - return res, nil - } - - // Count completed vs in-progress migrations. - completed := 0 - inProgress := 0 - for i := range vmigList.Items { - if vmigList.Items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseCompleted { - completed++ - } else { - inProgress++ - } - } - - // Build the set of existing VM names so we can detect missing ones. - // On first entry len == 0; after a cluster-state pause some Failed CRs were - // deleted by handleFailedVolumeMigrations and need to be recreated here. - existingVMNames := make(map[string]struct{}, len(vmigList.Items)) - for i := range vmigList.Items { - existingVMNames[vmigList.Items[i].Name] = struct{}{} - } - - // Create VMs for any PV that does not already have one. Only enters when - // there are missing VMs (first entry or resume-after-pause); otherwise the - // completion check below handles the rest. - if len(vmigList.Items) == 0 || r.hasMissingVolumeMigrations(ctx, apiClient, clusterUUID, nodeUUID, snCR, existingVMNames) { - return r.createMissingVolumeMigrations(ctx, apiClient, clusterUUID, snCR, vmigList.Items, existingVMNames) - } - - // All existing CRs are completed — update counters, clean up and advance. - if inProgress == 0 && completed == len(vmigList.Items) { - // Write final counters before deleting the CRs so the status reflects - // the true end state (all migrated, none pending). - finalPatch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.VolumesMigrated = completed - snCR.Status.ActionStatus.VolumesPending = 0 - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, finalPatch); err != nil { - log.Error(err, "drain: failed to patch final migration counters") - } - - for i := range vmigList.Items { - vm := &vmigList.Items[i] - if err := r.Delete(ctx, vm); err != nil { - log.Error(err, "drain: failed to delete completed VolumeMigration", "name", vm.Name) - } - } - r.Recorder.Eventf(snCR, corev1.EventTypeNormal, "MigrationCompleted", - "all %d volume migrations completed", completed) - - if err := advanceDrainSubPhase(ctx, r, snCR, drainSubPhaseVerifying); err != nil { - log.Error(err, "drain: failed to advance to Verifying") - return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil - } - return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil - } - - // Migrations still in progress — update counters. - patch := client.MergeFrom(snCR.DeepCopy()) - total := len(vmigList.Items) - snCR.Status.ActionStatus.VolumesMigrated = completed - snCR.Status.ActionStatus.VolumesPending = inProgress - snCR.Status.ActionStatus.Message = fmt.Sprintf("Migrating: %d of %d volumes migrated", completed, total) - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch migration progress counters") - } - return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil -} - -// hasMissingVolumeMigrations returns true if any PV-managed volume on the node -// does not yet have a corresponding VolumeMigration CR in existingVMNames. -// Used to detect the resume-after-pause case where failed CRs were deleted. -func (r *StorageNodeSetReconciler) hasMissingVolumeMigrations( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID, nodeUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, - existingVMNames map[string]struct{}, -) bool { - vols, err := listNodeVolumes(ctx, apiClient, clusterUUID, nodeUUID) - if err != nil { - return false - } - sf, err := resolveSystemVolumeFilter(snCR, r) - if err != nil { - return false - } - pvm, _, _, pvByVol, _, err := matchVolumesToPVs(ctx, r, vols, sf) - if err != nil { - return false - } - for _, volUUID := range pvm { - if pvName, ok := pvByVol[volUUID]; ok { - if _, exists := existingVMNames[drainMigrationName(nodeUUID, pvName)]; !exists { - return true - } - } - } - return false -} - -// createMissingVolumeMigrations lists PV-managed volumes on the node, skips any -// that already have a VolumeMigration CR in existingVMNames, and creates new CRs -// for the rest using round-robin target assignment. -func (r *StorageNodeSetReconciler) createMissingVolumeMigrations( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, - existingItems []simplyblockv1alpha1.VolumeMigration, - existingVMNames map[string]struct{}, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - volumes, err := listNodeVolumes(ctx, apiClient, clusterUUID, nodeUUID) - if err != nil { - log.Error(err, "drain: failed to list volumes for migration creation") - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil - } - - sysFilter, err := resolveSystemVolumeFilter(snCR, r) - if err != nil { - log.Error(err, "drain: invalid systemVolumeFilterRegex") - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil - } - - pvManaged, _, _, pvNameByVolumeUUID, pvcFetchFailed, err := matchVolumesToPVs(ctx, r, volumes, sysFilter) - if err != nil { - log.Error(err, "drain: matchVolumesToPVs failed during migration creation") - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil - } - // A PV-backed volume appeared as unmanaged because its PVC GET failed - // transiently. Retrying avoids silently skipping the volume, which would - // leave it on the node and stall Verifying indefinitely. - if pvcFetchFailed { - log.Info("drain: PVC fetch failed for a PV-backed volume — retrying to avoid skipping it") - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil - } - - if len(pvManaged) == 0 && len(existingItems) == 0 { - if err := advanceDrainSubPhase(ctx, r, snCR, drainSubPhaseVerifying); err != nil { - log.Error(err, "drain: failed to advance to Verifying (no migrations needed)") - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil - } - return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil - } - - pvNames := make([]string, 0, len(pvManaged)) - for _, volUUID := range pvManaged { - pvName, ok := pvNameByVolumeUUID[volUUID] - if !ok { - continue - } - if _, exists := existingVMNames[drainMigrationName(nodeUUID, pvName)]; !exists { - pvNames = append(pvNames, pvName) - } - } - if len(pvNames) == 0 { - return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil - } - - targetByPV, err := roundRobinTargetNodes(ctx, apiClient, clusterUUID, nodeUUID, pvNames) - if err != nil { - log.Error(err, "drain: no available target nodes for migration") - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "DrainNoMigrationTarget", - "drain stalled: no online storage node available as migration target for node %s — will retry when a peer node is online", - nodeUUID) - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil - } - - createdCount := 0 - for _, volUUID := range pvManaged { - pvName, ok := pvNameByVolumeUUID[volUUID] - if !ok { - continue - } - migName := drainMigrationName(nodeUUID, pvName) - if _, exists := existingVMNames[migName]; exists { - continue - } - vmig := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: migName, - Namespace: snCR.Namespace, - Labels: map[string]string{"storage.simplyblock.io/drain-node": nodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: pvName, - TargetNodeUUID: targetByPV[pvName], - }, - } - if err := controllerutil.SetControllerReference(snCR, vmig, r.Scheme); err != nil { - log.Error(err, "drain: failed to set controller reference on VolumeMigration", "name", migName) - continue - } - if err := r.Create(ctx, vmig); err != nil { - log.Error(err, "drain: failed to create VolumeMigration", "name", migName) - continue - } - r.Recorder.Eventf(snCR, corev1.EventTypeNormal, "MigrationCreated", - "created VolumeMigration %s for PV %s", migName, pvName) - createdCount++ - } - - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.VolumesPending = createdCount - snCR.Status.ActionStatus.VolumesMigrated = 0 - snCR.Status.ActionStatus.Message = fmt.Sprintf("Migrating: 0 of %d volumes migrated", createdCount) - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch VolumesPending counter") - } - return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil -} - -// drainVerify checks that the node holds no non-system volumes before removing it. -func (r *StorageNodeSetReconciler) drainVerify( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - // Fetch pools and node volumes in a single pass — drainVerify may also need - // the pool list for system volume cleanup, so we reuse it rather than - // making a second GetStoragePools call. - pools, volumes, err := fetchPoolVolumes(ctx, apiClient, clusterUUID, nodeUUID) - if err != nil { - log.Error(err, "drain: failed to list volumes during verification", "nodeUUID", nodeUUID) - return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil - } - - sysFilter, err := resolveSystemVolumeFilter(snCR, r) - if err != nil { - log.Error(err, "drain: invalid systemVolumeFilterRegex") - return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil - } - - var nonSystem, systemVols []string - for _, vol := range volumes { - if sysFilter.MatchString(vol.Name) { - systemVols = append(systemVols, vol.UUID) - } else { - nonSystem = append(nonSystem, vol.UUID) - } - } - - if len(nonSystem) > 0 { - log.Info("drain: node still has non-system volumes, requeuing", - "nodeUUID", nodeUUID, "volumeUUIDs", strings.Join(nonSystem, ", ")) - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "DrainVerifyPending", - "node %s still has %d non-system volume(s) after migration (%s); waiting for backend to confirm empty", - nodeUUID, len(nonSystem), strings.Join(nonSystem, ", ")) - return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil - } - - // If system/benchmark volumes remain, delete them now. The backend rejects - // node removal if any LVols are still present, even system ones. Requeue - // after issuing the deletes so the next reconcile confirms they are gone - // before advancing to Removing. - if len(systemVols) > 0 { - // Build pool lookup from the pools already fetched above — no extra API call. - poolByVol := make(map[string]string) - for _, pool := range pools { - vols, err := apiClient.GetPoolVolumes(ctx, clusterUUID, pool.UUID) - if err != nil { - continue - } - for _, v := range vols { - poolByVol[v.UUID] = pool.UUID - } - } - for _, volUUID := range systemVols { - poolUUID, ok := poolByVol[volUUID] - if !ok { - log.Info("drain: system volume pool not found, skipping", "volUUID", volUUID) - continue - } - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-pools/%s/volumes/%s/", - clusterUUID, poolUUID, volUUID) - _, delStatus, delErr := apiClient.Do(ctx, http.MethodDelete, endpoint, nil) - delClass := webapi.ClassifyError(delErr, delStatus) - switch { - case delErr == nil && (delStatus == http.StatusOK || delStatus == http.StatusNoContent || delStatus == http.StatusNotFound): - log.Info("drain: deleted system volume", "volUUID", volUUID, "status", delStatus) - case delClass.Retryable: - log.Error(delErr, "drain: transient error deleting system volume, will retry on next reconcile", - "volUUID", volUUID, "status", delStatus) - default: - // Permanent rejection — resume and fail the drain. - return r.resumeAndFail(ctx, apiClient, clusterUUID, snCR, - fmt.Sprintf("system volume %s delete rejected by backend (status %d)", volUUID, delStatus)) - } - } - // Requeue to confirm the volumes are gone before advancing to Removing. - return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil - } - - // Node is empty — advance to Removing. - if err := advanceDrainSubPhase(ctx, r, snCR, drainSubPhaseRemoving); err != nil { - log.Error(err, "drain: failed to advance to Removing") - return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil - } - return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil -} - -// drainRemove sends the DELETE request to remove the storage node from the cluster. -func (r *StorageNodeSetReconciler) drainRemove( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - endpoint := fmt.Sprintf( - "/api/v2/clusters/%s/storage-nodes/%s?force_remove=false", - clusterUUID, - nodeUUID, - ) - - _, status, err := apiClient.Do(ctx, http.MethodDelete, endpoint, nil) - - // 200, 204: success. 404: node already gone — treat as success. - if err == nil && (status == http.StatusOK || status == http.StatusNoContent || status == http.StatusNotFound) { - r.Recorder.Eventf(snCR, corev1.EventTypeNormal, "NodeRemoved", - "storage node %s removed successfully", nodeUUID) - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.State = utils.ActionStateSuccess - snCR.Status.ActionStatus.SubPhase = "" - snCR.Status.ActionStatus.VolumesPending = 0 - snCR.Status.ActionStatus.Message = "node removed successfully" - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch success status after remove") - } - return ctrl.Result{}, nil - } - - class := webapi.ClassifyError(err, status) - if class.Retryable { - log.Error(err, "drain: transient error on node DELETE, retrying", - "nodeUUID", nodeUUID, "status", status) - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - - // Permanent error — resume the node and mark failed. - return r.resumeAndFail(ctx, apiClient, clusterUUID, snCR, - fmt.Sprintf("DELETE node returned status %d", status)) -} - -// resumeAndFail attempts a best-effort resume of the suspended node, then marks -// the action as failed. -func (r *StorageNodeSetReconciler) resumeAndFail( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - snCR *simplyblockv1alpha1.StorageNodeSet, - reason string, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - nodeUUID := snCR.Spec.NodeUUID - - resumeEndpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s/resume", clusterUUID, nodeUUID) - _, resumeStatus, resumeErr := apiClient.Do(ctx, http.MethodPost, resumeEndpoint, nil) - resumeClass := webapi.ClassifyError(resumeErr, resumeStatus) - if resumeClass.Retryable { - // Transient resume failure — requeue so the node gets another resume - // attempt rather than being left suspended indefinitely. - log.Error(resumeErr, "drain: transient error resuming node, will retry", - "nodeUUID", nodeUUID, "status", resumeStatus) - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.Message = fmt.Sprintf("resume pending after failure: %s", reason) - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch status during resume retry") - } - return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil - } - if resumeErr != nil || (resumeStatus >= http.StatusBadRequest && resumeStatus != http.StatusNotFound) { - log.Error(resumeErr, "drain: permanent error resuming node — node may remain suspended", - "nodeUUID", nodeUUID, "status", resumeStatus) - } - - r.Recorder.Eventf(snCR, corev1.EventTypeWarning, "NodeResumed", - "drain failed, attempted resume of node %s: %s", nodeUUID, reason) - - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.State = utils.ActionStateFailed - snCR.Status.ActionStatus.SubPhase = "" - snCR.Status.ActionStatus.Message = reason - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "drain: failed to patch failed status") - } - return ctrl.Result{}, nil -} - -// advanceDrainSubPhase patches only the SubPhase and Message fields of the -// ActionStatus. -func advanceDrainSubPhase( - ctx context.Context, - r *StorageNodeSetReconciler, - snCR *simplyblockv1alpha1.StorageNodeSet, - newPhase string, -) error { - patch := client.MergeFrom(snCR.DeepCopy()) - snCR.Status.ActionStatus.SubPhase = newPhase - snCR.Status.ActionStatus.Message = fmt.Sprintf("drain: entering phase %s", newPhase) - snCR.Status.ActionStatus.UpdatedAt = metav1.Now() - return r.Status().Patch(ctx, snCR, patch) -} - // fetchPoolVolumes fetches all pools and returns (pools, nodeVolumes, err). // Callers that need both the pool list (e.g. for cleanup) and the node volumes // should call this once and reuse the returned pools, avoiding a second @@ -1038,7 +119,7 @@ func listNodeVolumes( // pvcFetchFailed to avoid silently skipping volumes that would stall Verifying. func matchVolumesToPVs( ctx context.Context, - r *StorageNodeSetReconciler, + c client.Client, volumes []webapi.VolumeInfo, sysFilter *regexp.Regexp, ) (pvManaged, pinned, unmanaged []string, pvNameByVolumeUUID map[string]string, pvcFetchFailed bool, err error) { @@ -1048,7 +129,7 @@ func matchVolumesToPVs( // Build a map: volumeUUID → pvName for all simplyblock PVs. var pvList corev1.PersistentVolumeList - if err = r.List(ctx, &pvList); err != nil { + if err = c.List(ctx, &pvList); err != nil { return nil, nil, nil, nil, false, fmt.Errorf("matchVolumesToPVs: list PVs: %w", err) } @@ -1091,7 +172,7 @@ func matchVolumesToPVs( } var pvc corev1.PersistentVolumeClaim - if err := r.Get(ctx, types.NamespacedName{ + if err := c.Get(ctx, types.NamespacedName{ Namespace: pv.Spec.ClaimRef.Namespace, Name: pv.Spec.ClaimRef.Name, }, &pvc); err != nil { diff --git a/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go index 90946508..5b561bbd 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go @@ -49,72 +49,6 @@ func newDrainReconciler(t *testing.T, objects ...client.Object) *StorageNodeSetR } } -func newDrainSN(nodeUUID, subPhase, state string) *simplyblockv1alpha1.StorageNodeSet { //nolint:unparam - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-drain", Namespace: drainTestNS}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: drainTestCluster, - Action: utils.NodeActionRemove, - NodeUUID: nodeUUID, - }, - } - if subPhase != "" || state != "" { - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: nodeUUID, - State: state, - SubPhase: subPhase, - } - } - return sn -} - -// ── performDrainAndRemove: terminal state early return ──────────────────────── - -func TestDrainSkipsWhenAlreadySuccess(t *testing.T) { - sn := newDrainSN(drainTestNodeUUID, "", utils.ActionStateSuccess) - r := newDrainReconciler(t, sn) - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient("http://127.0.0.1:1"), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("expected no error for success state, got %v", err) - } - if res.RequeueAfter != 0 { - t.Fatalf("expected zero requeue for success state, got %v", res.RequeueAfter) - } -} - -func TestDrainSkipsWhenAlreadyFailed(t *testing.T) { - sn := newDrainSN(drainTestNodeUUID, "", utils.ActionStateFailed) - r := newDrainReconciler(t, sn) - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient("http://127.0.0.1:1"), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("expected no error for failed state, got %v", err) - } - if res.RequeueAfter != 0 { - t.Fatalf("expected zero requeue for failed state, got %v", res.RequeueAfter) - } -} - -func TestDrainSkipsSuccessEvenWhenSubPhaseIsEmpty(t *testing.T) { - // Regression: stale reconcile reads SubPhase="" after success and must not - // re-initialize the drain (which would restart it on an already-removed node). - sn := newDrainSN(drainTestNodeUUID, "", utils.ActionStateSuccess) - r := newDrainReconciler(t, sn) - - // Run twice — second call must also return immediately without API calls. - for i := 0; i < 2; i++ { - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient("http://127.0.0.1:1"), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("iteration %d: unexpected error %v", i, err) - } - if res.RequeueAfter != 0 { - t.Fatalf("iteration %d: unexpected requeue %v", i, res.RequeueAfter) - } - } -} - // ── roundRobinTargetNodes ───────────────────────────────────────────────────── func TestRoundRobinDistributesEvenly(t *testing.T) { @@ -236,7 +170,7 @@ func TestMatchVolumesToPVs_PVManaged(t *testing.T) { r := newDrainReconciler(t, pv, pvc) vols := []webapi.VolumeInfo{{UUID: "vol-1111", Name: "pvc-something"}} - pvManaged, pinned, unmanaged, byUUID, _, err := matchVolumesToPVs(context.Background(), r, vols, regexp.MustCompile("^never-matches$")) + pvManaged, pinned, unmanaged, byUUID, _, err := matchVolumesToPVs(context.Background(), r.Client, vols, regexp.MustCompile("^never-matches$")) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -257,7 +191,7 @@ func TestMatchVolumesToPVs_Pinned(t *testing.T) { r := newDrainReconciler(t, pv, pvc) vols := []webapi.VolumeInfo{{UUID: "vol-2222", Name: "pvc-something"}} - pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r, vols, regexp.MustCompile("^never-matches$")) + pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r.Client, vols, regexp.MustCompile("^never-matches$")) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -273,7 +207,7 @@ func TestMatchVolumesToPVs_Unmanaged(t *testing.T) { r := newDrainReconciler(t) // no PVs in cluster vols := []webapi.VolumeInfo{{UUID: "vol-orphan", Name: "manually-created"}} - pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r, vols, regexp.MustCompile("^never-matches$")) + pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r.Client, vols, regexp.MustCompile("^never-matches$")) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -289,7 +223,7 @@ func TestMatchVolumesToPVs_SystemVolumeSkipped(t *testing.T) { r := newDrainReconciler(t) // no PVs — if not filtered, would be unmanaged vols := []webapi.VolumeInfo{{UUID: "vol-bench", Name: "sb-fio-baseline-xyz"}} - pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r, vols, defaultSystemVolumeFilter) + pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r.Client, vols, defaultSystemVolumeFilter) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -298,103 +232,9 @@ func TestMatchVolumesToPVs_SystemVolumeSkipped(t *testing.T) { } } -func TestDrainValidateBothPinnedAndUnmanagedSurfacedTogether(t *testing.T) { - // When a node has both pinned and unmanaged volumes, drainValidate must emit - // BOTH warning events in a single reconcile — not short-circuit after pinned. - // This prevents the user having to fix one issue, retry, then discover the next. - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - // Backend returns two volumes: one PV-managed+pinned, one unmanaged. - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/", - webapimock.RouteResponse{Status: http.StatusOK, Body: `[{"id":"pool-1","name":"p1"}]`}, - ) - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/pool-1/volumes/", - webapimock.RouteResponse{Status: http.StatusOK, Body: `[ - {"id":"vol-pinned","name":"pvc-a","storage_node_id":"` + drainTestNodeUUID + `"}, - {"id":"vol-orphan","name":"manually-created","storage_node_id":"` + drainTestNodeUUID + `"} - ]`}, - ) - // Cluster not rebalancing. - rebalancing := false - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Rebalancing: &rebalancing}, - } - pv := newPV("pv-pin", "vol-pinned", drainTestClusterUUID, "pool-1") - pvc := newPVC("pv-pin-pvc", drainTestNS, true) // pinned annotation set - - fakeRecorder := record.NewFakeRecorder(8) - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseValidating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - } - - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, cluster, sn, pv, pvc) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: fakeRecorder, - } - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when blocking volumes present") - } - - // Drain all buffered events. - close(fakeRecorder.Events) - var reasons []string - for e := range fakeRecorder.Events { - // Event format: "Warning PinnedVolumeBlocking ..." - parts := strings.SplitN(e, " ", 3) - if len(parts) >= 2 { - reasons = append(reasons, parts[1]) - } - } - - hasPinned := contains(reasons, "PinnedVolumeBlocking") - hasUnmanaged := contains(reasons, "UnmanagedVolumeBlocking") - - if !hasPinned { - t.Error("expected PinnedVolumeBlocking event to be emitted") - } - if !hasUnmanaged { - t.Error("expected UnmanagedVolumeBlocking event to be emitted in the same reconcile") - } - if !hasPinned || !hasUnmanaged { - t.Errorf("emitted events: %v — both blockers must surface together so the user can fix everything at once", reasons) - } - - // Message must mention both issues. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - msg := "" - if updated.Status.ActionStatus != nil { - msg = updated.Status.ActionStatus.Message - } - if !strings.Contains(msg, "pinned") || !strings.Contains(msg, "unmanaged") { - t.Errorf("status message should mention both blockers, got: %q", msg) - } -} - func TestMatchVolumesToPVs_EmptyNodeSkipsMigration(t *testing.T) { r := newDrainReconciler(t) - pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r, nil, defaultSystemVolumeFilter) + pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r.Client, nil, defaultSystemVolumeFilter) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -409,7 +249,7 @@ func TestMatchVolumesToPVs_OnlySystemVolumes(t *testing.T) { {UUID: "v1", Name: "sb-fio-baseline-read"}, {UUID: "v2", Name: "sb-fio-baseline-write"}, } - pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r, vols, defaultSystemVolumeFilter) + pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r.Client, vols, defaultSystemVolumeFilter) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -418,392 +258,6 @@ func TestMatchVolumesToPVs_OnlySystemVolumes(t *testing.T) { } } -// ── drainValidate: reconciler-level user-visible behaviour ─────────────────── -// -// These tests call drainValidate (not matchVolumesToPVs) and assert what the -// user actually sees: events emitted on the StorageNodeSet CR and the status -// message. Testing the classification helper alone is insufficient — it does -// not verify that the reconciler surfaces the result to the user. - -// newValidateSetup builds a mock HTTP server and reconciler for drainValidate -// tests. volumes is the JSON array the backend returns for the pool's volumes. -func newValidateSetup( - t *testing.T, - volumes string, - k8sObjects ...client.Object, -) (*StorageNodeSetReconciler, *record.FakeRecorder, *webapimock.SpecServer, *simplyblockv1alpha1.StorageNodeSet) { - t.Helper() - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - t.Cleanup(mock.Close) - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/", - webapimock.RouteResponse{Status: http.StatusOK, Body: `[{"id":"pool-1","name":"p1"}]`}, - ) - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/pool-1/volumes/", - webapimock.RouteResponse{Status: http.StatusOK, Body: volumes}, - ) - - rebalancing := false - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Rebalancing: &rebalancing}, - } - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseValidating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - } - - recorder := record.NewFakeRecorder(8) - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - allObjs := append([]client.Object{cluster, sn}, k8sObjects...) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, allObjs...) - r := &StorageNodeSetReconciler{Client: cl, Scheme: scheme, Recorder: recorder} - return r, recorder, mock, sn -} - -// collectEvents drains all buffered events and returns a map of reason→count. -func collectEvents(rec *record.FakeRecorder) map[string]int { - close(rec.Events) - reasons := map[string]int{} - for e := range rec.Events { - parts := strings.SplitN(e, " ", 3) - if len(parts) >= 2 { - reasons[parts[1]]++ - } - } - return reasons -} - -func TestDrainValidatePinnedVolumeEmitsEventAndBlocks(t *testing.T) { - // User sees: PinnedVolumeBlocking event; status message mentions "pinned"; - // drain does not advance past Validating. - pv := newPV("pv-x", "vol-pinned", drainTestClusterUUID, "pool-1") - pvc := newPVC("pv-x-pvc", drainTestNS, true) - r, rec, mock, sn := newValidateSetup(t, - `[{"id":"vol-pinned","name":"pvc-x","storage_node_id":"`+drainTestNodeUUID+`"}]`, - pv, pvc, - ) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when pinned volume blocks drain") - } - - events := collectEvents(rec) - if events["PinnedVolumeBlocking"] == 0 { - t.Error("expected PinnedVolumeBlocking event; user must be told why drain is blocked") - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - msg := "" - if updated.Status.ActionStatus != nil { - msg = updated.Status.ActionStatus.Message - } - if !strings.Contains(msg, "pinned") { - t.Errorf("status message must mention 'pinned' so user knows what to fix; got: %q", msg) - } - if updated.Status.ActionStatus != nil && updated.Status.ActionStatus.SubPhase != drainSubPhaseValidating { - t.Errorf("drain must stay in Validating when blocked; got SubPhase=%q", updated.Status.ActionStatus.SubPhase) - } -} - -func TestDrainValidateUnmanagedVolumeEmitsEventAndBlocks(t *testing.T) { - // User sees: UnmanagedVolumeBlocking event; status message mentions "unmanaged"; - // drain does not advance past Validating. - r, rec, mock, sn := newValidateSetup(t, - `[{"id":"vol-orphan","name":"manually-created","storage_node_id":"`+drainTestNodeUUID+`"}]`, - // no PV in cluster → volume is unmanaged - ) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when unmanaged volume blocks drain") - } - - events := collectEvents(rec) - if events["UnmanagedVolumeBlocking"] == 0 { - t.Error("expected UnmanagedVolumeBlocking event; user must be told why drain is blocked") - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - msg := "" - if updated.Status.ActionStatus != nil { - msg = updated.Status.ActionStatus.Message - } - if !strings.Contains(msg, "unmanaged") { - t.Errorf("status message must mention 'unmanaged' so user knows what to fix; got: %q", msg) - } -} - -func TestDrainValidateEmptyNodeAdvancesToSuspending(t *testing.T) { - // User sees: drain proceeds without any blocking event. - // Empty node must not block; it advances straight to Suspending. - r, rec, mock, sn := newValidateSetup(t, `[]`) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // Should requeue immediately (advance to Suspending), not with the blocking interval. - if res.RequeueAfter > drainRequeueBlocking { - t.Errorf("unexpected long requeue for empty node: %v (expected ≤ %v)", res.RequeueAfter, drainRequeueBlocking) - } - - events := collectEvents(rec) - if events["PinnedVolumeBlocking"] > 0 || events["UnmanagedVolumeBlocking"] > 0 { - t.Errorf("empty node must not emit blocking events; got %v", events) - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil || updated.Status.ActionStatus.SubPhase != drainSubPhaseSuspending { - got := "" - if updated.Status.ActionStatus != nil { - got = updated.Status.ActionStatus.SubPhase - } - t.Errorf("empty node must advance to Suspending; got SubPhase=%q", got) - } -} - -func TestDrainValidateSystemOnlyNodeAdvancesToSuspending(t *testing.T) { - // User sees: drain proceeds without blocking even though volumes exist — - // system volumes are silently filtered and the drain advances normally. - r, rec, mock, sn := newValidateSetup(t, `[ - {"id":"v1","name":"sb-fio-baseline-read","storage_node_id":"`+drainTestNodeUUID+`"}, - {"id":"v2","name":"sb-fio-baseline-write","storage_node_id":"`+drainTestNodeUUID+`"} - ]`) - defer mock.Close() - - _, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - events := collectEvents(rec) - if events["PinnedVolumeBlocking"] > 0 || events["UnmanagedVolumeBlocking"] > 0 { - t.Errorf("system-volume-only node must not emit blocking events; got %v", events) - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil || updated.Status.ActionStatus.SubPhase != drainSubPhaseSuspending { - got := "" - if updated.Status.ActionStatus != nil { - got = updated.Status.ActionStatus.SubPhase - } - t.Errorf("system-only node must advance to Suspending; got SubPhase=%q", got) - } -} - -// ── drainMigrate: VolumeMigration idempotency ───────────────────────────────── - -func TestDrainMigrateDoesNotRecreateExistingCRs(t *testing.T) { - // Existing VolumeMigration CR for a drain — operator restart must not - // create a duplicate with the same drain label. - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseMigrating, utils.ActionStateRunning) - existingVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-existing-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseRunning, - }, - } - r := newDrainReconciler(t, sn, existingVM) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - // No pool/volume listing needed — should use existing CRs. - - // Call drainMigrate. It should see the existing CR and not call the API - // to create new ones. - if _, err := r.drainMigrate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainMigrate: %v", err) - } - - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VolumeMigration: %v", err) - } - if len(vmList.Items) != 1 { - t.Errorf("expected exactly 1 VolumeMigration (no duplicate created), got %d", len(vmList.Items)) - } -} - -func TestDrainMigrateFailedCRDeletedAndRetriedWithNewTarget(t *testing.T) { - // Any VolumeMigration failure deletes the Failed CR and requeues so - // createMissingVolumeMigrations can recreate it with a fresh target. - // The node stays suspended — resumeAndFail is NOT called. - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseMigrating, utils.ActionStateRunning) - failedVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-failed-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseFailed, - ErrorMessage: "ContinueMigration: status 400: target node not online", - }, - } - r := newDrainReconciler(t, sn, failedVM) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.drainMigrate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("drainMigrate: %v", err) - } - // Must requeue for retry, not zero (which would mean terminal). - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when VM failed and retry is in progress") - } - - // State must still be running — the node is NOT resumed and drain NOT failed. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus != nil && updated.Status.ActionStatus.State == utils.ActionStateFailed { - t.Error("drain must not be marked failed — should retry with a new target") - } - - // The Failed VM must be deleted so the next reconcile recreates it. - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VMs: %v", err) - } - for _, vm := range vmList.Items { - if vm.Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed { - t.Errorf("Failed VM %q must be deleted so it can be recreated with a different target", vm.Name) - } - } -} - -func TestDrainMigrateFailedCRDeletedWhenClusterPaused(t *testing.T) { - // When a VolumeMigration fails AND the cluster is not ready, the drain - // should pause AND delete the Failed VM so it can be recreated with a fresh - // target assignment once the cluster recovers — not just defer the failure. - rebalancing := true - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{ - Status: utils.ClusterStatusActive, - Rebalancing: &rebalancing, - }, - } - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseMigrating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseMigrating, - } - failedVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-failed-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseFailed, - ErrorMessage: "target node in_shutdown", - }, - } - - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - &simplyblockv1alpha1.VolumeMigration{}, - }, cluster, sn, failedVM) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: record.NewFakeRecorder(8), - } - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.drainMigrate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("drainMigrate: %v", err) - } - // Must requeue (pause), not fail. - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when cluster is paused due to rebalancing") - } - // State must still be running — NOT failed. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus != nil && updated.Status.ActionStatus.State == utils.ActionStateFailed { - t.Error("drain must not be marked failed when pausing due to cluster state") - } - // The Failed VM must have been deleted so the drain can recreate it on resume. - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VMs: %v", err) - } - for _, vm := range vmList.Items { - if vm.Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed { - t.Errorf("Failed VolumeMigration %q must be deleted during cluster-state pause so it can be recreated on resume", vm.Name) - } - } -} - // ── drainMigrationName ──────────────────────────────────────────────────────── func TestDrainMigrationNameNoCollisionOnLongPVNames(t *testing.T) { @@ -857,295 +311,3 @@ func TestDrainMigrationNameIsDNSValid(t *testing.T) { } } -// ── drainHandleCancellation: stale cache guard ──────────────────────────────── - -func TestDrainCancellationDeletesInFlightVolumeMigrationCRs(t *testing.T) { - // When the user cancels mid-drain, all owned VolumeMigration CRs must be - // deleted so a subsequent drain starts fresh and emits MigrationCreated events. - // Without this, the re-applied drain silently reuses the old CRs and the user - // sees no indication that migration restarted. - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseMigrating, utils.ActionStateRunning) - sn.Spec.Action = "" // user cleared the action - sn.Spec.NodeUUID = "" - inFlightVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-inflight-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseRunning, - }, - } - r := newDrainReconciler(t, sn, inFlightVM) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - // Node is online (already resumed or never suspended in this test). - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-nodes/"+drainTestNodeUUID, - webapimock.RouteResponse{Status: http.StatusOK, Body: `{"status":"online"}`}, - ) - - if _, err := r.drainHandleCancellation(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainHandleCancellation: %v", err) - } - - // The in-flight VolumeMigration CR must have been deleted. - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VolumeMigration: %v", err) - } - if len(vmList.Items) != 0 { - t.Errorf("expected in-flight VolumeMigration CRs to be deleted on cancel, got %d remaining", len(vmList.Items)) - } - - // ActionStatus must also be cleared. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus != nil { - t.Errorf("expected ActionStatus cleared after cancellation, got %+v", updated.Status.ActionStatus) - } -} - -func TestDrainCancellationSkipsWhenActionStillActive(t *testing.T) { - // If the live CR still has action=remove, a stale reconcile must not resume. - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseMigrating, utils.ActionStateRunning) - r := newDrainReconciler(t, sn) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - // No resume endpoint registered — if resume were called the mock would return 404 - // which would log an error. We verify the CR is unchanged. - - // The passed-in sn has action=remove in the live store (seeded in fake client), - // so drainHandleCancellation's re-fetch will see it and exit early. - if _, err := r.drainHandleCancellation(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainHandleCancellation: %v", err) - } -} - -// ── drainValidate: cluster rebalancing blocks drain ─────────────────────────── - -func TestDrainValidateBlocksWhenClusterRebalancing(t *testing.T) { - rebalancing := true - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Rebalancing: &rebalancing}, - } - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseValidating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - } - - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, cluster, sn) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: record.NewFakeRecorder(32), - } - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // Should requeue — not advance to Suspending. - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when cluster is rebalancing") - } - // SubPhase must still be Validating. - if sn.Status.ActionStatus.SubPhase != drainSubPhaseValidating { - t.Errorf("expected SubPhase=Validating, got %q", sn.Status.ActionStatus.SubPhase) - } -} - -// ── drainClusterPauseCheck: pause during active sub-phases ──────────────────── - -// newPauseCheckReconciler builds a reconciler seeded with the given StorageCluster -// and a StorageNodeSet in the provided sub-phase, for drainClusterPauseCheck tests. -func newPauseCheckReconciler(t *testing.T, cluster *simplyblockv1alpha1.StorageCluster, subPhase string) (*StorageNodeSetReconciler, *simplyblockv1alpha1.StorageNodeSet) { - t.Helper() - sn := newDrainSN(drainTestNodeUUID, subPhase, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: subPhase, - } - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, cluster, sn) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: record.NewFakeRecorder(32), - } - return r, sn -} - -func TestDrainPausesWhenClusterInactive(t *testing.T) { - // When the cluster transitions to a non-active status (e.g. "degraded") - // during an active drain sub-phase, performDrainAndRemove must pause and - // requeue rather than advancing the state machine. - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Status: "degraded"}, - } - - for _, subPhase := range []string{ - drainSubPhaseSuspending, - drainSubPhaseMigrating, - drainSubPhaseVerifying, - drainSubPhaseRemoving, - } { - t.Run("subPhase="+subPhase, func(t *testing.T) { - r, sn := newPauseCheckReconciler(t, cluster, subPhase) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Errorf("expected non-zero requeue when cluster is inactive") - } - - // SubPhase must not have advanced. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil { - t.Fatal("expected actionStatus to remain set") - } - if updated.Status.ActionStatus.SubPhase != subPhase { - t.Errorf("sub-phase must not advance during pause: got %q want %q", - updated.Status.ActionStatus.SubPhase, subPhase) - } - // Message should mention the pause. - if !strings.Contains(updated.Status.ActionStatus.Message, "paused") { - t.Errorf("expected message to mention 'paused', got %q", - updated.Status.ActionStatus.Message) - } - }) - } -} - -func TestDrainPausesWhenClusterRebalancingDuringActiveDrain(t *testing.T) { - // When the cluster starts rebalancing after the drain has already advanced - // past Validating, the drain must pause at its current sub-phase. - rebalancing := true - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{ - Status: utils.ClusterStatusActive, - Rebalancing: &rebalancing, - }, - } - - for _, subPhase := range []string{ - drainSubPhaseSuspending, - drainSubPhaseMigrating, - } { - t.Run("subPhase="+subPhase, func(t *testing.T) { - r, sn := newPauseCheckReconciler(t, cluster, subPhase) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Errorf("expected non-zero requeue when cluster is rebalancing") - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil { - t.Fatal("expected actionStatus to remain set") - } - if updated.Status.ActionStatus.SubPhase != subPhase { - t.Errorf("sub-phase must not advance during pause: got %q want %q", - updated.Status.ActionStatus.SubPhase, subPhase) - } - if !strings.Contains(updated.Status.ActionStatus.Message, "rebalancing") { - t.Errorf("expected message to mention 'rebalancing', got %q", - updated.Status.ActionStatus.Message) - } - }) - } -} - -func TestDrainContinuesWhenClusterActiveAndNotRebalancing(t *testing.T) { - // When the cluster is active and not rebalancing, drainClusterPauseCheck - // must return (false, "") so the drain proceeds normally. - rebalancing := false - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{ - Status: utils.ClusterStatusActive, - Rebalancing: &rebalancing, - }, - } - r, sn := newPauseCheckReconciler(t, cluster, drainSubPhaseMigrating) - - paused, reason := r.drainClusterPauseCheck(context.Background(), sn) - if paused { - t.Errorf("expected drain to continue when cluster is active and not rebalancing, got reason=%q", reason) - } -} - -// ── Rapid action toggling ───────────────────────────────────────────────────── - -func TestRapidActionToggleDoesNotLeakState(t *testing.T) { - // Set action=remove, cancel immediately (Validating phase, before any suspend). - // After cancel the ActionStatus must be nil so re-applying remove starts fresh. - sn := newDrainSN(drainTestNodeUUID, drainSubPhaseValidating, utils.ActionStateRunning) - // Simulate user clearing spec.action. - sn.Spec.Action = "" - sn.Spec.NodeUUID = "" - r := newDrainReconciler(t, sn) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - if _, err := r.drainHandleCancellation(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainHandleCancellation: %v", err) - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - // ActionStatus cleared — re-applying action=remove will start a fresh drain. - if updated.Status.ActionStatus != nil { - t.Errorf("expected ActionStatus to be nil after cancel at Validating, got %+v", updated.Status.ActionStatus) - } -} diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go new file mode 100644 index 00000000..937b7789 --- /dev/null +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -0,0 +1,269 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +// reconcileStorageNodeCRs is the Phase-1 bridge that creates, updates, and +// deletes StorageNode CRs to match StorageNodeSet.spec.workerNodes × +// spec.socketsToUse. The StorageNodeSet is the single source of truth — this +// function only creates and garbage-collects; the StorageNodeReconciler owns +// the per-node provisioning and status-sync loops. +// +// Called from StorageNodeSetReconciler.Reconcile after fleet infrastructure +// (DaemonSet, RBAC, Services) has been reconciled. + +import ( + "context" + "fmt" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" +) + +// reconcileStorageNodeCRs creates a StorageNode CR for every (worker, socket) +// pair in the StorageNodeSet spec and deletes any owned StorageNode CRs that no +// longer correspond to a configured worker/socket. Overrides are synced from +// spec.nodeConfigs on every call. +func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( + ctx context.Context, + sns *simplyblockv1alpha1.StorageNodeSet, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Build the expected set of (worker, socket) pairs. + sockets := effectiveSockets(sns) + type workerSocket struct{ worker, socket string } + expected := make(map[workerSocket]struct{}, len(sns.Spec.WorkerNodes)*len(sockets)) + for _, worker := range sns.Spec.WorkerNodes { + for _, socket := range sockets { + expected[workerSocket{worker, socket}] = struct{}{} + } + } + + // List all StorageNode CRs owned by this StorageNodeSet. + var owned simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &owned, + client.InNamespace(sns.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": sns.Name}, + ); err != nil { + return ctrl.Result{}, fmt.Errorf("listing owned StorageNode CRs: %w", err) + } + + // Delete stale CRs (worker removed from spec.workerNodes or socket removed). + for i := range owned.Items { + sn := &owned.Items[i] + key := workerSocket{sn.Spec.WorkerNode, socketLabel(sn.Spec.SocketIndex)} + if _, ok := expected[key]; ok { + continue + } + if err := r.Delete(ctx, sn); err != nil && !apierrors.IsNotFound(err) { + log.Error(err, "failed to delete stale StorageNode CR", "name", sn.Name) + } else { + log.Info("deleted stale StorageNode CR", "name", sn.Name, + "worker", sn.Spec.WorkerNode, "socket", socketLabel(sn.Spec.SocketIndex)) + } + } + + // Create or sync each expected (worker, socket). + for _, worker := range sns.Spec.WorkerNodes { + for _, socket := range sockets { + if err := r.ensureStorageNodeCR(ctx, sns, worker, socket); err != nil { + log.Error(err, "failed to ensure StorageNode CR", "worker", worker, "socket", socket) + } + } + } + + // Aggregate status: TotalNodes / OnlineNodes / OfflineNodes. + if err := r.aggregateStorageNodeStatus(ctx, sns); err != nil { + log.Error(err, "failed to aggregate StorageNode status into StorageNodeSet") + } + + return ctrl.Result{}, nil +} + +// ensureStorageNodeCR creates or patches a StorageNode CR for (worker, socket). +func (r *StorageNodeSetReconciler) ensureStorageNodeCR( + ctx context.Context, + sns *simplyblockv1alpha1.StorageNodeSet, + worker, socket string, +) error { + log := logf.FromContext(ctx) + name := storageNodeCRName(sns.Name, worker, socket) + + var existing simplyblockv1alpha1.StorageNode + err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: sns.Namespace}, &existing) + + if apierrors.IsNotFound(err) { + // Create. + sn := buildStorageNodeCR(sns, name, worker, socket) + if setErr := controllerutil.SetControllerReference(sns, sn, r.Scheme); setErr != nil { + return fmt.Errorf("setting owner reference on StorageNode %s: %w", name, setErr) + } + if createErr := r.Create(ctx, sn); createErr != nil { + return fmt.Errorf("creating StorageNode %s: %w", name, createErr) + } + log.Info("created StorageNode CR", "name", name, "worker", worker, "socket", socket) + return nil + } + if err != nil { + return fmt.Errorf("getting StorageNode %s: %w", name, err) + } + + // Sync overrides from nodeConfigs — the StorageNodeSet is the source of truth. + overrides, hasConfig := sns.Spec.NodeConfigs[worker] + desired := existing.Spec.Overrides + if hasConfig { + desired = &overrides + } + + patch := client.MergeFrom(existing.DeepCopy()) + existing.Spec.Overrides = desired + if err := r.Patch(ctx, &existing, patch); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("patching StorageNode overrides %s: %w", name, err) + } + return nil +} + +// aggregateStorageNodeStatus rolls up online/offline counts from owned +// StorageNode CRs into StorageNodeSet.status. +func (r *StorageNodeSetReconciler) aggregateStorageNodeStatus( + ctx context.Context, + sns *simplyblockv1alpha1.StorageNodeSet, +) error { + var owned simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &owned, + client.InNamespace(sns.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": sns.Name}, + ); err != nil { + return err + } + + total, online, offline := len(owned.Items), 0, 0 + for _, sn := range owned.Items { + switch sn.Status.Status { + case "online": + online++ + case "offline", "suspended": + offline++ + } + } + + patch := client.MergeFrom(sns.DeepCopy()) + sns.Status.TotalNodes = total + sns.Status.OnlineNodes = online + sns.Status.OfflineNodes = offline + return r.Status().Patch(ctx, sns, patch) +} + +// storageNodeCRName builds a deterministic, DNS-label-safe name for a StorageNode CR. +// Pattern: {sns-name}-{sanitised-worker}-{socket}, truncated to 63 chars with +// an FNV-32 suffix to prevent collisions on long names. +func storageNodeCRName(snsName, worker, socket string) string { + // Sanitise worker hostname: lowercase, replace non-alnum with '-'. + sanitised := sanitiseDNSLabel(worker) + raw := snsName + "-" + sanitised + "-" + socket + raw = strings.ToLower(raw) + + const maxLen = 63 + if len(raw) <= maxLen { + return raw + } + // Append 7-char FNV hash suffix before truncation. + h := fnv32Hash(worker + socket) + suffix := fmt.Sprintf("-%06x", h) + keep := maxLen - len(suffix) + if keep < 1 { + keep = 1 + } + return raw[:keep] + suffix +} + +// sanitiseDNSLabel replaces characters not valid in a DNS label with '-' and +// strips leading/trailing hyphens. +func sanitiseDNSLabel(s string) string { + var b strings.Builder + for _, c := range strings.ToLower(s) { + if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '.' { + b.WriteRune(c) + } else { + b.WriteByte('-') + } + } + return strings.Trim(b.String(), "-.") +} + +// buildStorageNodeCR constructs a new StorageNode CR for the given worker and socket. +func buildStorageNodeCR( + sns *simplyblockv1alpha1.StorageNodeSet, + name, worker, socket string, +) *simplyblockv1alpha1.StorageNode { + var socketIndex *int32 + if socket != "0" && socket != "" { + // Parse the socket string to an int32; default to nil (= socket 0) on failure. + var idx int32 + if n, err := fmt.Sscanf(socket, "%d", &idx); n == 1 && err == nil { + socketIndex = &idx + } + } + + sn := &simplyblockv1alpha1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sns.Namespace, + Labels: map[string]string{ + "storage.simplyblock.io/storagenodeset": sns.Name, + "storage.simplyblock.io/worker": sanitiseDNSLabel(worker), + }, + }, + Spec: simplyblockv1alpha1.StorageNodeSpec{ + StorageNodeSetRef: sns.Name, + WorkerNode: worker, + SocketIndex: socketIndex, + }, + } + + if overrides, ok := sns.Spec.NodeConfigs[worker]; ok { + sn.Spec.Overrides = &overrides + } + + return sn +} + +// effectiveSockets returns the list of socket identifiers to use. When +// SocketsToUse is empty, a single socket "0" is assumed. +func effectiveSockets(sns *simplyblockv1alpha1.StorageNodeSet) []string { + if len(sns.Spec.SocketsToUse) == 0 { + return []string{"0"} + } + return sns.Spec.SocketsToUse +} + +// socketLabel converts a *int32 socket index back to the string representation +// used in storageNodeCRName, for matching against expected keys. +func socketLabel(idx *int32) string { + if idx == nil { + return "0" + } + return fmt.Sprintf("%d", *idx) +} diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go new file mode 100644 index 00000000..246ef33f --- /dev/null +++ b/operator/internal/controller/storagenode_controller.go @@ -0,0 +1,480 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" + "github.com/simplyblock/simplyblock-operator/internal/webapi" +) + +const ( + storageNodeFinalizer = "storage.simplyblock.io/storagenode-finalizer" + storageNodeSyncInterval = 30 * time.Second +) + +// StorageNodeReconciler reconciles StorageNode objects. +// It owns the per-node provisioning loop: node-add POST, online polling, status +// sync, and triggering a StorageNodeOps(action=remove) on deletion. +type StorageNodeReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder + TLSEnabled bool + TLSMutualEnabled bool +} + +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodes/finalizers,verbs=update + +func (r *StorageNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var sn simplyblockv1alpha1.StorageNode + if err := r.Get(ctx, req.NamespacedName, &sn); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Fetch the parent StorageNodeSet for fleet config. + var sns simplyblockv1alpha1.StorageNodeSet + if err := r.Get(ctx, types.NamespacedName{ + Name: sn.Spec.StorageNodeSetRef, + Namespace: sn.Namespace, + }, &sns); err != nil { + if apierrors.IsNotFound(err) { + log.Info("parent StorageNodeSet not found, requeuing", "ref", sn.Spec.StorageNodeSetRef) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + return ctrl.Result{}, err + } + + // Resolve cluster UUID early — needed for both provisioning and status sync. + clusterUUID, err := utils.ResolveClusterUUID(ctx, r.Client, sn.Namespace, sns.Spec.ClusterName) + if err != nil { + log.Info("cluster UUID not ready yet, requeuing", "cluster", sns.Spec.ClusterName) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // Handle deletion. + if !sn.DeletionTimestamp.IsZero() { + return r.handleDeletion(ctx, &sn, &sns) + } + + // Ensure finalizer. + if !controllerutil.ContainsFinalizer(&sn, storageNodeFinalizer) { + controllerutil.AddFinalizer(&sn, storageNodeFinalizer) + if err := r.Update(ctx, &sn); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + // Sync overrides from the parent StorageNodeSet. + if err := r.syncOverrides(ctx, &sn, &sns); err != nil { + return ctrl.Result{}, err + } + + apiClient := webapi.NewClient() + + // Node not yet provisioned → post to backend. + if sn.Status.UUID == "" { + return r.provisionNode(ctx, &sn, &sns, clusterUUID, apiClient) + } + + // Node provisioned → sync status periodically. + return r.syncStatus(ctx, &sn, clusterUUID, apiClient) +} + +// syncOverrides propagates StorageNodeSet.spec.nodeConfigs[worker] into +// StorageNode.spec.overrides. The StorageNodeSet is the single source of truth. +func (r *StorageNodeReconciler) syncOverrides( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + sns *simplyblockv1alpha1.StorageNodeSet, +) error { + overrides, ok := sns.Spec.NodeConfigs[sn.Spec.WorkerNode] + if !ok { + return nil + } + patch := client.MergeFrom(sn.DeepCopy()) + sn.Spec.Overrides = &overrides + if err := r.Patch(ctx, sn, patch); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("syncing overrides for %s: %w", sn.Name, err) + } + return nil +} + +// provisionNode posts the node to the backend API and begins polling for online status. +func (r *StorageNodeReconciler) provisionNode( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + sns *simplyblockv1alpha1.StorageNodeSet, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // Guard: if enableFailureDomains is set on the cluster, failureDomain must be populated. + if err := r.checkFailureDomain(ctx, sn, sns); err != nil { + r.Recorder.Event(sn, "Warning", "FailureDomainMissing", err.Error()) + log.Info("blocking node-add: "+err.Error(), "node", sn.Name) + return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + } + + // Wait until the node's API endpoint is reachable. + if err := checkNodeInfoReachable(ctx, sn.Spec.WorkerNode, sn.Namespace, r.TLSEnabled, r.TLSMutualEnabled); err != nil { + log.V(1).Info("storage node API not reachable yet, requeuing", + "worker", sn.Spec.WorkerNode, "error", err.Error()) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // Build effective config: fleet defaults merged with per-node overrides. + eff := effectiveNodeConfig(sn, sns) + + nodeAddress := utils.StorageNodeSetAPIAddress(sn.Spec.WorkerNode, sn.Namespace) + params := utils.StorageNodeSetAddParams{ + NodeAddress: nodeAddress, + InterfaceName: sns.Spec.MgmtIfname, + SPDKImage: eff.SpdkImage, + SPDKProxyImage: eff.SpdkProxyImage, + DataNics: sns.Spec.DataIfname, + Namespace: sn.Namespace, + JMPercent: journalManagerPercentPerDevice(sns), + Partitions: utils.IntPtrOrDefault(sns.Spec.Partitions, 1), + HaJMCount: journalManagerCount(sns), + CRName: sns.Name, + CRNameSpace: sns.Namespace, + CRPlural: "storagenodesets", + Format4K: utils.BoolPtrOrFalse(sns.Spec.ForceFormat4K), + SpdkSystemMemory: eff.SpdkSystemMemory, + FailureDomain: effectiveFailureDomain(sn, sns), + } + + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes", clusterUUID) + body, status, err := apiClient.Do(ctx, http.MethodPost, endpoint, params) + if err != nil || status >= 300 { + if err == nil { + err = fmt.Errorf("unexpected status %d", status) + } + log.Error(err, "storage node add failed", "status", status, "response", string(body)) + return ctrl.Result{RequeueAfter: 20 * time.Second}, nil + } + + log.Info("storage node add POST sent", "endpoint", endpoint, "status", status) + + // Mark PostedAt so the next reconcile can poll online status. + now := metav1.Now() + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.PostedAt = &now + if err := r.Status().Patch(ctx, sn, patch); err != nil { + log.Error(err, "failed to patch PostedAt") + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil +} + +// syncStatus fetches the current node status from the backend and updates StorageNode.status. +func (r *StorageNodeReconciler) syncStatus( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s", clusterUUID, sn.Status.UUID) + body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) + if err != nil || status >= 300 { + if err == nil { + err = fmt.Errorf("status %d", status) + } + log.Error(err, "failed to GET node status", "uuid", sn.Status.UUID) + return ctrl.Result{RequeueAfter: storageNodeSyncInterval}, nil + } + + var resp SNODEAPIResponse + if err := json.Unmarshal(body, &resp); err != nil { + log.Error(err, "failed to unmarshal node status response") + return ctrl.Result{RequeueAfter: storageNodeSyncInterval}, nil + } + + cpu := int32(resp.CPU) + volumes := int32(resp.Volumes) + rpcPort := int32(resp.RPC_PORT) + lvolPort := int32(resp.LVOL_PORT) + nvmfPort := int32(resp.NVMF_PORT) + + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.Status = resp.Status + sn.Status.Health = resp.Health + sn.Status.CPU = &cpu + sn.Status.Volumes = &volumes + sn.Status.MgmtIp = resp.IP + sn.Status.Hostname = resp.Hostname + sn.Status.RpcPort = &rpcPort + sn.Status.LvolPort = &lvolPort + sn.Status.NvmfPort = &nvmfPort + + if err := r.Status().Patch(ctx, sn, patch); err != nil { + log.Error(err, "failed to patch StorageNode status") + } + return ctrl.Result{RequeueAfter: storageNodeSyncInterval}, nil +} + +// checkFailureDomain returns an error if the parent cluster has +// enableFailureDomains=true but this node has no failureDomain set. +func (r *StorageNodeReconciler) checkFailureDomain( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + sns *simplyblockv1alpha1.StorageNodeSet, +) error { + var cluster simplyblockv1alpha1.StorageCluster + if err := r.Get(ctx, types.NamespacedName{ + Name: sns.Spec.ClusterName, + Namespace: sn.Namespace, + }, &cluster); err != nil { + return nil // can't determine; don't block + } + if cluster.Spec.EnableFailureDomains == nil || !*cluster.Spec.EnableFailureDomains { + return nil + } + if effectiveFailureDomain(sn, sns) > 0 { + return nil + } + return fmt.Errorf( + "failureDomain not set for worker %q; add nodeConfigs[%s].failureDomain to StorageNodeSet %q", + sn.Spec.WorkerNode, sn.Spec.WorkerNode, sns.Name, + ) +} + +// effectiveNodeConfig returns the merged config for a node: fleet defaults +// overridden by any per-node values from StorageNode.spec.overrides. +func effectiveNodeConfig(sn *simplyblockv1alpha1.StorageNode, sns *simplyblockv1alpha1.StorageNodeSet) simplyblockv1alpha1.StorageNodeOverrides { + eff := simplyblockv1alpha1.StorageNodeOverrides{ + SpdkImage: sns.Spec.SpdkImage, + SpdkProxyImage: sns.Spec.SpdkProxyImage, + MaxLogicalVolumeCount: sns.Spec.MaxLogicalVolumeCount, + MaxSize: sns.Spec.MaxSize, + CorePercentage: sns.Spec.CorePercentage, + SpdkSystemMemory: sns.Spec.SpdkSystemMemory, + JournalManagerSpec: sns.Spec.JournalManagerSpec, + PcieAllowList: sns.Spec.PcieAllowList, + PcieDenyList: sns.Spec.PcieDenyList, + PcieModel: sns.Spec.PcieModel, + DriveSizeRange: sns.Spec.DriveSizeRange, + DeviceNames: sns.Spec.DeviceNames, + EnableCpuTopology: sns.Spec.EnableCpuTopology, + ReservedSystemCPU: sns.Spec.ReservedSystemCPU, + UbuntuHost: sns.Spec.UbuntuHost, + } + if sn.Spec.Overrides == nil { + return eff + } + o := sn.Spec.Overrides + if o.SpdkImage != "" { + eff.SpdkImage = o.SpdkImage + } + if o.SpdkProxyImage != "" { + eff.SpdkProxyImage = o.SpdkProxyImage + } + if o.MaxLogicalVolumeCount != nil { + eff.MaxLogicalVolumeCount = o.MaxLogicalVolumeCount + } + if o.MaxSize != "" { + eff.MaxSize = o.MaxSize + } + if o.CorePercentage != nil { + eff.CorePercentage = o.CorePercentage + } + if o.SpdkSystemMemory != "" { + eff.SpdkSystemMemory = o.SpdkSystemMemory + } + if o.JournalManagerSpec != nil { + eff.JournalManagerSpec = o.JournalManagerSpec + } + if len(o.PcieAllowList) > 0 { + eff.PcieAllowList = o.PcieAllowList + } + if len(o.PcieDenyList) > 0 { + eff.PcieDenyList = o.PcieDenyList + } + if o.PcieModel != "" { + eff.PcieModel = o.PcieModel + } + if o.DriveSizeRange != "" { + eff.DriveSizeRange = o.DriveSizeRange + } + if len(o.DeviceNames) > 0 { + eff.DeviceNames = o.DeviceNames + } + if o.EnableCpuTopology != nil { + eff.EnableCpuTopology = o.EnableCpuTopology + } + if o.ReservedSystemCPU != "" { + eff.ReservedSystemCPU = o.ReservedSystemCPU + } + if o.UbuntuHost != nil { + eff.UbuntuHost = o.UbuntuHost + } + if o.FailureDomain != nil { + eff.FailureDomain = o.FailureDomain + } + return eff +} + +// effectiveFailureDomain returns the failure domain for the node: +// StorageNode.spec.overrides.failureDomain takes precedence over +// StorageNodeSet.spec.nodeFailureDomains[worker]. +func effectiveFailureDomain(sn *simplyblockv1alpha1.StorageNode, sns *simplyblockv1alpha1.StorageNodeSet) int { + if sn.Spec.Overrides != nil && sn.Spec.Overrides.FailureDomain != nil { + return int(*sn.Spec.Overrides.FailureDomain) + } + if v, ok := sns.Spec.NodeFailureDomains[sn.Spec.WorkerNode]; ok { + return int(v) + } + return 0 +} + +// handleDeletion ensures a StorageNodeOps(action=remove) exists for this node +// if it is online, then removes the finalizer once the ops CR completes. +func (r *StorageNodeReconciler) handleDeletion( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + _ *simplyblockv1alpha1.StorageNodeSet, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + // If the node was never provisioned, skip ops and remove finalizer immediately. + if sn.Status.UUID == "" { + controllerutil.RemoveFinalizer(sn, storageNodeFinalizer) + return ctrl.Result{}, r.Update(ctx, sn) + } + + if sn.Status.Status == utils.NodeStatusSuspended || + sn.Status.Status == utils.ClusterStatusActive || + sn.Status.Status == "online" { + if err := r.ensureRemoveOps(ctx, sn); err != nil { + return ctrl.Result{}, err + } + } + + // If an ops is still active, requeue and wait. + if sn.Status.ActiveOpsRef != "" { + log.Info("waiting for StorageNodeOps to complete before finalizer removal", + "ops", sn.Status.ActiveOpsRef) + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + } + + controllerutil.RemoveFinalizer(sn, storageNodeFinalizer) + return ctrl.Result{}, r.Update(ctx, sn) +} + +// ensureRemoveOps creates a StorageNodeOps(action=remove) for this StorageNode +// if one does not already exist. +func (r *StorageNodeReconciler) ensureRemoveOps( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, +) error { + opsName := sn.Name + "-remove" + var existing simplyblockv1alpha1.StorageNodeOps + err := r.Get(ctx, types.NamespacedName{Name: opsName, Namespace: sn.Namespace}, &existing) + if err == nil { + return nil // already exists + } + if !apierrors.IsNotFound(err) { + return err + } + + ops := simplyblockv1alpha1.StorageNodeOps{} + ops.Name = opsName + ops.Namespace = sn.Namespace + ops.Spec.StorageNodeRef = sn.Name + ops.Spec.Action = "remove" + if err := controllerutil.SetControllerReference(sn, &ops, r.Scheme); err != nil { + return err + } + return r.Create(ctx, &ops) +} + +// storageNodeSetToStorageNodeRequests maps a StorageNodeSet change to all +// owned StorageNode reconcile requests. +func (r *StorageNodeReconciler) storageNodeSetToStorageNodeRequests( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(obj.GetNamespace()), + client.MatchingFields{"spec.storageNodeSetRef": obj.GetName()}, + ); err != nil { + return nil + } + reqs := make([]reconcile.Request, len(snList.Items)) + for i, sn := range snList.Items { + reqs[i] = reconcile.Request{NamespacedName: types.NamespacedName{ + Name: sn.Name, + Namespace: sn.Namespace, + }} + } + return reqs +} + +// SetupWithManager registers the StorageNodeReconciler with the controller manager. +func (r *StorageNodeReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &simplyblockv1alpha1.StorageNode{}, + "spec.storageNodeSetRef", + func(obj client.Object) []string { + sn := obj.(*simplyblockv1alpha1.StorageNode) + return []string{sn.Spec.StorageNodeSetRef} + }, + ); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&simplyblockv1alpha1.StorageNode{}). + Named("storagenode"). + Watches( + &simplyblockv1alpha1.StorageNodeSet{}, + handler.EnqueueRequestsFromMapFunc(r.storageNodeSetToStorageNodeRequests), + ). + Owns(&simplyblockv1alpha1.StorageNodeOps{}). + Complete(r) +} diff --git a/operator/internal/controller/storagenode_controller_unit_test.go b/operator/internal/controller/storagenode_controller_unit_test.go new file mode 100644 index 00000000..e4324944 --- /dev/null +++ b/operator/internal/controller/storagenode_controller_unit_test.go @@ -0,0 +1,300 @@ +package controller + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" +) + +// ── helpers ────────────────────────────────────────────────────────────────── + +const ( + snTestNS = "test" + snTestCluster = "cluster-a" + snTestWorker = "worker-1.example.com" +) + +func newSNReconciler(t *testing.T, objects ...client.Object) *StorageNodeReconciler { + t.Helper() + scheme := newTestScheme(t, + simplyblockv1alpha1.AddToScheme, + corev1.AddToScheme, + ) + cl := newTestClient(t, scheme, + []client.Object{ + &simplyblockv1alpha1.StorageNode{}, + &simplyblockv1alpha1.StorageNodeOps{}, + &simplyblockv1alpha1.StorageCluster{}, + &simplyblockv1alpha1.StorageNodeSet{}, + }, + objects..., + ) + return &StorageNodeReconciler{ + Client: cl, + Scheme: scheme, + Recorder: record.NewFakeRecorder(16), + } +} + +func newStorageNodeSet(name, ns, cluster string, nodeConfigs map[string]simplyblockv1alpha1.StorageNodeOverrides) *simplyblockv1alpha1.StorageNodeSet { + return &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + ClusterName: cluster, + WorkerNodes: []string{snTestWorker}, + NodeConfigs: nodeConfigs, + }, + } +} + +func newStorageNode(name, ns, snsRef, worker string) *simplyblockv1alpha1.StorageNode { + return &simplyblockv1alpha1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: simplyblockv1alpha1.StorageNodeSpec{ + StorageNodeSetRef: snsRef, + WorkerNode: worker, + }, + } +} + +// ── TestSyncOverrides ───────────────────────────────────────────────────────── + +func TestSyncOverrides_PropagatesNodeConfigs(t *testing.T) { + maxLvol := int32(99) + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, map[string]simplyblockv1alpha1.StorageNodeOverrides{ + snTestWorker: {MaxLogicalVolumeCount: &maxLvol, SpdkSystemMemory: "8G"}, + }) + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + r := newSNReconciler(t, sns, sn) + + if err := r.syncOverrides(context.Background(), sn, sns); err != nil { + t.Fatalf("syncOverrides returned error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + if err := r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: snTestNS}, &updated); err != nil { + t.Fatalf("failed to fetch updated StorageNode: %v", err) + } + if updated.Spec.Overrides == nil { + t.Fatal("expected Overrides to be set") + } + if updated.Spec.Overrides.SpdkSystemMemory != "8G" { + t.Errorf("SpdkSystemMemory: got %q want %q", updated.Spec.Overrides.SpdkSystemMemory, "8G") + } + if *updated.Spec.Overrides.MaxLogicalVolumeCount != 99 { + t.Errorf("MaxLogicalVolumeCount: got %d want 99", *updated.Spec.Overrides.MaxLogicalVolumeCount) + } +} + +func TestSyncOverrides_NoopWhenWorkerNotInNodeConfigs(t *testing.T) { + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + r := newSNReconciler(t, sns, sn) + + if err := r.syncOverrides(context.Background(), sn, sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: snTestNS}, &updated) + if updated.Spec.Overrides != nil { + t.Error("expected Overrides to remain nil when worker not in nodeConfigs") + } +} + +// ── TestEffectiveNodeConfig ─────────────────────────────────────────────────── + +func TestEffectiveNodeConfig_OverridesTakePrecedence(t *testing.T) { + fleetMem := "4G" + overrideMem := "16G" + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{SpdkSystemMemory: fleetMem}, + } + sn := &simplyblockv1alpha1.StorageNode{ + Spec: simplyblockv1alpha1.StorageNodeSpec{ + Overrides: &simplyblockv1alpha1.StorageNodeOverrides{SpdkSystemMemory: overrideMem}, + }, + } + eff := effectiveNodeConfig(sn, sns) + if eff.SpdkSystemMemory != overrideMem { + t.Errorf("expected override %q, got %q", overrideMem, eff.SpdkSystemMemory) + } +} + +func TestEffectiveNodeConfig_FallsBackToFleetDefault(t *testing.T) { + fleetMem := "4G" + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{SpdkSystemMemory: fleetMem}, + } + sn := &simplyblockv1alpha1.StorageNode{} // no overrides + eff := effectiveNodeConfig(sn, sns) + if eff.SpdkSystemMemory != fleetMem { + t.Errorf("expected fleet default %q, got %q", fleetMem, eff.SpdkSystemMemory) + } +} + +// ── TestEffectiveFailureDomain ──────────────────────────────────────────────── + +func TestEffectiveFailureDomain_OverrideTakesPrecedenceOverMap(t *testing.T) { + fd := int32(3) + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + NodeFailureDomains: map[string]int32{snTestWorker: 1}, + }, + } + sn := &simplyblockv1alpha1.StorageNode{ + Spec: simplyblockv1alpha1.StorageNodeSpec{ + WorkerNode: snTestWorker, + Overrides: &simplyblockv1alpha1.StorageNodeOverrides{FailureDomain: &fd}, + }, + } + if got := effectiveFailureDomain(sn, sns); got != 3 { + t.Errorf("expected 3 from override, got %d", got) + } +} + +func TestEffectiveFailureDomain_FallsBackToMap(t *testing.T) { + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + NodeFailureDomains: map[string]int32{snTestWorker: 2}, + }, + } + sn := &simplyblockv1alpha1.StorageNode{ + Spec: simplyblockv1alpha1.StorageNodeSpec{WorkerNode: snTestWorker}, + } + if got := effectiveFailureDomain(sn, sns); got != 2 { + t.Errorf("expected 2 from map, got %d", got) + } +} + +func TestEffectiveFailureDomain_ZeroWhenNotSet(t *testing.T) { + sns := &simplyblockv1alpha1.StorageNodeSet{} + sn := &simplyblockv1alpha1.StorageNode{ + Spec: simplyblockv1alpha1.StorageNodeSpec{WorkerNode: snTestWorker}, + } + if got := effectiveFailureDomain(sn, sns); got != 0 { + t.Errorf("expected 0, got %d", got) + } +} + +// ── TestCheckFailureDomain ──────────────────────────────────────────────────── + +func TestCheckFailureDomain_BlocksWhenEnabledAndNotSet(t *testing.T) { + enabled := true + cluster := &simplyblockv1alpha1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: snTestCluster, Namespace: snTestNS}, + Spec: simplyblockv1alpha1.StorageClusterSpec{EnableFailureDomains: &enabled}, + } + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + r := newSNReconciler(t, cluster, sns, sn) + + err := r.checkFailureDomain(context.Background(), sn, sns) + if err == nil { + t.Fatal("expected error when failureDomain not set and enableFailureDomains=true") + } +} + +func TestCheckFailureDomain_AllowsWhenFailureDomainSet(t *testing.T) { + enabled := true + fd := int32(1) + cluster := &simplyblockv1alpha1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: snTestCluster, Namespace: snTestNS}, + Spec: simplyblockv1alpha1.StorageClusterSpec{EnableFailureDomains: &enabled}, + } + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + sn.Spec.Overrides = &simplyblockv1alpha1.StorageNodeOverrides{FailureDomain: &fd} + r := newSNReconciler(t, cluster, sns, sn) + + if err := r.checkFailureDomain(context.Background(), sn, sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestCheckFailureDomain_SkipsWhenFeatureDisabled(t *testing.T) { + disabled := false + cluster := &simplyblockv1alpha1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: snTestCluster, Namespace: snTestNS}, + Spec: simplyblockv1alpha1.StorageClusterSpec{EnableFailureDomains: &disabled}, + } + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) // no failureDomain + r := newSNReconciler(t, cluster, sns, sn) + + if err := r.checkFailureDomain(context.Background(), sn, sns); err != nil { + t.Fatalf("expected no error when feature disabled, got: %v", err) + } +} + +// ── TestEnsureRemoveOps ─────────────────────────────────────────────────────── + +func TestEnsureRemoveOps_CreatesOpsWhenMissing(t *testing.T) { + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + sn.Status.UUID = "uuid-1" + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + r := newSNReconciler(t, sn, sns) + + if err := r.ensureRemoveOps(context.Background(), sn); err != nil { + t.Fatalf("ensureRemoveOps returned error: %v", err) + } + + var ops simplyblockv1alpha1.StorageNodeOps + if err := r.Get(context.Background(), types.NamespacedName{ + Name: "sn-1-remove", Namespace: snTestNS, + }, &ops); err != nil { + t.Fatalf("expected StorageNodeOps to be created: %v", err) + } + if ops.Spec.Action != "remove" { + t.Errorf("expected action=remove, got %q", ops.Spec.Action) + } + if ops.Spec.StorageNodeRef != "sn-1" { + t.Errorf("expected storageNodeRef=sn-1, got %q", ops.Spec.StorageNodeRef) + } +} + +func TestEnsureRemoveOps_IdempotentWhenAlreadyExists(t *testing.T) { + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + sn.Status.UUID = "uuid-1" + existingOps := &simplyblockv1alpha1.StorageNodeOps{ + ObjectMeta: metav1.ObjectMeta{Name: "sn-1-remove", Namespace: snTestNS}, + Spec: simplyblockv1alpha1.StorageNodeOpsSpec{StorageNodeRef: "sn-1", Action: "remove"}, + } + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + r := newSNReconciler(t, sn, sns, existingOps) + + // Should not return an error on second call. + if err := r.ensureRemoveOps(context.Background(), sn); err != nil { + t.Fatalf("ensureRemoveOps should be idempotent, got: %v", err) + } +} + +// ── TestHandleDeletion ──────────────────────────────────────────────────────── + +func TestHandleDeletion_RemovesFinalizerWhenNeverProvisioned(t *testing.T) { + sn := newStorageNode("sn-1", snTestNS, "sns", snTestWorker) + sn.Finalizers = []string{storageNodeFinalizer} + // status.UUID is empty — node was never provisioned + sns := newStorageNodeSet("sns", snTestNS, snTestCluster, nil) + r := newSNReconciler(t, sn, sns) + + _, err := r.handleDeletion(context.Background(), sn, sns) + if err != nil { + t.Fatalf("handleDeletion returned error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: snTestNS}, &updated) + for _, f := range updated.Finalizers { + if f == storageNodeFinalizer { + t.Error("finalizer should have been removed for unprovisioned node") + } + } +} diff --git a/operator/internal/controller/storagenodeops_controller.go b/operator/internal/controller/storagenodeops_controller.go new file mode 100644 index 00000000..fe5caa20 --- /dev/null +++ b/operator/internal/controller/storagenodeops_controller.go @@ -0,0 +1,931 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "regexp" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" + "github.com/simplyblock/simplyblock-operator/internal/webapi" +) + +// StorageNodeOpsReconciler drives all imperative StorageNode operations. +// It replaces the existing action-handling in StorageNodeSetReconciler and +// owns VolumeMigration CRs during drain (action=remove). +type StorageNodeOpsReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodeops,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodeops/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodeops/finalizers,verbs=update +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodes,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodes/status,verbs=get;update;patch + +func (r *StorageNodeOpsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var ops simplyblockv1alpha1.StorageNodeOps + if err := r.Get(ctx, req.NamespacedName, &ops); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Terminal — nothing left to do. + if ops.Status.Phase == simplyblockv1alpha1.StorageNodeOpsPhaseSucceeded || + ops.Status.Phase == simplyblockv1alpha1.StorageNodeOpsPhaseFailed { + return ctrl.Result{}, nil + } + + // Fetch the target StorageNode. + var sn simplyblockv1alpha1.StorageNode + if err := r.Get(ctx, types.NamespacedName{ + Name: ops.Spec.StorageNodeRef, + Namespace: ops.Namespace, + }, &sn); err != nil { + if apierrors.IsNotFound(err) { + return r.failOps(ctx, &ops, "target StorageNode not found") + } + return ctrl.Result{}, err + } + + // Fetch the parent StorageNodeSet for cluster config. + var sns simplyblockv1alpha1.StorageNodeSet + if err := r.Get(ctx, types.NamespacedName{ + Name: sn.Spec.StorageNodeSetRef, + Namespace: sn.Namespace, + }, &sns); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Resolve cluster UUID. + clusterUUID, err := utils.ResolveClusterUUID(ctx, r.Client, sn.Namespace, sns.Spec.ClusterName) + if err != nil { + log.Info("cluster UUID not ready, requeuing", "cluster", sns.Spec.ClusterName) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + apiClient := webapi.NewClient() + + // Mutual exclusion: only one ops may run per StorageNode at a time. + if ops.Status.Phase == "" || ops.Status.Phase == simplyblockv1alpha1.StorageNodeOpsPhasePending { + return r.acquireLock(ctx, &ops, &sn) + } + + // Cluster pause check for drain operations. + if ops.Spec.Action == "remove" { + if res, paused := r.clusterPauseCheck(ctx, &ops, clusterUUID, apiClient); paused { + return res, nil + } + } + + log.Info("dispatching ops", "action", ops.Spec.Action, "subPhase", ops.Status.SubPhase) + return r.dispatch(ctx, &ops, &sn, &sns, clusterUUID, apiClient) +} + +// acquireLock attempts to set StorageNode.status.activeOpsRef to this ops. +// Requeues if another ops holds the lock. +func (r *StorageNodeOpsReconciler) acquireLock( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + if sn.Status.ActiveOpsRef != "" && sn.Status.ActiveOpsRef != ops.Name { + log.Info("another ops is active, requeuing", "activeOps", sn.Status.ActiveOpsRef) + return ctrl.Result{RequeueAfter: 15 * time.Second}, nil + } + + snPatch := client.MergeFrom(sn.DeepCopy()) + sn.Status.ActiveOpsRef = ops.Name + if err := r.Status().Patch(ctx, sn, snPatch); err != nil { + return ctrl.Result{}, fmt.Errorf("setting activeOpsRef: %w", err) + } + + now := metav1.Now() + opsPatch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning + ops.Status.StartedAt = &now + if ops.Spec.Action == "remove" { + ops.Status.SubPhase = simplyblockv1alpha1.StorageNodeOpsSubPhaseValidating + } + if err := r.Status().Patch(ctx, ops, opsPatch); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil +} + +// dispatch routes the ops to the correct handler. +func (r *StorageNodeOpsReconciler) dispatch( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + sns *simplyblockv1alpha1.StorageNodeSet, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + switch ops.Spec.Action { + case "remove": + return r.runDrain(ctx, ops, sn, clusterUUID, apiClient) + case "shutdown", "restart", "suspend", "resume": + return r.runSimpleAction(ctx, ops, sn, sns, clusterUUID, apiClient) + default: + return r.failOps(ctx, ops, fmt.Sprintf("unknown action %q", ops.Spec.Action)) + } +} + +// runSimpleAction handles shutdown / restart / suspend / resume by posting to +// the backend and polling until the node reaches its terminal status. +func (r *StorageNodeOpsReconciler) runSimpleAction( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + _ *simplyblockv1alpha1.StorageNodeSet, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + action := ops.Spec.Action + + // POST the action if not yet triggered. + if !ops.Status.Triggered { + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s/%s", + clusterUUID, nodeUUID, action) + body := map[string]interface{}{} + if ops.Spec.Force != nil && *ops.Spec.Force { + body["force"] = true + } + if action == "restart" && ops.Spec.ReattachVolume != nil { + body["reattach_volume"] = *ops.Spec.ReattachVolume + } + _, status, err := apiClient.Do(ctx, http.MethodPost, endpoint, body) + if err != nil || status >= 300 { + if err == nil { + err = fmt.Errorf("status %d", status) + } + log.Error(err, "action POST failed", "action", action, "nodeUUID", nodeUUID) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Triggered = true + ops.Status.Message = fmt.Sprintf("%s request sent, waiting for node", action) + if err := r.Status().Patch(ctx, ops, patch); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + + // Poll node status until terminal. + terminalStatus := map[string]string{ + "suspend": utils.NodeStatusSuspended, + "resume": "online", + "restart": "online", + "shutdown": "offline", + } + want := terminalStatus[action] + + currentStatus, err := getNodeBackendStatus(ctx, apiClient, clusterUUID, nodeUUID) + if err != nil { + log.Error(err, "failed to get node status during action poll") + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + if currentStatus == want { + return r.succeedOps(ctx, ops, sn) + } + log.Info("waiting for node to reach terminal status", + "want", want, "current", currentStatus, "action", action) + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil +} + +// ───────────────────────────────────────────────────────────────────────────── +// Drain state machine (action=remove) +// Phases: Validating → Suspending → Migrating → Verifying → Removing +// ───────────────────────────────────────────────────────────────────────────── + +func (r *StorageNodeOpsReconciler) runDrain( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + switch ops.Status.SubPhase { + case simplyblockv1alpha1.StorageNodeOpsSubPhaseValidating: + return r.drainValidate(ctx, ops, sn, clusterUUID, apiClient) + case simplyblockv1alpha1.StorageNodeOpsSubPhaseSuspending: + return r.drainSuspend(ctx, ops, sn, clusterUUID, apiClient) + case simplyblockv1alpha1.StorageNodeOpsSubPhaseMigrating: + return r.drainMigrate(ctx, ops, sn, clusterUUID, apiClient) + case simplyblockv1alpha1.StorageNodeOpsSubPhaseVerifying: + return r.drainVerify(ctx, ops, sn, clusterUUID, apiClient) + case simplyblockv1alpha1.StorageNodeOpsSubPhaseRemoving: + return r.drainRemove(ctx, ops, sn, clusterUUID, apiClient) + default: + return r.failOps(ctx, ops, fmt.Sprintf("unknown drain sub-phase %q", ops.Status.SubPhase)) + } +} + +func (r *StorageNodeOpsReconciler) drainValidate( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + volumes, err := listNodeVolumes(ctx, apiClient, clusterUUID, nodeUUID) + if err != nil { + log.Error(err, "drain: failed to list volumes during validation") + return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil + } + + sysFilter, err := r.resolveOpsSystemVolumeFilter(ops) + if err != nil { + return r.failOps(ctx, ops, "invalid systemVolumeFilterRegex: "+err.Error()) + } + + _, pinned, unmanaged, _, _, err := matchVolumesToPVs(ctx, r.Client, volumes, sysFilter) + if err != nil { + log.Error(err, "drain: matchVolumesToPVs failed during validation") + return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil + } + + if len(pinned) > 0 { + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "PinnedVolumeBlocking", + "drain blocked: %d pinned volume(s) on node %s — remove the %s annotation to proceed", + len(pinned), nodeUUID, simplyblockv1alpha1.AnnotationPinnedVolume) + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Message = fmt.Sprintf("blocked: %d pinned volume(s) — remove simplyblock.io/pinned-volume annotation", len(pinned)) + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + } + + if len(unmanaged) > 0 { + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "UnmanagedVolumeBlocking", + "drain blocked: %d unmanaged volume(s) on node %s — remove them manually", + len(unmanaged), nodeUUID) + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Message = fmt.Sprintf("blocked: %d unmanaged volume(s) — remove manually", len(unmanaged)) + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: 60 * time.Second}, nil + } + + return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseSuspending) +} + +func (r *StorageNodeOpsReconciler) drainSuspend( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + if !ops.Status.Triggered { + currentStatus, err := getNodeBackendStatus(ctx, apiClient, clusterUUID, nodeUUID) + if err != nil { + log.Error(err, "drain: could not read node status before suspend, retrying") + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + if currentStatus == utils.NodeStatusSuspended { + log.Info("drain: node already suspended, advancing without POST") + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Triggered = true + ops.Status.Message = "node already suspended" + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil + } + + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s/suspend", clusterUUID, nodeUUID) + _, status, err := apiClient.Do(ctx, http.MethodPost, endpoint, nil) + if err != nil || status >= 300 { + if err == nil { + err = fmt.Errorf("suspend API returned status %d", status) + } + log.Error(err, "drain: suspend POST failed") + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Triggered = true + ops.Status.Message = "suspend request sent, waiting for node to suspend" + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + + // Poll node status. + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s", clusterUUID, nodeUUID) + body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) + if err != nil || status >= 300 { + if err == nil { + err = fmt.Errorf("status %d", status) + } + log.Error(err, "drain: failed to GET node status during suspend poll") + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + var nodeResp utils.NodeStatusResponse + if err := json.Unmarshal(body, &nodeResp); err != nil { + log.Error(err, "drain: failed to unmarshal node status") + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + if nodeResp.Status != utils.NodeStatusSuspended { + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainSuspendPending", + "waiting for node %s to suspend (current status: %s)", nodeUUID, nodeResp.Status) + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseMigrating) +} + +func (r *StorageNodeOpsReconciler) drainMigrate( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + var vmigList simplyblockv1alpha1.VolumeMigrationList + if err := r.List(ctx, &vmigList, + client.InNamespace(ops.Namespace), + client.MatchingLabels{"storage.simplyblock.io/drain-node": nodeUUID}, + ); err != nil { + log.Error(err, "drain: failed to list VolumeMigration CRs") + return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil + } + + // Handle failed migrations. + if res, handled := r.handleFailedVolumeMigrations(ctx, ops, sn, clusterUUID, apiClient, vmigList.Items); handled { + return res, nil + } + + completed, inProgress := 0, 0 + for i := range vmigList.Items { + if vmigList.Items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseCompleted { + completed++ + } else { + inProgress++ + } + } + + existingVMNames := make(map[string]struct{}, len(vmigList.Items)) + for i := range vmigList.Items { + existingVMNames[vmigList.Items[i].Name] = struct{}{} + } + + if len(vmigList.Items) == 0 || r.hasMissingVolumeMigrationsOps(ctx, apiClient, clusterUUID, nodeUUID, ops, existingVMNames) { + return r.createMissingVolumeMigrationsOps(ctx, apiClient, clusterUUID, ops, sn, vmigList.Items, existingVMNames) + } + + if inProgress == 0 && completed == len(vmigList.Items) { + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.VolumesMigrated = completed + ops.Status.VolumesPending = 0 + _ = r.Status().Patch(ctx, ops, patch) + + for i := range vmigList.Items { + vm := &vmigList.Items[i] + if err := r.Delete(ctx, vm); err != nil { + log.Error(err, "drain: failed to delete completed VolumeMigration", "name", vm.Name) + } + } + r.Recorder.Eventf(ops, corev1.EventTypeNormal, "MigrationCompleted", + "all %d volume migrations completed", completed) + return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseVerifying) + } + + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.VolumesMigrated = completed + ops.Status.VolumesPending = inProgress + ops.Status.Message = fmt.Sprintf("Migrating: %d of %d volumes migrated", completed, len(vmigList.Items)) + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil +} + +func (r *StorageNodeOpsReconciler) handleFailedVolumeMigrations( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, + items []simplyblockv1alpha1.VolumeMigration, +) (ctrl.Result, bool) { + log := logf.FromContext(ctx) + var failed []simplyblockv1alpha1.VolumeMigration + for i := range items { + if items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed || + items[i].Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseAborted { + failed = append(failed, items[i]) + } + } + if len(failed) == 0 { + return ctrl.Result{}, false + } + + // Check if the cluster is paused — if so, delete and wait. + if res, paused := r.clusterPauseCheck(ctx, ops, clusterUUID, apiClient); paused { + for i := range failed { + _ = r.Delete(ctx, &failed[i]) + } + log.Info("drain: cluster not ready, deleted failed VMs and pausing", "count", len(failed)) + return res, true + } + + // Cluster ready: delete failed CRs and let createMissingVolumeMigrationsOps recreate them. + for i := range failed { + vm := &failed[i] + if err := r.Delete(ctx, vm); err != nil { + log.Error(err, "drain: failed to delete failed VolumeMigration", "name", vm.Name) + continue + } + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "MigrationRetry", + "VolumeMigration %s failed, deleted and will retry with new target", vm.Name) + } + return ctrl.Result{RequeueAfter: drainRequeueImmediate}, true +} + +func (r *StorageNodeOpsReconciler) hasMissingVolumeMigrationsOps( + ctx context.Context, + apiClient *webapi.Client, + clusterUUID, nodeUUID string, + ops *simplyblockv1alpha1.StorageNodeOps, + existingVMNames map[string]struct{}, +) bool { + vols, err := listNodeVolumes(ctx, apiClient, clusterUUID, nodeUUID) + if err != nil { + return false + } + sf, err := r.resolveOpsSystemVolumeFilter(ops) + if err != nil { + return false + } + pvm, _, _, pvByVol, _, err := matchVolumesToPVs(ctx, r.Client, vols, sf) + if err != nil { + return false + } + for _, volUUID := range pvm { + if pvName, ok := pvByVol[volUUID]; ok { + if _, exists := existingVMNames[drainMigrationName(nodeUUID, pvName)]; !exists { + return true + } + } + } + return false +} + +func (r *StorageNodeOpsReconciler) createMissingVolumeMigrationsOps( + ctx context.Context, + apiClient *webapi.Client, + clusterUUID string, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + existingItems []simplyblockv1alpha1.VolumeMigration, + existingVMNames map[string]struct{}, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + volumes, err := listNodeVolumes(ctx, apiClient, clusterUUID, nodeUUID) + if err != nil { + log.Error(err, "drain: failed to list volumes for migration creation") + return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil + } + + sysFilter, err := r.resolveOpsSystemVolumeFilter(ops) + if err != nil { + return r.failOps(ctx, ops, "invalid systemVolumeFilterRegex: "+err.Error()) + } + + pvManaged, _, _, pvNameByVolumeUUID, pvcFetchFailed, err := matchVolumesToPVs(ctx, r.Client, volumes, sysFilter) + if err != nil { + log.Error(err, "drain: matchVolumesToPVs failed") + return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil + } + if pvcFetchFailed { + log.Info("drain: PVC fetch failed — retrying to avoid skipping volumes") + return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil + } + + if len(pvManaged) == 0 && len(existingItems) == 0 { + return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseVerifying) + } + + pvNames := make([]string, 0, len(pvManaged)) + for _, volUUID := range pvManaged { + pvName, ok := pvNameByVolumeUUID[volUUID] + if !ok { + continue + } + if _, exists := existingVMNames[drainMigrationName(nodeUUID, pvName)]; !exists { + pvNames = append(pvNames, pvName) + } + } + if len(pvNames) == 0 { + return ctrl.Result{RequeueAfter: drainRequeueMigrate}, nil + } + + targetByPV, err := roundRobinTargetNodes(ctx, apiClient, clusterUUID, nodeUUID, pvNames) + if err != nil { + log.Error(err, "drain: no available target nodes for migration") + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainNoMigrationTarget", + "drain stalled: no online storage node available as migration target for node %s", nodeUUID) + return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil + } + + createdCount := 0 + for _, volUUID := range pvManaged { + pvName, ok := pvNameByVolumeUUID[volUUID] + if !ok { + continue + } + migName := drainMigrationName(nodeUUID, pvName) + if _, exists := existingVMNames[migName]; exists { + continue + } + vmig := &simplyblockv1alpha1.VolumeMigration{ + ObjectMeta: metav1.ObjectMeta{ + Name: migName, + Namespace: ops.Namespace, + Labels: map[string]string{"storage.simplyblock.io/drain-node": nodeUUID}, + }, + Spec: simplyblockv1alpha1.VolumeMigrationSpec{ + PVName: pvName, + TargetNodeUUID: targetByPV[pvName], + }, + } + if err := controllerutil.SetControllerReference(ops, vmig, r.Scheme); err != nil { + log.Error(err, "drain: failed to set controller reference", "name", migName) + continue + } + if err := r.Create(ctx, vmig); err != nil { + log.Error(err, "drain: failed to create VolumeMigration", "name", migName) + continue + } + createdCount++ + } + + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.VolumesPending = createdCount + ops.Status.VolumesMigrated = 0 + ops.Status.Message = fmt.Sprintf("Migrating: 0 of %d volumes migrated", createdCount) + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil +} + +func (r *StorageNodeOpsReconciler) drainVerify( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + pools, volumes, err := fetchPoolVolumes(ctx, apiClient, clusterUUID, nodeUUID) + if err != nil { + log.Error(err, "drain: failed to list volumes during verification") + return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil + } + + sysFilter, err := r.resolveOpsSystemVolumeFilter(ops) + if err != nil { + return r.failOps(ctx, ops, "invalid systemVolumeFilterRegex: "+err.Error()) + } + + var nonSystem, systemVols []string + for _, vol := range volumes { + if sysFilter.MatchString(vol.Name) { + systemVols = append(systemVols, vol.UUID) + } else { + nonSystem = append(nonSystem, vol.UUID) + } + } + + if len(nonSystem) > 0 { + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainVerifyPending", + "node %s still has %d non-system volume(s) after migration; waiting for backend to confirm empty", + nodeUUID, len(nonSystem)) + return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil + } + + if len(systemVols) > 0 { + poolByVol := make(map[string]string) + for _, pool := range pools { + vols, err := apiClient.GetPoolVolumes(ctx, clusterUUID, pool.UUID) + if err != nil { + continue + } + for _, v := range vols { + poolByVol[v.UUID] = pool.UUID + } + } + for _, volUUID := range systemVols { + poolUUID, ok := poolByVol[volUUID] + if !ok { + continue + } + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-pools/%s/volumes/%s/", + clusterUUID, poolUUID, volUUID) + _, delStatus, delErr := apiClient.Do(ctx, http.MethodDelete, endpoint, nil) + delClass := webapi.ClassifyError(delErr, delStatus) + switch { + case delErr == nil && (delStatus == http.StatusOK || delStatus == http.StatusNoContent || delStatus == http.StatusNotFound): + log.Info("drain: deleted system volume", "volUUID", volUUID) + case delClass.Retryable: + log.Error(delErr, "drain: transient error deleting system volume, retrying", "volUUID", volUUID) + default: + return r.resumeAndFail(ctx, ops, sn, apiClient, clusterUUID, + fmt.Sprintf("system volume %s delete rejected by backend (status %d)", volUUID, delStatus)) + } + } + return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil + } + + return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseRemoving) +} + +func (r *StorageNodeOpsReconciler) drainRemove( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s?force_remove=false", + clusterUUID, nodeUUID) + _, status, err := apiClient.Do(ctx, http.MethodDelete, endpoint, nil) + + if err == nil && (status == http.StatusOK || status == http.StatusNoContent || status == http.StatusNotFound) { + r.Recorder.Eventf(ops, corev1.EventTypeNormal, "NodeRemoved", + "storage node %s removed successfully", nodeUUID) + return r.succeedOps(ctx, ops, sn) + } + + class := webapi.ClassifyError(err, status) + if class.Retryable { + log.Error(err, "drain: transient error on node DELETE, retrying", "status", status) + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + return r.resumeAndFail(ctx, ops, sn, apiClient, clusterUUID, + fmt.Sprintf("DELETE node returned status %d", status)) +} + +func (r *StorageNodeOpsReconciler) resumeAndFail( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, + apiClient *webapi.Client, + clusterUUID, reason string, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + nodeUUID := sn.Status.UUID + + resumeEndpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s/resume", clusterUUID, nodeUUID) + _, resumeStatus, resumeErr := apiClient.Do(ctx, http.MethodPost, resumeEndpoint, nil) + resumeClass := webapi.ClassifyError(resumeErr, resumeStatus) + if resumeClass.Retryable { + log.Error(resumeErr, "drain: transient error resuming node, will retry", "status", resumeStatus) + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Message = fmt.Sprintf("resume pending after failure: %s", reason) + _ = r.Status().Patch(ctx, ops, patch) + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil + } + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "NodeResumed", + "drain failed, attempted resume of node %s: %s", nodeUUID, reason) + return r.failOps(ctx, ops, reason) +} + +// clusterPauseCheck returns (requeue, true) if the cluster is not ready for drain operations. +func (r *StorageNodeOpsReconciler) clusterPauseCheck( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + clusterUUID string, + _ *webapi.Client, +) (ctrl.Result, bool) { + log := logf.FromContext(ctx) + + // Resolve the StorageNode to get the namespace and cluster name. + var sn simplyblockv1alpha1.StorageNode + if err := r.Get(ctx, types.NamespacedName{Name: ops.Spec.StorageNodeRef, Namespace: ops.Namespace}, &sn); err != nil { + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, false + } + var sns simplyblockv1alpha1.StorageNodeSet + if err := r.Get(ctx, types.NamespacedName{Name: sn.Spec.StorageNodeSetRef, Namespace: sn.Namespace}, &sns); err != nil { + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, false + } + + clusterCR, err := utils.ResolveClusterCR(ctx, r.Client, ops.Namespace, sns.Spec.ClusterName) + if err != nil { + log.Error(err, "drain: could not resolve cluster CR") + return ctrl.Result{RequeueAfter: drainRequeueSuspend}, false + } + + var reason string + if clusterCR.Status.Status != "" && clusterCR.Status.Status != utils.ClusterStatusActive { + reason = fmt.Sprintf("cluster status is %q (not active)", clusterCR.Status.Status) + } else if clusterCR.Status.Rebalancing != nil && *clusterCR.Status.Rebalancing { + reason = "cluster is rebalancing" + } + + if reason == "" { + return ctrl.Result{}, false + } + + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Message = "drain paused: " + reason + _ = r.Status().Patch(ctx, ops, patch) + r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainPaused", + "drain paused: %s — will resume when cluster is active", reason) + log.Info("drain: pausing — cluster not ready", "reason", reason) + return ctrl.Result{RequeueAfter: 60 * time.Second}, true +} + +// advanceSubPhase patches ops.status.subPhase and requeues immediately. +func (r *StorageNodeOpsReconciler) advanceSubPhase( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + next simplyblockv1alpha1.StorageNodeOpsSubPhase, +) (ctrl.Result, error) { + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.SubPhase = next + ops.Status.Triggered = false + ops.Status.Message = fmt.Sprintf("entering phase %s", next) + if err := r.Status().Patch(ctx, ops, patch); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: drainRequeueImmediate}, nil +} + +// succeedOps marks the ops as Succeeded and releases the lock on the StorageNode. +func (r *StorageNodeOpsReconciler) succeedOps( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + sn *simplyblockv1alpha1.StorageNode, +) (ctrl.Result, error) { + now := metav1.Now() + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseSucceeded + ops.Status.SubPhase = "" + ops.Status.CompletedAt = &now + if err := r.Status().Patch(ctx, ops, patch); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.releaseLock(ctx, sn, ops.Name) +} + +// failOps marks the ops as Failed with the given reason and releases the lock. +func (r *StorageNodeOpsReconciler) failOps( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + reason string, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + log.Error(nil, "ops failed", "ops", ops.Name, "reason", reason) + r.Recorder.Event(ops, "Warning", "OpsFailed", reason) + + now := metav1.Now() + patch := client.MergeFrom(ops.DeepCopy()) + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseFailed + ops.Status.SubPhase = "" + ops.Status.Message = reason + ops.Status.CompletedAt = &now + if err := r.Status().Patch(ctx, ops, patch); err != nil { + return ctrl.Result{}, err + } + + var sn simplyblockv1alpha1.StorageNode + if err := r.Get(ctx, types.NamespacedName{ + Name: ops.Spec.StorageNodeRef, + Namespace: ops.Namespace, + }, &sn); err == nil { + _ = r.releaseLock(ctx, &sn, ops.Name) + } + return ctrl.Result{}, nil +} + +// releaseLock clears StorageNode.status.activeOpsRef if it still points to opsName. +func (r *StorageNodeOpsReconciler) releaseLock( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + opsName string, +) error { + if sn.Status.ActiveOpsRef != opsName { + return nil + } + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.ActiveOpsRef = "" + return r.Status().Patch(ctx, sn, patch) +} + +// resolveOpsSystemVolumeFilter compiles the system volume filter regex from the ops, +// falling back to the default pattern. +func (r *StorageNodeOpsReconciler) resolveOpsSystemVolumeFilter( + ops *simplyblockv1alpha1.StorageNodeOps, +) (*regexp.Regexp, error) { + pattern := simplyblockv1alpha1.DefaultSystemVolumeFilterRegex + if ops.Spec.Drain != nil && ops.Spec.Drain.SystemVolumeFilterRegex != nil { + pattern = *ops.Spec.Drain.SystemVolumeFilterRegex + } + return regexp.Compile(pattern) +} + +// storageNodeToOpsRequests maps a StorageNode change to any pending +// StorageNodeOps that targets it, so ops waiting on lock acquisition requeue +// immediately when activeOpsRef is cleared rather than waiting for the poll timer. +func (r *StorageNodeOpsReconciler) storageNodeToOpsRequests( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + var opsList simplyblockv1alpha1.StorageNodeOpsList + if err := r.List(ctx, &opsList, + client.InNamespace(obj.GetNamespace()), + client.MatchingFields{"spec.storageNodeRef": obj.GetName()}, + ); err != nil { + return nil + } + reqs := make([]reconcile.Request, 0, len(opsList.Items)) + for _, ops := range opsList.Items { + if ops.Status.Phase == simplyblockv1alpha1.StorageNodeOpsPhasePending || + ops.Status.Phase == "" { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: ops.Name, + Namespace: ops.Namespace, + }}) + } + } + return reqs +} + +// SetupWithManager registers the StorageNodeOpsReconciler with the controller manager. +func (r *StorageNodeOpsReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Index StorageNodeOps by their target StorageNode for efficient watch lookups. + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &simplyblockv1alpha1.StorageNodeOps{}, + "spec.storageNodeRef", + func(obj client.Object) []string { + ops := obj.(*simplyblockv1alpha1.StorageNodeOps) + return []string{ops.Spec.StorageNodeRef} + }, + ); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&simplyblockv1alpha1.StorageNodeOps{}). + Named("storagenodeops"). + Watches( + &simplyblockv1alpha1.StorageNode{}, + handler.EnqueueRequestsFromMapFunc(r.storageNodeToOpsRequests), + ). + Owns(&simplyblockv1alpha1.VolumeMigration{}). + Complete(r) +} diff --git a/operator/internal/controller/storagenodeops_controller_unit_test.go b/operator/internal/controller/storagenodeops_controller_unit_test.go new file mode 100644 index 00000000..6b90eab3 --- /dev/null +++ b/operator/internal/controller/storagenodeops_controller_unit_test.go @@ -0,0 +1,310 @@ +package controller + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" +) + +// ── helpers ─────────────────────────────────────────────────────────────────── + +const ( + opsTestNS = "test" + opsTestCluster = "cluster-a" + opsTestWorker = "worker-1.example.com" + opsTestNodeUUID = "aaaa0000-0000-0000-0000-000000000001" +) + +func newOpsReconciler(t *testing.T, objects ...client.Object) *StorageNodeOpsReconciler { + t.Helper() + scheme := newTestScheme(t, + simplyblockv1alpha1.AddToScheme, + corev1.AddToScheme, + ) + cl := newTestClient(t, scheme, + []client.Object{ + &simplyblockv1alpha1.StorageNode{}, + &simplyblockv1alpha1.StorageNodeOps{}, + &simplyblockv1alpha1.StorageNodeSet{}, + &simplyblockv1alpha1.StorageCluster{}, + &simplyblockv1alpha1.VolumeMigration{}, + }, + objects..., + ) + return &StorageNodeOpsReconciler{ + Client: cl, + Scheme: scheme, + Recorder: record.NewFakeRecorder(16), + } +} + +func newTestStorageNode(name, ns, snsRef, worker, uuid string) *simplyblockv1alpha1.StorageNode { + sn := &simplyblockv1alpha1.StorageNode{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: simplyblockv1alpha1.StorageNodeSpec{ + StorageNodeSetRef: snsRef, + WorkerNode: worker, + }, + } + sn.Status.UUID = uuid + return sn +} + +func newTestStorageNodeOps(name, ns, snRef, action string) *simplyblockv1alpha1.StorageNodeOps { + return &simplyblockv1alpha1.StorageNodeOps{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + Spec: simplyblockv1alpha1.StorageNodeOpsSpec{ + StorageNodeRef: snRef, + Action: action, + }, + } +} + +// ── TestAcquireLock ─────────────────────────────────────────────────────────── + +func TestAcquireLock_SetsActiveOpsRefAndTransitionsToRunning(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + r := newOpsReconciler(t, sn, ops) + + _, err := r.acquireLock(context.Background(), ops, sn) + if err != nil { + t.Fatalf("acquireLock returned error: %v", err) + } + + // Check StorageNode.status.activeOpsRef was set. + var updatedSN simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updatedSN) + if updatedSN.Status.ActiveOpsRef != "ops-1" { + t.Errorf("activeOpsRef: got %q want ops-1", updatedSN.Status.ActiveOpsRef) + } + + // Check ops phase was set to Running. + var updatedOps simplyblockv1alpha1.StorageNodeOps + _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updatedOps) + if updatedOps.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseRunning { + t.Errorf("phase: got %q want Running", updatedOps.Status.Phase) + } +} + +func TestAcquireLock_RequeuesWhenAnotherOpsActive(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + sn.Status.ActiveOpsRef = "ops-other" + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + r := newOpsReconciler(t, sn, ops) + + result, err := r.acquireLock(context.Background(), ops, sn) + if err != nil { + t.Fatalf("acquireLock returned error: %v", err) + } + if result.RequeueAfter == 0 { + t.Error("expected requeue when another ops is active") + } + + // StorageNode.activeOpsRef must NOT be changed. + var updatedSN simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updatedSN) + if updatedSN.Status.ActiveOpsRef != "ops-other" { + t.Errorf("activeOpsRef should not change: got %q", updatedSN.Status.ActiveOpsRef) + } +} + +func TestAcquireLock_RemoveDrainSetsValidatingSubPhase(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + ops := newTestStorageNodeOps("ops-drain", opsTestNS, "sn-1", "remove") + r := newOpsReconciler(t, sn, ops) + + _, err := r.acquireLock(context.Background(), ops, sn) + if err != nil { + t.Fatalf("acquireLock returned error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNodeOps + _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-drain", Namespace: opsTestNS}, &updated) + if updated.Status.SubPhase != simplyblockv1alpha1.StorageNodeOpsSubPhaseValidating { + t.Errorf("subPhase: got %q want Validating", updated.Status.SubPhase) + } +} + +// ── TestSucceedOps ──────────────────────────────────────────────────────────── + +func TestSucceedOps_SetsPhaseAndClearsLock(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + sn.Status.ActiveOpsRef = "ops-1" + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning + r := newOpsReconciler(t, sn, ops) + + _, err := r.succeedOps(context.Background(), ops, sn) + if err != nil { + t.Fatalf("succeedOps returned error: %v", err) + } + + var updatedOps simplyblockv1alpha1.StorageNodeOps + _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updatedOps) + if updatedOps.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseSucceeded { + t.Errorf("phase: got %q want Succeeded", updatedOps.Status.Phase) + } + if updatedOps.Status.CompletedAt == nil { + t.Error("expected CompletedAt to be set") + } + + var updatedSN simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updatedSN) + if updatedSN.Status.ActiveOpsRef != "" { + t.Errorf("activeOpsRef should be cleared, got %q", updatedSN.Status.ActiveOpsRef) + } +} + +// ── TestFailOps ─────────────────────────────────────────────────────────────── + +func TestFailOps_SetsPhaseAndClearsLock(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + sn.Status.ActiveOpsRef = "ops-1" + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning + r := newOpsReconciler(t, sn, ops) + + _, err := r.failOps(context.Background(), ops, "something went wrong") + if err != nil { + t.Fatalf("failOps returned error: %v", err) + } + + var updatedOps simplyblockv1alpha1.StorageNodeOps + _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updatedOps) + if updatedOps.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseFailed { + t.Errorf("phase: got %q want Failed", updatedOps.Status.Phase) + } + if updatedOps.Status.Message != "something went wrong" { + t.Errorf("message: got %q", updatedOps.Status.Message) + } + + var updatedSN simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updatedSN) + if updatedSN.Status.ActiveOpsRef != "" { + t.Errorf("activeOpsRef should be cleared after failure, got %q", updatedSN.Status.ActiveOpsRef) + } +} + +// ── TestReleaseLock ─────────────────────────────────────────────────────────── + +func TestReleaseLock_OnlyClearsIfOwner(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + sn.Status.ActiveOpsRef = "ops-other" + r := newOpsReconciler(t, sn) + + // Releasing with a different name should be a no-op. + if err := r.releaseLock(context.Background(), sn, "ops-1"); err != nil { + t.Fatalf("releaseLock returned error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updated) + if updated.Status.ActiveOpsRef != "ops-other" { + t.Error("releaseLock should not clear a lock it does not own") + } +} + +// ── TestAdvanceSubPhase ─────────────────────────────────────────────────────── + +func TestAdvanceSubPhase_UpdatesSubPhaseAndResetsTrigger(t *testing.T) { + ops := newTestStorageNodeOps("ops-drain", opsTestNS, "sn-1", "remove") + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning + ops.Status.SubPhase = simplyblockv1alpha1.StorageNodeOpsSubPhaseValidating + ops.Status.Triggered = true + r := newOpsReconciler(t, ops) + + _, err := r.advanceSubPhase(context.Background(), ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseSuspending) + if err != nil { + t.Fatalf("advanceSubPhase returned error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNodeOps + _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-drain", Namespace: opsTestNS}, &updated) + if updated.Status.SubPhase != simplyblockv1alpha1.StorageNodeOpsSubPhaseSuspending { + t.Errorf("subPhase: got %q want Suspending", updated.Status.SubPhase) + } + if updated.Status.Triggered { + t.Error("Triggered should be reset to false on phase advance") + } +} + +// ── TestDispatch ────────────────────────────────────────────────────────────── + +func TestDispatch_UnknownActionFails(t *testing.T) { + sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: opsTestNS}, + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: opsTestCluster}, + } + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "bogus-action") + ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning + r := newOpsReconciler(t, sn, sns, ops) + + _, err := r.dispatch(context.Background(), ops, sn, sns, "cluster-uuid", nil) + if err != nil { + t.Fatalf("dispatch returned unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNodeOps + _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updated) + if updated.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseFailed { + t.Errorf("expected Failed for unknown action, got %q", updated.Status.Phase) + } +} + +// ── TestResolveOpsSystemVolumeFilter ───────────────────────────────────────── + +func TestResolveOpsSystemVolumeFilter_UsesDefaultWhenNoDrain(t *testing.T) { + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "remove") + r := newOpsReconciler(t, ops) + + re, err := r.resolveOpsSystemVolumeFilter(ops) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // Default pattern matches sb-fio-baseline-* names. + if !re.MatchString("sb-fio-baseline-read") { + t.Error("default filter should match sb-fio-baseline-read") + } + if re.MatchString("user-volume") { + t.Error("default filter should not match user volumes") + } +} + +func TestResolveOpsSystemVolumeFilter_UsesCustomPattern(t *testing.T) { + custom := "^bench-.*" + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "remove") + ops.Spec.Drain = &simplyblockv1alpha1.DrainOpsSpec{SystemVolumeFilterRegex: &custom} + r := newOpsReconciler(t, ops) + + re, err := r.resolveOpsSystemVolumeFilter(ops) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !re.MatchString("bench-read") { + t.Error("custom filter should match bench-read") + } + if re.MatchString("sb-fio-baseline-read") { + t.Error("custom filter should not match sb-fio-baseline-read") + } +} + +func TestResolveOpsSystemVolumeFilter_InvalidPatternReturnsError(t *testing.T) { + bad := "[" + ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "remove") + ops.Spec.Drain = &simplyblockv1alpha1.DrainOpsSpec{SystemVolumeFilterRegex: &bad} + r := newOpsReconciler(t, ops) + + _, err := r.resolveOpsSystemVolumeFilter(ops) + if err == nil { + t.Fatal("expected error for invalid regex pattern") + } +} From 4e55fe56141f31da05acf21863595c106190723f Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 11:50:06 +0100 Subject: [PATCH 03/52] test: fix drain unit tests after StorageNodeSet action field removal --- atlas-lib/nvme/sysfs_test.go | 18 +- operator/dist/install.yaml | 207 +++-- .../simplyblockstoragenodeset_controller.go | 1 - .../simplyblockstoragenodeset_drain.go | 1 - ...mplyblockstoragenodeset_drain_unit_test.go | 843 ------------------ .../storagenodeops_controller_unit_test.go | 6 +- 6 files changed, 145 insertions(+), 931 deletions(-) diff --git a/atlas-lib/nvme/sysfs_test.go b/atlas-lib/nvme/sysfs_test.go index 4938488c..c54b45a1 100644 --- a/atlas-lib/nvme/sysfs_test.go +++ b/atlas-lib/nvme/sysfs_test.go @@ -54,15 +54,15 @@ func vm17Fixture(t *testing.T) string { ns + "/dev": "259:1", ns + "/queue/logical_block_size": "4096", // controller nvme0 (path A -> vm19) - "class/nvme/nvme0/subsysnqn": nqn, - "class/nvme/nvme0/transport": "tcp", - "class/nvme/nvme0/state": "live", - "class/nvme/nvme0/cntrltype": "io", - "class/nvme/nvme0/cntlid": "1", - "class/nvme/nvme0/address": "traddr=192.168.10.69,trsvcid=4426,src_addr=192.168.10.67", - "class/nvme/nvme0/numa_node": "-1", - "class/nvme/nvme0/queue_count": "15", - "class/nvme/nvme0/dev": "238:0", + "class/nvme/nvme0/subsysnqn": nqn, + "class/nvme/nvme0/transport": "tcp", + "class/nvme/nvme0/state": "live", + "class/nvme/nvme0/cntrltype": "io", + "class/nvme/nvme0/cntlid": "1", + "class/nvme/nvme0/address": "traddr=192.168.10.69,trsvcid=4426,src_addr=192.168.10.67", + "class/nvme/nvme0/numa_node": "-1", + "class/nvme/nvme0/queue_count": "15", + "class/nvme/nvme0/dev": "238:0", // controller nvme1 (path B -> vm17) "class/nvme/nvme1/subsysnqn": nqn, "class/nvme/nvme1/transport": "tcp", diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 7a25c9b4..7b2a50c6 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -1795,15 +1795,6 @@ spec: spec: description: spec defines the desired state of StorageNodeSet properties: - action: - description: Action triggers an imperative node operation. - enum: - - shutdown - - restart - - suspend - - resume - - remove - type: string clusterImage: description: |- ClusterImage is the container image used for storage-node workloads. @@ -1899,9 +1890,6 @@ spec: enableCpuTopology: description: EnableCpuTopology enables topology-aware CPU handling. type: boolean - force: - description: Force enables forced action execution where supported. - type: boolean forceFormat4K: description: ForceFormat4K forces 4K blocksize formatting of the NVMe device where supported. @@ -2015,6 +2003,122 @@ spec: x-kubernetes-validations: - message: field is immutable rule: self == oldSelf + nodeConfigs: + additionalProperties: + description: |- + StorageNodeOverrides holds per-node configuration that overrides the parent + StorageNodeSet fleet defaults for a specific worker node. Populated by the + StorageNodeSetReconciler from StorageNodeSet.spec.nodeConfigs[workerNode] on + every reconcile. The StorageNodeSet is the single source of truth — users + should not edit this struct directly on the StorageNode. + + Fields here mirror the configurable (non-immutable, non-infrastructure) fields + of StorageNodeSetSpec. When a field is set here it takes precedence over the + fleet default; when omitted the fleet default applies. + properties: + corePercentage: + description: CorePercentage overrides the percentage of cores + allocated to SPDK for this node (0-99). + format: int32 + type: integer + deviceNames: + description: |- + DeviceNames explicitly defines the NVMe namespace names to use on this node + (e.g. ["nvme0n1","nvme1n1"]). + items: + type: string + type: array + driveSizeRange: + description: DriveSizeRange overrides the drive size range filter + for this node. + type: string + enableCpuTopology: + description: EnableCpuTopology overrides topology-aware CPU + handling for this node. + type: boolean + failureDomain: + description: |- + FailureDomain is the failure-domain group index (≥ 1) for this node. + Required when the parent StorageCluster has enableFailureDomains=true. + Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + format: int32 + minimum: 1 + type: integer + journalManager: + description: JournalManagerSpec overrides journal manager tuning + for this node. + properties: + count: + description: Count is the number of journal managers to + configure. + format: int32 + type: integer + percentPerDevice: + description: PercentPerDevice is the journal manager capacity + percentage per device. + format: int32 + type: integer + type: object + maxLogicalVolumeCount: + description: MaxLogicalVolumeCount overrides the maximum number + of logical volumes for this node. + format: int32 + type: integer + maxSize: + description: MaxSize overrides the maximum allocatable size + of huge pages for this node. + type: string + pcieAllowList: + description: PcieAllowList overrides the list of PCI addresses + allowed for use on this node. + items: + type: string + type: array + pcieDenyList: + description: PcieDenyList overrides the list of PCI addresses + excluded from use on this node. + items: + type: string + type: array + pcieModel: + description: PcieModel overrides the PCI model filter for this + node. + type: string + reservedSystemCPU: + description: ReservedSystemCPU overrides the CPUs reserved for + system workloads on this node. + type: string + skipKubeletConfiguration: + description: |- + SkipKubeletConfiguration overrides whether kubelet configuration changes are + skipped for this node. + type: boolean + spdkImage: + description: SpdkImage overrides the SPDK image for this node + (e.g. for phased rollouts). + type: string + spdkProxyImage: + description: SpdkProxyImage overrides the SPDK proxy image for + this node. + type: string + spdkSystemMemory: + description: |- + SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + (e.g. "4G", "512M"). + pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ + type: string + ubuntuHost: + description: UbuntuHost overrides the Ubuntu host OS flag for + this node. + type: boolean + type: object + description: |- + NodeConfigs allows per-worker-node configuration overrides keyed by the + Kubernetes worker node name. Entries are propagated to the corresponding + StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. + The StorageNodeSet is the single source of truth for all per-node config, + including failure domain assignment via nodeConfigs[worker].failureDomain. + type: object nodeFailureDomains: additionalProperties: format: int32 @@ -2027,9 +2131,6 @@ spec: the same group index so the control plane can spread erasure-coding chunks across independent fault groups. type: object - nodeUUID: - description: NodeUUID is required when action is specified - type: string nodesPerSocket: description: NodesPerSocket defines how many storage nodes are created per NUMA socket. @@ -2071,10 +2172,6 @@ spec: pcieModel: description: PcieModel filters devices by PCI model. type: string - reattachVolume: - description: ReattachVolume reattaches volumes during restart where - supported by the backend. - type: boolean reservedSystemCPU: description: ReservedSystemCPU defines CPUs reserved for system workloads. type: string @@ -2105,12 +2202,6 @@ spec: When omitted the backend default is used. pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ type: string - systemVolumeFilterRegex: - description: |- - SystemVolumeFilterRegex is a Go regular expression matched against backend - volume names. Matching volumes are excluded from drain migration and from - the final verification check. Defaults to "^sb-fio-baseline-.*". - type: string tolerations: description: Tolerations configures pod tolerations for storage-node pods. @@ -2155,9 +2246,6 @@ spec: ubuntuHost: description: UbuntuHost indicates the node host OS is Ubuntu. type: boolean - workerNode: - description: WorkerNode is a single worker node used by action flows. - type: string workerNodes: description: WorkerNodes is the set of Kubernetes worker nodes to manage. @@ -2179,52 +2267,6 @@ spec: status: description: status defines the observed state of StorageNodeSet properties: - actionStatus: - description: ActionStatus tracks the latest action execution status. - properties: - action: - description: Action is the requested action name. - type: string - message: - description: Message is a human-readable action result or error. - type: string - nodeUUID: - description: NodeUUID is the target node UUID for the action. - type: string - observedGeneration: - description: ObservedGeneration is the resource generation observed - by this status. - format: int64 - type: integer - state: - type: string - subPhase: - description: SubPhase tracks the active drain step within the - remove action. - enum: - - Validating - - Suspending - - Migrating - - Verifying - - Removing - type: string - triggered: - description: Triggered indicates whether the underlying backend - action has been fired. - type: boolean - updatedAt: - description: UpdatedAt is the timestamp of the last status transition. - format: date-time - type: string - volumesMigrated: - description: VolumesMigrated is the count of volumes successfully - migrated so far. - type: integer - volumesPending: - description: VolumesPending is the count of volumes still awaiting - migration. - type: integer - type: object drainCoordination: description: DrainCoordination tracks the upgrade-drain state per worker node. @@ -2355,6 +2397,14 @@ spec: type: integer type: object type: array + offlineNodes: + description: OfflineNodes is the count of StorageNode CRs with status + "offline" or "suspended". + type: integer + onlineNodes: + description: OnlineNodes is the count of StorageNode CRs with status + "online". + type: integer pendingNodeAdds: additionalProperties: format: date-time @@ -2374,6 +2424,9 @@ spec: a FailedScheduling event during node add. Used to emit a recovery event when the node subsequently comes online. type: object + totalNodes: + description: TotalNodes is the total number of owned StorageNode CRs. + type: integer type: object required: - spec @@ -2935,6 +2988,8 @@ rules: - snapshotreplications - storagebackups - storageclusters + - storagenodeops + - storagenodes - storagenodesets - tasks - volumemigrations @@ -2956,6 +3011,8 @@ rules: - snapshotreplications/finalizers - storagebackups/finalizers - storageclusters/finalizers + - storagenodeops/finalizers + - storagenodes/finalizers - storagenodesets/finalizers - tasks/finalizers - volumemigrations/finalizers @@ -2972,6 +3029,8 @@ rules: - snapshotreplications/status - storagebackups/status - storageclusters/status + - storagenodeops/status + - storagenodes/status - storagenodesets/status - tasks/status - volumemigrations/status diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 2764c654..e68151fb 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -1700,4 +1700,3 @@ func getNodeStatus( } return resp.Status, nil } - diff --git a/operator/internal/controller/simplyblockstoragenodeset_drain.go b/operator/internal/controller/simplyblockstoragenodeset_drain.go index 808a2ce0..3e80daa7 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_drain.go +++ b/operator/internal/controller/simplyblockstoragenodeset_drain.go @@ -40,7 +40,6 @@ import ( // is malformed — intentional fast-fail for a hardcoded value. var defaultSystemVolumeFilter = regexp.MustCompile(simplyblockv1alpha1.DefaultSystemVolumeFilterRegex) - // Requeue intervals used by the drain state machine. const ( drainRequeueImmediate = 1 * time.Second diff --git a/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go index 5cf175a7..a5d8c662 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_drain_unit_test.go @@ -49,74 +49,6 @@ func newDrainReconciler(t *testing.T, objects ...client.Object) *StorageNodeSetR } } -func newDrainSN(subPhase, state string) *simplyblockv1alpha1.StorageNodeSet { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{Name: "sn-drain", Namespace: drainTestNS}, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: drainTestCluster, - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - }, - } - if subPhase != "" || state != "" { - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: state, - SubPhase: subPhase, - } - } - return sn -} - -// ── performDrainAndRemove: terminal state early return ──────────────────────── - -func TestDrainSkipsWhenAlreadySuccess(t *testing.T) { - sn := newDrainSN("", utils.ActionStateSuccess) - r := newDrainReconciler(t, sn) - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient("http://127.0.0.1:1"), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("expected no error for success state, got %v", err) - } - if res.RequeueAfter != 0 { - t.Fatalf("expected zero requeue for success state, got %v", res.RequeueAfter) - } -} - -func TestDrainSkipsWhenAlreadyFailed(t *testing.T) { - sn := newDrainSN("", utils.ActionStateFailed) - r := newDrainReconciler(t, sn) - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient("http://127.0.0.1:1"), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("expected no error for failed state, got %v", err) - } - if res.RequeueAfter != 0 { - t.Fatalf("expected zero requeue for failed state, got %v", res.RequeueAfter) - } -} - -func TestDrainSkipsSuccessEvenWhenSubPhaseIsEmpty(t *testing.T) { - // Regression: stale reconcile reads SubPhase="" after success and must not - // re-initialize the drain (which would restart it on an already-removed node). - sn := newDrainSN("", utils.ActionStateSuccess) - r := newDrainReconciler(t, sn) - - // Run twice — second call must also return immediately without API calls. - for i := 0; i < 2; i++ { - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient("http://127.0.0.1:1"), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("iteration %d: unexpected error %v", i, err) - } - if res.RequeueAfter != 0 { - t.Fatalf("iteration %d: unexpected requeue %v", i, res.RequeueAfter) - } - } -} - -// ── roundRobinTargetNodes ───────────────────────────────────────────────────── - func TestRoundRobinDistributesEvenly(t *testing.T) { mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) defer mock.Close() @@ -298,100 +230,6 @@ func TestMatchVolumesToPVs_SystemVolumeSkipped(t *testing.T) { } } -func TestDrainValidateBothPinnedAndUnmanagedSurfacedTogether(t *testing.T) { - // When a node has both pinned and unmanaged volumes, drainValidate must emit - // BOTH warning events in a single reconcile — not short-circuit after pinned. - // This prevents the user having to fix one issue, retry, then discover the next. - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - // Backend returns two volumes: one PV-managed+pinned, one unmanaged. - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/", - webapimock.RouteResponse{Status: http.StatusOK, Body: `[{"id":"pool-1","name":"p1"}]`}, - ) - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/pool-1/volumes/", - webapimock.RouteResponse{Status: http.StatusOK, Body: `[ - {"id":"vol-pinned","name":"pvc-a","storage_node_id":"` + drainTestNodeUUID + `"}, - {"id":"vol-orphan","name":"manually-created","storage_node_id":"` + drainTestNodeUUID + `"} - ]`}, - ) - // Cluster not rebalancing. - rebalancing := false - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Rebalancing: &rebalancing}, - } - pv := newPV("pv-pin", "vol-pinned") - pvc := newPVC("pv-pin-pvc", true) // pinned annotation set - - fakeRecorder := record.NewFakeRecorder(8) - sn := newDrainSN(drainSubPhaseValidating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - } - - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, cluster, sn, pv, pvc) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: fakeRecorder, - } - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when blocking volumes present") - } - - // Drain all buffered events. - close(fakeRecorder.Events) - var reasons []string - for e := range fakeRecorder.Events { - // Event format: "Warning PinnedVolumeBlocking ..." - parts := strings.SplitN(e, " ", 3) - if len(parts) >= 2 { - reasons = append(reasons, parts[1]) - } - } - - hasPinned := contains(reasons, "PinnedVolumeBlocking") - hasUnmanaged := contains(reasons, "UnmanagedVolumeBlocking") - - if !hasPinned { - t.Error("expected PinnedVolumeBlocking event to be emitted") - } - if !hasUnmanaged { - t.Error("expected UnmanagedVolumeBlocking event to be emitted in the same reconcile") - } - if !hasPinned || !hasUnmanaged { - t.Errorf("emitted events: %v — both blockers must surface together so the user can fix everything at once", reasons) - } - - // Message must mention both issues. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - msg := "" - if updated.Status.ActionStatus != nil { - msg = updated.Status.ActionStatus.Message - } - if !strings.Contains(msg, "pinned") || !strings.Contains(msg, "unmanaged") { - t.Errorf("status message should mention both blockers, got: %q", msg) - } -} - func TestMatchVolumesToPVs_EmptyNodeSkipsMigration(t *testing.T) { r := newDrainReconciler(t) pvManaged, pinned, unmanaged, _, _, err := matchVolumesToPVs(context.Background(), r.Client, nil, defaultSystemVolumeFilter) @@ -418,394 +256,6 @@ func TestMatchVolumesToPVs_OnlySystemVolumes(t *testing.T) { } } -// ── drainValidate: reconciler-level user-visible behaviour ─────────────────── -// -// These tests call drainValidate (not matchVolumesToPVs) and assert what the -// user actually sees: events emitted on the StorageNodeSet CR and the status -// message. Testing the classification helper alone is insufficient — it does -// not verify that the reconciler surfaces the result to the user. - -// newValidateSetup builds a mock HTTP server and reconciler for drainValidate -// tests. volumes is the JSON array the backend returns for the pool's volumes. -func newValidateSetup( - t *testing.T, - volumes string, - k8sObjects ...client.Object, -) (*StorageNodeSetReconciler, *record.FakeRecorder, *webapimock.SpecServer, *simplyblockv1alpha1.StorageNodeSet) { - t.Helper() - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - t.Cleanup(mock.Close) - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/", - webapimock.RouteResponse{Status: http.StatusOK, Body: `[{"id":"pool-1","name":"p1"}]`}, - ) - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-pools/pool-1/volumes/", - webapimock.RouteResponse{Status: http.StatusOK, Body: volumes}, - ) - - rebalancing := false - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Rebalancing: &rebalancing}, - } - sn := newDrainSN(drainSubPhaseValidating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - } - - recorder := record.NewFakeRecorder(8) - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - allObjs := append([]client.Object{cluster, sn}, k8sObjects...) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, allObjs...) - r := &StorageNodeSetReconciler{Client: cl, Scheme: scheme, Recorder: recorder} - return r, recorder, mock, sn -} - -// collectEvents drains all buffered events and returns a map of reason→count. -func collectEvents(rec *record.FakeRecorder) map[string]int { - close(rec.Events) - reasons := map[string]int{} - for e := range rec.Events { - parts := strings.SplitN(e, " ", 3) - if len(parts) >= 2 { - reasons[parts[1]]++ - } - } - return reasons -} - -func TestDrainValidatePinnedVolumeEmitsEventAndBlocks(t *testing.T) { - // User sees: PinnedVolumeBlocking event; status message mentions "pinned"; - // drain does not advance past Validating. - pv := newPV("pv-x", "vol-pinned") - pvc := newPVC("pv-x-pvc", true) - r, rec, mock, sn := newValidateSetup(t, - `[{"id":"vol-pinned","name":"pvc-x","storage_node_id":"`+drainTestNodeUUID+`"}]`, - pv, pvc, - ) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when pinned volume blocks drain") - } - - events := collectEvents(rec) - if events["PinnedVolumeBlocking"] == 0 { - t.Error("expected PinnedVolumeBlocking event; user must be told why drain is blocked") - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - msg := "" - if updated.Status.ActionStatus != nil { - msg = updated.Status.ActionStatus.Message - } - if !strings.Contains(msg, "pinned") { - t.Errorf("status message must mention 'pinned' so user knows what to fix; got: %q", msg) - } - if updated.Status.ActionStatus != nil && updated.Status.ActionStatus.SubPhase != drainSubPhaseValidating { - t.Errorf("drain must stay in Validating when blocked; got SubPhase=%q", updated.Status.ActionStatus.SubPhase) - } -} - -func TestDrainValidateUnmanagedVolumeEmitsEventAndBlocks(t *testing.T) { - // User sees: UnmanagedVolumeBlocking event; status message mentions "unmanaged"; - // drain does not advance past Validating. - r, rec, mock, sn := newValidateSetup(t, - `[{"id":"vol-orphan","name":"manually-created","storage_node_id":"`+drainTestNodeUUID+`"}]`, - // no PV in cluster → volume is unmanaged - ) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when unmanaged volume blocks drain") - } - - events := collectEvents(rec) - if events["UnmanagedVolumeBlocking"] == 0 { - t.Error("expected UnmanagedVolumeBlocking event; user must be told why drain is blocked") - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - msg := "" - if updated.Status.ActionStatus != nil { - msg = updated.Status.ActionStatus.Message - } - if !strings.Contains(msg, "unmanaged") { - t.Errorf("status message must mention 'unmanaged' so user knows what to fix; got: %q", msg) - } -} - -func TestDrainValidateEmptyNodeAdvancesToSuspending(t *testing.T) { - // User sees: drain proceeds without any blocking event. - // Empty node must not block; it advances straight to Suspending. - r, rec, mock, sn := newValidateSetup(t, `[]`) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // Should requeue immediately (advance to Suspending), not with the blocking interval. - if res.RequeueAfter > drainRequeueBlocking { - t.Errorf("unexpected long requeue for empty node: %v (expected ≤ %v)", res.RequeueAfter, drainRequeueBlocking) - } - - events := collectEvents(rec) - if events["PinnedVolumeBlocking"] > 0 || events["UnmanagedVolumeBlocking"] > 0 { - t.Errorf("empty node must not emit blocking events; got %v", events) - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil || updated.Status.ActionStatus.SubPhase != drainSubPhaseSuspending { - got := "" - if updated.Status.ActionStatus != nil { - got = updated.Status.ActionStatus.SubPhase - } - t.Errorf("empty node must advance to Suspending; got SubPhase=%q", got) - } -} - -func TestDrainValidateSystemOnlyNodeAdvancesToSuspending(t *testing.T) { - // User sees: drain proceeds without blocking even though volumes exist — - // system volumes are silently filtered and the drain advances normally. - r, rec, mock, sn := newValidateSetup(t, `[ - {"id":"v1","name":"sb-fio-baseline-read","storage_node_id":"`+drainTestNodeUUID+`"}, - {"id":"v2","name":"sb-fio-baseline-write","storage_node_id":"`+drainTestNodeUUID+`"} - ]`) - defer mock.Close() - - _, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - events := collectEvents(rec) - if events["PinnedVolumeBlocking"] > 0 || events["UnmanagedVolumeBlocking"] > 0 { - t.Errorf("system-volume-only node must not emit blocking events; got %v", events) - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil || updated.Status.ActionStatus.SubPhase != drainSubPhaseSuspending { - got := "" - if updated.Status.ActionStatus != nil { - got = updated.Status.ActionStatus.SubPhase - } - t.Errorf("system-only node must advance to Suspending; got SubPhase=%q", got) - } -} - -// ── drainMigrate: VolumeMigration idempotency ───────────────────────────────── - -func TestDrainMigrateDoesNotRecreateExistingCRs(t *testing.T) { - // Existing VolumeMigration CR for a drain — operator restart must not - // create a duplicate with the same drain label. - sn := newDrainSN(drainSubPhaseMigrating, utils.ActionStateRunning) - existingVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-existing-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseRunning, - }, - } - r := newDrainReconciler(t, sn, existingVM) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - // No pool/volume listing needed — should use existing CRs. - - // Call drainMigrate. It should see the existing CR and not call the API - // to create new ones. - if _, err := r.drainMigrate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainMigrate: %v", err) - } - - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VolumeMigration: %v", err) - } - if len(vmList.Items) != 1 { - t.Errorf("expected exactly 1 VolumeMigration (no duplicate created), got %d", len(vmList.Items)) - } -} - -func TestDrainMigrateFailedCRDeletedAndRetriedWithNewTarget(t *testing.T) { - // Any VolumeMigration failure deletes the Failed CR and requeues so - // createMissingVolumeMigrations can recreate it with a fresh target. - // The node stays suspended — resumeAndFail is NOT called. - sn := newDrainSN(drainSubPhaseMigrating, utils.ActionStateRunning) - failedVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-failed-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseFailed, - ErrorMessage: "ContinueMigration: status 400: target node not online", - }, - } - r := newDrainReconciler(t, sn, failedVM) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.drainMigrate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("drainMigrate: %v", err) - } - // Must requeue for retry, not zero (which would mean terminal). - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when VM failed and retry is in progress") - } - - // State must still be running — the node is NOT resumed and drain NOT failed. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus != nil && updated.Status.ActionStatus.State == utils.ActionStateFailed { - t.Error("drain must not be marked failed — should retry with a new target") - } - - // The Failed VM must be deleted so the next reconcile recreates it. - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VMs: %v", err) - } - for _, vm := range vmList.Items { - if vm.Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed { - t.Errorf("Failed VM %q must be deleted so it can be recreated with a different target", vm.Name) - } - } -} - -func TestDrainMigrateFailedCRDeletedWhenClusterPaused(t *testing.T) { - // When a VolumeMigration fails AND the cluster is not ready, the drain - // should pause AND delete the Failed VM so it can be recreated with a fresh - // target assignment once the cluster recovers — not just defer the failure. - rebalancing := true - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{ - Status: utils.ClusterStatusActive, - Rebalancing: &rebalancing, - }, - } - sn := newDrainSN(drainSubPhaseMigrating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseMigrating, - } - failedVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-failed-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseFailed, - ErrorMessage: "target node in_shutdown", - }, - } - - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - &simplyblockv1alpha1.VolumeMigration{}, - }, cluster, sn, failedVM) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: record.NewFakeRecorder(8), - } - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.drainMigrate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("drainMigrate: %v", err) - } - // Must requeue (pause), not fail. - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when cluster is paused due to rebalancing") - } - // State must still be running — NOT failed. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus != nil && updated.Status.ActionStatus.State == utils.ActionStateFailed { - t.Error("drain must not be marked failed when pausing due to cluster state") - } - // The Failed VM must have been deleted so the drain can recreate it on resume. - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VMs: %v", err) - } - for _, vm := range vmList.Items { - if vm.Status.Phase == simplyblockv1alpha1.VolumeMigrationPhaseFailed { - t.Errorf("Failed VolumeMigration %q must be deleted during cluster-state pause so it can be recreated on resume", vm.Name) - } - } -} - -// ── drainMigrationName ──────────────────────────────────────────────────────── - func TestDrainMigrationNameNoCollisionOnLongPVNames(t *testing.T) { // Two PV names that share a 60+ char common prefix must produce distinct CR // names after sanitisation and truncation (collision guard via FNV suffix). @@ -856,296 +306,3 @@ func TestDrainMigrationNameIsDNSValid(t *testing.T) { } } } - -// ── drainHandleCancellation: stale cache guard ──────────────────────────────── - -func TestDrainCancellationDeletesInFlightVolumeMigrationCRs(t *testing.T) { - // When the user cancels mid-drain, all owned VolumeMigration CRs must be - // deleted so a subsequent drain starts fresh and emits MigrationCreated events. - // Without this, the re-applied drain silently reuses the old CRs and the user - // sees no indication that migration restarted. - sn := newDrainSN(drainSubPhaseMigrating, utils.ActionStateRunning) - sn.Spec.Action = "" // user cleared the action - sn.Spec.NodeUUID = "" - inFlightVM := &simplyblockv1alpha1.VolumeMigration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "drain-inflight-pvc-a", - Namespace: drainTestNS, - Labels: map[string]string{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - }, - Spec: simplyblockv1alpha1.VolumeMigrationSpec{ - PVName: "pv-a", - TargetNodeUUID: drainTestNodeUUID2, - }, - Status: simplyblockv1alpha1.VolumeMigrationStatus{ - Phase: simplyblockv1alpha1.VolumeMigrationPhaseRunning, - }, - } - r := newDrainReconciler(t, sn, inFlightVM) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - // Node is online (already resumed or never suspended in this test). - mock.Register(http.MethodGet, - "/api/v2/clusters/"+drainTestClusterUUID+"/storage-nodes/"+drainTestNodeUUID, - webapimock.RouteResponse{Status: http.StatusOK, Body: `{"status":"online"}`}, - ) - - if _, err := r.drainHandleCancellation(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainHandleCancellation: %v", err) - } - - // The in-flight VolumeMigration CR must have been deleted. - var vmList simplyblockv1alpha1.VolumeMigrationList - if err := r.List(context.Background(), &vmList, - client.InNamespace(drainTestNS), - client.MatchingLabels{"storage.simplyblock.io/drain-node": drainTestNodeUUID}, - ); err != nil { - t.Fatalf("list VolumeMigration: %v", err) - } - if len(vmList.Items) != 0 { - t.Errorf("expected in-flight VolumeMigration CRs to be deleted on cancel, got %d remaining", len(vmList.Items)) - } - - // ActionStatus must also be cleared. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus != nil { - t.Errorf("expected ActionStatus cleared after cancellation, got %+v", updated.Status.ActionStatus) - } -} - -func TestDrainCancellationSkipsWhenActionStillActive(t *testing.T) { - // If the live CR still has action=remove, a stale reconcile must not resume. - sn := newDrainSN(drainSubPhaseMigrating, utils.ActionStateRunning) - r := newDrainReconciler(t, sn) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - // No resume endpoint registered — if resume were called the mock would return 404 - // which would log an error. We verify the CR is unchanged. - - // The passed-in sn has action=remove in the live store (seeded in fake client), - // so drainHandleCancellation's re-fetch will see it and exit early. - if _, err := r.drainHandleCancellation(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainHandleCancellation: %v", err) - } -} - -// ── drainValidate: cluster rebalancing blocks drain ─────────────────────────── - -func TestDrainValidateBlocksWhenClusterRebalancing(t *testing.T) { - rebalancing := true - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Rebalancing: &rebalancing}, - } - sn := newDrainSN(drainSubPhaseValidating, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: drainSubPhaseValidating, - } - - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, cluster, sn) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: record.NewFakeRecorder(32), - } - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.drainValidate(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // Should requeue — not advance to Suspending. - if res.RequeueAfter == 0 { - t.Error("expected non-zero requeue when cluster is rebalancing") - } - // SubPhase must still be Validating. - if sn.Status.ActionStatus.SubPhase != drainSubPhaseValidating { - t.Errorf("expected SubPhase=Validating, got %q", sn.Status.ActionStatus.SubPhase) - } -} - -// ── drainClusterPauseCheck: pause during active sub-phases ──────────────────── - -// newPauseCheckReconciler builds a reconciler seeded with the given StorageCluster -// and a StorageNodeSet in the provided sub-phase, for drainClusterPauseCheck tests. -func newPauseCheckReconciler(t *testing.T, cluster *simplyblockv1alpha1.StorageCluster, subPhase string) (*StorageNodeSetReconciler, *simplyblockv1alpha1.StorageNodeSet) { - t.Helper() - sn := newDrainSN(subPhase, utils.ActionStateRunning) - sn.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ - Action: utils.NodeActionRemove, - NodeUUID: drainTestNodeUUID, - State: utils.ActionStateRunning, - SubPhase: subPhase, - } - scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme, corev1.AddToScheme) - cl := newTestClient(t, scheme, []client.Object{ - &simplyblockv1alpha1.StorageNodeSet{}, - &simplyblockv1alpha1.StorageCluster{}, - }, cluster, sn) - r := &StorageNodeSetReconciler{ - Client: cl, - Scheme: scheme, - Recorder: record.NewFakeRecorder(32), - } - return r, sn -} - -func TestDrainPausesWhenClusterInactive(t *testing.T) { - // When the cluster transitions to a non-active status (e.g. "degraded") - // during an active drain sub-phase, performDrainAndRemove must pause and - // requeue rather than advancing the state machine. - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{Status: "degraded"}, - } - - for _, subPhase := range []string{ - drainSubPhaseSuspending, - drainSubPhaseMigrating, - drainSubPhaseVerifying, - drainSubPhaseRemoving, - } { - t.Run("subPhase="+subPhase, func(t *testing.T) { - r, sn := newPauseCheckReconciler(t, cluster, subPhase) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Errorf("expected non-zero requeue when cluster is inactive") - } - - // SubPhase must not have advanced. - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil { - t.Fatal("expected actionStatus to remain set") - } - if updated.Status.ActionStatus.SubPhase != subPhase { - t.Errorf("sub-phase must not advance during pause: got %q want %q", - updated.Status.ActionStatus.SubPhase, subPhase) - } - // Message should mention the pause. - if !strings.Contains(updated.Status.ActionStatus.Message, "paused") { - t.Errorf("expected message to mention 'paused', got %q", - updated.Status.ActionStatus.Message) - } - }) - } -} - -func TestDrainPausesWhenClusterRebalancingDuringActiveDrain(t *testing.T) { - // When the cluster starts rebalancing after the drain has already advanced - // past Validating, the drain must pause at its current sub-phase. - rebalancing := true - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{ - Status: utils.ClusterStatusActive, - Rebalancing: &rebalancing, - }, - } - - for _, subPhase := range []string{ - drainSubPhaseSuspending, - drainSubPhaseMigrating, - } { - t.Run("subPhase="+subPhase, func(t *testing.T) { - r, sn := newPauseCheckReconciler(t, cluster, subPhase) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - res, err := r.performDrainAndRemove(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.RequeueAfter == 0 { - t.Errorf("expected non-zero requeue when cluster is rebalancing") - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - if updated.Status.ActionStatus == nil { - t.Fatal("expected actionStatus to remain set") - } - if updated.Status.ActionStatus.SubPhase != subPhase { - t.Errorf("sub-phase must not advance during pause: got %q want %q", - updated.Status.ActionStatus.SubPhase, subPhase) - } - if !strings.Contains(updated.Status.ActionStatus.Message, "rebalancing") { - t.Errorf("expected message to mention 'rebalancing', got %q", - updated.Status.ActionStatus.Message) - } - }) - } -} - -func TestDrainContinuesWhenClusterActiveAndNotRebalancing(t *testing.T) { - // When the cluster is active and not rebalancing, drainClusterPauseCheck - // must return (false, "") so the drain proceeds normally. - rebalancing := false - cluster := &simplyblockv1alpha1.StorageCluster{ - ObjectMeta: metav1.ObjectMeta{Name: drainTestCluster, Namespace: drainTestNS}, - Status: simplyblockv1alpha1.StorageClusterStatus{ - Status: utils.ClusterStatusActive, - Rebalancing: &rebalancing, - }, - } - r, sn := newPauseCheckReconciler(t, cluster, drainSubPhaseMigrating) - - paused, reason := r.drainClusterPauseCheck(context.Background(), sn) - if paused { - t.Errorf("expected drain to continue when cluster is active and not rebalancing, got reason=%q", reason) - } -} - -// ── Rapid action toggling ───────────────────────────────────────────────────── - -func TestRapidActionToggleDoesNotLeakState(t *testing.T) { - // Set action=remove, cancel immediately (Validating phase, before any suspend). - // After cancel the ActionStatus must be nil so re-applying remove starts fresh. - sn := newDrainSN(drainSubPhaseValidating, utils.ActionStateRunning) - // Simulate user clearing spec.action. - sn.Spec.Action = "" - sn.Spec.NodeUUID = "" - r := newDrainReconciler(t, sn) - - mock := webapimock.NewSpecServerFromFile(t, "../../openapi.json", true) - defer mock.Close() - - if _, err := r.drainHandleCancellation(context.Background(), webapi.NewClient(mock.URL()), drainTestClusterUUID, sn); err != nil { - t.Fatalf("drainHandleCancellation: %v", err) - } - - updated := &simplyblockv1alpha1.StorageNodeSet{} - if err := r.Get(context.Background(), client.ObjectKeyFromObject(sn), updated); err != nil { - t.Fatalf("get: %v", err) - } - // ActionStatus cleared — re-applying action=remove will start a fresh drain. - if updated.Status.ActionStatus != nil { - t.Errorf("expected ActionStatus to be nil after cancel at Validating, got %+v", updated.Status.ActionStatus) - } -} diff --git a/operator/internal/controller/storagenodeops_controller_unit_test.go b/operator/internal/controller/storagenodeops_controller_unit_test.go index 6b90eab3..fa89842e 100644 --- a/operator/internal/controller/storagenodeops_controller_unit_test.go +++ b/operator/internal/controller/storagenodeops_controller_unit_test.go @@ -16,9 +16,9 @@ import ( // ── helpers ─────────────────────────────────────────────────────────────────── const ( - opsTestNS = "test" - opsTestCluster = "cluster-a" - opsTestWorker = "worker-1.example.com" + opsTestNS = "test" + opsTestCluster = "cluster-a" + opsTestWorker = "worker-1.example.com" opsTestNodeUUID = "aaaa0000-0000-0000-0000-000000000001" ) From 2d9327b1bcb27534b99af760bf0a81da81e6db90 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 12:08:51 +0100 Subject: [PATCH 04/52] fix: resolve all golangci-lint issues in new StorageNode and StorageNodeOps controllers --- operator/api/v1alpha1/storagenodeset_types.go | 2 - ...torage.simplyblock.io_storagenodesets.yaml | 8 ---- .../simplyblockstoragecluster_actions.go | 10 +++-- .../simplyblockstoragenodeset_controller.go | 26 +---------- .../simplyblockstoragenodeset_storagenode.go | 7 ++- .../controller/storagenode_controller.go | 2 +- .../storagenode_controller_unit_test.go | 2 + .../controller/storagenodeops_controller.go | 8 ++-- .../storagenodeops_controller_unit_test.go | 44 ++++++++++--------- 9 files changed, 42 insertions(+), 67 deletions(-) diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index 17d06e1e..b03c7bf3 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -321,8 +321,6 @@ type ActionStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:validation:XValidation:rule="!(has(self.spec.action) && self.spec.action != \"\" && (!has(self.spec.nodeUUID) || self.spec.nodeUUID == \"\"))",message="nodeUUID is required when action is specified" -// +kubebuilder:validation:XValidation:rule="(has(self.spec.action) && self.spec.action != \"\") || (has(self.spec.maxLogicalVolumeCount) && has(self.spec.workerNodes) && size(self.spec.workerNodes) > 0 && has(self.spec.mgmtIfname) && self.spec.mgmtIfname != \"\")",message="maxLogicalVolumeCount, workerNodes, and mgmtIfname are required when action is not specified" // +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)",message="all nodeFailureDomains values must be >= 1 (failure-domain group index)" // +operator-sdk:csv:customresourcedefinitions:displayName="Storage Node",resources={{ServiceAccount,v1,simplyblock-storage-node},{Service,v1,simplyblock-storage-node},{DaemonSet,v1,simplyblock-storage-node},{ClusterRole,v1,simplyblock-storage-node},{ClusterRoleBinding,v1,simplyblock-storage-node}} // StorageNodeSet is the Schema for the storagenodesets API diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index f1b72900..074a89ac 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -676,14 +676,6 @@ spec: - spec type: object x-kubernetes-validations: - - message: nodeUUID is required when action is specified - rule: '!(has(self.spec.action) && self.spec.action != "" && (!has(self.spec.nodeUUID) - || self.spec.nodeUUID == ""))' - - message: maxLogicalVolumeCount, workerNodes, and mgmtIfname are required - when action is not specified - rule: (has(self.spec.action) && self.spec.action != "") || (has(self.spec.maxLogicalVolumeCount) - && has(self.spec.workerNodes) && size(self.spec.workerNodes) > 0 && has(self.spec.mgmtIfname) - && self.spec.mgmtIfname != "") - message: all nodeFailureDomains values must be >= 1 (failure-domain group index) rule: '!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, diff --git a/operator/internal/controller/simplyblockstoragecluster_actions.go b/operator/internal/controller/simplyblockstoragecluster_actions.go index 0c38068b..014da654 100644 --- a/operator/internal/controller/simplyblockstoragecluster_actions.go +++ b/operator/internal/controller/simplyblockstoragecluster_actions.go @@ -34,6 +34,10 @@ import ( "github.com/simplyblock/simplyblock-operator/internal/webapi" ) +// clusterRestartPhaseShutdown is the sub-phase message stored in +// ActionStatus.Message during the shutdown leg of a cluster restart. +const clusterRestartPhaseShutdown = "shutdown" + // failAction marks the current ActionStatus as failed with the given error. func (r *StorageClusterReconciler) failAction( ctx context.Context, @@ -229,7 +233,7 @@ func (r *StorageClusterReconciler) reconcileRestart( clusterCR.Status.ActionStatus = &simplyblockv1alpha1.ActionStatus{ Action: utils.ClusterActionRestart, State: utils.ActionStateRunning, - Message: "shutdown", // restart sub-phase + Message: clusterRestartPhaseShutdown, // restart sub-phase ObservedGeneration: clusterCR.Generation, } return ctrl.Result{Requeue: true}, r.Status().Update(ctx, clusterCR) @@ -246,7 +250,7 @@ func (r *StorageClusterReconciler) reconcileRestart( if !clusterCR.Status.ActionStatus.Triggered { var apiEndpoint string - if phase == "shutdown" { + if phase == clusterRestartPhaseShutdown { apiEndpoint = fmt.Sprintf("/api/v2/clusters/%s/shutdown", clusterUUID) } else { apiEndpoint = fmt.Sprintf("/api/v2/clusters/%s/start", clusterUUID) @@ -285,7 +289,7 @@ func (r *StorageClusterReconciler) reconcileRestart( clusterCR.Status.Status = resp.Status - if phase == "shutdown" && resp.Status == utils.ClusterStatusSuspended { + if phase == clusterRestartPhaseShutdown && resp.Status == utils.ClusterStatusSuspended { clusterCR.Status.ActionStatus.Message = "start" clusterCR.Status.ActionStatus.Triggered = false if err := r.Status().Update(ctx, clusterCR); err != nil { diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index e68151fb..c84cebc0 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -196,7 +196,7 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Phase-1 bridge: create/sync/delete owned StorageNode CRs to match // spec.workerNodes × spec.socketsToUse. The StorageNodeReconciler owns // the per-node provisioning; this only manages CR lifecycle. - if _, err := r.reconcileStorageNodeCRs(ctx, snCR); err != nil { + if err := r.reconcileStorageNodeCRs(ctx, snCR); err != nil { log.Error(err, "failed to reconcile StorageNode CRs") } @@ -1282,7 +1282,7 @@ func checkNodeInfoReachable(ctx context.Context, nodeName, namespace string, tls return nil } -func waitForNodeInfoReachable( +func waitForNodeInfoReachable( //nolint:unparam ctx context.Context, nodeName string, namespace string, @@ -1678,25 +1678,3 @@ func journalManagerCount( } return utils.IntPtrOrDefault(snCR.Spec.JournalManagerSpec.Count, 3) } - -// getNodeStatus reads the current backend status string for a storage node. -func getNodeStatus( - ctx context.Context, - apiClient *webapi.Client, - clusterUUID string, - nodeUUID string, -) (string, error) { - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s", clusterUUID, nodeUUID) - body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) - if err != nil { - return "", err - } - if status >= 300 { - return "", fmt.Errorf("unexpected status %d: %s", status, string(body)) - } - var resp utils.NodeStatusResponse - if err := json.Unmarshal(body, &resp); err != nil { - return "", err - } - return resp.Status, nil -} diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 937b7789..058f1d35 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -33,7 +33,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -48,7 +47,7 @@ import ( func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( ctx context.Context, sns *simplyblockv1alpha1.StorageNodeSet, -) (ctrl.Result, error) { +) error { log := logf.FromContext(ctx) // Build the expected set of (worker, socket) pairs. @@ -67,7 +66,7 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( client.InNamespace(sns.Namespace), client.MatchingFields{"spec.storageNodeSetRef": sns.Name}, ); err != nil { - return ctrl.Result{}, fmt.Errorf("listing owned StorageNode CRs: %w", err) + return fmt.Errorf("listing owned StorageNode CRs: %w", err) } // Delete stale CRs (worker removed from spec.workerNodes or socket removed). @@ -99,7 +98,7 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( log.Error(err, "failed to aggregate StorageNode status into StorageNodeSet") } - return ctrl.Result{}, nil + return nil } // ensureStorageNodeCR creates or patches a StorageNode CR for (worker, socket). diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 246ef33f..b41dc3a0 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -424,7 +424,7 @@ func (r *StorageNodeReconciler) ensureRemoveOps( ops.Name = opsName ops.Namespace = sn.Namespace ops.Spec.StorageNodeRef = sn.Name - ops.Spec.Action = "remove" + ops.Spec.Action = utils.NodeActionRemove if err := controllerutil.SetControllerReference(sn, &ops, r.Scheme); err != nil { return err } diff --git a/operator/internal/controller/storagenode_controller_unit_test.go b/operator/internal/controller/storagenode_controller_unit_test.go index e4324944..c8710184 100644 --- a/operator/internal/controller/storagenode_controller_unit_test.go +++ b/operator/internal/controller/storagenode_controller_unit_test.go @@ -43,6 +43,7 @@ func newSNReconciler(t *testing.T, objects ...client.Object) *StorageNodeReconci } } +//nolint:unparam func newStorageNodeSet(name, ns, cluster string, nodeConfigs map[string]simplyblockv1alpha1.StorageNodeOverrides) *simplyblockv1alpha1.StorageNodeSet { return &simplyblockv1alpha1.StorageNodeSet{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, @@ -54,6 +55,7 @@ func newStorageNodeSet(name, ns, cluster string, nodeConfigs map[string]simplybl } } +//nolint:unparam func newStorageNode(name, ns, snsRef, worker string) *simplyblockv1alpha1.StorageNode { return &simplyblockv1alpha1.StorageNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, diff --git a/operator/internal/controller/storagenodeops_controller.go b/operator/internal/controller/storagenodeops_controller.go index fe5caa20..8428c296 100644 --- a/operator/internal/controller/storagenodeops_controller.go +++ b/operator/internal/controller/storagenodeops_controller.go @@ -111,7 +111,7 @@ func (r *StorageNodeOpsReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Cluster pause check for drain operations. if ops.Spec.Action == "remove" { - if res, paused := r.clusterPauseCheck(ctx, &ops, clusterUUID, apiClient); paused { + if res, paused := r.clusterPauseCheck(ctx, &ops, apiClient); paused { return res, nil } } @@ -399,7 +399,7 @@ func (r *StorageNodeOpsReconciler) drainMigrate( } // Handle failed migrations. - if res, handled := r.handleFailedVolumeMigrations(ctx, ops, sn, clusterUUID, apiClient, vmigList.Items); handled { + if res, handled := r.handleFailedVolumeMigrations(ctx, ops, clusterUUID, apiClient, vmigList.Items); handled { return res, nil } @@ -449,7 +449,6 @@ func (r *StorageNodeOpsReconciler) drainMigrate( func (r *StorageNodeOpsReconciler) handleFailedVolumeMigrations( ctx context.Context, ops *simplyblockv1alpha1.StorageNodeOps, - sn *simplyblockv1alpha1.StorageNode, clusterUUID string, apiClient *webapi.Client, items []simplyblockv1alpha1.VolumeMigration, @@ -467,7 +466,7 @@ func (r *StorageNodeOpsReconciler) handleFailedVolumeMigrations( } // Check if the cluster is paused — if so, delete and wait. - if res, paused := r.clusterPauseCheck(ctx, ops, clusterUUID, apiClient); paused { + if res, paused := r.clusterPauseCheck(ctx, ops, apiClient); paused { for i := range failed { _ = r.Delete(ctx, &failed[i]) } @@ -747,7 +746,6 @@ func (r *StorageNodeOpsReconciler) resumeAndFail( func (r *StorageNodeOpsReconciler) clusterPauseCheck( ctx context.Context, ops *simplyblockv1alpha1.StorageNodeOps, - clusterUUID string, _ *webapi.Client, ) (ctrl.Result, bool) { log := logf.FromContext(ctx) diff --git a/operator/internal/controller/storagenodeops_controller_unit_test.go b/operator/internal/controller/storagenodeops_controller_unit_test.go index fa89842e..09800499 100644 --- a/operator/internal/controller/storagenodeops_controller_unit_test.go +++ b/operator/internal/controller/storagenodeops_controller_unit_test.go @@ -20,6 +20,8 @@ const ( opsTestCluster = "cluster-a" opsTestWorker = "worker-1.example.com" opsTestNodeUUID = "aaaa0000-0000-0000-0000-000000000001" + opsTestOpsName = "ops-1" + opsTestOtherOps = "ops-other" ) func newOpsReconciler(t *testing.T, objects ...client.Object) *StorageNodeOpsReconciler { @@ -45,6 +47,7 @@ func newOpsReconciler(t *testing.T, objects ...client.Object) *StorageNodeOpsRec } } +//nolint:unparam func newTestStorageNode(name, ns, snsRef, worker, uuid string) *simplyblockv1alpha1.StorageNode { sn := &simplyblockv1alpha1.StorageNode{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, @@ -57,6 +60,7 @@ func newTestStorageNode(name, ns, snsRef, worker, uuid string) *simplyblockv1alp return sn } +//nolint:unparam func newTestStorageNodeOps(name, ns, snRef, action string) *simplyblockv1alpha1.StorageNodeOps { return &simplyblockv1alpha1.StorageNodeOps{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, @@ -71,7 +75,7 @@ func newTestStorageNodeOps(name, ns, snRef, action string) *simplyblockv1alpha1. func TestAcquireLock_SetsActiveOpsRefAndTransitionsToRunning(t *testing.T) { sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "suspend") r := newOpsReconciler(t, sn, ops) _, err := r.acquireLock(context.Background(), ops, sn) @@ -82,13 +86,13 @@ func TestAcquireLock_SetsActiveOpsRefAndTransitionsToRunning(t *testing.T) { // Check StorageNode.status.activeOpsRef was set. var updatedSN simplyblockv1alpha1.StorageNode _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updatedSN) - if updatedSN.Status.ActiveOpsRef != "ops-1" { + if updatedSN.Status.ActiveOpsRef != opsTestOpsName { t.Errorf("activeOpsRef: got %q want ops-1", updatedSN.Status.ActiveOpsRef) } // Check ops phase was set to Running. var updatedOps simplyblockv1alpha1.StorageNodeOps - _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updatedOps) + _ = r.Get(context.Background(), types.NamespacedName{Name: opsTestOpsName, Namespace: opsTestNS}, &updatedOps) if updatedOps.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseRunning { t.Errorf("phase: got %q want Running", updatedOps.Status.Phase) } @@ -96,8 +100,8 @@ func TestAcquireLock_SetsActiveOpsRefAndTransitionsToRunning(t *testing.T) { func TestAcquireLock_RequeuesWhenAnotherOpsActive(t *testing.T) { sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) - sn.Status.ActiveOpsRef = "ops-other" - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + sn.Status.ActiveOpsRef = opsTestOtherOps + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "suspend") r := newOpsReconciler(t, sn, ops) result, err := r.acquireLock(context.Background(), ops, sn) @@ -111,7 +115,7 @@ func TestAcquireLock_RequeuesWhenAnotherOpsActive(t *testing.T) { // StorageNode.activeOpsRef must NOT be changed. var updatedSN simplyblockv1alpha1.StorageNode _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updatedSN) - if updatedSN.Status.ActiveOpsRef != "ops-other" { + if updatedSN.Status.ActiveOpsRef != opsTestOtherOps { t.Errorf("activeOpsRef should not change: got %q", updatedSN.Status.ActiveOpsRef) } } @@ -137,8 +141,8 @@ func TestAcquireLock_RemoveDrainSetsValidatingSubPhase(t *testing.T) { func TestSucceedOps_SetsPhaseAndClearsLock(t *testing.T) { sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) - sn.Status.ActiveOpsRef = "ops-1" - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + sn.Status.ActiveOpsRef = opsTestOpsName + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "suspend") ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning r := newOpsReconciler(t, sn, ops) @@ -148,7 +152,7 @@ func TestSucceedOps_SetsPhaseAndClearsLock(t *testing.T) { } var updatedOps simplyblockv1alpha1.StorageNodeOps - _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updatedOps) + _ = r.Get(context.Background(), types.NamespacedName{Name: opsTestOpsName, Namespace: opsTestNS}, &updatedOps) if updatedOps.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseSucceeded { t.Errorf("phase: got %q want Succeeded", updatedOps.Status.Phase) } @@ -167,8 +171,8 @@ func TestSucceedOps_SetsPhaseAndClearsLock(t *testing.T) { func TestFailOps_SetsPhaseAndClearsLock(t *testing.T) { sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) - sn.Status.ActiveOpsRef = "ops-1" - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "suspend") + sn.Status.ActiveOpsRef = opsTestOpsName + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "suspend") ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning r := newOpsReconciler(t, sn, ops) @@ -178,7 +182,7 @@ func TestFailOps_SetsPhaseAndClearsLock(t *testing.T) { } var updatedOps simplyblockv1alpha1.StorageNodeOps - _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updatedOps) + _ = r.Get(context.Background(), types.NamespacedName{Name: opsTestOpsName, Namespace: opsTestNS}, &updatedOps) if updatedOps.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseFailed { t.Errorf("phase: got %q want Failed", updatedOps.Status.Phase) } @@ -197,17 +201,17 @@ func TestFailOps_SetsPhaseAndClearsLock(t *testing.T) { func TestReleaseLock_OnlyClearsIfOwner(t *testing.T) { sn := newTestStorageNode("sn-1", opsTestNS, "sns", opsTestWorker, opsTestNodeUUID) - sn.Status.ActiveOpsRef = "ops-other" + sn.Status.ActiveOpsRef = opsTestOtherOps r := newOpsReconciler(t, sn) // Releasing with a different name should be a no-op. - if err := r.releaseLock(context.Background(), sn, "ops-1"); err != nil { + if err := r.releaseLock(context.Background(), sn, opsTestOpsName); err != nil { t.Fatalf("releaseLock returned error: %v", err) } var updated simplyblockv1alpha1.StorageNode _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: opsTestNS}, &updated) - if updated.Status.ActiveOpsRef != "ops-other" { + if updated.Status.ActiveOpsRef != opsTestOtherOps { t.Error("releaseLock should not clear a lock it does not own") } } @@ -244,7 +248,7 @@ func TestDispatch_UnknownActionFails(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: opsTestNS}, Spec: simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: opsTestCluster}, } - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "bogus-action") + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "bogus-action") ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning r := newOpsReconciler(t, sn, sns, ops) @@ -254,7 +258,7 @@ func TestDispatch_UnknownActionFails(t *testing.T) { } var updated simplyblockv1alpha1.StorageNodeOps - _ = r.Get(context.Background(), types.NamespacedName{Name: "ops-1", Namespace: opsTestNS}, &updated) + _ = r.Get(context.Background(), types.NamespacedName{Name: opsTestOpsName, Namespace: opsTestNS}, &updated) if updated.Status.Phase != simplyblockv1alpha1.StorageNodeOpsPhaseFailed { t.Errorf("expected Failed for unknown action, got %q", updated.Status.Phase) } @@ -263,7 +267,7 @@ func TestDispatch_UnknownActionFails(t *testing.T) { // ── TestResolveOpsSystemVolumeFilter ───────────────────────────────────────── func TestResolveOpsSystemVolumeFilter_UsesDefaultWhenNoDrain(t *testing.T) { - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "remove") + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "remove") r := newOpsReconciler(t, ops) re, err := r.resolveOpsSystemVolumeFilter(ops) @@ -281,7 +285,7 @@ func TestResolveOpsSystemVolumeFilter_UsesDefaultWhenNoDrain(t *testing.T) { func TestResolveOpsSystemVolumeFilter_UsesCustomPattern(t *testing.T) { custom := "^bench-.*" - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "remove") + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "remove") ops.Spec.Drain = &simplyblockv1alpha1.DrainOpsSpec{SystemVolumeFilterRegex: &custom} r := newOpsReconciler(t, ops) @@ -299,7 +303,7 @@ func TestResolveOpsSystemVolumeFilter_UsesCustomPattern(t *testing.T) { func TestResolveOpsSystemVolumeFilter_InvalidPatternReturnsError(t *testing.T) { bad := "[" - ops := newTestStorageNodeOps("ops-1", opsTestNS, "sn-1", "remove") + ops := newTestStorageNodeOps(opsTestOpsName, opsTestNS, "sn-1", "remove") ops.Spec.Drain = &simplyblockv1alpha1.DrainOpsSpec{SystemVolumeFilterRegex: &bad} r := newOpsReconciler(t, ops) From cf5ca36c2238db061497a814cbdaf42936f0e8c0 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 12:18:20 +0100 Subject: [PATCH 05/52] fix: resolve remaining golangci-lint issues in StorageNodeOps controller --- operator/dist/install.yaml | 8 -------- .../controller/simplyblockstoragenodeset_controller.go | 6 +++--- .../internal/controller/storagenodeops_controller.go | 9 ++++----- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 7b2a50c6..32ec2697 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2432,14 +2432,6 @@ spec: - spec type: object x-kubernetes-validations: - - message: nodeUUID is required when action is specified - rule: '!(has(self.spec.action) && self.spec.action != "" && (!has(self.spec.nodeUUID) - || self.spec.nodeUUID == ""))' - - message: maxLogicalVolumeCount, workerNodes, and mgmtIfname are required - when action is not specified - rule: (has(self.spec.action) && self.spec.action != "") || (has(self.spec.maxLogicalVolumeCount) - && has(self.spec.workerNodes) && size(self.spec.workerNodes) > 0 && has(self.spec.mgmtIfname) - && self.spec.mgmtIfname != "") - message: all nodeFailureDomains values must be >= 1 (failure-domain group index) rule: '!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index c84cebc0..9bdd5e11 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -1282,11 +1282,11 @@ func checkNodeInfoReachable(ctx context.Context, nodeName, namespace string, tls return nil } -func waitForNodeInfoReachable( //nolint:unparam +func waitForNodeInfoReachable( ctx context.Context, nodeName string, - namespace string, - tlsEnabled, tlsMutualEnabled bool, + namespace string, //nolint:unparam + tlsEnabled, tlsMutualEnabled bool, //nolint:unparam ) error { log := logf.FromContext(ctx) diff --git a/operator/internal/controller/storagenodeops_controller.go b/operator/internal/controller/storagenodeops_controller.go index 8428c296..85c7bda1 100644 --- a/operator/internal/controller/storagenodeops_controller.go +++ b/operator/internal/controller/storagenodeops_controller.go @@ -110,7 +110,7 @@ func (r *StorageNodeOpsReconciler) Reconcile(ctx context.Context, req ctrl.Reque } // Cluster pause check for drain operations. - if ops.Spec.Action == "remove" { + if ops.Spec.Action == utils.NodeActionRemove { if res, paused := r.clusterPauseCheck(ctx, &ops, apiClient); paused { return res, nil } @@ -144,7 +144,7 @@ func (r *StorageNodeOpsReconciler) acquireLock( opsPatch := client.MergeFrom(ops.DeepCopy()) ops.Status.Phase = simplyblockv1alpha1.StorageNodeOpsPhaseRunning ops.Status.StartedAt = &now - if ops.Spec.Action == "remove" { + if ops.Spec.Action == utils.NodeActionRemove { ops.Status.SubPhase = simplyblockv1alpha1.StorageNodeOpsSubPhaseValidating } if err := r.Status().Patch(ctx, ops, opsPatch); err != nil { @@ -163,7 +163,7 @@ func (r *StorageNodeOpsReconciler) dispatch( apiClient *webapi.Client, ) (ctrl.Result, error) { switch ops.Spec.Action { - case "remove": + case utils.NodeActionRemove: return r.runDrain(ctx, ops, sn, clusterUUID, apiClient) case "shutdown", "restart", "suspend", "resume": return r.runSimpleAction(ctx, ops, sn, sns, clusterUUID, apiClient) @@ -399,7 +399,7 @@ func (r *StorageNodeOpsReconciler) drainMigrate( } // Handle failed migrations. - if res, handled := r.handleFailedVolumeMigrations(ctx, ops, clusterUUID, apiClient, vmigList.Items); handled { + if res, handled := r.handleFailedVolumeMigrations(ctx, ops, apiClient, vmigList.Items); handled { return res, nil } @@ -449,7 +449,6 @@ func (r *StorageNodeOpsReconciler) drainMigrate( func (r *StorageNodeOpsReconciler) handleFailedVolumeMigrations( ctx context.Context, ops *simplyblockv1alpha1.StorageNodeOps, - clusterUUID string, apiClient *webapi.Client, items []simplyblockv1alpha1.VolumeMigration, ) (ctrl.Result, bool) { From 74b9474fcc39aebec09245270fab950b4a723d02 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 13:03:09 +0100 Subject: [PATCH 06/52] formatted file internal/controller/simplyblockstoragenodeset_controller.go --- .../internal/controller/simplyblockstoragenodeset_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 9bdd5e11..7e1f18ab 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -1285,7 +1285,7 @@ func checkNodeInfoReachable(ctx context.Context, nodeName, namespace string, tls func waitForNodeInfoReachable( ctx context.Context, nodeName string, - namespace string, //nolint:unparam + namespace string, //nolint:unparam tlsEnabled, tlsMutualEnabled bool, //nolint:unparam ) error { log := logf.FromContext(ctx) From a9cd6a8d085f04391410fd2965a5fe67cf1d9cb7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 15:21:14 +0100 Subject: [PATCH 07/52] fix: stop duplicate storage node POSTs by bridging UUID from StorageNodeSet status in Phase 1 --- .../controller/storagenode_controller.go | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index b41dc3a0..a22a759c 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -110,17 +110,51 @@ func (r *StorageNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - apiClient := webapi.NewClient() - - // Node not yet provisioned → post to backend. + // Phase 1 bridge: the StorageNodeSetReconciler still owns provisioning. + // Sync the UUID from StorageNodeSet.status.nodes[] so this reconciler + // can see when the node is online without re-POSTing. if sn.Status.UUID == "" { - return r.provisionNode(ctx, &sn, &sns, clusterUUID, apiClient) + if err := r.syncUUIDFromNodeSet(ctx, &sn, &sns); err != nil { + return ctrl.Result{}, err + } + // Re-read after patch — if UUID is still empty, old reconciler hasn't + // provisioned yet; requeue and wait. + if sn.Status.UUID == "" { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } } + apiClient := webapi.NewClient() + // Node provisioned → sync status periodically. return r.syncStatus(ctx, &sn, clusterUUID, apiClient) } +// syncUUIDFromNodeSet copies the backend UUID from StorageNodeSet.status.nodes[] +// into StorageNode.status.uuid. This is the Phase 1 bridge: the old +// StorageNodeSetReconciler owns provisioning and tracks UUIDs in its own status; +// the StorageNodeReconciler reads that status so it doesn't re-POST. +func (r *StorageNodeReconciler) syncUUIDFromNodeSet( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + sns *simplyblockv1alpha1.StorageNodeSet, +) error { + for _, ns := range sns.Status.Nodes { + if ns.Hostname != sn.Spec.WorkerNode || ns.UUID == "" { + continue + } + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.UUID = ns.UUID + sn.Status.Status = ns.Status + sn.Status.Health = ns.Health + if err := r.Status().Patch(ctx, sn, patch); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("syncing UUID for StorageNode %s: %w", sn.Name, err) + } + return nil + } + return nil +} + // syncOverrides propagates StorageNodeSet.spec.nodeConfigs[worker] into // StorageNode.spec.overrides. The StorageNodeSet is the single source of truth. func (r *StorageNodeReconciler) syncOverrides( From e58c8431c5ea0b2a3fd4d20e66fe872156e6933f Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 16:11:08 +0100 Subject: [PATCH 08/52] operator: move storage node provisioning ownership to StorageNodeReconciler with per-node override support --- .../simplyblockstoragenodeset_controller.go | 262 ++---------------- ...lockstoragenodeset_controller_unit_test.go | 27 -- .../simplyblockstoragenodeset_storagenode.go | 43 +++ .../controller/storagenode_controller.go | 118 ++++++-- 4 files changed, 165 insertions(+), 285 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 7e1f18ab..47f0783b 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -262,164 +262,16 @@ func (r *StorageNodeSetReconciler) reconcileWorkerNode( return ctrl.Result{RequeueAfter: time.Second * 10}, nil } - // PendingNodeAdds is the authoritative guard against duplicate POSTs. - // It is a separate map field so patches to Status.Nodes by - // syncTrackedNodesStatus can never inadvertently delete it. - // The legacy UUID=="" placeholder check is kept as a fallback for - // CRs that existed before PendingNodeAdds was introduced. - _, isPending := snCR.Status.PendingNodeAdds[nodeName] - if !isPending { - for _, n := range snCR.Status.Nodes { - if n.Hostname == nodeName && n.UUID == "" { - isPending = true - break - } - } + // StorageNodeReconciler is now the sole owner of provisioning. Skip this + // node if its StorageNode CR already has PostedAt set — the new reconciler + // has taken over and we must not double-POST. + if r.storageNodeAlreadyPosted(ctx, snCR.Namespace, nodeName) { + return r.pollNodeOnline(ctx, apiClient, clusterUUID, ip, nodeName, expectedPerHost, snCR) } - if !isPending { - // Proactive: check if nodes already exist on backend (e.g. from Helm deployment) - // before sending a POST that would either fail or create a duplicate. - // For multi-socket deployments (expectedPerHost > 1) a single POST creates all - // NUMA-socket nodes at once, so we skip POST if ANY node for this IP already - // exists. pollNodeOnline then waits for all expectedPerHost nodes to be online. - allNodes, lookupErr := listStorageNodesForCluster(ctx, apiClient, clusterUUID) - if lookupErr == nil { - existingCount := 0 - for _, n := range allNodes { - if n.IP == ip { - existingCount++ - } - } - if existingCount > 0 { - log.Info("Storage node(s) already exist on backend, adopting", "node", nodeName, "count", existingCount) - if err := r.Get(ctx, req.NamespacedName, snCR); err != nil { - return ctrl.Result{}, err - } - var matchingNodes []SNODEAPIResponse - for _, n := range allNodes { - if n.IP == ip { - matchingNodes = append(matchingNodes, n) - } - } - adoptStorageNodeStatus(snCR, nodeName, matchingNodes) - if err := r.Status().Update(ctx, snCR); err != nil { - log.Error(err, "Failed to set node status for adoption") - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - log.Info("Storage node(s) adopted from backend", "node", nodeName, "count", existingCount) - return ctrl.Result{}, nil - } - } - // Persist the pending marker BEFORE the POST so that every future - // reconcile — including those triggered while sbcli is retrying - // internally after a failure — sees the marker and skips the POST. - patch := client.MergeFrom(snCR.DeepCopy()) - if snCR.Status.PendingNodeAdds == nil { - snCR.Status.PendingNodeAdds = make(map[string]metav1.Time) - } - snCR.Status.PendingNodeAdds[nodeName] = metav1.Now() - if err := r.Status().Patch(ctx, snCR, patch); err != nil { - log.Error(err, "Failed to persist pending node add marker before POST, retrying", "node", nodeName) - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - - if res, err := r.postStorageNodeSet(ctx, req, snCR, nodeName, ip, clusterUUID, apiClient); err != nil || res.RequeueAfter > 0 { - // POST failed — clear the pending marker so the next reconcile - // retries the POST rather than waiting on a node that was never created. - clearPatch := client.MergeFrom(snCR.DeepCopy()) - delete(snCR.Status.PendingNodeAdds, nodeName) - if patchErr := r.Status().Patch(ctx, snCR, clearPatch); patchErr != nil { - log.Error(patchErr, "Failed to clear pending node add marker after POST failure", "node", nodeName) - } - return res, err - } - } - - return r.pollNodeOnline(ctx, apiClient, clusterUUID, ip, nodeName, expectedPerHost, snCR) -} - -// postStorageNodeSet calls the backend storage-node creation API and records the -// placeholder status entry. -func (r *StorageNodeSetReconciler) postStorageNodeSet( - ctx context.Context, - req ctrl.Request, - snCR *simplyblockv1alpha1.StorageNodeSet, - nodeName, ip, clusterUUID string, - apiClient *webapi.Client, -) (ctrl.Result, error) { - log := logf.FromContext(ctx) - - if err := checkNodeInfoReachable(ctx, nodeName, snCR.Namespace, r.TLSEnabled, r.TLSMutualEnabled); err != nil { - log.V(1).Info("Storage node API not reachable yet, requeueing", - "node", nodeName, - "ip", ip, - "error", err.Error(), - ) - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - - nodeAddress := utils.StorageNodeSetAPIAddress(nodeName, snCR.Namespace) - params := utils.StorageNodeSetAddParams{ - NodeAddress: nodeAddress, - InterfaceName: snCR.Spec.MgmtIfname, - SPDKImage: snCR.Spec.SpdkImage, - SPDKProxyImage: snCR.Spec.SpdkProxyImage, - SPDKDebug: false, - IdDeviceByNQN: false, - DataNics: snCR.Spec.DataIfname, - Namespace: snCR.Namespace, - JMPercent: journalManagerPercentPerDevice(snCR), - Partitions: utils.IntPtrOrDefault(snCR.Spec.Partitions, 1), - IOBufSmallPoolCount: 0, - IOBufLargePoolCount: 0, - HaJMCount: journalManagerCount(snCR), - CRName: snCR.Name, - CRNameSpace: snCR.Namespace, - CRPlural: "storagenodesets", - Format4K: utils.BoolPtrOrFalse(snCR.Spec.ForceFormat4K), - SpdkSystemMemory: snCR.Spec.SpdkSystemMemory, - FailureDomain: nodeFailureDomain(snCR, nodeName), - } - - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes", clusterUUID) - - jsonParams, err := json.MarshalIndent(params, "", " ") - if err != nil { - log.Error(err, "Failed to marshal params") - } else { - log.Info("Sending Storage Node Add Request", - "endpoint", endpoint, - "request_body", string(jsonParams), - ) - } - - body, status, err := apiClient.Do(ctx, http.MethodPost, endpoint, params) - if err != nil || status >= 300 { - if err == nil { - err = fmt.Errorf("unexpected status %d", status) - } - log.Error(err, "StorageNodeSet creation failed", "status", status, "response", string(body)) - return ctrl.Result{RequeueAfter: 20 * time.Second}, nil - } - - log.Info("SNODE API call", - "endpoint", endpoint, - "status", status, - "response", string(body), - ) - - if err := r.Get(ctx, req.NamespacedName, snCR); err != nil { - return ctrl.Result{}, err - } - - ensureNodeStatus(snCR, nodeName, ip) - - if err := r.Status().Update(ctx, snCR); err != nil { - log.Error(err, "Failed to update storage node status") - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil - } - return ctrl.Result{}, nil + // StorageNodeReconciler is the sole owner of provisioning. If it hasn't + // POSTed yet, requeue and wait — never POST from here. + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } // SetupWithManager sets up the controller with the Manager. @@ -1174,73 +1026,6 @@ func getNodeInternalIP(ctx context.Context, c client.Client, nodeName string) (s return "", fmt.Errorf("node %s has no InternalIP", nodeName) } -// listStorageNodesForCluster fetches all backend storage nodes for the given cluster. -func listStorageNodesForCluster(ctx context.Context, apiClient *webapi.Client, clusterUUID string) ([]SNODEAPIResponse, error) { - endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/", clusterUUID) - body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) - if err != nil || status >= 300 { - return nil, fmt.Errorf("list storage nodes failed, status %d: %v", status, err) - } - var nodes []SNODEAPIResponse - if err := json.Unmarshal(body, &nodes); err != nil { - return nil, fmt.Errorf("failed to parse storage node list: %w", err) - } - return nodes, nil -} - -func ensureNodeStatus( - snCR *simplyblockv1alpha1.StorageNodeSet, - nodeName, ip string, -) *simplyblockv1alpha1.NodeStatus { - - for i := range snCR.Status.Nodes { - if snCR.Status.Nodes[i].Hostname == nodeName { - return &snCR.Status.Nodes[i] - } - } - - now := metav1.Now() - snCR.Status.Nodes = append(snCR.Status.Nodes, simplyblockv1alpha1.NodeStatus{ - Hostname: nodeName, - MgmtIp: ip, - Status: "in_creation", - PostedAt: &now, - }) - - return &snCR.Status.Nodes[len(snCR.Status.Nodes)-1] -} - -func adoptStorageNodeStatus(snCR *simplyblockv1alpha1.StorageNodeSet, nodeName string, nodes []SNODEAPIResponse) { - for _, res := range nodes { - entry := simplyblockv1alpha1.NodeStatus{ - Hostname: nodeName, - UUID: res.UUID, - Status: res.Status, - MgmtIp: res.IP, - Health: res.Health, - Devices: fmt.Sprintf("%d/%d", res.DevicesCount, res.OnlineDevicesCount), - CPU: utils.IntToInt32Ptr(res.CPU), - Memory: utils.HumanBytes(res.Memory, "iec"), - Volumes: utils.IntToInt32Ptr(res.Volumes), - RpcPort: utils.IntToInt32Ptr(res.RPC_PORT), - LvolPort: utils.IntToInt32Ptr(res.LVOL_PORT), - NvmfPort: utils.IntToInt32Ptr(res.NVMF_PORT), - } - matched := false - for i := range snCR.Status.Nodes { - n := &snCR.Status.Nodes[i] - if n.Hostname == nodeName && (n.UUID == res.UUID || n.UUID == "") { - *n = entry - matched = true - break - } - } - if !matched { - snCR.Status.Nodes = append(snCR.Status.Nodes, entry) - } - } -} - func checkNodeInfoReachable(ctx context.Context, nodeName, namespace string, tlsEnabled, tlsMutualEnabled bool) error { scheme := "http" httpClient := &http.Client{Timeout: 3 * time.Second} @@ -1394,23 +1179,30 @@ func (r *StorageNodeSetReconciler) nodeOnlineRequeueOrTimeout( log := logf.FromContext(ctx) timeout := time.Duration(waitForNodeOnlineRetries) * waitForNodeOnlineWaitInterval - // PendingNodeAdds is the primary source for the post timestamp. - // Fall back to the legacy UUID=="" PostedAt for backward compatibility. - if postedAt, ok := snCR.Status.PendingNodeAdds[nodeName]; ok { - if time.Since(postedAt.Time) <= timeout { - if time.Since(postedAt.Time) >= spdkPodEventDelay { - r.recordSpdkPodEvents(ctx, snCR, nodeName) - } - return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil - } + // Read the post timestamp from the StorageNode CR (set by StorageNodeReconciler). + // Fall back to PendingNodeAdds (legacy) and status.nodes[].PostedAt for + // deployments that pre-date the StorageNodeReconciler. + var postedAt *metav1.Time + if t := r.storageNodePostedAt(ctx, snCR.Namespace, nodeName); t != nil { + postedAt = t + } else if t2, ok := snCR.Status.PendingNodeAdds[nodeName]; ok { + postedAt = &t2 } else { for i := range snCR.Status.Nodes { n := &snCR.Status.Nodes[i] if n.Hostname == nodeName && n.UUID == "" && n.PostedAt != nil { - if time.Since(n.PostedAt.Time) <= timeout { - return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil - } + postedAt = n.PostedAt + break + } + } + } + + if postedAt != nil { + if time.Since(postedAt.Time) <= timeout { + if time.Since(postedAt.Time) >= spdkPodEventDelay { + r.recordSpdkPodEvents(ctx, snCR, nodeName) } + return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil } } diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go index 2de97d7e..207cef21 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go @@ -32,33 +32,6 @@ const ( caVolumeName = "certificate-authority" ) -func TestEnsureNodeStatus(t *testing.T) { - cr := &simplyblockv1alpha1.StorageNodeSet{} - - s := ensureNodeStatus(cr, "node-a", mgmtIP) - if s == nil { - t.Fatalf("ensureNodeStatus returned nil") - } - if s.Hostname != "node-a" || s.MgmtIp != mgmtIP || s.Status != "in_creation" { - t.Fatalf("unexpected initial node status: %#v", *s) - } - if len(cr.Status.Nodes) != 1 { - t.Fatalf("expected one node, got %d", len(cr.Status.Nodes)) - } - - s2 := ensureNodeStatus(cr, "node-a", "10.0.0.99") - if s2 == nil { - t.Fatalf("ensureNodeStatus second call returned nil") - } - if len(cr.Status.Nodes) != 1 { - t.Fatalf("should not append duplicate node entry") - } - // existing value should be retained - if s2.MgmtIp != mgmtIP { - t.Fatalf("expected existing node status to be reused, got %#v", *s2) - } -} - func TestStorageNodeSetFinalizerLifecycleHelpers(t *testing.T) { now := metav1.NewTime(time.Now()) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 058f1d35..49577a34 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -249,6 +249,49 @@ func buildStorageNodeCR( return sn } +// storageNodePostedAt returns the PostedAt timestamp from the StorageNode CR +// for the given worker, or nil if not found / not yet set. +func (r *StorageNodeSetReconciler) storageNodePostedAt( + ctx context.Context, + namespace, workerNode string, +) *metav1.Time { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(namespace), + client.MatchingFields{"spec.workerNode": workerNode}, + ); err != nil { + return nil + } + for _, sn := range snList.Items { + if sn.Status.PostedAt != nil { + return sn.Status.PostedAt + } + } + return nil +} + +// storageNodeAlreadyPosted returns true if the StorageNode CR for the given +// worker node already has PostedAt set, meaning StorageNodeReconciler has taken +// over provisioning and this reconciler must not duplicate the POST. +func (r *StorageNodeSetReconciler) storageNodeAlreadyPosted( + ctx context.Context, + namespace, workerNode string, +) bool { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(namespace), + client.MatchingFields{"spec.workerNode": workerNode}, + ); err != nil { + return false + } + for _, sn := range snList.Items { + if sn.Status.PostedAt != nil { + return true + } + } + return false +} + // effectiveSockets returns the list of socket identifiers to use. When // SocketsToUse is empty, a single socket "0" is assumed. func effectiveSockets(sns *simplyblockv1alpha1.StorageNodeSet) []string { diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index a22a759c..fbc28104 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -110,22 +110,20 @@ func (r *StorageNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - // Phase 1 bridge: the StorageNodeSetReconciler still owns provisioning. - // Sync the UUID from StorageNodeSet.status.nodes[] so this reconciler - // can see when the node is online without re-POSTing. + apiClient := webapi.NewClient() + if sn.Status.UUID == "" { + // Check if the old StorageNodeSetReconciler already provisioned this node + // (present in status.nodes[]) — adopt its UUID to avoid a double POST. if err := r.syncUUIDFromNodeSet(ctx, &sn, &sns); err != nil { return ctrl.Result{}, err } - // Re-read after patch — if UUID is still empty, old reconciler hasn't - // provisioned yet; requeue and wait. if sn.Status.UUID == "" { - return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + // Not yet provisioned — this reconciler is now the sole owner of the POST. + return r.provisionNode(ctx, &sn, &sns, clusterUUID, apiClient) } } - apiClient := webapi.NewClient() - // Node provisioned → sync status periodically. return r.syncStatus(ctx, &sn, clusterUUID, apiClient) } @@ -174,7 +172,9 @@ func (r *StorageNodeReconciler) syncOverrides( return nil } -// provisionNode posts the node to the backend API and begins polling for online status. +// provisionNode posts the node to the backend API. StorageNodeReconciler is the +// sole owner of provisioning — the old StorageNodeSetReconciler skips nodes +// whose StorageNode CR already has PostedAt set. func (r *StorageNodeReconciler) provisionNode( ctx context.Context, sn *simplyblockv1alpha1.StorageNode, @@ -184,40 +184,57 @@ func (r *StorageNodeReconciler) provisionNode( ) (ctrl.Result, error) { log := logf.FromContext(ctx) - // Guard: if enableFailureDomains is set on the cluster, failureDomain must be populated. + // POST already sent — poll until the UUID appears via syncUUIDFromNodeSet. + if sn.Status.PostedAt != nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // Respect MaxParallelNodeAdds: count sibling StorageNode CRs in this set + // that are in-flight (PostedAt set, UUID not yet assigned) and block if the + // limit is reached. This replicates the old PendingNodeAdds gate. + if sns.Spec.MaxParallelNodeAdds != nil { + inFlight, err := r.countInFlightNodes(ctx, sn.Namespace, sn.Spec.StorageNodeSetRef, sn.Name) + if err == nil && inFlight >= int(*sns.Spec.MaxParallelNodeAdds) { + log.Info("parallel node add limit reached, requeuing", + "inFlight", inFlight, "max", *sns.Spec.MaxParallelNodeAdds) + return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil + } + } + + // Guard: failure domain must be set if the feature is enabled. if err := r.checkFailureDomain(ctx, sn, sns); err != nil { r.Recorder.Event(sn, "Warning", "FailureDomainMissing", err.Error()) log.Info("blocking node-add: "+err.Error(), "node", sn.Name) return ctrl.Result{RequeueAfter: 60 * time.Second}, nil } - // Wait until the node's API endpoint is reachable. + // Wait until the node's SPDK API endpoint is reachable. if err := checkNodeInfoReachable(ctx, sn.Spec.WorkerNode, sn.Namespace, r.TLSEnabled, r.TLSMutualEnabled); err != nil { log.V(1).Info("storage node API not reachable yet, requeuing", "worker", sn.Spec.WorkerNode, "error", err.Error()) return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } - // Build effective config: fleet defaults merged with per-node overrides. + // Merge fleet defaults with per-node overrides — overrides always win. eff := effectiveNodeConfig(sn, sns) nodeAddress := utils.StorageNodeSetAPIAddress(sn.Spec.WorkerNode, sn.Namespace) params := utils.StorageNodeSetAddParams{ NodeAddress: nodeAddress, - InterfaceName: sns.Spec.MgmtIfname, - SPDKImage: eff.SpdkImage, - SPDKProxyImage: eff.SpdkProxyImage, - DataNics: sns.Spec.DataIfname, + InterfaceName: sns.Spec.MgmtIfname, // fleet-only (immutable) + SPDKImage: eff.SpdkImage, // per-node override + SPDKProxyImage: eff.SpdkProxyImage, // per-node override + DataNics: sns.Spec.DataIfname, // fleet-only Namespace: sn.Namespace, - JMPercent: journalManagerPercentPerDevice(sns), - Partitions: utils.IntPtrOrDefault(sns.Spec.Partitions, 1), - HaJMCount: journalManagerCount(sns), + JMPercent: journalManagerPercentPerDeviceFromSpec(eff.JournalManagerSpec), // per-node override + Partitions: utils.IntPtrOrDefault(sns.Spec.Partitions, 1), // fleet-only (immutable) + HaJMCount: journalManagerCountFromSpec(eff.JournalManagerSpec), // per-node override CRName: sns.Name, CRNameSpace: sns.Namespace, CRPlural: "storagenodesets", - Format4K: utils.BoolPtrOrFalse(sns.Spec.ForceFormat4K), - SpdkSystemMemory: eff.SpdkSystemMemory, - FailureDomain: effectiveFailureDomain(sn, sns), + Format4K: utils.BoolPtrOrFalse(sns.Spec.ForceFormat4K), // fleet-only (immutable) + SpdkSystemMemory: eff.SpdkSystemMemory, // per-node override + FailureDomain: effectiveFailureDomain(sn, sns), // per-node override } endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes", clusterUUID) @@ -232,7 +249,6 @@ func (r *StorageNodeReconciler) provisionNode( log.Info("storage node add POST sent", "endpoint", endpoint, "status", status) - // Mark PostedAt so the next reconcile can poll online status. now := metav1.Now() patch := client.MergeFrom(sn.DeepCopy()) sn.Status.PostedAt = &now @@ -242,6 +258,50 @@ func (r *StorageNodeReconciler) provisionNode( return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } +// countInFlightNodes returns how many sibling StorageNode CRs in the same +// StorageNodeSet have PostedAt set but no UUID yet (i.e. add is in progress). +// The calling node is excluded from the count. +func (r *StorageNodeReconciler) countInFlightNodes( + ctx context.Context, + namespace, snsRef, excludeName string, +) (int, error) { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(namespace), + client.MatchingFields{"spec.storageNodeSetRef": snsRef}, + ); err != nil { + return 0, err + } + count := 0 + for _, sn := range snList.Items { + if sn.Name == excludeName { + continue + } + if sn.Status.PostedAt != nil && sn.Status.UUID == "" { + count++ + } + } + return count, nil +} + +// journalManagerPercentPerDeviceFromSpec returns JM percent from the effective +// JournalManagerSpec, defaulting to 3 when nil. +func journalManagerPercentPerDeviceFromSpec(spec *simplyblockv1alpha1.JournalManagerSpec) int { + if spec == nil { + return 3 + } + return utils.IntPtrOrDefault(spec.PercentPerDevice, 3) +} + +// journalManagerCountFromSpec returns JM count from the effective +// JournalManagerSpec, defaulting to 3 when nil. +func journalManagerCountFromSpec(spec *simplyblockv1alpha1.JournalManagerSpec) int { + if spec == nil { + return 3 + } + return utils.IntPtrOrDefault(spec.Count, 3) +} + // syncStatus fetches the current node status from the backend and updates StorageNode.status. func (r *StorageNodeReconciler) syncStatus( ctx context.Context, @@ -502,6 +562,18 @@ func (r *StorageNodeReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &simplyblockv1alpha1.StorageNode{}, + "spec.workerNode", + func(obj client.Object) []string { + sn := obj.(*simplyblockv1alpha1.StorageNode) + return []string{sn.Spec.WorkerNode} + }, + ); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&simplyblockv1alpha1.StorageNode{}). Named("storagenode"). From b65b6a8a6cd054549eccb7b564c21558cc9eb7fc Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 16:47:00 +0100 Subject: [PATCH 09/52] fix: always set socketIndex on StorageNode CRs and remove unused functions from StorageNodeSet reconciler --- .../simplyblockstoragenodeset_controller.go | 36 ++----------------- ...lockstoragenodeset_controller_unit_test.go | 7 ++-- .../simplyblockstoragenodeset_storagenode.go | 11 +++--- .../controller/storagenode_controller.go | 20 +++++------ 4 files changed, 19 insertions(+), 55 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 47f0783b..d496091e 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -200,7 +200,7 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque log.Error(err, "failed to reconcile StorageNode CRs") } - if res, err := r.reconcileWorkerNodes(ctx, req, snCR, clusterUUID, apiClient, expectedPerHost); err != nil || res.RequeueAfter > 0 { + if res, err := r.reconcileWorkerNodes(ctx, snCR, clusterUUID, apiClient, expectedPerHost); err != nil || res.RequeueAfter > 0 { return res, err } @@ -236,7 +236,6 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque // reconcileWorkerNode handles provisioning and online-wait for a single worker node. func (r *StorageNodeSetReconciler) reconcileWorkerNode( ctx context.Context, - req ctrl.Request, snCR *simplyblockv1alpha1.StorageNodeSet, nodeName, clusterUUID string, apiClient *webapi.Client, @@ -829,7 +828,6 @@ func (r *StorageNodeSetReconciler) recordSpdkPodEvents( // always populates it before the CR is stored — it is safe to dereference directly. func (r *StorageNodeSetReconciler) reconcileWorkerNodes( ctx context.Context, - req ctrl.Request, snCR *simplyblockv1alpha1.StorageNodeSet, clusterUUID string, apiClient *webapi.Client, @@ -867,7 +865,7 @@ func (r *StorageNodeSetReconciler) reconcileWorkerNodes( continue } } - res, err := r.reconcileWorkerNode(ctx, req, snCR, nodeName, clusterUUID, apiClient, expectedPerHost) + res, err := r.reconcileWorkerNode(ctx, snCR, nodeName, clusterUUID, apiClient, expectedPerHost) if err != nil { return ctrl.Result{}, err } @@ -883,7 +881,7 @@ func (r *StorageNodeSetReconciler) reconcileWorkerNodes( } for _, nodeName := range sequentialWorkers { - res, err := r.reconcileWorkerNode(ctx, req, snCR, nodeName, clusterUUID, apiClient, expectedPerHost) + res, err := r.reconcileWorkerNode(ctx, snCR, nodeName, clusterUUID, apiClient, expectedPerHost) if err != nil { return ctrl.Result{}, err } @@ -1442,31 +1440,3 @@ func maybeActivateCluster( return nil } - -// nodeFailureDomain returns the failure-domain group for a specific worker node, -// looked up from the StorageNodeSet's NodeFailureDomains map. Returns 0 (omitted) -// when no entry is configured, which the backend interprets as no failure domain. -func nodeFailureDomain(snCR *simplyblockv1alpha1.StorageNodeSet, nodeName string) int { - if snCR.Spec.NodeFailureDomains == nil { - return 0 - } - return int(snCR.Spec.NodeFailureDomains[nodeName]) -} - -func journalManagerPercentPerDevice( - snCR *simplyblockv1alpha1.StorageNodeSet, -) int { - if snCR.Spec.JournalManagerSpec == nil { - return 3 - } - return utils.IntPtrOrDefault(snCR.Spec.JournalManagerSpec.PercentPerDevice, 3) -} - -func journalManagerCount( - snCR *simplyblockv1alpha1.StorageNodeSet, -) int { - if snCR.Spec.JournalManagerSpec == nil { - return 3 - } - return utils.IntPtrOrDefault(snCR.Spec.JournalManagerSpec.Count, 3) -} diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go index 207cef21..2a55c8a9 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go @@ -2234,7 +2234,6 @@ func TestPendingNodeAddsBlocksDuplicatePost(t *testing.T) { r := newStorageNodeSetStateTestReconciler(t, sn, node) res, err := r.reconcileWorkerNode( context.Background(), - ctrl.Request{NamespacedName: client.ObjectKeyFromObject(sn)}, sn, workerName, clusterUUID, webapi.NewClient(srv.URL), 1, ) if err != nil { @@ -2283,7 +2282,6 @@ func TestPendingNodeAddsLegacyPlaceholderBlocksPost(t *testing.T) { r := newStorageNodeSetStateTestReconciler(t, sn, node) _, err := r.reconcileWorkerNode( context.Background(), - ctrl.Request{NamespacedName: client.ObjectKeyFromObject(sn)}, sn, workerName, clusterUUID, webapi.NewClient(srv.URL), 1, ) if err != nil { @@ -2332,10 +2330,9 @@ func TestParallelNodeAddContinuesPastPendingWorker(t *testing.T) { r := newStorageNodeSetStateTestReconciler(t, sn, node1, node2) apiClient := webapi.NewClient(srv.URL) - req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(sn)} // worker-1: pending — must return RequeueAfter without touching worker-2. - res1, err := r.reconcileWorkerNode(context.Background(), req, sn, "worker-1", clusterUUID, apiClient, 1) + res1, err := r.reconcileWorkerNode(context.Background(), sn, "worker-1", clusterUUID, apiClient, 1) if err != nil { t.Fatalf("worker-1: unexpected error: %v", err) } @@ -2352,7 +2349,7 @@ func TestParallelNodeAddContinuesPastPendingWorker(t *testing.T) { // PendingNodeAdds["worker-2"] before attempting the POST. checkNodeInfoReachable // will fail (no real snode API in tests), so the marker is cleared and // RequeueAfter is returned — but worker-2 WAS reached and processed. - res2, err := r.reconcileWorkerNode(context.Background(), req, sn, "worker-2", clusterUUID, apiClient, 1) + res2, err := r.reconcileWorkerNode(context.Background(), sn, "worker-2", clusterUUID, apiClient, 1) if err != nil { t.Fatalf("worker-2: unexpected error: %v", err) } diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 49577a34..74abf56c 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -217,14 +217,11 @@ func buildStorageNodeCR( sns *simplyblockv1alpha1.StorageNodeSet, name, worker, socket string, ) *simplyblockv1alpha1.StorageNode { - var socketIndex *int32 - if socket != "0" && socket != "" { - // Parse the socket string to an int32; default to nil (= socket 0) on failure. - var idx int32 - if n, err := fmt.Sscanf(socket, "%d", &idx); n == 1 && err == nil { - socketIndex = &idx - } + var idx int32 + if socket != "" { + fmt.Sscanf(socket, "%d", &idx) //nolint:errcheck } + socketIndex := &idx sn := &simplyblockv1alpha1.StorageNode{ ObjectMeta: metav1.ObjectMeta{ diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index fbc28104..f874a838 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -221,20 +221,20 @@ func (r *StorageNodeReconciler) provisionNode( nodeAddress := utils.StorageNodeSetAPIAddress(sn.Spec.WorkerNode, sn.Namespace) params := utils.StorageNodeSetAddParams{ NodeAddress: nodeAddress, - InterfaceName: sns.Spec.MgmtIfname, // fleet-only (immutable) - SPDKImage: eff.SpdkImage, // per-node override - SPDKProxyImage: eff.SpdkProxyImage, // per-node override - DataNics: sns.Spec.DataIfname, // fleet-only + InterfaceName: sns.Spec.MgmtIfname, + SPDKImage: eff.SpdkImage, + SPDKProxyImage: eff.SpdkProxyImage, + DataNics: sns.Spec.DataIfname, Namespace: sn.Namespace, - JMPercent: journalManagerPercentPerDeviceFromSpec(eff.JournalManagerSpec), // per-node override - Partitions: utils.IntPtrOrDefault(sns.Spec.Partitions, 1), // fleet-only (immutable) - HaJMCount: journalManagerCountFromSpec(eff.JournalManagerSpec), // per-node override + JMPercent: journalManagerPercentPerDeviceFromSpec(eff.JournalManagerSpec), + Partitions: utils.IntPtrOrDefault(sns.Spec.Partitions, 1), + HaJMCount: journalManagerCountFromSpec(eff.JournalManagerSpec), CRName: sns.Name, CRNameSpace: sns.Namespace, CRPlural: "storagenodesets", - Format4K: utils.BoolPtrOrFalse(sns.Spec.ForceFormat4K), // fleet-only (immutable) - SpdkSystemMemory: eff.SpdkSystemMemory, // per-node override - FailureDomain: effectiveFailureDomain(sn, sns), // per-node override + Format4K: utils.BoolPtrOrFalse(sns.Spec.ForceFormat4K), + SpdkSystemMemory: eff.SpdkSystemMemory, + FailureDomain: effectiveFailureDomain(sn, sns), } endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes", clusterUUID) From 603760e1ad6932ec5d330414644f31e517111e75 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 22:42:20 +0100 Subject: [PATCH 10/52] operator: add Total/Online/Offline printcolumns to StorageNodeSet --- operator/api/v1alpha1/storagenodeset_types.go | 4 ++++ .../storage.simplyblock.io_storagenodesets.yaml | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index b03c7bf3..ac3d4e9e 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -321,6 +321,10 @@ type ActionStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Total",type=integer,JSONPath=".status.totalNodes" +// +kubebuilder:printcolumn:name="Online",type=integer,JSONPath=".status.onlineNodes" +// +kubebuilder:printcolumn:name="Offline",type=integer,JSONPath=".status.offlineNodes" +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp" // +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)",message="all nodeFailureDomains values must be >= 1 (failure-domain group index)" // +operator-sdk:csv:customresourcedefinitions:displayName="Storage Node",resources={{ServiceAccount,v1,simplyblock-storage-node},{Service,v1,simplyblock-storage-node},{DaemonSet,v1,simplyblock-storage-node},{ClusterRole,v1,simplyblock-storage-node},{ClusterRoleBinding,v1,simplyblock-storage-node}} // StorageNodeSet is the Schema for the storagenodesets API diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index 074a89ac..1179bd45 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -14,7 +14,20 @@ spec: singular: storagenodeset scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.totalNodes + name: Total + type: integer + - jsonPath: .status.onlineNodes + name: Online + type: integer + - jsonPath: .status.offlineNodes + name: Offline + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: StorageNodeSet is the Schema for the storagenodesets API From 03c01ce97c66ebd4bf0fe99aa67769292a799775 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 22:50:01 +0100 Subject: [PATCH 11/52] operator: add granular status counts (suspended/creating/removed) to StorageNodeSet with wide column display --- operator/api/v1alpha1/storagenodeset_types.go | 16 +++++++-- ...torage.simplyblock.io_storagenodesets.yaml | 33 ++++++++++++++++--- .../simplyblockstoragenodeset_storagenode.go | 15 +++++++-- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index ac3d4e9e..d3d6074c 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -234,9 +234,18 @@ type StorageNodeSetStatus struct { // OnlineNodes is the count of StorageNode CRs with status "online". // +optional OnlineNodes int `json:"onlineNodes,omitempty"` - // OfflineNodes is the count of StorageNode CRs with status "offline" or "suspended". + // OfflineNodes is the count of StorageNode CRs with status "offline". // +optional OfflineNodes int `json:"offlineNodes,omitempty"` + // SuspendedNodes is the count of StorageNode CRs with status "suspended". + // +optional + SuspendedNodes int `json:"suspendedNodes,omitempty"` + // CreatingNodes is the count of StorageNode CRs with status "in_creation". + // +optional + CreatingNodes int `json:"creatingNodes,omitempty"` + // RemovedNodes is the count of StorageNode CRs with status "removed". + // +optional + RemovedNodes int `json:"removedNodes,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=status,displayName="Nodes" // Nodes is the observed state of each managed storage node. @@ -323,8 +332,11 @@ type ActionStatus struct { // +kubebuilder:subresource:status // +kubebuilder:printcolumn:name="Total",type=integer,JSONPath=".status.totalNodes" // +kubebuilder:printcolumn:name="Online",type=integer,JSONPath=".status.onlineNodes" -// +kubebuilder:printcolumn:name="Offline",type=integer,JSONPath=".status.offlineNodes" // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Offline",type=integer,JSONPath=".status.offlineNodes",priority=1 +// +kubebuilder:printcolumn:name="Suspended",type=integer,JSONPath=".status.suspendedNodes",priority=1 +// +kubebuilder:printcolumn:name="Creating",type=integer,JSONPath=".status.creatingNodes",priority=1 +// +kubebuilder:printcolumn:name="Removed",type=integer,JSONPath=".status.removedNodes",priority=1 // +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)",message="all nodeFailureDomains values must be >= 1 (failure-domain group index)" // +operator-sdk:csv:customresourcedefinitions:displayName="Storage Node",resources={{ServiceAccount,v1,simplyblock-storage-node},{Service,v1,simplyblock-storage-node},{DaemonSet,v1,simplyblock-storage-node},{ClusterRole,v1,simplyblock-storage-node},{ClusterRoleBinding,v1,simplyblock-storage-node}} // StorageNodeSet is the Schema for the storagenodesets API diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index 1179bd45..4e2a62d6 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -21,12 +21,25 @@ spec: - jsonPath: .status.onlineNodes name: Online type: integer - - jsonPath: .status.offlineNodes - name: Offline - type: integer - jsonPath: .metadata.creationTimestamp name: Age type: date + - jsonPath: .status.offlineNodes + name: Offline + priority: 1 + type: integer + - jsonPath: .status.suspendedNodes + name: Suspended + priority: 1 + type: integer + - jsonPath: .status.creatingNodes + name: Creating + priority: 1 + type: integer + - jsonPath: .status.removedNodes + name: Removed + priority: 1 + type: integer name: v1alpha1 schema: openAPIV3Schema: @@ -524,6 +537,10 @@ spec: status: description: status defines the observed state of StorageNodeSet properties: + creatingNodes: + description: CreatingNodes is the count of StorageNode CRs with status + "in_creation". + type: integer drainCoordination: description: DrainCoordination tracks the upgrade-drain state per worker node. @@ -656,7 +673,7 @@ spec: type: array offlineNodes: description: OfflineNodes is the count of StorageNode CRs with status - "offline" or "suspended". + "offline". type: integer onlineNodes: description: OnlineNodes is the count of StorageNode CRs with status @@ -673,6 +690,10 @@ spec: POSTs — it is a separate map field so patches to Status.Nodes never inadvertently delete it. type: object + removedNodes: + description: RemovedNodes is the count of StorageNode CRs with status + "removed". + type: integer schedulingFailedWorkers: additionalProperties: type: boolean @@ -681,6 +702,10 @@ spec: a FailedScheduling event during node add. Used to emit a recovery event when the node subsequently comes online. type: object + suspendedNodes: + description: SuspendedNodes is the count of StorageNode CRs with status + "suspended". + type: integer totalNodes: description: TotalNodes is the total number of owned StorageNode CRs. type: integer diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 74abf56c..390cff4c 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -158,20 +158,29 @@ func (r *StorageNodeSetReconciler) aggregateStorageNodeStatus( return err } - total, online, offline := len(owned.Items), 0, 0 + var online, offline, suspended, creating, removed int for _, sn := range owned.Items { switch sn.Status.Status { case "online": online++ - case "offline", "suspended": + case "offline": offline++ + case "suspended": + suspended++ + case "in_creation": + creating++ + case "removed": + removed++ } } patch := client.MergeFrom(sns.DeepCopy()) - sns.Status.TotalNodes = total + sns.Status.TotalNodes = len(owned.Items) sns.Status.OnlineNodes = online sns.Status.OfflineNodes = offline + sns.Status.SuspendedNodes = suspended + sns.Status.CreatingNodes = creating + sns.Status.RemovedNodes = removed return r.Status().Patch(ctx, sns, patch) } From 20d450b23d3cd5a67644cf7a40bd77d97796ebfe Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Tue, 14 Jul 2026 23:40:44 +0100 Subject: [PATCH 12/52] Ran make operator-build-installer --- operator/dist/install.yaml | 42 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 32ec2697..95b17b42 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -1770,7 +1770,33 @@ spec: singular: storagenodeset scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.totalNodes + name: Total + type: integer + - jsonPath: .status.onlineNodes + name: Online + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.offlineNodes + name: Offline + priority: 1 + type: integer + - jsonPath: .status.suspendedNodes + name: Suspended + priority: 1 + type: integer + - jsonPath: .status.creatingNodes + name: Creating + priority: 1 + type: integer + - jsonPath: .status.removedNodes + name: Removed + priority: 1 + type: integer + name: v1alpha1 schema: openAPIV3Schema: description: StorageNodeSet is the Schema for the storagenodesets API @@ -2267,6 +2293,10 @@ spec: status: description: status defines the observed state of StorageNodeSet properties: + creatingNodes: + description: CreatingNodes is the count of StorageNode CRs with status + "in_creation". + type: integer drainCoordination: description: DrainCoordination tracks the upgrade-drain state per worker node. @@ -2399,7 +2429,7 @@ spec: type: array offlineNodes: description: OfflineNodes is the count of StorageNode CRs with status - "offline" or "suspended". + "offline". type: integer onlineNodes: description: OnlineNodes is the count of StorageNode CRs with status @@ -2416,6 +2446,10 @@ spec: POSTs — it is a separate map field so patches to Status.Nodes never inadvertently delete it. type: object + removedNodes: + description: RemovedNodes is the count of StorageNode CRs with status + "removed". + type: integer schedulingFailedWorkers: additionalProperties: type: boolean @@ -2424,6 +2458,10 @@ spec: a FailedScheduling event during node add. Used to emit a recovery event when the node subsequently comes online. type: object + suspendedNodes: + description: SuspendedNodes is the count of StorageNode CRs with status + "suspended". + type: integer totalNodes: description: TotalNodes is the total number of owned StorageNode CRs. type: integer From 0998dd7f6fdd60eba77b09ee572aa530f98bad12 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 09:17:41 +0100 Subject: [PATCH 13/52] operator: validate nodeConfigs keys match workerNodes and workerNodes has no duplicates --- operator/api/v1alpha1/storagenodeset_types.go | 2 ++ .../crd/bases/storage.simplyblock.io_storagenodesets.yaml | 6 ++++++ operator/dist/install.yaml | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index d3d6074c..e0382767 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -338,6 +338,8 @@ type ActionStatus struct { // +kubebuilder:printcolumn:name="Creating",type=integer,JSONPath=".status.creatingNodes",priority=1 // +kubebuilder:printcolumn:name="Removed",type=integer,JSONPath=".status.removedNodes",priority=1 // +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)",message="all nodeFailureDomains values must be >= 1 (failure-domain group index)" +// +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, w == k))",message="nodeConfigs keys must match a workerNode entry" +// +kubebuilder:validation:XValidation:rule="!has(self.spec.workerNodes) || self.spec.workerNodes.all(w, self.spec.workerNodes.filter(x, x == w).size() == 1)",message="workerNodes must not contain duplicate entries" // +operator-sdk:csv:customresourcedefinitions:displayName="Storage Node",resources={{ServiceAccount,v1,simplyblock-storage-node},{Service,v1,simplyblock-storage-node},{DaemonSet,v1,simplyblock-storage-node},{ClusterRole,v1,simplyblock-storage-node},{ClusterRoleBinding,v1,simplyblock-storage-node}} // StorageNodeSet is the Schema for the storagenodesets API type StorageNodeSet struct { diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index 4e2a62d6..a4eab112 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -718,6 +718,12 @@ spec: index) rule: '!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)' + - message: nodeConfigs keys must match a workerNode entry + rule: '!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, + w == k))' + - message: workerNodes must not contain duplicate entries + rule: '!has(self.spec.workerNodes) || self.spec.workerNodes.all(w, self.spec.workerNodes.filter(x, + x == w).size() == 1)' served: true storage: true subresources: diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 95b17b42..8febab26 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2474,6 +2474,12 @@ spec: index) rule: '!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)' + - message: nodeConfigs keys must match a workerNode entry + rule: '!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, + w == k))' + - message: workerNodes must not contain duplicate entries + rule: '!has(self.spec.workerNodes) || self.spec.workerNodes.all(w, self.spec.workerNodes.filter(x, + x == w).size() == 1)' served: true storage: true subresources: From 9480926c8f5ae57123678f7401d2fd6ed74d8be7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 09:29:05 +0100 Subject: [PATCH 14/52] operator: bound CEL rule cost with MaxItems/MaxProperties on workerNodes and nodeConfigs --- operator/api/v1alpha1/storagenodeset_types.go | 2 ++ .../crd/bases/storage.simplyblock.io_storagenodesets.yaml | 2 ++ operator/dist/install.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index e0382767..3347b829 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -98,6 +98,7 @@ type StorageNodeSetSpec struct { DataIfname []string `json:"dataIfname,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Worker Nodes" // WorkerNodes is the set of Kubernetes worker nodes to manage. + // +kubebuilder:validation:MaxItems=200 WorkerNodes []string `json:"workerNodes,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OpenShift Cluster" // OpenShiftCluster indicates OpenShift-specific behavior should be enabled. @@ -166,6 +167,7 @@ type StorageNodeSetSpec struct { // +optional NodeFailureDomains map[string]int32 `json:"nodeFailureDomains,omitempty"` + // +kubebuilder:validation:MaxProperties=200 // NodeConfigs allows per-worker-node configuration overrides keyed by the // Kubernetes worker node name. Entries are propagated to the corresponding // StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index a4eab112..b375a1c6 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -388,6 +388,7 @@ spec: StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. The StorageNodeSet is the single source of truth for all per-node config, including failure domain assignment via nodeConfigs[worker].failureDomain. + maxProperties: 200 type: object nodeFailureDomains: additionalProperties: @@ -521,6 +522,7 @@ spec: manage. items: type: string + maxItems: 200 type: array required: - clusterName diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 8febab26..20ec1d96 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2144,6 +2144,7 @@ spec: StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. The StorageNodeSet is the single source of truth for all per-node config, including failure domain assignment via nodeConfigs[worker].failureDomain. + maxProperties: 200 type: object nodeFailureDomains: additionalProperties: @@ -2277,6 +2278,7 @@ spec: manage. items: type: string + maxItems: 200 type: array required: - clusterName From 4522e12a07a142651abec86571955359d02ce43d Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 09:35:59 +0100 Subject: [PATCH 15/52] operator: enforce workerNodes uniqueness via listType=set instead of expensive CEL rule --- operator/api/v1alpha1/storagenodeset_types.go | 2 +- .../crd/bases/storage.simplyblock.io_storagenodesets.yaml | 4 +--- operator/dist/install.yaml | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index 3347b829..a25897aa 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -99,6 +99,7 @@ type StorageNodeSetSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Worker Nodes" // WorkerNodes is the set of Kubernetes worker nodes to manage. // +kubebuilder:validation:MaxItems=200 + // +listType=set WorkerNodes []string `json:"workerNodes,omitempty"` // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OpenShift Cluster" // OpenShiftCluster indicates OpenShift-specific behavior should be enabled. @@ -341,7 +342,6 @@ type ActionStatus struct { // +kubebuilder:printcolumn:name="Removed",type=integer,JSONPath=".status.removedNodes",priority=1 // +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)",message="all nodeFailureDomains values must be >= 1 (failure-domain group index)" // +kubebuilder:validation:XValidation:rule="!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, w == k))",message="nodeConfigs keys must match a workerNode entry" -// +kubebuilder:validation:XValidation:rule="!has(self.spec.workerNodes) || self.spec.workerNodes.all(w, self.spec.workerNodes.filter(x, x == w).size() == 1)",message="workerNodes must not contain duplicate entries" // +operator-sdk:csv:customresourcedefinitions:displayName="Storage Node",resources={{ServiceAccount,v1,simplyblock-storage-node},{Service,v1,simplyblock-storage-node},{DaemonSet,v1,simplyblock-storage-node},{ClusterRole,v1,simplyblock-storage-node},{ClusterRoleBinding,v1,simplyblock-storage-node}} // StorageNodeSet is the Schema for the storagenodesets API type StorageNodeSet struct { diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index b375a1c6..aeba2fd9 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -524,6 +524,7 @@ spec: type: string maxItems: 200 type: array + x-kubernetes-list-type: set required: - clusterName type: object @@ -723,9 +724,6 @@ spec: - message: nodeConfigs keys must match a workerNode entry rule: '!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, w == k))' - - message: workerNodes must not contain duplicate entries - rule: '!has(self.spec.workerNodes) || self.spec.workerNodes.all(w, self.spec.workerNodes.filter(x, - x == w).size() == 1)' served: true storage: true subresources: diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 20ec1d96..f48023fc 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2280,6 +2280,7 @@ spec: type: string maxItems: 200 type: array + x-kubernetes-list-type: set required: - clusterName type: object @@ -2479,9 +2480,6 @@ spec: - message: nodeConfigs keys must match a workerNode entry rule: '!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, w == k))' - - message: workerNodes must not contain duplicate entries - rule: '!has(self.spec.workerNodes) || self.spec.workerNodes.all(w, self.spec.workerNodes.filter(x, - x == w).size() == 1)' served: true storage: true subresources: From dc3e22017a3026c7814395cc96d5c171c20e1a09 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 10:03:26 +0100 Subject: [PATCH 16/52] operator: implement per-node ConfigMap and DaemonSet init container for node-specific configuration --- .../simplyblockstoragenodeset_controller.go | 5 + ...lockstoragenodeset_controller_unit_test.go | 12 +- ...simplyblockstoragenodeset_pernodeconfig.go | 187 ++++++++++++++++++ operator/internal/utils/storage_nodeset_ds.go | 115 +++++++---- 4 files changed, 279 insertions(+), 40 deletions(-) create mode 100644 operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index d496091e..27a2915a 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -200,6 +200,11 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque log.Error(err, "failed to reconcile StorageNode CRs") } + // Reconcile per-node ConfigMap used by the DaemonSet init container. + if err := r.reconcilePerNodeConfigMap(ctx, snCR); err != nil { + log.Error(err, "failed to reconcile per-node ConfigMap") + } + if res, err := r.reconcileWorkerNodes(ctx, snCR, clusterUUID, apiClient, expectedPerHost); err != nil || res.RequeueAfter > 0 { return res, err } diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go index 2a55c8a9..98ac5db9 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go @@ -291,10 +291,10 @@ func TestStorageNodeSetDaemonSetReconcileTLSEnabled(t *testing.T) { t.Fatalf("expected projected sources for secret and ca configmap, got secret=%v ca=%v", gotSecret, gotCA) } - if len(ds.Spec.Template.Spec.InitContainers) != 1 { - t.Fatalf("expected single init container") + if len(ds.Spec.Template.Spec.InitContainers) != 2 { + t.Fatalf("expected 2 init containers (node-env-writer + s-node-api-config-generator)") } - checkTLSMounts(t, "init container", ds.Spec.Template.Spec.InitContainers[0].VolumeMounts) + checkTLSMounts(t, "init container", ds.Spec.Template.Spec.InitContainers[1].VolumeMounts) if len(ds.Spec.Template.Spec.Containers) != 1 { t.Fatalf("expected single main container") } @@ -351,10 +351,10 @@ func TestStorageNodeSetDaemonSetReconcileTLSCertManagerProvider(t *testing.T) { t.Fatalf("expected Secret volume referencing simplyblock-storage-node-api-tls, got %#v", tlsVol.Secret) } - if len(ds.Spec.Template.Spec.InitContainers) != 1 { - t.Fatalf("expected single init container") + if len(ds.Spec.Template.Spec.InitContainers) != 2 { + t.Fatalf("expected 2 init containers (node-env-writer + s-node-api-config-generator)") } - checkTLSMounts(t, "init container", ds.Spec.Template.Spec.InitContainers[0].VolumeMounts) + checkTLSMounts(t, "init container", ds.Spec.Template.Spec.InitContainers[1].VolumeMounts) if len(ds.Spec.Template.Spec.Containers) != 1 { t.Fatalf("expected single main container") } diff --git a/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go b/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go new file mode 100644 index 00000000..15402b32 --- /dev/null +++ b/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go @@ -0,0 +1,187 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +// reconcilePerNodeConfigMap creates or updates a single ConfigMap that holds +// per-worker-node effective configuration values. The DaemonSet init container +// mounts this ConfigMap and sources the file matching its hostname so that +// fields like maxLogicalVolumeCount, corePercentage, deviceNames, etc. differ +// per node without requiring a separate DaemonSet per node. +// +// ConfigMap structure: +// +// data: +// vm02.example.com: | +// MAX_LVOL=20 +// MAX_SIZE= +// CORES_PERCENTAGE=50 +// RESERVED_SYSTEM_CPUS=0,1 +// CPU_TOPOLOGY_ENABLED=true +// ... +// vm03.example.com: | +// MAX_LVOL=25 +// ... + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" +) + +// PerNodeConfigMapName returns the name of the per-node ConfigMap for a StorageNodeSet. +func PerNodeConfigMapName(snsName string) string { + return snsName + "-per-node-config" +} + +// reconcilePerNodeConfigMap creates or updates the per-node ConfigMap with the +// effective (fleet defaults merged with nodeConfigs overrides) values for every +// worker in the StorageNodeSet. +func (r *StorageNodeSetReconciler) reconcilePerNodeConfigMap( + ctx context.Context, + sns *simplyblockv1alpha1.StorageNodeSet, +) error { + log := logf.FromContext(ctx) + name := PerNodeConfigMapName(sns.Name) + + data := make(map[string]string, len(sns.Spec.WorkerNodes)) + for _, worker := range sns.Spec.WorkerNodes { + data[worker] = buildPerNodeEnvFile(sns, worker) + } + + var existing corev1.ConfigMap + err := r.Get(ctx, client.ObjectKey{Name: name, Namespace: sns.Namespace}, &existing) + + if apierrors.IsNotFound(err) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: sns.Namespace, + }, + Data: data, + } + if setErr := controllerutil.SetControllerReference(sns, cm, r.Scheme); setErr != nil { + return fmt.Errorf("setting owner reference on per-node ConfigMap: %w", setErr) + } + if createErr := r.Create(ctx, cm); createErr != nil { + return fmt.Errorf("creating per-node ConfigMap: %w", createErr) + } + log.Info("created per-node ConfigMap", "name", name) + return nil + } + if err != nil { + return fmt.Errorf("getting per-node ConfigMap: %w", err) + } + + // Update if data changed. + patch := client.MergeFrom(existing.DeepCopy()) + existing.Data = data + if patchErr := r.Patch(ctx, &existing, patch); patchErr != nil { + return fmt.Errorf("patching per-node ConfigMap: %w", patchErr) + } + return nil +} + +// buildPerNodeEnvFile returns a shell-sourceable env file string with the +// effective per-node values for the given worker, merging fleet defaults from +// the StorageNodeSet spec with any nodeConfigs overrides. +func buildPerNodeEnvFile(sns *simplyblockv1alpha1.StorageNodeSet, worker string) string { + // Start with fleet defaults. + eff := simplyblockv1alpha1.StorageNodeOverrides{ + MaxLogicalVolumeCount: sns.Spec.MaxLogicalVolumeCount, + MaxSize: sns.Spec.MaxSize, + CorePercentage: sns.Spec.CorePercentage, + SpdkSystemMemory: sns.Spec.SpdkSystemMemory, + JournalManagerSpec: sns.Spec.JournalManagerSpec, + PcieAllowList: sns.Spec.PcieAllowList, + PcieDenyList: sns.Spec.PcieDenyList, + PcieModel: sns.Spec.PcieModel, + DriveSizeRange: sns.Spec.DriveSizeRange, + DeviceNames: sns.Spec.DeviceNames, + EnableCpuTopology: sns.Spec.EnableCpuTopology, + ReservedSystemCPU: sns.Spec.ReservedSystemCPU, + } + + // Apply per-node overrides if present. + if o, ok := sns.Spec.NodeConfigs[worker]; ok { + if o.MaxLogicalVolumeCount != nil { + eff.MaxLogicalVolumeCount = o.MaxLogicalVolumeCount + } + if o.MaxSize != "" { + eff.MaxSize = o.MaxSize + } + if o.CorePercentage != nil { + eff.CorePercentage = o.CorePercentage + } + if o.SpdkSystemMemory != "" { + eff.SpdkSystemMemory = o.SpdkSystemMemory + } + if o.JournalManagerSpec != nil { + eff.JournalManagerSpec = o.JournalManagerSpec + } + if len(o.PcieAllowList) > 0 { + eff.PcieAllowList = o.PcieAllowList + } + if len(o.PcieDenyList) > 0 { + eff.PcieDenyList = o.PcieDenyList + } + if o.PcieModel != "" { + eff.PcieModel = o.PcieModel + } + if o.DriveSizeRange != "" { + eff.DriveSizeRange = o.DriveSizeRange + } + if len(o.DeviceNames) > 0 { + eff.DeviceNames = o.DeviceNames + } + if o.EnableCpuTopology != nil { + eff.EnableCpuTopology = o.EnableCpuTopology + } + if o.ReservedSystemCPU != "" { + eff.ReservedSystemCPU = o.ReservedSystemCPU + } + } + + var b strings.Builder + b.WriteString(fmt.Sprintf("MAX_LVOL=%s\n", utils.Int32PtrToString(eff.MaxLogicalVolumeCount))) + b.WriteString(fmt.Sprintf("MAX_SIZE=%s\n", eff.MaxSize)) + b.WriteString(fmt.Sprintf("CORES_PERCENTAGE=%s\n", utils.Int32PtrToString(eff.CorePercentage))) + b.WriteString(fmt.Sprintf("RESERVED_SYSTEM_CPUS=%s\n", eff.ReservedSystemCPU)) + b.WriteString(fmt.Sprintf("CPU_TOPOLOGY_ENABLED=%s\n", utils.BoolPtrToString(eff.EnableCpuTopology))) + b.WriteString(fmt.Sprintf("PCI_ALLOWED=%s\n", strings.Join(eff.PcieAllowList, ","))) + b.WriteString(fmt.Sprintf("PCI_BLOCKED=%s\n", strings.Join(eff.PcieDenyList, ","))) + b.WriteString(fmt.Sprintf("NVME_DEVICES=%s\n", strings.Join(eff.DeviceNames, ","))) + b.WriteString(fmt.Sprintf("DEVICE_MODEL=%s\n", eff.PcieModel)) + b.WriteString(fmt.Sprintf("SIZE_RANGE=%s\n", eff.DriveSizeRange)) + if eff.JournalManagerSpec != nil { + b.WriteString(fmt.Sprintf("JM_PERCENT=%s\n", utils.Int32PtrToString(eff.JournalManagerSpec.PercentPerDevice))) + b.WriteString(fmt.Sprintf("HA_JM_COUNT=%s\n", utils.Int32PtrToString(eff.JournalManagerSpec.Count))) + } else { + b.WriteString("JM_PERCENT=\n") + b.WriteString("HA_JM_COUNT=\n") + } + return b.String() +} diff --git a/operator/internal/utils/storage_nodeset_ds.go b/operator/internal/utils/storage_nodeset_ds.go index d71cbeb9..29f4eee9 100644 --- a/operator/internal/utils/storage_nodeset_ds.go +++ b/operator/internal/utils/storage_nodeset_ds.go @@ -50,38 +50,33 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna } image := sn.Spec.ClusterImage - initCmd := []string{ - "sudo", "-E", - "python3", - "simplyblock_web/node_configure.py", - "--max-lvol=" + Int32PtrToString(sn.Spec.MaxLogicalVolumeCount), - "--max-size=" + sn.Spec.MaxSize, - } - if len(sn.Spec.PcieAllowList) > 0 { - initCmd = append(initCmd, "--pci-allowed="+JoinList(sn.Spec.PcieAllowList)) - } - if len(sn.Spec.PcieDenyList) > 0 { - initCmd = append(initCmd, "--pci-blocked="+JoinList(sn.Spec.PcieDenyList)) - } - if len(sn.Spec.DeviceNames) > 0 { - initCmd = append(initCmd, "--nvme-devices="+JoinList(sn.Spec.DeviceNames)) - } + // Build the fleet-level (non-overridable) args that are always appended. + // Per-node args (max-lvol, cores-percentage, pci-*, device-*, size-range) + // are read at runtime from the per-node ConfigMap via the init script. + fleetArgs := "" if len(sn.Spec.SocketsToUse) > 0 { - initCmd = append(initCmd, "--sockets-to-use="+JoinList(sn.Spec.SocketsToUse)) + fleetArgs += " --sockets-to-use=" + JoinList(sn.Spec.SocketsToUse) } if sn.Spec.NodesPerSocket != nil { - initCmd = append(initCmd, "--nodes-per-socket="+Int32PtrToString(sn.Spec.NodesPerSocket)) - } - if sn.Spec.PcieModel != "" { - initCmd = append(initCmd, "--device-model="+sn.Spec.PcieModel) - } - if sn.Spec.DriveSizeRange != "" { - initCmd = append(initCmd, "--size-range="+sn.Spec.DriveSizeRange) - } - if sn.Spec.CorePercentage != nil { - initCmd = append(initCmd, "--cores-percentage="+Int32PtrToString(sn.Spec.CorePercentage)) - } + fleetArgs += " --nodes-per-socket=" + Int32PtrToString(sn.Spec.NodesPerSocket) + } + + // The init container sources the per-node env file (written by node-env-writer) + // so that node_configure.py receives per-node values for each pod. + initScript := `set -e +[ -f /etc/node-env/env.sh ] && . /etc/node-env/env.sh +ARGS="--max-lvol=${MAX_LVOL:-0} --max-size=${MAX_SIZE:-}" +[ -n "${CORES_PERCENTAGE}" ] && ARGS="${ARGS} --cores-percentage=${CORES_PERCENTAGE}" +[ -n "${PCI_ALLOWED}" ] && ARGS="${ARGS} --pci-allowed=${PCI_ALLOWED}" +[ -n "${PCI_BLOCKED}" ] && ARGS="${ARGS} --pci-blocked=${PCI_BLOCKED}" +[ -n "${NVME_DEVICES}" ] && ARGS="${ARGS} --nvme-devices=${NVME_DEVICES}" +[ -n "${DEVICE_MODEL}" ] && ARGS="${ARGS} --device-model=${DEVICE_MODEL}" +[ -n "${SIZE_RANGE}" ] && ARGS="${ARGS} --size-range=${SIZE_RANGE}" +ARGS="${ARGS}` + fleetArgs + `" +eval sudo -E python3 simplyblock_web/node_configure.py ${ARGS} +` + initCmd := []string{"sh", "-c", initScript} imagePullPolicy := sn.Spec.ImagePullPolicy if imagePullPolicy == "" { @@ -91,22 +86,20 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna mainEnv := []corev1.EnvVar{ {Name: "UBUNTU_HOST", Value: BoolPtrToString(sn.Spec.UbuntuHost)}, {Name: "OPENSHIFT_CLUSTER", Value: BoolPtrToString(sn.Spec.OpenShiftCluster)}, - {Name: "CPU_TOPOLOGY_ENABLED", Value: BoolPtrToString(sn.Spec.EnableCpuTopology)}, {Name: "SKIP_KUBELET_CONFIGURATION", Value: BoolPtrToString(sn.Spec.SkipKubeletConfiguration)}, {Name: "SIMPLY_BLOCK_DOCKER_IMAGE", Value: image}, {Name: "HOSTNAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }}, } + // CPU_TOPOLOGY_ENABLED and RESERVED_SYSTEM_CPUS are now per-node: sourced + // from /etc/node-env/env.sh at container start via the command wrapper. if sn.Spec.MaxParallelNodeAdds != nil { mainEnv = append(mainEnv, corev1.EnvVar{Name: "MAX_PARALLEL_NODE_ADDS", Value: fmt.Sprintf("%d", *sn.Spec.MaxParallelNodeAdds)}) } if sn.Spec.OpenShiftMachineConfigPool != "" { mainEnv = append(mainEnv, corev1.EnvVar{Name: "OPENSHIFT_MCP", Value: sn.Spec.OpenShiftMachineConfigPool}) } - if sn.Spec.ReservedSystemCPU != "" { - mainEnv = append(mainEnv, corev1.EnvVar{Name: "RESERVED_SYSTEM_CPUS", Value: sn.Spec.ReservedSystemCPU}) - } if tlsMutualEnabled { mainEnv = append(mainEnv, corev1.EnvVar{Name: "SB_TLS_SERVE", Value: "true"}, @@ -123,7 +116,26 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna ) } + perNodeConfigMapName := sn.Name + "-per-node-config" + volumes := []corev1.Volume{ + // per-node-config: ConfigMap with one key per worker hostname. + // Written by reconcilePerNodeConfigMap; read by node-env-writer init container. + { + Name: "per-node-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: perNodeConfigMapName}, + Optional: BoolPtr(true), + }, + }, + }, + // node-env: emptyDir shared between init containers and the main container. + // node-env-writer writes /etc/node-env/env.sh; others source it. + { + Name: "node-env", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, { Name: "dev-vol", VolumeSource: corev1.VolumeSource{ @@ -175,10 +187,13 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna }, } + nodeEnvMount := corev1.VolumeMount{Name: "node-env", MountPath: "/etc/node-env"} + initMounts := []corev1.VolumeMount{ {Name: "etc-simplyblock", MountPath: "/etc/simplyblock"}, {Name: "host-modules", MountPath: "/lib/modules", ReadOnly: true}, {Name: "host-mnt", MountPath: "/mnt"}, + nodeEnvMount, } mainMounts := []corev1.VolumeMount{ @@ -186,6 +201,7 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna {Name: "etc-simplyblock", MountPath: "/etc/simplyblock"}, {Name: "host-sys", MountPath: "/sys"}, {Name: "var-run-simplyblock", MountPath: "/var/run/simplyblock"}, + nodeEnvMount, } readinessProbe := &corev1.Probe{ @@ -263,6 +279,33 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna Volumes: volumes, InitContainers: []corev1.Container{ + // node-env-writer: copies the per-node env file from the + // ConfigMap (keyed by hostname) into the shared node-env volume + // so both the config-generator and the main container can source it. + { + Name: "node-env-writer", + Image: "busybox:1.36", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"sh", "-c", + `mkdir -p /etc/node-env +if [ -f /etc/per-node-config/${HOSTNAME} ]; then + cp /etc/per-node-config/${HOSTNAME} /etc/node-env/env.sh +else + touch /etc/node-env/env.sh +fi`, + }, + Env: []corev1.EnvVar{ + {Name: "HOSTNAME", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "per-node-config", MountPath: "/etc/per-node-config", ReadOnly: true}, + nodeEnvMount, + }, + }, + // s-node-api-config-generator: runs node_configure.py with + // per-node values sourced from /etc/node-env/env.sh. { Name: "s-node-api-config-generator", Image: image, @@ -284,8 +327,12 @@ func BuildStorageNodeSetDaemonSet(sn *simplyblockv1alpha1.StorageNodeSet, tlsEna Name: "s-node-api-container", Image: image, ImagePullPolicy: imagePullPolicy, - Command: []string{ - "sudo", "-E", "python3", "simplyblock_web/node_webapp.py", "storage_node_k8s", + // Source the per-node env file so that CPU_TOPOLOGY_ENABLED + // and RESERVED_SYSTEM_CPUS pick up per-node values before + // the main process starts. + Command: []string{"sh", "-c", + `[ -f /etc/node-env/env.sh ] && . /etc/node-env/env.sh +exec sudo -E python3 simplyblock_web/node_webapp.py storage_node_k8s`, }, SecurityContext: &corev1.SecurityContext{Privileged: BoolPtr(true)}, Resources: effectiveResources(sn.Spec.ContainerResources, defaultContainerResources), From 60fa7aa28c6bb488fe64a195440a1a743fc352de Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 10:07:56 +0100 Subject: [PATCH 17/52] fix: replace WriteString(fmt.Sprintf) with fmt.Fprintf in buildPerNodeEnvFile --- ...simplyblockstoragenodeset_pernodeconfig.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go b/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go index 15402b32..73b29f31 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go +++ b/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go @@ -166,19 +166,19 @@ func buildPerNodeEnvFile(sns *simplyblockv1alpha1.StorageNodeSet, worker string) } var b strings.Builder - b.WriteString(fmt.Sprintf("MAX_LVOL=%s\n", utils.Int32PtrToString(eff.MaxLogicalVolumeCount))) - b.WriteString(fmt.Sprintf("MAX_SIZE=%s\n", eff.MaxSize)) - b.WriteString(fmt.Sprintf("CORES_PERCENTAGE=%s\n", utils.Int32PtrToString(eff.CorePercentage))) - b.WriteString(fmt.Sprintf("RESERVED_SYSTEM_CPUS=%s\n", eff.ReservedSystemCPU)) - b.WriteString(fmt.Sprintf("CPU_TOPOLOGY_ENABLED=%s\n", utils.BoolPtrToString(eff.EnableCpuTopology))) - b.WriteString(fmt.Sprintf("PCI_ALLOWED=%s\n", strings.Join(eff.PcieAllowList, ","))) - b.WriteString(fmt.Sprintf("PCI_BLOCKED=%s\n", strings.Join(eff.PcieDenyList, ","))) - b.WriteString(fmt.Sprintf("NVME_DEVICES=%s\n", strings.Join(eff.DeviceNames, ","))) - b.WriteString(fmt.Sprintf("DEVICE_MODEL=%s\n", eff.PcieModel)) - b.WriteString(fmt.Sprintf("SIZE_RANGE=%s\n", eff.DriveSizeRange)) + fmt.Fprintf(&b, "MAX_LVOL=%s\n", utils.Int32PtrToString(eff.MaxLogicalVolumeCount)) + fmt.Fprintf(&b, "MAX_SIZE=%s\n", eff.MaxSize) + fmt.Fprintf(&b, "CORES_PERCENTAGE=%s\n", utils.Int32PtrToString(eff.CorePercentage)) + fmt.Fprintf(&b, "RESERVED_SYSTEM_CPUS=%s\n", eff.ReservedSystemCPU) + fmt.Fprintf(&b, "CPU_TOPOLOGY_ENABLED=%s\n", utils.BoolPtrToString(eff.EnableCpuTopology)) + fmt.Fprintf(&b, "PCI_ALLOWED=%s\n", strings.Join(eff.PcieAllowList, ",")) + fmt.Fprintf(&b, "PCI_BLOCKED=%s\n", strings.Join(eff.PcieDenyList, ",")) + fmt.Fprintf(&b, "NVME_DEVICES=%s\n", strings.Join(eff.DeviceNames, ",")) + fmt.Fprintf(&b, "DEVICE_MODEL=%s\n", eff.PcieModel) + fmt.Fprintf(&b, "SIZE_RANGE=%s\n", eff.DriveSizeRange) if eff.JournalManagerSpec != nil { - b.WriteString(fmt.Sprintf("JM_PERCENT=%s\n", utils.Int32PtrToString(eff.JournalManagerSpec.PercentPerDevice))) - b.WriteString(fmt.Sprintf("HA_JM_COUNT=%s\n", utils.Int32PtrToString(eff.JournalManagerSpec.Count))) + fmt.Fprintf(&b, "JM_PERCENT=%s\n", utils.Int32PtrToString(eff.JournalManagerSpec.PercentPerDevice)) + fmt.Fprintf(&b, "HA_JM_COUNT=%s\n", utils.Int32PtrToString(eff.JournalManagerSpec.Count)) } else { b.WriteString("JM_PERCENT=\n") b.WriteString("HA_JM_COUNT=\n") From d40c13cb44664931c96a44479dee34318968348c Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 10:27:01 +0100 Subject: [PATCH 18/52] refactor: extract node-env-writer shell script into named variable for consistency --- operator/internal/utils/storage_nodeset_ds.go | 24 +++++++++++-------- .../internal/utils/storage_nodeset_ds_test.go | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/operator/internal/utils/storage_nodeset_ds.go b/operator/internal/utils/storage_nodeset_ds.go index 29f4eee9..396602dd 100644 --- a/operator/internal/utils/storage_nodeset_ds.go +++ b/operator/internal/utils/storage_nodeset_ds.go @@ -78,6 +78,16 @@ eval sudo -E python3 simplyblock_web/node_configure.py ${ARGS} ` initCmd := []string{"sh", "-c", initScript} + // nodeEnvWriterScript copies the per-node env file from the mounted ConfigMap + // (keyed by hostname) into the shared node-env emptyDir volume. + nodeEnvWriterScript := `mkdir -p /etc/node-env +if [ -f /etc/per-node-config/${HOSTNAME} ]; then + cp /etc/per-node-config/${HOSTNAME} /etc/node-env/env.sh +else + touch /etc/node-env/env.sh +fi` + nodeEnvWriterCmd := []string{"sh", "-c", nodeEnvWriterScript} + imagePullPolicy := sn.Spec.ImagePullPolicy if imagePullPolicy == "" { imagePullPolicy = corev1.PullAlways @@ -284,21 +294,15 @@ eval sudo -E python3 simplyblock_web/node_configure.py ${ARGS} // so both the config-generator and the main container can source it. { Name: "node-env-writer", - Image: "busybox:1.36", - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"sh", "-c", - `mkdir -p /etc/node-env -if [ -f /etc/per-node-config/${HOSTNAME} ]; then - cp /etc/per-node-config/${HOSTNAME} /etc/node-env/env.sh -else - touch /etc/node-env/env.sh -fi`, - }, + Image: image, + ImagePullPolicy: imagePullPolicy, + Command: nodeEnvWriterCmd, Env: []corev1.EnvVar{ {Name: "HOSTNAME", ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, }}, }, + Resources: effectiveResources(sn.Spec.InitContainerResources, defaultInitContainerResources), VolumeMounts: []corev1.VolumeMount{ {Name: "per-node-config", MountPath: "/etc/per-node-config", ReadOnly: true}, nodeEnvMount, diff --git a/operator/internal/utils/storage_nodeset_ds_test.go b/operator/internal/utils/storage_nodeset_ds_test.go index 7fbf1c7e..1447053f 100644 --- a/operator/internal/utils/storage_nodeset_ds_test.go +++ b/operator/internal/utils/storage_nodeset_ds_test.go @@ -109,7 +109,7 @@ func TestBuildStorageNodeSetDaemonSetUserResourcesOverrideDefaults(t *testing.T) t.Errorf("main container: expected user memory limit 4Gi, got %v", mainMem.String()) } - init := ds.Spec.Template.Spec.InitContainers[0] + init := ds.Spec.Template.Spec.InitContainers[1] // [0]=node-env-writer, [1]=s-node-api-config-generator initMem := init.Resources.Limits[corev1.ResourceMemory] if initMem.String() != "128Mi" { t.Errorf("init container: expected user memory limit 128Mi, got %v", initMem.String()) From 8090876565e43f21446416e9eaa7b34f649d933d Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 11:05:29 +0100 Subject: [PATCH 19/52] fix: create per-node ConfigMap before DaemonSet to prevent empty env on pod start --- .../controller/simplyblockstoragenodeset_controller.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 27a2915a..cc493cc0 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -179,6 +179,12 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } + // Reconcile per-node ConfigMap BEFORE the DaemonSet so that pods never + // start without the ConfigMap already present. + if err := r.reconcilePerNodeConfigMap(ctx, snCR); err != nil { + log.Error(err, "failed to reconcile per-node ConfigMap") + } + if err := r.reconcileDaemonSet(ctx, snCR); err != nil { return ctrl.Result{}, err } @@ -200,10 +206,6 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque log.Error(err, "failed to reconcile StorageNode CRs") } - // Reconcile per-node ConfigMap used by the DaemonSet init container. - if err := r.reconcilePerNodeConfigMap(ctx, snCR); err != nil { - log.Error(err, "failed to reconcile per-node ConfigMap") - } if res, err := r.reconcileWorkerNodes(ctx, snCR, clusterUUID, apiClient, expectedPerHost); err != nil || res.RequeueAfter > 0 { return res, err From a5de793c04fbc4aa1ab42e4f7f3cbf13be45bc9b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 11:08:21 +0100 Subject: [PATCH 20/52] formatted file internal/controller/simplyblockstoragenodeset_controller.go --- .../internal/controller/simplyblockstoragenodeset_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index cc493cc0..6c406b63 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -206,7 +206,6 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque log.Error(err, "failed to reconcile StorageNode CRs") } - if res, err := r.reconcileWorkerNodes(ctx, snCR, clusterUUID, apiClient, expectedPerHost); err != nil || res.RequeueAfter > 0 { return res, err } From 1fd41bece562af4c37b9e94c8b9605feb06decbc Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 11:32:00 +0100 Subject: [PATCH 21/52] operator: mirror worker-specific events from StorageNodeOps and StorageNodeSet onto StorageNode CR --- .../simplyblockstoragenodeset_controller.go | 3 +++ .../simplyblockstoragenodeset_storagenode.go | 23 +++++++++++++++++ .../controller/storagenodeops_controller.go | 25 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 6c406b63..78370598 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -815,6 +815,7 @@ func (r *StorageNodeSetReconciler) recordSpdkPodEvents( r.Recorder.Eventf(snCR, corev1.EventTypeWarning, latest.Reason, "worker %s: %s", nodeName, latest.Message) + r.emitOnStorageNodeForWorker(ctx, snCR, nodeName, corev1.EventTypeWarning, latest.Reason, latest.Message) // Persist the flag so the recovery event is emitted correctly even if the // operator restarts before the node comes online. @@ -1296,6 +1297,8 @@ func onAllSocketNodesOnline( if snCR.Status.SchedulingFailedWorkers[nodeName] { r.Recorder.Eventf(snCR, corev1.EventTypeNormal, "NodeOnline", "worker %s: SPDK pod is now online after previous scheduling failure", nodeName) + r.emitOnStorageNodeForWorker(ctx, snCR, nodeName, corev1.EventTypeNormal, "NodeOnline", + fmt.Sprintf("SPDK pod is now online after previous scheduling failure on %s", nodeName)) delete(snCR.Status.SchedulingFailedWorkers, nodeName) changed = true } diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 390cff4c..9ddc551a 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -255,6 +255,29 @@ func buildStorageNodeCR( return sn } +// emitOnStorageNodeForWorker emits an event on the StorageNode CR for the given +// worker, mirroring events that are emitted on the StorageNodeSet. +func (r *StorageNodeSetReconciler) emitOnStorageNodeForWorker( + ctx context.Context, + sns *simplyblockv1alpha1.StorageNodeSet, + workerNode string, + eventType, reason, message string, +) { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(sns.Namespace), + client.MatchingFields{"spec.workerNode": workerNode}, + ); err != nil { + return + } + for i := range snList.Items { + if snList.Items[i].Spec.StorageNodeSetRef == sns.Name { + r.Recorder.Event(&snList.Items[i], eventType, reason, message) + return + } + } +} + // storageNodePostedAt returns the PostedAt timestamp from the StorageNode CR // for the given worker, or nil if not found / not yet set. func (r *StorageNodeSetReconciler) storageNodePostedAt( diff --git a/operator/internal/controller/storagenodeops_controller.go b/operator/internal/controller/storagenodeops_controller.go index 85c7bda1..4f192b51 100644 --- a/operator/internal/controller/storagenodeops_controller.go +++ b/operator/internal/controller/storagenodeops_controller.go @@ -296,6 +296,7 @@ func (r *StorageNodeOpsReconciler) drainValidate( r.Recorder.Eventf(ops, corev1.EventTypeWarning, "PinnedVolumeBlocking", "drain blocked: %d pinned volume(s) on node %s — remove the %s annotation to proceed", len(pinned), nodeUUID, simplyblockv1alpha1.AnnotationPinnedVolume) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "PinnedVolumeBlocking", fmt.Sprintf("drain blocked: %d pinned volume(s) on node %s — remove the %s annotation to proceed", len(pinned), nodeUUID, simplyblockv1alpha1.AnnotationPinnedVolume)) patch := client.MergeFrom(ops.DeepCopy()) ops.Status.Message = fmt.Sprintf("blocked: %d pinned volume(s) — remove simplyblock.io/pinned-volume annotation", len(pinned)) _ = r.Status().Patch(ctx, ops, patch) @@ -306,6 +307,7 @@ func (r *StorageNodeOpsReconciler) drainValidate( r.Recorder.Eventf(ops, corev1.EventTypeWarning, "UnmanagedVolumeBlocking", "drain blocked: %d unmanaged volume(s) on node %s — remove them manually", len(unmanaged), nodeUUID) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "UnmanagedVolumeBlocking", fmt.Sprintf("drain blocked: %d unmanaged volume(s) on node %s — remove them manually", len(unmanaged), nodeUUID)) patch := client.MergeFrom(ops.DeepCopy()) ops.Status.Message = fmt.Sprintf("blocked: %d unmanaged volume(s) — remove manually", len(unmanaged)) _ = r.Status().Patch(ctx, ops, patch) @@ -374,6 +376,7 @@ func (r *StorageNodeOpsReconciler) drainSuspend( if nodeResp.Status != utils.NodeStatusSuspended { r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainSuspendPending", "waiting for node %s to suspend (current status: %s)", nodeUUID, nodeResp.Status) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "DrainSuspendPending", fmt.Sprintf("waiting for node %s to suspend (current status: %s)", nodeUUID, nodeResp.Status)) return ctrl.Result{RequeueAfter: drainRequeueSuspend}, nil } return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseMigrating) @@ -435,6 +438,7 @@ func (r *StorageNodeOpsReconciler) drainMigrate( } r.Recorder.Eventf(ops, corev1.EventTypeNormal, "MigrationCompleted", "all %d volume migrations completed", completed) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeNormal, "MigrationCompleted", fmt.Sprintf("all %d volume migrations completed", completed)) return r.advanceSubPhase(ctx, ops, simplyblockv1alpha1.StorageNodeOpsSubPhaseVerifying) } @@ -482,6 +486,7 @@ func (r *StorageNodeOpsReconciler) handleFailedVolumeMigrations( } r.Recorder.Eventf(ops, corev1.EventTypeWarning, "MigrationRetry", "VolumeMigration %s failed, deleted and will retry with new target", vm.Name) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "MigrationRetry", fmt.Sprintf("VolumeMigration %s failed, deleted and will retry with new target", vm.Name)) } return ctrl.Result{RequeueAfter: drainRequeueImmediate}, true } @@ -571,6 +576,7 @@ func (r *StorageNodeOpsReconciler) createMissingVolumeMigrationsOps( log.Error(err, "drain: no available target nodes for migration") r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainNoMigrationTarget", "drain stalled: no online storage node available as migration target for node %s", nodeUUID) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "DrainNoMigrationTarget", fmt.Sprintf("drain stalled: no online storage node available as migration target for node %s", nodeUUID)) return ctrl.Result{RequeueAfter: drainRequeueMigrateNew}, nil } @@ -648,6 +654,7 @@ func (r *StorageNodeOpsReconciler) drainVerify( r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainVerifyPending", "node %s still has %d non-system volume(s) after migration; waiting for backend to confirm empty", nodeUUID, len(nonSystem)) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "DrainVerifyPending", fmt.Sprintf("node %s still has %d non-system volume(s) after migration; waiting for backend to confirm empty", nodeUUID, len(nonSystem))) return ctrl.Result{RequeueAfter: drainRequeueVerify}, nil } @@ -704,6 +711,7 @@ func (r *StorageNodeOpsReconciler) drainRemove( if err == nil && (status == http.StatusOK || status == http.StatusNoContent || status == http.StatusNotFound) { r.Recorder.Eventf(ops, corev1.EventTypeNormal, "NodeRemoved", "storage node %s removed successfully", nodeUUID) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeNormal, "NodeRemoved", fmt.Sprintf("storage node %s removed successfully", nodeUUID)) return r.succeedOps(ctx, ops, sn) } @@ -738,6 +746,7 @@ func (r *StorageNodeOpsReconciler) resumeAndFail( } r.Recorder.Eventf(ops, corev1.EventTypeWarning, "NodeResumed", "drain failed, attempted resume of node %s: %s", nodeUUID, reason) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "NodeResumed", fmt.Sprintf("drain failed, attempted resume of node %s: %s", nodeUUID, reason)) return r.failOps(ctx, ops, reason) } @@ -781,6 +790,7 @@ func (r *StorageNodeOpsReconciler) clusterPauseCheck( _ = r.Status().Patch(ctx, ops, patch) r.Recorder.Eventf(ops, corev1.EventTypeWarning, "DrainPaused", "drain paused: %s — will resume when cluster is active", reason) + r.emitOnStorageNode(ctx, ops, corev1.EventTypeWarning, "DrainPaused", fmt.Sprintf("drain paused: %s — will resume when cluster is active", reason)) log.Info("drain: pausing — cluster not ready", "reason", reason) return ctrl.Result{RequeueAfter: 60 * time.Second}, true } @@ -827,6 +837,7 @@ func (r *StorageNodeOpsReconciler) failOps( log := logf.FromContext(ctx) log.Error(nil, "ops failed", "ops", ops.Name, "reason", reason) r.Recorder.Event(ops, "Warning", "OpsFailed", reason) + r.emitOnStorageNode(ctx, ops, "Warning", "OpsFailed", reason) now := metav1.Now() patch := client.MergeFrom(ops.DeepCopy()) @@ -848,6 +859,20 @@ func (r *StorageNodeOpsReconciler) failOps( return ctrl.Result{}, nil } +// emitOnStorageNode emits an event on the StorageNode that this ops targets, +// mirroring events that are also emitted on the StorageNodeOps CR itself. +func (r *StorageNodeOpsReconciler) emitOnStorageNode( + ctx context.Context, + ops *simplyblockv1alpha1.StorageNodeOps, + eventType, reason, message string, +) { + var sn simplyblockv1alpha1.StorageNode + if err := r.Get(ctx, types.NamespacedName{Name: ops.Spec.StorageNodeRef, Namespace: ops.Namespace}, &sn); err != nil { + return + } + r.Recorder.Event(&sn, eventType, reason, message) +} + // releaseLock clears StorageNode.status.activeOpsRef if it still points to opsName. func (r *StorageNodeOpsReconciler) releaseLock( ctx context.Context, From d89dba5bee89ab21915732e55b33fefb4073e0e4 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 12:14:10 +0100 Subject: [PATCH 22/52] operator: support manually created StorageNode CRs referencing an existing StorageNodeSet --- ...simplyblockstoragenodeset_pernodeconfig.go | 24 +++++++++++++++++++ .../simplyblockstoragenodeset_storagenode.go | 8 +++++++ 2 files changed, 32 insertions(+) diff --git a/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go b/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go index 73b29f31..cd211eec 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go +++ b/operator/internal/controller/simplyblockstoragenodeset_pernodeconfig.go @@ -72,6 +72,30 @@ func (r *StorageNodeSetReconciler) reconcilePerNodeConfigMap( data[worker] = buildPerNodeEnvFile(sns, worker) } + // Also include manually created StorageNode CRs that reference this StorageNodeSet + // but whose worker is not in spec.workerNodes. Their overrides are merged with + // fleet defaults so the DaemonSet init container gets the right per-node config. + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(sns.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": sns.Name}, + ); err == nil { + for _, sn := range snList.Items { + if _, ok := data[sn.Spec.WorkerNode]; ok { + continue // already covered by spec.workerNodes + } + // Manually created: use its overrides on top of fleet defaults. + snsCopy := sns.DeepCopy() + if sn.Spec.Overrides != nil { + if snsCopy.Spec.NodeConfigs == nil { + snsCopy.Spec.NodeConfigs = make(map[string]simplyblockv1alpha1.StorageNodeOverrides) + } + snsCopy.Spec.NodeConfigs[sn.Spec.WorkerNode] = *sn.Spec.Overrides + } + data[sn.Spec.WorkerNode] = buildPerNodeEnvFile(snsCopy, sn.Spec.WorkerNode) + } + } + var existing corev1.ConfigMap err := r.Get(ctx, client.ObjectKey{Name: name, Namespace: sns.Namespace}, &existing) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 9ddc551a..e2e46d6d 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -70,12 +70,20 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( } // Delete stale CRs (worker removed from spec.workerNodes or socket removed). + // Manually created StorageNode CRs (no controller OwnerReference pointing to + // this StorageNodeSet) are preserved — they can reference the fleet config + // without being listed in spec.workerNodes. for i := range owned.Items { sn := &owned.Items[i] key := workerSocket{sn.Spec.WorkerNode, socketLabel(sn.Spec.SocketIndex)} if _, ok := expected[key]; ok { continue } + // Skip if this CR was not created by the operator (no controller owner). + owner := metav1.GetControllerOf(sn) + if owner == nil || owner.Name != sns.Name { + continue + } if err := r.Delete(ctx, sn); err != nil && !apierrors.IsNotFound(err) { log.Error(err, "failed to delete stale StorageNode CR", "name", sn.Name) } else { From e6e79612c16587467de3f69696d0c38c837fc6ea Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 12:31:57 +0100 Subject: [PATCH 23/52] operator: label manually created StorageNode worker nodes so DaemonSet schedules pods on them --- .../simplyblockstoragenodeset_controller.go | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 78370598..65844df0 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -421,7 +421,27 @@ func (r *StorageNodeSetReconciler) ensureFinalizer( } func (r *StorageNodeSetReconciler) labelWorkerNodes(ctx context.Context, sn *simplyblockv1alpha1.StorageNodeSet) error { - for _, nodeName := range sn.Spec.WorkerNodes { + // Collect all workers: spec.workerNodes plus any manually created StorageNode CRs + // that reference this StorageNodeSet but are not in spec.workerNodes. + workers := make(map[string]struct{}, len(sn.Spec.WorkerNodes)) + for _, w := range sn.Spec.WorkerNodes { + workers[w] = struct{}{} + } + + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(sn.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": sn.Name}, + ); err == nil { + for _, snCR := range snList.Items { + workers[snCR.Spec.WorkerNode] = struct{}{} + } + } + + key := "io.simplyblock.node-type" + value := "simplyblock-storage-plane-" + sn.Spec.ClusterName + + for nodeName := range workers { var node corev1.Node if err := r.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { return err @@ -431,9 +451,6 @@ func (r *StorageNodeSetReconciler) labelWorkerNodes(ctx context.Context, sn *sim node.Labels = map[string]string{} } - key := "io.simplyblock.node-type" - value := "simplyblock-storage-plane-" + sn.Spec.ClusterName - if node.Labels[key] == value { continue } From 81f12baa1c0b194d249ab5740e33f440a463eec0 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 12:50:49 +0100 Subject: [PATCH 24/52] operator: include manually created StorageNode workers in EndpointSlice for DNS resolution --- .../simplyblockstoragenodeset_controller.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 65844df0..c79c1783 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -608,6 +608,7 @@ func (r *StorageNodeSetReconciler) reconcileEndpointSlice( ) error { log := logf.FromContext(ctx) + // Start with workers from spec.workerNodes. nodeIPs := make(map[string]string) for _, nodeName := range snCR.Spec.WorkerNodes { ip, err := getNodeInternalIP(ctx, r.Client, nodeName) @@ -618,6 +619,26 @@ func (r *StorageNodeSetReconciler) reconcileEndpointSlice( nodeIPs[nodeName] = ip } + // Also include workers from manually created StorageNode CRs so their + // per-node DNS hostname resolves and checkNodeInfoReachable succeeds. + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(snCR.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": snCR.Name}, + ); err == nil { + for _, sn := range snList.Items { + if _, ok := nodeIPs[sn.Spec.WorkerNode]; ok { + continue // already covered + } + ip, err := getNodeInternalIP(ctx, r.Client, sn.Spec.WorkerNode) + if err != nil { + log.Error(err, "failed to get IP for manual StorageNode worker, skipping", "worker", sn.Spec.WorkerNode) + continue + } + nodeIPs[sn.Spec.WorkerNode] = ip + } + } + return r.applyStorageNodeSetEndpointSlice(ctx, snCR, nodeIPs) } From 8c6b4194e1956ee4ed255d642f18fb2b9de71ad3 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 13:29:59 +0100 Subject: [PATCH 25/52] operator: poll backend by IP/socket to resolve UUID for manually created StorageNode CRs and surface them in StorageNodeSet status --- .../simplyblockstoragenodeset_controller.go | 6 + .../simplyblockstoragenodeset_storagenode.go | 78 +++++++++++++ .../controller/storagenode_controller.go | 105 +++++++++++++++++- 3 files changed, 186 insertions(+), 3 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index c79c1783..c7fe6bb9 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -214,6 +214,12 @@ func (r *StorageNodeSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque log.Error(err, "Failed to sync storage node status") } + // Sync manually created StorageNode CRs (not in spec.workerNodes) into + // StorageNodeSet.status.nodes[] so their status is visible in the fleet view. + if err := r.syncManualStorageNodeStatus(ctx, snCR); err != nil { + log.Error(err, "Failed to sync manual StorageNode status") + } + // On every reconcile, check whether the cluster is still unready and if // the activation conditions are now met. This catches cases where the // operator restarted after nodes came online but before activation fired, diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index e2e46d6d..a5baa487 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -286,6 +286,84 @@ func (r *StorageNodeSetReconciler) emitOnStorageNodeForWorker( } } +// syncManualStorageNodeStatus merges manually created StorageNode CRs (those +// without a controller OwnerReference pointing to this StorageNodeSet, i.e. +// not in spec.workerNodes) into StorageNodeSet.status.nodes[] so their status +// is visible in the fleet view alongside operator-managed nodes. +func (r *StorageNodeSetReconciler) syncManualStorageNodeStatus( + ctx context.Context, + sns *simplyblockv1alpha1.StorageNodeSet, +) error { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(sns.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": sns.Name}, + ); err != nil { + return err + } + + // Build a set of workers already covered by spec.workerNodes. + managed := make(map[string]struct{}, len(sns.Spec.WorkerNodes)) + for _, w := range sns.Spec.WorkerNodes { + managed[w] = struct{}{} + } + + changed := false + patch := client.MergeFrom(sns.DeepCopy()) + + for _, sn := range snList.Items { + if _, ok := managed[sn.Spec.WorkerNode]; ok { + continue // already tracked by reconcileWorkerNodes + } + if sn.Status.UUID == "" { + continue // not yet provisioned + } + + // Check if this node is already in status.nodes[]. + found := false + for i := range sns.Status.Nodes { + if sns.Status.Nodes[i].UUID == sn.Status.UUID { + // Update in-place if fields changed. + n := &sns.Status.Nodes[i] + if n.Status != sn.Status.Status || n.Health != sn.Status.Health { + n.Status = sn.Status.Status + n.Health = sn.Status.Health + n.MgmtIp = sn.Status.MgmtIp + n.Hostname = sn.Status.Hostname + n.CPU = sn.Status.CPU + n.Volumes = sn.Status.Volumes + n.RpcPort = sn.Status.RpcPort + n.LvolPort = sn.Status.LvolPort + n.NvmfPort = sn.Status.NvmfPort + changed = true + } + found = true + break + } + } + if !found { + sns.Status.Nodes = append(sns.Status.Nodes, simplyblockv1alpha1.NodeStatus{ + Hostname: sn.Spec.WorkerNode, + UUID: sn.Status.UUID, + Status: sn.Status.Status, + Health: sn.Status.Health, + MgmtIp: sn.Status.MgmtIp, + CPU: sn.Status.CPU, + Volumes: sn.Status.Volumes, + RpcPort: sn.Status.RpcPort, + LvolPort: sn.Status.LvolPort, + NvmfPort: sn.Status.NvmfPort, + }) + changed = true + } + } + + if !changed { + return nil + } + return r.Status().Patch(ctx, sns, patch) +} + // storageNodePostedAt returns the PostedAt timestamp from the StorageNode CR // for the given worker, or nil if not found / not yet set. func (r *StorageNodeSetReconciler) storageNodePostedAt( diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index f874a838..65b6e002 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net/http" + "slices" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -113,13 +114,24 @@ func (r *StorageNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) apiClient := webapi.NewClient() if sn.Status.UUID == "" { - // Check if the old StorageNodeSetReconciler already provisioned this node - // (present in status.nodes[]) — adopt its UUID to avoid a double POST. + // Fast path: old StorageNodeSetReconciler recorded UUID in status.nodes[]. if err := r.syncUUIDFromNodeSet(ctx, &sn, &sns); err != nil { return ctrl.Result{}, err } + + if sn.Status.UUID == "" && sn.Status.PostedAt != nil { + // POST already sent but UUID not in status.nodes[] — this happens for + // manually created StorageNodes whose worker is not in spec.workerNodes. + // Poll the backend by worker IP to retrieve the UUID directly. + if err := r.pollUUIDFromBackend(ctx, &sn, clusterUUID, apiClient); err != nil { + return ctrl.Result{}, err + } + if sn.Status.UUID == "" { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + } + if sn.Status.UUID == "" { - // Not yet provisioned — this reconciler is now the sole owner of the POST. return r.provisionNode(ctx, &sn, &sns, clusterUUID, apiClient) } } @@ -128,6 +140,93 @@ func (r *StorageNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) return r.syncStatus(ctx, &sn, clusterUUID, apiClient) } +// pollUUIDFromBackend lists all backend nodes for the cluster, finds the ones +// matching the worker's internal IP, and assigns the UUID to this StorageNode +// based on its socketIndex. For multi-socket workers (multiple nodes per IP), +// backend nodes are sorted by RPC port (ascending) and matched by position +// to the socketIndex — socket 0 → lowest RPC port, socket 1 → next, etc. +// Called every 10s while PostedAt is set but UUID is still empty; stops as +// soon as the UUID is assigned. +func (r *StorageNodeReconciler) pollUUIDFromBackend( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + clusterUUID string, + apiClient *webapi.Client, +) error { + log := logf.FromContext(ctx) + + ip, err := getNodeInternalIP(ctx, r.Client, sn.Spec.WorkerNode) + if err != nil { + log.V(1).Info("pollUUIDFromBackend: could not get worker IP, retrying", + "worker", sn.Spec.WorkerNode, "error", err.Error()) + return nil + } + + endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/", clusterUUID) + body, httpStatus, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) + if err != nil || httpStatus >= 300 { + return nil // transient — requeue silently + } + + var allNodes []SNODEAPIResponse + if err := json.Unmarshal(body, &allNodes); err != nil { + return nil + } + + // Collect all backend nodes for this worker's IP. + // Multi-socket: one backend node per socket, each with a different RPC port. + var matching []SNODEAPIResponse + for _, n := range allNodes { + if n.IP == ip && n.UUID != "" { + matching = append(matching, n) + } + } + if len(matching) == 0 { + return nil // node not yet visible on backend — requeue + } + + // Sort by RPC port ascending: socket 0 → lowest port, socket 1 → next, etc. + slices.SortFunc(matching, func(a, b SNODEAPIResponse) int { + return a.RPC_PORT - b.RPC_PORT + }) + + socketIdx := 0 + if sn.Spec.SocketIndex != nil { + socketIdx = int(*sn.Spec.SocketIndex) + } + if socketIdx >= len(matching) { + log.V(1).Info("pollUUIDFromBackend: socket not yet online", + "worker", sn.Spec.WorkerNode, "socketIndex", socketIdx, "found", len(matching)) + return nil + } + n := matching[socketIdx] + + cpu := int32(n.CPU) + volumes := int32(n.Volumes) + rpcPort := int32(n.RPC_PORT) + lvolPort := int32(n.LVOL_PORT) + nvmfPort := int32(n.NVMF_PORT) + + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.UUID = n.UUID + sn.Status.Status = n.Status + sn.Status.Health = n.Health + sn.Status.MgmtIp = n.IP + sn.Status.Hostname = n.Hostname + sn.Status.CPU = &cpu + sn.Status.Volumes = &volumes + sn.Status.RpcPort = &rpcPort + sn.Status.LvolPort = &lvolPort + sn.Status.NvmfPort = &nvmfPort + if err := r.Status().Patch(ctx, sn, patch); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("pollUUIDFromBackend: %w", err) + } + log.Info("pollUUIDFromBackend: UUID assigned", + "worker", sn.Spec.WorkerNode, "socketIndex", socketIdx, + "uuid", n.UUID, "status", n.Status) + return nil +} + // syncUUIDFromNodeSet copies the backend UUID from StorageNodeSet.status.nodes[] // into StorageNode.status.uuid. This is the Phase 1 bridge: the old // StorageNodeSetReconciler owns provisioning and tracks UUIDs in its own status; From 19d95d4a7a559264d4b2553803210ffc61ceeb5b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:06:50 +0100 Subject: [PATCH 26/52] operator: enforce parallel node add limits and FDB sequential constraint in StorageNodeReconciler --- .../controller/storagenode_controller.go | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 65b6e002..3d66c4ef 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -24,6 +24,7 @@ import ( "slices" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -300,6 +301,17 @@ func (r *StorageNodeReconciler) provisionNode( } } + // FDB workers must be added sequentially to avoid simultaneous reboots that + // would reduce FDB fault tolerance. If this worker hosts an FDB pod and any + // other FDB worker in the same StorageNodeSet is currently in-flight, block. + if r.isWorkerFDB(ctx, sn.Namespace, sn.Spec.WorkerNode) { + if blocked, err := r.isFDBWorkerBlocked(ctx, sn, sns); err == nil && blocked { + log.Info("FDB worker: another FDB node is in-flight, requeuing sequentially", + "worker", sn.Spec.WorkerNode) + return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil + } + } + // Guard: failure domain must be set if the feature is enabled. if err := r.checkFailureDomain(ctx, sn, sns); err != nil { r.Recorder.Event(sn, "Warning", "FailureDomainMissing", err.Error()) @@ -357,6 +369,50 @@ func (r *StorageNodeReconciler) provisionNode( return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } +// isWorkerFDB returns true if the given worker node currently hosts at least +// one FDB pod. +func (r *StorageNodeReconciler) isWorkerFDB(ctx context.Context, namespace, workerNode string) bool { + var podList corev1.PodList + if err := r.List(ctx, &podList, + client.InNamespace(namespace), + client.HasLabels{utils.LabelFDBClusterName}, + client.MatchingFields{"spec.nodeName": workerNode}, + ); err != nil { + return false + } + return len(podList.Items) > 0 +} + +// isFDBWorkerBlocked returns true if any sibling StorageNode in the same +// StorageNodeSet is an FDB worker currently in-flight (PostedAt set, UUID +// empty). Used to enforce sequential adds for FDB nodes. +func (r *StorageNodeReconciler) isFDBWorkerBlocked( + ctx context.Context, + sn *simplyblockv1alpha1.StorageNode, + sns *simplyblockv1alpha1.StorageNodeSet, +) (bool, error) { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(sn.Namespace), + client.MatchingFields{"spec.storageNodeSetRef": sn.Spec.StorageNodeSetRef}, + ); err != nil { + return false, err + } + for _, sibling := range snList.Items { + if sibling.Name == sn.Name { + continue + } + if sibling.Status.PostedAt == nil || sibling.Status.UUID != "" { + continue + } + // Sibling is in-flight — check if it's also an FDB worker. + if r.isWorkerFDB(ctx, sn.Namespace, sibling.Spec.WorkerNode) { + return true, nil + } + } + return false, nil +} + // countInFlightNodes returns how many sibling StorageNode CRs in the same // StorageNodeSet have PostedAt set but no UUID yet (i.e. add is in progress). // The calling node is excluded from the count. @@ -673,6 +729,22 @@ func (r *StorageNodeReconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + // Index Pods by spec.nodeName for efficient FDB worker detection. + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &corev1.Pod{}, + "spec.nodeName", + func(obj client.Object) []string { + pod := obj.(*corev1.Pod) + if pod.Spec.NodeName == "" { + return nil + } + return []string{pod.Spec.NodeName} + }, + ); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&simplyblockv1alpha1.StorageNode{}). Named("storagenode"). From 5f8c3611b4df24f2e99fb91261b74ae02cf235b4 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:15:43 +0100 Subject: [PATCH 27/52] docs: update design document to Phase 2 complete and extract test plan to docs/tests/ --- .../design-storagenodeset-storagenode.md | 358 +++--------------- .../docs/tests/test-plan-storagenode-ops.md | 157 ++++++++ 2 files changed, 212 insertions(+), 303 deletions(-) create mode 100644 operator/docs/tests/test-plan-storagenode-ops.md diff --git a/operator/docs/designs/design-storagenodeset-storagenode.md b/operator/docs/designs/design-storagenodeset-storagenode.md index eb1bbbf1..09ea5611 100644 --- a/operator/docs/designs/design-storagenodeset-storagenode.md +++ b/operator/docs/designs/design-storagenodeset-storagenode.md @@ -1,8 +1,8 @@ # Design Document: StorageNodeSet / StorageNode / StorageNodeOps Three-Tier Model -**Status:** Phase 1 Complete +**Status:** Phase 2 Complete **Author:** Israel Geoffrey -**Date:** 2026-07-14 +**Date:** 2026-07-15 --- @@ -335,12 +335,47 @@ Implemented on 2026-07-14. Deviations from the original plan: - CRD YAMLs generated and deployed to the helm chart. - 28 unit tests added covering both new reconcilers. -### Phase 2 — Move provisioning to StorageNodeReconciler - -- `StorageNodeReconciler` takes over node-add POST and status sync from - `StorageNodeSetReconciler.reconcileWorkerNodes`. The `provisionNode` and `syncStatus` - stubs are already in place; they need to consume the existing `postStorageNodeSet` and - `pollNodeOnline` logic refactored to read from `StorageNode` instead of `StorageNodeSet`. +### Phase 2 — Complete ✓ + +Implemented on 2026-07-15. + +- **`StorageNodeReconciler` is now the sole owner of provisioning.** `postStorageNodeSet` + and related helpers were removed from `StorageNodeSetReconciler`. `reconcileWorkerNode` + no longer POSTs — it only polls for online status after `StorageNodeReconciler` has sent + the POST. +- **`provisionNode`** merges fleet defaults with per-node `StorageNodeOverrides` for every + API parameter (`SpdkImage`, `SpdkSystemMemory`, `JMPercent`, `HaJMCount`, `FailureDomain`, + `PcieAllowList`, `DeviceNames`, etc.). +- **Parallel node add** enforced via `countInFlightNodes` (counts siblings with + `PostedAt != nil && UUID == ""`). **FDB sequential constraint** preserved via + `isFDBWorkerBlocked` — any FDB worker waits until no other FDB sibling is in-flight. +- **`pollUUIDFromBackend`** polls the backend by worker IP + socket index to retrieve the UUID + for manually created `StorageNode` CRs (whose worker is not in `spec.workerNodes` and + therefore never appears in `StorageNodeSet.status.nodes[]`). Supports multi-socket by + sorting backend nodes by RPC port and matching by socket index position. +- **Manually created `StorageNode` CRs** — users can create a `StorageNode` CR referencing + an existing `StorageNodeSet` without adding the worker to `spec.workerNodes`. The operator: + - Does NOT delete it (no OwnerReference = not stale) + - Labels the Kubernetes node so the DaemonSet pod is scheduled on it + - Adds the worker to the storage-node-api EndpointSlice for DNS resolution + - Adds the worker to the per-node ConfigMap for DaemonSet init container config + - Provisions the backend node via `provisionNode` + - Syncs the status back into `StorageNodeSet.status.nodes[]` via `syncManualStorageNodeStatus` +- **Per-node ConfigMap** (`{sns}-per-node-config`) created before the DaemonSet, containing + a shell-sourceable env file per worker. The `node-env-writer` init container copies the + node's section into a shared `emptyDir`; the config-generator and main containers source it. +- **`StorageNodeOps` drives all actions** (shutdown, restart, suspend, resume, remove/drain). + Events are mirrored from `StorageNodeOps` and `StorageNodeSet` onto the specific `StorageNode` + CR so `kubectl describe storagenode` shows the full event history. +- **`StorageNodeSet` status** now includes `TotalNodes`, `OnlineNodes`, `OfflineNodes`, + `SuspendedNodes`, `CreatingNodes`, `RemovedNodes`. Wide columns (`-o wide`) show the + breakdown; default view shows `Total` and `Online`. +- **Validation**: `spec.workerNodes` enforced unique via `+listType=set`; `spec.nodeConfigs` + keys must match a `workerNodes` entry (CEL rule with `MaxItems=200`, `MaxProperties=200` + to bound cost). +- **42+ unit tests** covering both new reconcilers, drain state machine helpers, + and StorageNodeSet status aggregation. See full test plan: + [`docs/tests/test-plan-storagenode-ops.md`](../tests/test-plan-storagenode-ops.md) ### Phase 3 — Remove legacy provisioning fields @@ -355,308 +390,25 @@ Implemented on 2026-07-14. Deviations from the original plan: **Q1: StorageNodeOps retention policy** How long should completed `StorageNodeOps` CRs be retained? Similar to `ttlSecondsAfterFinished` -on Jobs, or retained indefinitely for audit? +on Jobs, or retained indefinitely for audit? Not yet implemented. **Q2: Concurrent ops on different nodes of the same set** Multiple `StorageNodeOps` targeting different `StorageNode` CRs in the same `StorageNodeSet` -should be allowed. How many concurrent drains are permitted given FTT constraints? +should be allowed. How many concurrent drains are permitted given FTT constraints? Currently +unlimited — one per node simultaneously. FDB sequential constraint is enforced. -**Q3: StorageNode naming** -Name pattern: `{storagenodeset-name}-{sanitised-worker}-{socket}`. Needs to be deterministic -and DNS-label safe. +**Q3: StorageNode naming** — Resolved ✓ +Pattern: `{storagenodeset-name}-{sanitised-worker}-{socket}`. Implemented in +`storageNodeCRName()` with FNV-32 collision guard on truncation. -**Q4: Adoption of pre-existing backend nodes** -If a node already exists in the backend (from a prior deployment), the `StorageNode` CR should -be created with `status.uuid` pre-populated rather than triggering a new node-add. +**Q4: Adoption of pre-existing backend nodes** — Resolved ✓ +`syncUUIDFromNodeSet` reads existing UUIDs from `StorageNodeSet.status.nodes[]`. +`pollUUIDFromBackend` handles manually created `StorageNode` CRs and multi-socket workers. **Q5: Who triggers the drain StorageNodeOps on scale-down?** -When `StorageNodeSet.spec.workerNodes` shrinks, the controller must decide whether to trigger -`action=remove` or simply delete the `StorageNode` CR. Explicit `StorageNodeOps(action=remove)` -is safer. - ---- - -## 9. Architecture Diagrams - -**Ownership Diagram** — Shows the ownership hierarchy from StorageCluster through StorageNodeSet down to individual StorageNode and StorageNodeOps resources, including how VolumeMigration CRs are created during drain operations. - -``` -StorageCluster - spec.enableFailureDomains = true - │ - │ (referenced by clusterName) - ▼ -StorageNodeSet (fleet / template) - spec.nodeFailureDomains: - worker-A: 1 - worker-B: 2 - worker-C: 1 - spec.nodeConfigs: - worker-A: { maxLogicalVolumeCount: 50 } - spec.workerNodes: [worker-A, worker-B, worker-C] - │ - │ owns (OwnerReference) - ├──────────────────────────────────────────────────────┐ - ▼ ▼ -StorageNode StorageNode - name: sns-worker-a-0 name: sns-worker-b-0 - spec.workerNode: worker-A spec.workerNode: worker-B - spec.socketIndex: 0 spec.socketIndex: 0 - spec.failureDomain: 1 ◄── propagated spec.failureDomain: 2 ◄── propagated - spec.overrides: status.uuid: - maxLogicalVolumeCount: 50 status.activeOpsRef: "ops-restart-b" - status.uuid: │ - status.activeOpsRef: "ops-remove-a" │ targeted by - │ ▼ - │ targeted by StorageNodeOps - ▼ name: ops-restart-b -StorageNodeOps spec.storageNodeRef: sns-worker-b-0 - name: ops-remove-a spec.action: restart - spec.storageNodeRef: sns-worker-a-0 status.phase: Running - spec.action: remove status.subPhase: "" - spec.drain: (no VolumeMigration CRs) - systemVolumeFilterRegex: "^sb-fio-.*" - status.phase: Running - status.subPhase: Migrating - status.volumesMigrated: 3 - status.volumesPending: 2 - │ - │ owns (during drain only) - ├──────────────────────────┐ - ▼ ▼ -VolumeMigration VolumeMigration - name: vm-lvol-aaa name: vm-lvol-bbb - status.phase: Completed status.phase: Running - (deleted on Verifying) (in-flight) - - ┌─────────────────────────────────────────────────────────┐ - │ StorageNode (worker-C, socket 0) │ - │ spec.failureDomain: 1 │ - │ status.activeOpsRef: "" (no active operation) │ - └─────────────────────────────────────────────────────────┘ -``` +When a worker is removed from `spec.workerNodes`, the `StorageNodeSetReconciler` deletes the +owned `StorageNode` CR. The `StorageNodeReconciler` finalizer creates a `StorageNodeOps(action=remove)` +if the node is online before allowing the CR to be deleted. --- -**Reconciler Flow** — Illustrates the three controllers and what each reconciler watches, manages, and produces. - -``` - ┌───────────────────────────────────────────────────────────────────────────────────┐ - │ Kubernetes Controller Manager │ - └───────────────────────────────────────────────────────────────────────────────────┘ - │ │ │ - ▼ ▼ ▼ - ┌─────────────────────┐ ┌──────────────────────┐ ┌──────────────────────────┐ - │ StorageNodeSet │ │ StorageNode │ │ StorageNodeOps │ - │ Reconciler │ │ Reconciler (new) │ │ Reconciler (new) │ - ├─────────────────────┤ ├──────────────────────┤ ├──────────────────────────┤ - │ WATCHES │ │ WATCHES │ │ WATCHES │ - │ • StorageNodeSet │ │ • StorageNode │ │ • StorageNodeOps │ - │ • Pod (SPDK ready) │ │ • StorageNodeSet │ │ • StorageNode │ - │ • Node (labels) │ │ (read config) │ │ • VolumeMigration │ - ├─────────────────────┤ ├──────────────────────┤ ├──────────────────────────┤ - │ MANAGES │ │ MANAGES │ │ MANAGES │ - │ • DaemonSet │ │ • backend node-add │ │ • backend actions: │ - │ • ServiceAccount │ │ POST (if uuid=="") │ │ shutdown / restart │ - │ • ClusterRole / │ │ • status sync (30s) │ │ suspend / resume │ - │ ClusterRoleBinding│ │ • pollNodeOnline │ │ remove/drain │ - │ • Service │ │ • creates │ │ • VolumeMigration CRs │ - │ • EndpointSlices │ │ StorageNodeOps │ │ (drain only) │ - │ • TLS Certificates │ │ (action=remove) │ │ • StorageNode.status │ - │ • StorageNode CRs │ │ on deletion │ │ .activeOpsRef │ - │ (create/delete) │ │ • StorageNode.status │ │ │ - │ • status aggregate │ │ (uuid, health...) │ │ MUTUAL EXCLUSION │ - │ (totalNodes, │ │ │ │ checks activeOpsRef │ - │ online/offline) │ │ READS EFFECTIVE CFG │ │ before starting; │ - │ • nodeConfigs sync │ │ StorageNodeSet.spec │ │ requeues if set │ - │ → StorageNode │ │ defaults merged with │ │ │ - │ .spec.overrides │ │ StorageNode.spec │ │ ON COMPLETION │ - │ • failureDomain │ │ .overrides │ │ Phase=Succeeded/Failed │ - │ sync │ │ │ │ clears activeOpsRef │ - └─────────────────────┘ └──────────────────────┘ └──────────────────────────┘ - │ creates/deletes │ │ owns - ▼ ▼ ▼ - StorageNode CRs backend API calls VolumeMigration CRs - (one per worker/socket) /api/v2/clusters/... (drain sub-phase only) -``` - ---- - -**Drain State Machine** — The five sequential sub-phases of the remove/drain operation, including the cluster pause guard, migration retry loop, and cancellation path. - -``` - StorageNodeOps(action=remove) created - │ - ▼ - ╔══════════════════════════════════════════════════════╗ - ║ ENTRY: SubPhase=Validating, Triggered=false ║ - ╚══════════════════════════════════════════════════════╝ - │ - ▼ - ┌───────────────────────────────────────────────────────────────┐ - │ [1] VALIDATING │ - │ • Fetch all volumes on the node from backend API │ - │ • Identify pinned / unmanaged volumes (no matching PVC) │ - │ ─────────────────────────────────────────────────────────────│ - │ EXIT → SUSPENDING : no pinned or unmanaged volumes │ - │ BLOCK (requeue 60s): pinned or unmanaged volumes present │ - └───────────────────────────────────────────────────────────────┘ - │ - ┌─────────────────────────▼──────────────────────────────────────────┐ - │ CLUSTER PAUSE GUARD (all phases after Validating) │ - │ if cluster.status != "active" OR rebalancing == true │ - │ → emit DrainPaused event, requeue 60s │ - │ → auto-resumes when cluster is active and not rebalancing │ - └─────────────────────────┬──────────────────────────────────────────┘ - │ - ▼ - ┌───────────────────────────────────────────────────────────────┐ - │ [2] SUSPENDING │ - │ • GET node status; if already suspended → skip POST │ - │ • POST /storage-nodes/{uuid}/suspend, set Triggered=true │ - │ • Poll GET every 10s │ - │ ─────────────────────────────────────────────────────────────│ - │ EXIT → MIGRATING : node.status == "suspended" │ - │ RETRY (10s) : node not yet suspended │ - └───────────────────────────────────────────────────────────────┘ - │ - ▼ - ┌───────────────────────────────────────────────────────────────────────────┐ - │ [3] MIGRATING │ - │ • createMissingVolumeMigrations: one CR per PV-managed volume │ - │ with round-robin target node assignment │ - │ • Track VolumesMigrated / VolumesPending │ - │ ┌────────────────────────────────────────────────────────────┐ │ - │ │ FAILURE RETRY LOOP │ │ - │ │ VolumeMigration.phase == Failed/Aborted: │ │ - │ │ cluster not ready → delete CR, pause (DrainPaused) │ │ - │ │ cluster ready → delete CR, recreate with new target │ │ - │ └────────────────────────────────────────────────────────────┘ │ - │ ─────────────────────────────────────────────────────────────────────────│ - │ EXIT → VERIFYING : all migrations complete, no in-flight │ - │ RETRY (15s) : migrations still running │ - └───────────────────────────────────────────────────────────────────────────┘ - │ - ▼ - ┌───────────────────────────────────────────────────────────────┐ - │ [4] VERIFYING │ - │ • GET all volumes remaining on node │ - │ • Delete system volumes (match systemVolumeFilterRegex) │ - │ • Poll 30s until none remain │ - │ ─────────────────────────────────────────────────────────────│ - │ EXIT → REMOVING : no non-system volumes remain │ - │ RETRY (30s) : volumes still present │ - └───────────────────────────────────────────────────────────────┘ - │ - ▼ - ┌───────────────────────────────────────────────────────────────┐ - │ [5] REMOVING │ - │ • DELETE /api/v2/clusters/{uuid}/storage-nodes/{nodeUUID} │ - │ ─────────────────────────────────────────────────────────────│ - │ 200/204/404 → SUCCEEDED │ - │ error → FAILED (resume node best-effort) │ - └───────────────────────────────────────────────────────────────┘ - │ │ - ▼ ▼ - ╔══════════════════════╗ ╔═══════════════════════════════════╗ - ║ SUCCEEDED ║ ║ FAILED ║ - ║ Phase=Succeeded ║ ║ POST /resume (best-effort) ║ - ║ activeOpsRef="" ║ ║ Phase=Failed, activeOpsRef="" ║ - ╚══════════════════════╝ ╚═══════════════════════════════════╝ - - CANCELLATION (any phase after Suspending): - spec.action cleared → POST /resume if suspended - → delete all owned VolumeMigration CRs - → ActionStatus = nil -``` - ---- - -**Failure Domain Propagation** — How `enableFailureDomains` and `nodeFailureDomains` flow from StorageCluster and StorageNodeSet down to individual StorageNode specs and the backend API, including the blocking guard for missing entries. - -``` - ┌─────────────────────────────────────────────────┐ - │ StorageCluster │ - │ spec.enableFailureDomains: true │ - └─────────────────────────────────────────────────┘ - │ referenced by StorageNodeSet.spec.clusterName - ▼ - ┌─────────────────────────────────────────────────┐ - │ StorageNodeSet │ - │ spec.nodeFailureDomains: │ - │ worker-A: 1 ← rack-1 / AZ-1 │ - │ worker-B: 2 ← rack-2 / AZ-2 │ - │ worker-C: 1 ← rack-1 / AZ-1 │ - │ worker-D: ? ← MISSING │ - │ │ - │ CEL rule: all values >= 1 │ - └──────────┬──────────────┬──────────────┬────────┘ - │ sync │ sync │ MISSING - ▼ ▼ ▼ - ┌──────────────┐ ┌──────────────┐ ┌──────────────────────────┐ - │ StorageNode │ │ StorageNode │ │ StorageNode (worker-D) │ - │ worker-A, s0 │ │ worker-B, s0 │ │ │ - │ .failureDomain│ │ .failureDomain│ │ enableFailureDomains=T │ - │ = 1 │ │ = 2 │ │ no entry in map │ - └──────┬───────┘ └──────┬───────┘ │ → Warning event emitted │ - │ │ │ → node-add BLOCKED │ - ▼ ▼ │ → requeue until populated │ - ┌────────────────────────────────┐ └──────────────────────────┘ - │ Backend API POST │ - │ { "failure_domain": 1 } │ - │ { "failure_domain": 2 } │ - └────────────────────────────────┘ -``` - ---- - -**StorageNodeOps Lifecycle** — Full lifecycle from creation through mutual exclusion gate, action dispatch, sub-phases for `action=remove`, and terminal states with `activeOpsRef` cleanup. - -``` - kubectl apply -f ops.yaml (or auto-created on StorageNode deletion) - │ - ▼ - ┌─────────────────────────────────────────────────────────────┐ - │ PENDING — reconciler checks StorageNode.status.activeOpsRef│ - │ │ - │ activeOpsRef == "" activeOpsRef != "" │ - │ → set activeOpsRef = self → requeue (back-off) │ - │ → phase = Running (another op is running) │ - └──────────────────────┬──────────────────────────────────────┘ - │ - ▼ - ╔═════════════════════════════════════════════════════╗ - ║ status.phase: Running ║ - ╚═════════════════════════════════════════════════════╝ - │ - ┌──────────────┴───────────────────────────┐ - │ action dispatch │ - ▼ ▼ - ┌────────────────────────────┐ ┌──────────────────────────────────────┐ - │ shutdown / restart / │ │ remove │ - │ suspend / resume │ │ │ - │ │ │ Validating → Suspending │ - │ POST action to backend │ │ → Migrating │ - │ poll until terminal state │ │ (VolumeMigration CRs owned) │ - │ suspend → "suspended" │ │ → Verifying │ - │ resume → "online" │ │ → Removing │ - │ restart → "online" │ │ │ - │ shutdown → "offline" │ │ VolumesMigrated / VolumesPending │ - └─────────────┬──────────────┘ │ tracked in status │ - │ └──────────────────┬───────────────────┘ - │ │ - └──────────────────┬──────────────────┘ - │ - ┌──────────────┴──────────────┐ - ▼ ▼ - ╔═══════════════════════════╗ ╔═══════════════════════════════╗ - ║ Succeeded ║ ║ Failed ║ - ║ completedAt: ║ ║ message: ║ - ╚═══════════════════════════╝ ║ completedAt: ║ - │ ╚═══════════════════════════════╝ - └──────────────────┬──────────────────────────┘ - │ both terminal states: - ▼ - StorageNode.status.activeOpsRef = "" - (next StorageNodeOps may now acquire the lock) -``` diff --git a/operator/docs/tests/test-plan-storagenode-ops.md b/operator/docs/tests/test-plan-storagenode-ops.md new file mode 100644 index 00000000..884757a8 --- /dev/null +++ b/operator/docs/tests/test-plan-storagenode-ops.md @@ -0,0 +1,157 @@ +# Test Plan: StorageNode / StorageNodeOps Three-Tier Model + +Related design: [`designs/design-storagenodeset-storagenode.md`](../designs/design-storagenodeset-storagenode.md) +Regression script: [`regression_test/test_storagenode_ops.sh`](../../../../regression_test/test_storagenode_ops.sh) + +Each scenario is classified as **Unit** (no cluster needed), **Integration** (live cluster, +non-destructive), or **E2E** (live cluster, may remove/modify nodes). + +--- + +## Unit Tests — implemented + +### `storagenode_controller_unit_test.go` + +| Test | Scenario | +|---|---| +| `TestSyncOverrides_PropagatesNodeConfigs` | `nodeConfigs` entry propagated to `StorageNode.spec.overrides` | +| `TestSyncOverrides_NoopWhenWorkerNotInNodeConfigs` | No overrides when worker absent from `nodeConfigs` | +| `TestEffectiveNodeConfig_OverridesTakePrecedence` | Per-node override wins over fleet default | +| `TestEffectiveNodeConfig_FallsBackToFleetDefault` | Fleet default used when no override set | +| `TestEffectiveFailureDomain_OverrideTakesPrecedenceOverMap` | `overrides.failureDomain` beats `nodeFailureDomains` map | +| `TestEffectiveFailureDomain_FallsBackToMap` | `nodeFailureDomains[worker]` used when no override | +| `TestEffectiveFailureDomain_ZeroWhenNotSet` | Returns 0 when neither source is set | +| `TestCheckFailureDomain_BlocksWhenEnabledAndNotSet` | Node-add blocked when `enableFailureDomains=true` and no domain set | +| `TestCheckFailureDomain_AllowsWhenFailureDomainSet` | No error when failure domain is populated | +| `TestCheckFailureDomain_SkipsWhenFeatureDisabled` | No error when feature disabled | +| `TestEnsureRemoveOps_CreatesOpsWhenMissing` | `StorageNodeOps(action=remove)` created on deletion | +| `TestEnsureRemoveOps_IdempotentWhenAlreadyExists` | Second call is a no-op | +| `TestHandleDeletion_RemovesFinalizerWhenNeverProvisioned` | Unprovisioned node skips ops, finalizer removed immediately | + +### `storagenodeops_controller_unit_test.go` + +| Test | Scenario | +|---|---| +| `TestAcquireLock_SetsActiveOpsRefAndTransitionsToRunning` | Lock acquired, ops transitions to Running | +| `TestAcquireLock_RequeuesWhenAnotherOpsActive` | Lock blocked when `activeOpsRef` already set | +| `TestAcquireLock_RemoveDrainSetsValidatingSubPhase` | `action=remove` initialises sub-phase to Validating | +| `TestSucceedOps_SetsPhaseAndClearsLock` | Succeeded phase set, `activeOpsRef` cleared | +| `TestFailOps_SetsPhaseAndClearsLock` | Failed phase set with message, `activeOpsRef` cleared | +| `TestReleaseLock_OnlyClearsIfOwner` | Lock not cleared when called by non-owner | +| `TestAdvanceSubPhase_UpdatesSubPhaseAndResetsTrigger` | Sub-phase advances, `Triggered` reset to false | +| `TestDispatch_UnknownActionFails` | Unknown action immediately transitions to Failed | +| `TestResolveOpsSystemVolumeFilter_UsesDefaultWhenNoDrain` | Default regex matches `sb-fio-baseline-*` | +| `TestResolveOpsSystemVolumeFilter_UsesCustomPattern` | Custom regex overrides default | +| `TestResolveOpsSystemVolumeFilter_InvalidPatternReturnsError` | Invalid regex returns error | + +### `simplyblockstoragenodeset_drain_unit_test.go` (standalone helpers) + +| Test | Scenario | +|---|---| +| `TestRoundRobinDistributesEvenly` | Migration targets distributed evenly across online peers | +| `TestRoundRobinErrorsWhenNoTargetAvailable` | Error when no online peer available | +| `TestRoundRobinSkipsOfflineNodes` | Offline nodes excluded from round-robin | +| `TestMatchVolumesToPVs_PVManaged` | PV-managed volume classified correctly | +| `TestMatchVolumesToPVs_Pinned` | Pinned PVC classified correctly | +| `TestMatchVolumesToPVs_Unmanaged` | Unmanaged volume classified correctly | +| `TestMatchVolumesToPVs_SystemVolumeSkipped` | System volume excluded from all buckets | +| `TestMatchVolumesToPVs_EmptyNodeSkipsMigration` | Empty node produces no drain work | +| `TestMatchVolumesToPVs_OnlySystemVolumes` | System-volume-only node skips migration | +| `TestDrainMigrationNameNoCollisionOnLongPVNames` | Long PV names produce unique CR names | +| `TestDrainMigrationNameIsDNSValid` | Generated CR names are always valid DNS labels | + +--- + +## Integration Tests — `regression_test/test_storagenode_ops.sh` + +### StorageNode Lifecycle (non-destructive) + +| Test # | Scenario | Classification | +|---|---|---| +| 1 | `StorageNode` CRs created for all `spec.workerNodes` | Integration | +| 2 | `StorageNode` status fields: `uuid`, `status=online`, `socketIndex`, `health` all populated | Integration | +| 3 | `StorageNodeSet.status` aggregation: `totalNodes` / `onlineNodes` match worker count | Integration | +| 4 | `nodeConfigs` overrides reflected in `StorageNode.spec.overrides` | Integration | +| 10 | `StorageNodeSet` wide columns (`-o wide`) show `OFFLINE`, `SUSPENDED`, `CREATING`, `REMOVED` | Integration | + +### StorageNodeOps Lifecycle + +| Test # | Scenario | Classification | +|---|---|---| +| 6 | Mutual exclusion: second ops requeues while first holds `activeOpsRef` | Integration | +| 7 | Unknown action → `Failed` phase immediately | Integration | +| 8 | `activeOpsRef` cleared after Succeeded and Failed outcomes | Integration | +| 9 | Events mirrored onto `StorageNode` CR (`kubectl describe storagenode` shows events) | Integration | + +### Drain / Remove (destructive — removes a node) + +| Test # | Scenario | Classification | +|---|---|---| +| 5 | Happy path remove: drain completes, `StorageNode.status=removed`, `phase=Succeeded` | E2E | + +### Cluster Expansion + +| Test # | Scenario | Classification | +|---|---|---| +| 11 | Manually created `StorageNode` CR references existing `StorageNodeSet`: survives reconciler, worker labeled, EndpointSlice updated, pod scheduled, node provisions, status in `StorageNodeSet.status.nodes[]` | E2E | + +--- + +## Additional Test Scenarios (manual / to be automated) + +### Parallel Node Add + +| Scenario | Expected | Classification | +|---|---|---| +| Add 5 workers with `maxParallelNodeAdds=2` | At most 2 nodes in-flight simultaneously (`PostedAt` set, UUID empty) | E2E | +| Add FDB + non-FDB workers simultaneously | FDB workers sequential; non-FDB respect `MaxParallelNodeAdds` | E2E | +| Second FDB worker while first is in-flight | Second FDB worker requeues until first comes online | E2E | + +### Per-Node Configuration (DaemonSet) + +| Scenario | Expected | Classification | +|---|---|---| +| `nodeConfigs[worker].maxLogicalVolumeCount` differs per node | Per-node ConfigMap has correct `MAX_LVOL` per worker | Integration | +| `nodeConfigs[worker].spdkSystemMemory` differs per node | Init container receives correct `spdk_sys_mem` in POST | E2E | +| `nodeConfigs[worker].deviceNames` set | `--nvme-devices` arg in init container uses override | Integration | +| ConfigMap created before DaemonSet on fresh install | Init container finds non-empty env file; no `MAX_LVOL=0` failure | E2E | + +### Failure Domain + +| Scenario | Expected | Classification | +|---|---|---| +| `enableFailureDomains=true`, no `failureDomain` set | Node-add blocked with `FailureDomainMissing` event on `StorageNode` | Integration | +| `failureDomain` in `nodeConfigs` | Sent as `failure_domain` in POST; reflected in `StorageNode.spec.overrides` | E2E | +| `overrides.failureDomain` overrides `nodeFailureDomains` map | Per-node override wins | Integration | + +### Drain Robustness (see also `test-plan-drain-remove.md`) + +| Scenario | Expected | Classification | +|---|---|---| +| Pinned PVC blocks drain | `PinnedVolumeBlocking` event; drain stalls; unblocks after annotation removed | E2E | +| Cancel drain mid-Suspending | Node resumed, `activeOpsRef` cleared | E2E | +| Cancel drain mid-Migrating | In-flight `VolumeMigration` CRs deleted, node resumed | E2E | +| Cluster degraded during drain | `DrainPaused` event; resumes when cluster active | E2E | +| Operator restart mid-drain | Sub-phase (`SubPhase`) preserved; drain resumes from same phase | E2E | +| Remove empty node (no PVCs) | No `VolumeMigration` CRs created; completes directly | E2E | +| VolumeMigration fails | Failed CR deleted; new CR created with fresh round-robin target | E2E | + +### Validation + +| Scenario | Expected | Classification | +|---|---|---| +| Duplicate entry in `spec.workerNodes` | API server rejects with `duplicate value not allowed` | Integration | +| `nodeConfigs` key not in `spec.workerNodes` | API server rejects with `nodeConfigs keys must match a workerNode entry` | Integration | +| `nodeFailureDomains` value < 1 | API server rejects with validation message | Integration | + +--- + +## What Is Not Yet Covered + +| Gap | Reason | +|---|---| +| `StorageNodeOps` TTL / auto-cleanup of completed ops | Feature not yet implemented | +| Phase 3 migration (remove `status.nodes[]` from `StorageNodeSet`) | Not yet implemented | +| Scale-down triggered drain (shrink `spec.workerNodes` while nodes are online) | Not yet implemented | +| Multi-socket worker (2+ `StorageNode` CRs per worker) | Requires specific hardware in test environment | +| Node adoption on fresh operator install (backend node exists, no `StorageNode` CR) | `pollUUIDFromBackend` handles this but no explicit test | From e28aca22a793f4a80486862cc0515f525ef68c6b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:18:15 +0100 Subject: [PATCH 28/52] fix: remove unused sns parameter from isFDBWorkerBlocked --- operator/internal/controller/storagenode_controller.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 3d66c4ef..96454bd3 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -305,7 +305,7 @@ func (r *StorageNodeReconciler) provisionNode( // would reduce FDB fault tolerance. If this worker hosts an FDB pod and any // other FDB worker in the same StorageNodeSet is currently in-flight, block. if r.isWorkerFDB(ctx, sn.Namespace, sn.Spec.WorkerNode) { - if blocked, err := r.isFDBWorkerBlocked(ctx, sn, sns); err == nil && blocked { + if blocked, err := r.isFDBWorkerBlocked(ctx, sn); err == nil && blocked { log.Info("FDB worker: another FDB node is in-flight, requeuing sequentially", "worker", sn.Spec.WorkerNode) return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil @@ -389,7 +389,6 @@ func (r *StorageNodeReconciler) isWorkerFDB(ctx context.Context, namespace, work func (r *StorageNodeReconciler) isFDBWorkerBlocked( ctx context.Context, sn *simplyblockv1alpha1.StorageNode, - sns *simplyblockv1alpha1.StorageNodeSet, ) (bool, error) { var snList simplyblockv1alpha1.StorageNodeList if err := r.List(ctx, &snList, From 49862f32e03640d3db750dae6ce2f0e07e407a62 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:29:05 +0100 Subject: [PATCH 29/52] test: add 20 unit tests for StorageNode naming, per-node config, UUID sync, and manual node status --- .../docs/tests/test-plan-storagenode-ops.md | 25 ++ ...ockstoragenodeset_storagenode_unit_test.go | 410 ++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go diff --git a/operator/docs/tests/test-plan-storagenode-ops.md b/operator/docs/tests/test-plan-storagenode-ops.md index 884757a8..0cf322da 100644 --- a/operator/docs/tests/test-plan-storagenode-ops.md +++ b/operator/docs/tests/test-plan-storagenode-ops.md @@ -44,6 +44,31 @@ non-destructive), or **E2E** (live cluster, may remove/modify nodes). | `TestResolveOpsSystemVolumeFilter_UsesCustomPattern` | Custom regex overrides default | | `TestResolveOpsSystemVolumeFilter_InvalidPatternReturnsError` | Invalid regex returns error | +### `simplyblockstoragenodeset_storagenode_unit_test.go` + +| Test | Scenario | +|---|---| +| `TestStorageNodeCRName_SimpleCase` | Name is non-empty, ≤ 63 chars, lowercase | +| `TestStorageNodeCRName_TruncatesLongNames` | Long worker hostnames truncated to 63 chars | +| `TestStorageNodeCRName_CollisionGuard` | Two workers with shared long prefix produce distinct names | +| `TestStorageNodeCRName_IsDNSLabelSafe` | Generated name contains only valid DNS characters | +| `TestSanitiseDNSLabel_ReplacesInvalidChars` | Uppercase and underscores replaced | +| `TestSanitiseDNSLabel_StripsLeadingTrailingHyphens` | Leading/trailing hyphens stripped | +| `TestBuildPerNodeEnvFile_UsesFleetDefaults` | Fleet-level values appear in env file | +| `TestBuildPerNodeEnvFile_OverrideWinsOverFleet` | Per-node override beats fleet default | +| `TestBuildPerNodeEnvFile_WorkerNotInNodeConfigs_UsesFleet` | Worker absent from nodeConfigs gets fleet values | +| `TestBuildPerNodeEnvFile_ContainsAllRequiredKeys` | All required env keys present | +| `TestCountInFlightNodes_ZeroWhenNonePosted` | Returns 0 when no siblings have PostedAt set | +| `TestCountInFlightNodes_CountsSiblingsWithPostedAtAndNoUUID` | Counts only in-flight siblings (PostedAt set, UUID empty) | +| `TestCountInFlightNodes_ExcludesSelf` | Self not counted in in-flight total | +| `TestSyncUUIDFromNodeSet_CopiesUUIDWhenFound` | UUID and status copied from StorageNodeSet.status.nodes[] | +| `TestSyncUUIDFromNodeSet_NoopWhenWorkerNotInNodes` | UUID stays empty when worker absent from status.nodes[] | +| `TestSyncUUIDFromNodeSet_SkipsEmptyUUID` | Placeholder entry (UUID="") skipped | +| `TestSyncManualStorageNodeStatus_AddsManualNodeToSNSStatus` | Manual StorageNode UUID added to StorageNodeSet.status.nodes[] | +| `TestSyncManualStorageNodeStatus_SkipsUnprovisionedNodes` | Node without UUID not added | +| `TestSyncManualStorageNodeStatus_SkipsWorkerInSpecWorkerNodes` | Operator-managed nodes not duplicated | +| `TestSyncManualStorageNodeStatus_IdempotentOnSecondCall` | Second call does not duplicate the entry | + ### `simplyblockstoragenodeset_drain_unit_test.go` (standalone helpers) | Test | Scenario | diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go new file mode 100644 index 00000000..b836ce40 --- /dev/null +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go @@ -0,0 +1,410 @@ +package controller + +import ( + "context" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" +) + +const ( + snsTestNS = "test" + snsTestCluster = "cluster-a" +) + +func newSNSReconciler(t *testing.T, objects ...client.Object) *StorageNodeSetReconciler { + t.Helper() + scheme := newTestScheme(t, simplyblockv1alpha1.AddToScheme) + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource( + &simplyblockv1alpha1.StorageNode{}, + &simplyblockv1alpha1.StorageNodeSet{}, + ). + WithObjects(objects...). + WithIndex(&simplyblockv1alpha1.StorageNode{}, "spec.storageNodeSetRef", func(obj client.Object) []string { + sn := obj.(*simplyblockv1alpha1.StorageNode) + return []string{sn.Spec.StorageNodeSetRef} + }). + Build() + return &StorageNodeSetReconciler{ + Client: cl, + Scheme: scheme, + Recorder: record.NewFakeRecorder(16), + } +} + +// ── TestStorageNodeCRName ────────────────────────────────────────────────────── + +func TestStorageNodeCRName_SimpleCase(t *testing.T) { + name := storageNodeCRName("my-sns", "worker-a.example.com", "0") + if name == "" { + t.Fatal("expected non-empty name") + } + if len(name) > 63 { + t.Errorf("name exceeds 63 chars: %q (%d)", name, len(name)) + } + // Must be lowercase + if name != strings.ToLower(name) { + t.Errorf("name is not lowercase: %q", name) + } +} + +func TestStorageNodeCRName_TruncatesLongNames(t *testing.T) { + longWorker := "vm" + strings.Repeat("a", 60) + ".simplyblock3.localdomain" + name := storageNodeCRName("simplyblock-node", longWorker, "0") + if len(name) > 63 { + t.Errorf("truncated name still exceeds 63 chars: len=%d", len(name)) + } +} + +func TestStorageNodeCRName_CollisionGuard(t *testing.T) { + // Two workers sharing a long prefix must produce distinct names. + base := "vm" + strings.Repeat("x", 55) + ".example.com" + name1 := storageNodeCRName("sns", base+"1", "0") + name2 := storageNodeCRName("sns", base+"2", "0") + if name1 == name2 { + t.Errorf("collision: both workers mapped to %q", name1) + } +} + +func TestStorageNodeCRName_IsDNSLabelSafe(t *testing.T) { + name := storageNodeCRName("my-sns", "vm01.simplyblock3.localdomain", "0") + for _, c := range name { + if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '.') { + t.Errorf("invalid character %q in name %q", c, name) + } + } +} + +// ── TestSanitiseDNSLabel ─────────────────────────────────────────────────────── + +func TestSanitiseDNSLabel_ReplacesInvalidChars(t *testing.T) { + got := sanitiseDNSLabel("vm_01.EXAMPLE.com") + if strings.ContainsAny(got, "_ABCDEFGHIJKLMNOPQRSTUVWXYZ") { + t.Errorf("unsanitised result: %q", got) + } +} + +func TestSanitiseDNSLabel_StripsLeadingTrailingHyphens(t *testing.T) { + got := sanitiseDNSLabel("-bad-label-") + if strings.HasPrefix(got, "-") || strings.HasSuffix(got, "-") { + t.Errorf("result has leading/trailing hyphen: %q", got) + } +} + +// ── TestBuildPerNodeEnvFile ─────────────────────────────────────────────────── + +func TestBuildPerNodeEnvFile_UsesFleetDefaults(t *testing.T) { + maxLvol := int32(20) + corePercent := int32(50) + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + ClusterName: snsTestCluster, + MaxLogicalVolumeCount: &maxLvol, + CorePercentage: &corePercent, + SpdkSystemMemory: "4G", + }, + } + env := buildPerNodeEnvFile(sns, "worker-a.example.com") + if !strings.Contains(env, "MAX_LVOL=20") { + t.Errorf("missing MAX_LVOL=20 in env:\n%s", env) + } + if !strings.Contains(env, "CORES_PERCENTAGE=50") { + t.Errorf("missing CORES_PERCENTAGE=50 in env:\n%s", env) + } +} + +func TestBuildPerNodeEnvFile_OverrideWinsOverFleet(t *testing.T) { + fleetMax := int32(20) + overrideMax := int32(99) + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + ClusterName: snsTestCluster, + MaxLogicalVolumeCount: &fleetMax, + NodeConfigs: map[string]simplyblockv1alpha1.StorageNodeOverrides{ + "worker-b": {MaxLogicalVolumeCount: &overrideMax}, + }, + }, + } + env := buildPerNodeEnvFile(sns, "worker-b") + if !strings.Contains(env, "MAX_LVOL=99") { + t.Errorf("expected MAX_LVOL=99 (override), got:\n%s", env) + } +} + +func TestBuildPerNodeEnvFile_WorkerNotInNodeConfigs_UsesFleet(t *testing.T) { + maxLvol := int32(15) + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + ClusterName: snsTestCluster, + MaxLogicalVolumeCount: &maxLvol, + }, + } + env := buildPerNodeEnvFile(sns, "worker-not-configured") + if !strings.Contains(env, "MAX_LVOL=15") { + t.Errorf("expected fleet MAX_LVOL=15:\n%s", env) + } +} + +func TestBuildPerNodeEnvFile_ContainsAllRequiredKeys(t *testing.T) { + sns := &simplyblockv1alpha1.StorageNodeSet{ + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: snsTestCluster}, + } + env := buildPerNodeEnvFile(sns, "any-worker") + required := []string{"MAX_LVOL=", "MAX_SIZE=", "CORES_PERCENTAGE=", + "RESERVED_SYSTEM_CPUS=", "CPU_TOPOLOGY_ENABLED=", + "PCI_ALLOWED=", "PCI_BLOCKED=", "NVME_DEVICES=", + "DEVICE_MODEL=", "SIZE_RANGE=", "JM_PERCENT=", "HA_JM_COUNT="} + for _, key := range required { + if !strings.Contains(env, key) { + t.Errorf("missing key %q in env:\n%s", key, env) + } + } +} + +// ── TestCountInFlightNodes ──────────────────────────────────────────────────── + +func TestCountInFlightNodes_ZeroWhenNonePosted(t *testing.T) { + sn1 := newStorageNode("sn-1", snsTestNS, "sns", "worker-1.example.com") + sn2 := newStorageNode("sn-2", snsTestNS, "sns", "worker-2.example.com") + r := newSNReconciler(t, sn1, sn2) + + count, err := r.countInFlightNodes(context.Background(), snsTestNS, "sns", "sn-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if count != 0 { + t.Errorf("expected 0 in-flight, got %d", count) + } +} + +func TestCountInFlightNodes_CountsSiblingsWithPostedAtAndNoUUID(t *testing.T) { + now := metav1.Now() + sn1 := newStorageNode("sn-1", snsTestNS, "sns", "worker-1.example.com") + sn2 := newStorageNode("sn-2", snsTestNS, "sns", "worker-2.example.com") + sn2.Status.PostedAt = &now // sn-2 is in-flight + sn3 := newStorageNode("sn-3", snsTestNS, "sns", "worker-3.example.com") + sn3.Status.PostedAt = &now + sn3.Status.UUID = "already-online-uuid" // sn-3 is done + r := newSNReconciler(t, sn1, sn2, sn3) + + count, err := r.countInFlightNodes(context.Background(), snsTestNS, "sns", "sn-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if count != 1 { + t.Errorf("expected 1 in-flight (sn-2), got %d", count) + } +} + +func TestCountInFlightNodes_ExcludesSelf(t *testing.T) { + now := metav1.Now() + sn1 := newStorageNode("sn-1", snsTestNS, "sns", "worker-1.example.com") + sn1.Status.PostedAt = &now // self is in-flight + r := newSNReconciler(t, sn1) + + count, err := r.countInFlightNodes(context.Background(), snsTestNS, "sns", "sn-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if count != 0 { + t.Errorf("self should not be counted, got %d", count) + } +} + +// ── TestSyncUUIDFromNodeSet ─────────────────────────────────────────────────── + +func TestSyncUUIDFromNodeSet_CopiesUUIDWhenFound(t *testing.T) { + sn := newStorageNode("sn-1", snsTestNS, "sns", snTestWorker) + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Status: simplyblockv1alpha1.StorageNodeSetStatus{ + Nodes: []simplyblockv1alpha1.NodeStatus{ + {Hostname: snTestWorker, UUID: "backend-uuid-123", Status: "online", Health: true}, + }, + }, + } + r := newSNReconciler(t, sn, sns) + + if err := r.syncUUIDFromNodeSet(context.Background(), sn, sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: snsTestNS}, &updated) + if updated.Status.UUID != "backend-uuid-123" { + t.Errorf("UUID not synced: got %q", updated.Status.UUID) + } + if updated.Status.Status != "online" { + t.Errorf("status not synced: got %q", updated.Status.Status) + } +} + +func TestSyncUUIDFromNodeSet_NoopWhenWorkerNotInNodes(t *testing.T) { + sn := newStorageNode("sn-1", snsTestNS, "sns", snTestWorker) + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Status: simplyblockv1alpha1.StorageNodeSetStatus{ + Nodes: []simplyblockv1alpha1.NodeStatus{ + {Hostname: "other-worker.example.com", UUID: "other-uuid"}, + }, + }, + } + r := newSNReconciler(t, sn, sns) + + if err := r.syncUUIDFromNodeSet(context.Background(), sn, sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: snsTestNS}, &updated) + if updated.Status.UUID != "" { + t.Errorf("UUID should remain empty, got %q", updated.Status.UUID) + } +} + +func TestSyncUUIDFromNodeSet_SkipsEmptyUUID(t *testing.T) { + sn := newStorageNode("sn-1", snsTestNS, "sns", snTestWorker) + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Status: simplyblockv1alpha1.StorageNodeSetStatus{ + Nodes: []simplyblockv1alpha1.NodeStatus{ + {Hostname: snTestWorker, UUID: ""}, // placeholder, not yet online + }, + }, + } + r := newSNReconciler(t, sn, sns) + + if err := r.syncUUIDFromNodeSet(context.Background(), sn, sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNode + _ = r.Get(context.Background(), types.NamespacedName{Name: "sn-1", Namespace: snsTestNS}, &updated) + if updated.Status.UUID != "" { + t.Errorf("UUID should remain empty when node entry has empty UUID, got %q", updated.Status.UUID) + } +} + +// ── TestSyncManualStorageNodeStatus ─────────────────────────────────────────── + +func TestSyncManualStorageNodeStatus_AddsManualNodeToSNSStatus(t *testing.T) { + // A StorageNode without OwnerReference (manual) that has a UUID + sn := newStorageNode("manual-sn", snsTestNS, "sns", "manual-worker.example.com") + sn.Status.UUID = "manual-uuid-456" + sn.Status.Status = "online" + sn.Status.Health = true + + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: snsTestCluster}, + // WorkerNodes does NOT contain manual-worker + } + r := newSNSReconciler(t, sn, sns) + + if err := r.syncManualStorageNodeStatus(context.Background(), sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNodeSet + _ = r.Get(context.Background(), types.NamespacedName{Name: "sns", Namespace: snsTestNS}, &updated) + + found := false + for _, n := range updated.Status.Nodes { + if n.UUID == "manual-uuid-456" { + found = true + if n.Status != "online" { + t.Errorf("status not synced: got %q", n.Status) + } + } + } + if !found { + t.Error("manual StorageNode UUID not added to StorageNodeSet.status.nodes[]") + } +} + +func TestSyncManualStorageNodeStatus_SkipsUnprovisionedNodes(t *testing.T) { + sn := newStorageNode("manual-sn", snsTestNS, "sns", "manual-worker.example.com") + // UUID is empty — not yet provisioned + + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: snsTestCluster}, + } + r := newSNSReconciler(t, sn, sns) + + if err := r.syncManualStorageNodeStatus(context.Background(), sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNodeSet + _ = r.Get(context.Background(), types.NamespacedName{Name: "sns", Namespace: snsTestNS}, &updated) + if len(updated.Status.Nodes) != 0 { + t.Errorf("expected empty status.nodes[], got %d entries", len(updated.Status.Nodes)) + } +} + +func TestSyncManualStorageNodeStatus_SkipsWorkerInSpecWorkerNodes(t *testing.T) { + // Worker is in spec.workerNodes — it's operator-managed, not manual + sn := newStorageNode("managed-sn", snsTestNS, "sns", snTestWorker) + sn.Status.UUID = "managed-uuid" + + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ + ClusterName: snsTestCluster, + WorkerNodes: []string{snTestWorker}, // operator-managed + }, + } + r := newSNSReconciler(t, sn, sns) + + if err := r.syncManualStorageNodeStatus(context.Background(), sns); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updated simplyblockv1alpha1.StorageNodeSet + _ = r.Get(context.Background(), types.NamespacedName{Name: "sns", Namespace: snsTestNS}, &updated) + if len(updated.Status.Nodes) != 0 { + t.Errorf("operator-managed node should not be added by syncManualStorageNodeStatus") + } +} + +func TestSyncManualStorageNodeStatus_IdempotentOnSecondCall(t *testing.T) { + sn := newStorageNode("manual-sn", snsTestNS, "sns", "manual-worker.example.com") + sn.Status.UUID = "manual-uuid-789" + sn.Status.Status = "online" + + sns := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, + Spec: simplyblockv1alpha1.StorageNodeSetSpec{ClusterName: snsTestCluster}, + } + r := newSNSReconciler(t, sn, sns) + + // First call + _ = r.syncManualStorageNodeStatus(context.Background(), sns) + + // Re-fetch and call again + var sns2 simplyblockv1alpha1.StorageNodeSet + _ = r.Get(context.Background(), types.NamespacedName{Name: "sns", Namespace: snsTestNS}, &sns2) + _ = r.syncManualStorageNodeStatus(context.Background(), &sns2) + + var updated simplyblockv1alpha1.StorageNodeSet + _ = r.Get(context.Background(), types.NamespacedName{Name: "sns", Namespace: snsTestNS}, &updated) + count := 0 + for _, n := range updated.Status.Nodes { + if n.UUID == "manual-uuid-789" { + count++ + } + } + if count != 1 { + t.Errorf("expected exactly 1 entry for manual node, got %d (not idempotent)", count) + } +} From 4274354f4803f278e917006103f08adc9e7e9697 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:43:10 +0100 Subject: [PATCH 30/52] fix: disable RestartOnSecretRefresh to prevent cert rotator crash loop in e2e --- operator/internal/webhook/cert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/webhook/cert.go b/operator/internal/webhook/cert.go index c13ed3c7..036cbc93 100644 --- a/operator/internal/webhook/cert.go +++ b/operator/internal/webhook/cert.go @@ -80,7 +80,7 @@ func SetupWebhookCertificate(mgr ctrl.Manager, namespace, tlsProvider string) (c ExtraDNSNames: []string{dnsName + ".cluster.local"}, IsReady: ready, Webhooks: webhooks, - RestartOnSecretRefresh: true, + RestartOnSecretRefresh: false, RequireLeaderElection: true, }); err != nil { return nil, fmt.Errorf("add webhook cert rotator: %w", err) From ad81b1b27c65303d86564ac778882f1f472f1598 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:52:59 +0100 Subject: [PATCH 31/52] fix: resolve remaining lint issues and register field indexes in test fake clients --- .../simplyblockstoragenodeset_storagenode.go | 3 ++- ...ockstoragenodeset_storagenode_unit_test.go | 2 +- .../storagenode_controller_unit_test.go | 20 ++++++++++++++----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index a5baa487..7e2629dd 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -38,6 +38,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" ) // reconcileStorageNodeCRs creates a StorageNode CR for every (worker, socket) @@ -169,7 +170,7 @@ func (r *StorageNodeSetReconciler) aggregateStorageNodeStatus( var online, offline, suspended, creating, removed int for _, sn := range owned.Items { switch sn.Status.Status { - case "online": + case utils.NodeStatusOnline: online++ case "offline": offline++ diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go index b836ce40..110cfc3f 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go @@ -78,7 +78,7 @@ func TestStorageNodeCRName_CollisionGuard(t *testing.T) { func TestStorageNodeCRName_IsDNSLabelSafe(t *testing.T) { name := storageNodeCRName("my-sns", "vm01.simplyblock3.localdomain", "0") for _, c := range name { - if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '.') { + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '.' { t.Errorf("invalid character %q in name %q", c, name) } } diff --git a/operator/internal/controller/storagenode_controller_unit_test.go b/operator/internal/controller/storagenode_controller_unit_test.go index c8710184..6a94ca69 100644 --- a/operator/internal/controller/storagenode_controller_unit_test.go +++ b/operator/internal/controller/storagenode_controller_unit_test.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" ) @@ -27,15 +28,24 @@ func newSNReconciler(t *testing.T, objects ...client.Object) *StorageNodeReconci simplyblockv1alpha1.AddToScheme, corev1.AddToScheme, ) - cl := newTestClient(t, scheme, - []client.Object{ + cl := fake.NewClientBuilder(). + WithScheme(scheme). + WithStatusSubresource( &simplyblockv1alpha1.StorageNode{}, &simplyblockv1alpha1.StorageNodeOps{}, &simplyblockv1alpha1.StorageCluster{}, &simplyblockv1alpha1.StorageNodeSet{}, - }, - objects..., - ) + ). + WithObjects(objects...). + WithIndex(&simplyblockv1alpha1.StorageNode{}, "spec.storageNodeSetRef", func(obj client.Object) []string { + sn := obj.(*simplyblockv1alpha1.StorageNode) + return []string{sn.Spec.StorageNodeSetRef} + }). + WithIndex(&simplyblockv1alpha1.StorageNode{}, "spec.workerNode", func(obj client.Object) []string { + sn := obj.(*simplyblockv1alpha1.StorageNode) + return []string{sn.Spec.WorkerNode} + }). + Build() return &StorageNodeReconciler{ Client: cl, Scheme: scheme, From b7a7895ae88b9f7bab1a336502e1531a6337f540 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 14:58:09 +0100 Subject: [PATCH 32/52] fix: replace remaining online string literals with utils.NodeStatusOnline constant --- operator/internal/controller/storagenode_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 96454bd3..0f575bd1 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -635,7 +635,7 @@ func (r *StorageNodeReconciler) handleDeletion( if sn.Status.Status == utils.NodeStatusSuspended || sn.Status.Status == utils.ClusterStatusActive || - sn.Status.Status == "online" { + sn.Status.Status == utils.NodeStatusOnline { if err := r.ensureRemoveOps(ctx, sn); err != nil { return ctrl.Result{}, err } From 0b08155173603dda364e62b8125ffb53f3eab5b7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 15:09:56 +0100 Subject: [PATCH 33/52] fix: replace online string literals with utils.NodeStatusOnline constant throughout controller package --- ...simplyblockstoragenodeset_storagenode_unit_test.go | 11 ++++++----- .../internal/controller/storagenodeops_controller.go | 4 ++-- operator/internal/utils/objects.go | 3 ++- .../autobalancing/logical_volume_selector.go | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go index 110cfc3f..25f0464c 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" ) const ( @@ -228,7 +229,7 @@ func TestSyncUUIDFromNodeSet_CopiesUUIDWhenFound(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, Status: simplyblockv1alpha1.StorageNodeSetStatus{ Nodes: []simplyblockv1alpha1.NodeStatus{ - {Hostname: snTestWorker, UUID: "backend-uuid-123", Status: "online", Health: true}, + {Hostname: snTestWorker, UUID: "backend-uuid-123", Status: utils.NodeStatusOnline, Health: true}, }, }, } @@ -243,7 +244,7 @@ func TestSyncUUIDFromNodeSet_CopiesUUIDWhenFound(t *testing.T) { if updated.Status.UUID != "backend-uuid-123" { t.Errorf("UUID not synced: got %q", updated.Status.UUID) } - if updated.Status.Status != "online" { + if updated.Status.Status != utils.NodeStatusOnline { t.Errorf("status not synced: got %q", updated.Status.Status) } } @@ -300,7 +301,7 @@ func TestSyncManualStorageNodeStatus_AddsManualNodeToSNSStatus(t *testing.T) { // A StorageNode without OwnerReference (manual) that has a UUID sn := newStorageNode("manual-sn", snsTestNS, "sns", "manual-worker.example.com") sn.Status.UUID = "manual-uuid-456" - sn.Status.Status = "online" + sn.Status.Status = utils.NodeStatusOnline sn.Status.Health = true sns := &simplyblockv1alpha1.StorageNodeSet{ @@ -321,7 +322,7 @@ func TestSyncManualStorageNodeStatus_AddsManualNodeToSNSStatus(t *testing.T) { for _, n := range updated.Status.Nodes { if n.UUID == "manual-uuid-456" { found = true - if n.Status != "online" { + if n.Status != utils.NodeStatusOnline { t.Errorf("status not synced: got %q", n.Status) } } @@ -380,7 +381,7 @@ func TestSyncManualStorageNodeStatus_SkipsWorkerInSpecWorkerNodes(t *testing.T) func TestSyncManualStorageNodeStatus_IdempotentOnSecondCall(t *testing.T) { sn := newStorageNode("manual-sn", snsTestNS, "sns", "manual-worker.example.com") sn.Status.UUID = "manual-uuid-789" - sn.Status.Status = "online" + sn.Status.Status = utils.NodeStatusOnline sns := &simplyblockv1alpha1.StorageNodeSet{ ObjectMeta: metav1.ObjectMeta{Name: "sns", Namespace: snsTestNS}, diff --git a/operator/internal/controller/storagenodeops_controller.go b/operator/internal/controller/storagenodeops_controller.go index 4f192b51..8d8f755f 100644 --- a/operator/internal/controller/storagenodeops_controller.go +++ b/operator/internal/controller/storagenodeops_controller.go @@ -217,8 +217,8 @@ func (r *StorageNodeOpsReconciler) runSimpleAction( // Poll node status until terminal. terminalStatus := map[string]string{ "suspend": utils.NodeStatusSuspended, - "resume": "online", - "restart": "online", + "resume": utils.NodeStatusOnline, + "restart": utils.NodeStatusOnline, "shutdown": "offline", } want := terminalStatus[action] diff --git a/operator/internal/utils/objects.go b/operator/internal/utils/objects.go index a1fc56a2..d6f9ad75 100644 --- a/operator/internal/utils/objects.go +++ b/operator/internal/utils/objects.go @@ -15,6 +15,7 @@ import ( simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" "github.com/simplyblock/simplyblock-operator/internal/webapi" ) @@ -174,7 +175,7 @@ func CountOnlineHealthyNodes( ) int { count := 0 for _, n := range nodes { - if n.Status == "online" && n.Health { + if n.Status == utils.NodeStatusOnline && n.Health { count++ } } diff --git a/operator/internal/volumemigration/autobalancing/logical_volume_selector.go b/operator/internal/volumemigration/autobalancing/logical_volume_selector.go index 411bad60..54eb8853 100644 --- a/operator/internal/volumemigration/autobalancing/logical_volume_selector.go +++ b/operator/internal/volumemigration/autobalancing/logical_volume_selector.go @@ -153,7 +153,7 @@ func (lvs *LogicalVolumeSelector) FilterEligibleVolumes( if input.IsCoolingDown != nil && input.IsCoolingDown(vp.UUID) { continue } - if vp.Status != "online" { + if vp.Status != utils.NodeStatusOnline { continue } if vp.Migrating { From 56dc53b4cca8cdb69abbfd13d9d943fe4850cddf Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 15:17:09 +0100 Subject: [PATCH 34/52] fixed cyclomatic import --- operator/internal/utils/objects.go | 3 +-- operator/internal/webhook/cert.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/operator/internal/utils/objects.go b/operator/internal/utils/objects.go index d6f9ad75..336ec52f 100644 --- a/operator/internal/utils/objects.go +++ b/operator/internal/utils/objects.go @@ -15,7 +15,6 @@ import ( simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" - "github.com/simplyblock/simplyblock-operator/internal/utils" "github.com/simplyblock/simplyblock-operator/internal/webapi" ) @@ -175,7 +174,7 @@ func CountOnlineHealthyNodes( ) int { count := 0 for _, n := range nodes { - if n.Status == utils.NodeStatusOnline && n.Health { + if n.Status == NodeStatusOnline && n.Health { count++ } } diff --git a/operator/internal/webhook/cert.go b/operator/internal/webhook/cert.go index 036cbc93..c13ed3c7 100644 --- a/operator/internal/webhook/cert.go +++ b/operator/internal/webhook/cert.go @@ -80,7 +80,7 @@ func SetupWebhookCertificate(mgr ctrl.Manager, namespace, tlsProvider string) (c ExtraDNSNames: []string{dnsName + ".cluster.local"}, IsReady: ready, Webhooks: webhooks, - RestartOnSecretRefresh: false, + RestartOnSecretRefresh: true, RequireLeaderElection: true, }); err != nil { return nil, fmt.Errorf("add webhook cert rotator: %w", err) From ba18ed94d157ba30b2f0b54fd3751ec3e64436d7 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 16:21:00 +0100 Subject: [PATCH 35/52] operator: enforce FDB sequential node add at StorageNode CR creation level to fix cache lag race --- .../simplyblockstoragenodeset_storagenode.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 7e2629dd..c599f2cd 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -93,9 +93,44 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( } } + // Determine which workers host FDB pods — these must be added sequentially. + fdbWorkers := r.fdbWorkerSet(ctx, sns) + + // Track whether any FDB worker's StorageNode CR is still awaiting its UUID. + // If so, stop creating new FDB CRs — the StorageNodeReconciler will POST + // the current one and we wait until it comes online before creating the next. + fdbInFlight := false + for _, sn := range owned.Items { + if !fdbWorkers[sn.Spec.WorkerNode] { + continue + } + if sn.Status.UUID == "" { + fdbInFlight = true + break + } + } + // Create or sync each expected (worker, socket). for _, worker := range sns.Spec.WorkerNodes { for _, socket := range sockets { + // For FDB workers: only create the next CR when no other FDB + // worker is still being provisioned (UUID not yet assigned). + if fdbWorkers[worker] && fdbInFlight { + // Check if THIS worker already has a CR (already in progress or done). + crName := storageNodeCRName(sns.Name, worker, socket) + alreadyExists := false + for _, sn := range owned.Items { + if sn.Name == crName { + alreadyExists = true + break + } + } + if !alreadyExists { + log.Info("FDB worker: deferring StorageNode CR creation until previous FDB node is online", + "worker", worker) + continue // skip — revisit on next reconcile + } + } if err := r.ensureStorageNodeCR(ctx, sns, worker, socket); err != nil { log.Error(err, "failed to ensure StorageNode CR", "worker", worker, "socket", socket) } From 3c53f017b6d5f4752316e2fd95b7915039ed4eb6 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 16:54:18 +0100 Subject: [PATCH 36/52] operator: pre-populate StorageNode status from legacy StorageNodeSet.status.nodes[] on CR creation for backward compatibility --- .../simplyblockstoragenodeset_storagenode.go | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index c599f2cd..b3d7e38e 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -167,6 +167,34 @@ func (r *StorageNodeSetReconciler) ensureStorageNodeCR( return fmt.Errorf("creating StorageNode %s: %w", name, createErr) } log.Info("created StorageNode CR", "name", name, "worker", worker, "socket", socket) + + // Backward compatibility: pre-populate UUID and status from the legacy + // StorageNodeSet.status.nodes[] so nodes that were already provisioned + // before the three-tier model are adopted immediately (no re-POST). + for i := range sns.Status.Nodes { + ns := &sns.Status.Nodes[i] + if ns.Hostname != worker || ns.UUID == "" { + continue + } + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.UUID = ns.UUID + sn.Status.Status = ns.Status + sn.Status.Health = ns.Health + sn.Status.MgmtIp = ns.MgmtIp + sn.Status.Hostname = ns.Hostname + sn.Status.CPU = ns.CPU + sn.Status.Volumes = ns.Volumes + sn.Status.RpcPort = ns.RpcPort + sn.Status.LvolPort = ns.LvolPort + sn.Status.NvmfPort = ns.NvmfPort + if patchErr := r.Status().Patch(ctx, sn, patch); patchErr != nil { + log.Error(patchErr, "failed to pre-populate StorageNode status", "name", name) + } else { + log.Info("pre-populated StorageNode status from legacy nodes[]", + "name", name, "uuid", ns.UUID, "status", ns.Status) + } + break + } return nil } if err != nil { From 16f52a4d9e4579924be388d50801c3bf40b94392 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 23:48:14 +0100 Subject: [PATCH 37/52] fix: add StorageNode, StorageNodeOps, BackupImport, and ControlPlane CRDs to kustomization.yaml --- operator/config/crd/kustomization.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/operator/config/crd/kustomization.yaml b/operator/config/crd/kustomization.yaml index 51e9b364..3d476230 100644 --- a/operator/config/crd/kustomization.yaml +++ b/operator/config/crd/kustomization.yaml @@ -11,6 +11,10 @@ resources: - bases/storage.simplyblock.io_snapshotreplications.yaml - bases/storage.simplyblock.io_backuppolicies.yaml - bases/storage.simplyblock.io_volumemigrations.yaml +- bases/storage.simplyblock.io_storagenodes.yaml +- bases/storage.simplyblock.io_storagenodeops.yaml +- bases/storage.simplyblock.io_backupimports.yaml +- bases/storage.simplyblock.io_controlplanes.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: [] From 08612b9bf162f52a490938034bf416cfed817717 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Wed, 15 Jul 2026 23:50:36 +0100 Subject: [PATCH 38/52] Ran make operator-build-installer --- operator/dist/install.yaml | 643 +++++++++++++++++++++++++++++++++++++ 1 file changed, 643 insertions(+) diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index f48023fc..6670a96c 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -9,6 +9,130 @@ metadata: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: backupimports.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: BackupImport + listKind: BackupImportList + plural: backupimports + shortNames: + - bi + singular: backupimport + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .spec.sourceClusterName + name: Source + type: string + - jsonPath: .spec.targetClusterName + name: Target + type: string + - jsonPath: .status.storageBackupRef + name: BackupRef + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + BackupImport imports a completed backup from a source cluster into a target cluster, + creating a StorageBackup CR that can be referenced by a BackupRestore. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of BackupImport + properties: + sourceBackupID: + description: SourceBackupID is the UUID of the backup on the source + cluster to import. + pattern: ^[a-zA-Z0-9_-]{1,128}$ + type: string + sourceClusterName: + description: SourceClusterName is the StorageCluster CR name of the + cluster that owns the backup. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + targetClusterName: + description: TargetClusterName is the StorageCluster CR name of the + cluster to import into. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - sourceBackupID + - sourceClusterName + - targetClusterName + type: object + status: + description: status defines the observed state of BackupImport + properties: + completedAt: + description: CompletedAt is when the import completed. + format: date-time + type: string + importedBackupID: + description: ImportedBackupID is the backup UUID after successful + import into the target cluster. + type: string + message: + description: Message contains the latest reconciliation detail or + error. + type: string + phase: + description: Phase is the high-level lifecycle shown in kubectl output. + type: string + sourceClusterUUID: + description: SourceClusterUUID is the resolved UUID of the source + cluster. + type: string + storageBackupRef: + description: |- + StorageBackupRef is the name of the StorageBackup CR created in the target namespace + after a successful import. This CR can be referenced directly in a BackupRestore. + type: string + targetClusterUUID: + description: TargetClusterUUID is the resolved UUID of the target + cluster. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.21.0 @@ -550,6 +674,103 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: controlplanes.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: ControlPlane + listKind: ControlPlaneList + plural: controlplanes + singular: controlplane + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Initializing while FDB is not ready; Ready once the control plane + is operational + jsonPath: .status.phase + name: Phase + type: string + - description: Human-readable status detail + jsonPath: .status.message + name: Message + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ControlPlane is a singleton resource (one per namespace, named "simplyblock") + that reflects the readiness of the simplyblock control plane. It is created + automatically by the Helm chart and should not be created or deleted manually. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + ControlPlaneSpec holds configuration for the singleton ControlPlane resource + created by the Helm chart. + properties: + image: + description: |- + Image is the container image used for all simplyblock control-plane and + storage-node workloads (e.g. quay.io/simplyblock-io/simplyblock:26.2.2). + StorageNodeSet CRs that omit spec.clusterImage inherit this value. + Must reference one of the trusted registries (quay.io/simplyblock-io, docker.io/simplyblock, public.ecr.aws/simply-block); digest pinning (@sha256:...) is recommended. + pattern: ^($|(quay\.io/simplyblock-io|docker\.io/simplyblock|public\.ecr\.aws/simply-block)/[a-z0-9][a-z0-9._-]*:[a-zA-Z0-9][a-zA-Z0-9._-]*(@sha256:[a-f0-9]{64})?)$ + type: string + type: object + status: + description: |- + ControlPlaneStatus reflects the observed readiness of the simplyblock + control plane (FDB + management API). + properties: + lastChecked: + description: LastChecked is the timestamp of the most recent FDB health + probe. + format: date-time + type: string + message: + description: |- + Message contains a human-readable explanation of the current phase, + for example the FDB error returned by the health endpoint. + type: string + phase: + description: |- + Phase is Initializing while the control plane is not yet healthy, + and Ready once the FDB health check passes. + enum: + - Initializing + - Ready + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.21.0 @@ -1757,6 +1978,428 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: storagenodeops.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: StorageNodeOps + listKind: StorageNodeOpsList + plural: storagenodeops + shortNames: + - snops + singular: storagenodeops + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.storageNodeRef + name: Node + type: string + - jsonPath: .spec.action + name: Action + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.subPhase + name: SubPhase + type: string + - jsonPath: .status.message + name: Message + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StorageNodeOps is a one-shot operational CR targeting a single StorageNode. + Analogous to a Kubernetes Job — it drives an action (shutdown, restart, suspend, + resume, remove/drain) to completion and records the result. Only one + StorageNodeOps can be active per StorageNode at a time. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageNodeOpsSpec defines the desired state of a StorageNodeOps. + properties: + action: + description: Action is the operation to perform. Immutable. + enum: + - shutdown + - restart + - suspend + - resume + - remove + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + drain: + description: Drain configures the drain workflow. Only applicable + when action=remove. + properties: + systemVolumeFilterRegex: + description: |- + SystemVolumeFilterRegex is a Go regular expression matched against backend + volume names. Matching volumes are treated as system volumes: excluded from + drain migration and deleted inline during the Verifying phase. + Defaults to "^sb-fio-baseline-.*". + type: string + type: object + force: + description: Force enables forced execution where the backend supports + it. + type: boolean + reattachVolume: + description: |- + ReattachVolume reattaches volumes during restart. + Only applicable when action=restart. + type: boolean + storageNodeRef: + description: StorageNodeRef is the name of the target StorageNode. + Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - action + - storageNodeRef + type: object + status: + description: StorageNodeOpsStatus holds the observed state of a StorageNodeOps. + properties: + completedAt: + description: CompletedAt is when the operation finished (successfully + or not). + format: date-time + type: string + message: + description: Message is a human-readable description of the current + state or failure reason. + type: string + phase: + description: Phase is the high-level lifecycle phase. + enum: + - Pending + - Running + - Succeeded + - Failed + type: string + startedAt: + description: StartedAt is when the operation began. + format: date-time + type: string + subPhase: + description: SubPhase tracks the active drain step when action=remove + and phase=Running. + enum: + - Validating + - Suspending + - Migrating + - Verifying + - Removing + type: string + triggered: + description: |- + Triggered indicates the backend action POST has been sent (used during + Suspending to avoid duplicate POSTs across reconcile iterations). + type: boolean + volumesMigrated: + description: VolumesMigrated is the count of volumes successfully + migrated (drain only). + type: integer + volumesPending: + description: VolumesPending is the count of volumes awaiting migration + (drain only). + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: storagenodes.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: StorageNode + listKind: StorageNodeList + plural: storagenodes + shortNames: + - sn + singular: storagenode + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.workerNode + name: Worker + type: string + - jsonPath: .spec.socketIndex + name: Socket + type: integer + - jsonPath: .status.uuid + name: UUID + type: string + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.health + name: Health + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StorageNode is the Schema for a single backend storage node instance. + One StorageNode CR exists per (workerNode, socketIndex) pair and is owned + by the parent StorageNodeSet. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageNodeSpec defines the desired state of a StorageNode. + properties: + overrides: + description: |- + Overrides holds per-node configuration propagated from + StorageNodeSet.spec.nodeConfigs[workerNode] on every reconcile. + properties: + corePercentage: + description: CorePercentage overrides the percentage of cores + allocated to SPDK for this node (0-99). + format: int32 + type: integer + deviceNames: + description: |- + DeviceNames explicitly defines the NVMe namespace names to use on this node + (e.g. ["nvme0n1","nvme1n1"]). + items: + type: string + type: array + driveSizeRange: + description: DriveSizeRange overrides the drive size range filter + for this node. + type: string + enableCpuTopology: + description: EnableCpuTopology overrides topology-aware CPU handling + for this node. + type: boolean + failureDomain: + description: |- + FailureDomain is the failure-domain group index (≥ 1) for this node. + Required when the parent StorageCluster has enableFailureDomains=true. + Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + format: int32 + minimum: 1 + type: integer + journalManager: + description: JournalManagerSpec overrides journal manager tuning + for this node. + properties: + count: + description: Count is the number of journal managers to configure. + format: int32 + type: integer + percentPerDevice: + description: PercentPerDevice is the journal manager capacity + percentage per device. + format: int32 + type: integer + type: object + maxLogicalVolumeCount: + description: MaxLogicalVolumeCount overrides the maximum number + of logical volumes for this node. + format: int32 + type: integer + maxSize: + description: MaxSize overrides the maximum allocatable size of + huge pages for this node. + type: string + pcieAllowList: + description: PcieAllowList overrides the list of PCI addresses + allowed for use on this node. + items: + type: string + type: array + pcieDenyList: + description: PcieDenyList overrides the list of PCI addresses + excluded from use on this node. + items: + type: string + type: array + pcieModel: + description: PcieModel overrides the PCI model filter for this + node. + type: string + reservedSystemCPU: + description: ReservedSystemCPU overrides the CPUs reserved for + system workloads on this node. + type: string + skipKubeletConfiguration: + description: |- + SkipKubeletConfiguration overrides whether kubelet configuration changes are + skipped for this node. + type: boolean + spdkImage: + description: SpdkImage overrides the SPDK image for this node + (e.g. for phased rollouts). + type: string + spdkProxyImage: + description: SpdkProxyImage overrides the SPDK proxy image for + this node. + type: string + spdkSystemMemory: + description: |- + SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + (e.g. "4G", "512M"). + pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ + type: string + ubuntuHost: + description: UbuntuHost overrides the Ubuntu host OS flag for + this node. + type: boolean + type: object + socketIndex: + description: SocketIndex is the NUMA socket index (0-based). Immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + storageNodeSetRef: + description: StorageNodeSetRef is the name of the owning StorageNodeSet. + Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + workerNode: + description: WorkerNode is the Kubernetes node hostname this StorageNode + runs on. Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - storageNodeSetRef + - workerNode + type: object + x-kubernetes-validations: + - message: field socketIndex is immutable once set + rule: '!has(oldSelf.socketIndex) || has(self.socketIndex)' + status: + description: StorageNodeStatus holds the observed state of a StorageNode. + properties: + activeOpsRef: + description: |- + ActiveOpsRef is the name of the currently active StorageNodeOps CR targeting + this node. Empty when no operation is in progress. Used for mutual exclusion. + type: string + cpu: + description: CPU is the number of CPU cores allocated to the node. + format: int32 + type: integer + devices: + description: Devices is the device list reported by the backend. + type: string + health: + description: Health is the backend-reported node health flag. + type: boolean + hostname: + description: Hostname is the node hostname as reported by the backend. + type: string + lvolPort: + description: LvolPort is the lvol port of the node. + format: int32 + type: integer + memory: + description: Memory is the memory allocation reported by the backend. + type: string + mgmtIp: + description: MgmtIp is the management IP address of the node. + type: string + nvmfPort: + description: NvmfPort is the NVMe-oF port of the node. + format: int32 + type: integer + postedAt: + description: |- + PostedAt is the timestamp when the node-add POST was sent. + Used as a provisioning guard against duplicate POSTs. + format: date-time + type: string + rpcPort: + description: RpcPort is the RPC port of the node. + format: int32 + type: integer + status: + description: Status is the backend-reported node status (e.g. online, + suspended, offline). + type: string + uptime: + description: Uptime is the node uptime as reported by the backend. + type: string + uuid: + description: UUID is the backend storage node UUID. Set once after + node-add completes. + type: string + volumes: + description: Volumes is the number of logical volumes on this node. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.21.0 From 25283b31f5aa540cd720f331cb3a9afaa53e4e4e Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 16 Jul 2026 09:49:13 +0100 Subject: [PATCH 39/52] operator: nest StorageNode status into resources/ports groups per reviewer feedback --- operator/api/v1alpha1/storagenode_types.go | 69 +++++++------ .../api/v1alpha1/zz_generated.deepcopy.go | 91 +++++++++++++----- .../storage.simplyblock.io_storagenodes.yaml | 96 +++++++++++++------ operator/dist/install.yaml | 96 +++++++++++++------ .../simplyblockstoragenodeset_storagenode.go | 53 ++++++---- .../controller/storagenode_controller.go | 32 ++++--- 6 files changed, 296 insertions(+), 141 deletions(-) diff --git a/operator/api/v1alpha1/storagenode_types.go b/operator/api/v1alpha1/storagenode_types.go index ffd8376b..2abb6c44 100644 --- a/operator/api/v1alpha1/storagenode_types.go +++ b/operator/api/v1alpha1/storagenode_types.go @@ -129,6 +129,38 @@ type StorageNodeSpec struct { Overrides *StorageNodeOverrides `json:"overrides,omitempty"` } +// StorageNodeResources groups compute and storage resource fields reported by the backend. +type StorageNodeResources struct { + // CPU is the number of SPDK CPU cores allocated to this node. + // +optional + CPU *int32 `json:"cpu,omitempty"` + // Memory is the SPDK memory allocation reported by the backend. + // +optional + Memory string `json:"memory,omitempty"` + // Volumes is the current number of logical volumes on this node. + // +optional + Volumes *int32 `json:"volumes,omitempty"` + // Devices is the device summary (online/total) reported by the backend. + // +optional + Devices string `json:"devices,omitempty"` +} + +// StorageNodePorts groups the network port and address fields reported by the backend. +type StorageNodePorts struct { + // Management is the management IP address of the node. + // +optional + Management string `json:"management,omitempty"` + // NvmeOf is the NVMe-oF fabric port. + // +optional + NvmeOf *int32 `json:"nvmeof,omitempty"` + // Lvol is the logical-volume subsystem port. + // +optional + Lvol *int32 `json:"lvol,omitempty"` + // Rpc is the RPC/management API port. + // +optional + Rpc *int32 `json:"rpc,omitempty"` +} + // StorageNodeStatus holds the observed state of a StorageNode. type StorageNodeStatus struct { // UUID is the backend storage node UUID. Set once after node-add completes. @@ -143,26 +175,6 @@ type StorageNodeStatus struct { // +optional Health bool `json:"health,omitempty"` - // CPU is the number of CPU cores allocated to the node. - // +optional - CPU *int32 `json:"cpu,omitempty"` - - // Memory is the memory allocation reported by the backend. - // +optional - Memory string `json:"memory,omitempty"` - - // Volumes is the number of logical volumes on this node. - // +optional - Volumes *int32 `json:"volumes,omitempty"` - - // Devices is the device list reported by the backend. - // +optional - Devices string `json:"devices,omitempty"` - - // MgmtIp is the management IP address of the node. - // +optional - MgmtIp string `json:"mgmtIp,omitempty"` - // Hostname is the node hostname as reported by the backend. // +optional Hostname string `json:"hostname,omitempty"` @@ -171,17 +183,13 @@ type StorageNodeStatus struct { // +optional Uptime string `json:"uptime,omitempty"` - // RpcPort is the RPC port of the node. + // Resources groups compute and storage resource metrics. // +optional - RpcPort *int32 `json:"rpcPort,omitempty"` + Resources *StorageNodeResources `json:"resources,omitempty"` - // LvolPort is the lvol port of the node. + // Ports groups network connectivity fields (addresses and ports). // +optional - LvolPort *int32 `json:"lvolPort,omitempty"` - - // NvmfPort is the NVMe-oF port of the node. - // +optional - NvmfPort *int32 `json:"nvmfPort,omitempty"` + Ports *StorageNodePorts `json:"ports,omitempty"` // PostedAt is the timestamp when the node-add POST was sent. // Used as a provisioning guard against duplicate POSTs. @@ -192,6 +200,11 @@ type StorageNodeStatus struct { // this node. Empty when no operation is in progress. Used for mutual exclusion. // +optional ActiveOpsRef string `json:"activeOpsRef,omitempty"` + + // LatencyMetrics holds the fio-measured baseline NVMe-oF latency for this node, + // used by the volume rebalancer to make data-placement decisions. + // +optional + LatencyMetrics *NodeLatencyMetrics `json:"latencyMetrics,omitempty"` } // +kubebuilder:object:root=true diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 2ef09617..6082ffaf 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1813,6 +1813,61 @@ func (in *StorageNodeOverrides) DeepCopy() *StorageNodeOverrides { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodePorts) DeepCopyInto(out *StorageNodePorts) { + *out = *in + if in.NvmeOf != nil { + in, out := &in.NvmeOf, &out.NvmeOf + *out = new(int32) + **out = **in + } + if in.Lvol != nil { + in, out := &in.Lvol, &out.Lvol + *out = new(int32) + **out = **in + } + if in.Rpc != nil { + in, out := &in.Rpc, &out.Rpc + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodePorts. +func (in *StorageNodePorts) DeepCopy() *StorageNodePorts { + if in == nil { + return nil + } + out := new(StorageNodePorts) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageNodeResources) DeepCopyInto(out *StorageNodeResources) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + *out = new(int32) + **out = **in + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeResources. +func (in *StorageNodeResources) DeepCopy() *StorageNodeResources { + if in == nil { + return nil + } + out := new(StorageNodeResources) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageNodeSet) DeepCopyInto(out *StorageNodeSet) { *out = *in @@ -2073,35 +2128,25 @@ func (in *StorageNodeSpec) DeepCopy() *StorageNodeSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageNodeStatus) DeepCopyInto(out *StorageNodeStatus) { *out = *in - if in.CPU != nil { - in, out := &in.CPU, &out.CPU - *out = new(int32) - **out = **in - } - if in.Volumes != nil { - in, out := &in.Volumes, &out.Volumes - *out = new(int32) - **out = **in - } - if in.RpcPort != nil { - in, out := &in.RpcPort, &out.RpcPort - *out = new(int32) - **out = **in - } - if in.LvolPort != nil { - in, out := &in.LvolPort, &out.LvolPort - *out = new(int32) - **out = **in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(StorageNodeResources) + (*in).DeepCopyInto(*out) } - if in.NvmfPort != nil { - in, out := &in.NvmfPort, &out.NvmfPort - *out = new(int32) - **out = **in + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = new(StorageNodePorts) + (*in).DeepCopyInto(*out) } if in.PostedAt != nil { in, out := &in.PostedAt, &out.PostedAt *out = (*in).DeepCopy() } + if in.LatencyMetrics != nil { + in, out := &in.LatencyMetrics, &out.LatencyMetrics + *out = new(NodeLatencyMetrics) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeStatus. diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml index 3029de08..c4571803 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml @@ -199,43 +199,85 @@ spec: ActiveOpsRef is the name of the currently active StorageNodeOps CR targeting this node. Empty when no operation is in progress. Used for mutual exclusion. type: string - cpu: - description: CPU is the number of CPU cores allocated to the node. - format: int32 - type: integer - devices: - description: Devices is the device list reported by the backend. - type: string health: description: Health is the backend-reported node health flag. type: boolean hostname: description: Hostname is the node hostname as reported by the backend. type: string - lvolPort: - description: LvolPort is the lvol port of the node. - format: int32 - type: integer - memory: - description: Memory is the memory allocation reported by the backend. - type: string - mgmtIp: - description: MgmtIp is the management IP address of the node. - type: string - nvmfPort: - description: NvmfPort is the NVMe-oF port of the node. - format: int32 - type: integer + latencyMetrics: + description: |- + LatencyMetrics holds the fio-measured baseline NVMe-oF latency for this node, + used by the volume rebalancer to make data-placement decisions. + properties: + baselineMeasuredAt: + description: BaselineMeasuredAt is when the baseline was established. + format: date-time + type: string + baselineP50NS: + description: BaselineP50NS is the p50 write latency (nanoseconds) + from the initial empty-cluster benchmark. + format: int64 + type: integer + baselineP99NS: + description: BaselineP99NS is the p99 write latency (nanoseconds) + from the initial empty-cluster benchmark. + format: int64 + type: integer + nodeUUID: + description: NodeUUID is the backend storage node UUID. + type: string + required: + - nodeUUID + type: object + ports: + description: Ports groups network connectivity fields (addresses and + ports). + properties: + lvol: + description: Lvol is the logical-volume subsystem port. + format: int32 + type: integer + management: + description: Management is the management IP address of the node. + type: string + nvmeof: + description: NvmeOf is the NVMe-oF fabric port. + format: int32 + type: integer + rpc: + description: Rpc is the RPC/management API port. + format: int32 + type: integer + type: object postedAt: description: |- PostedAt is the timestamp when the node-add POST was sent. Used as a provisioning guard against duplicate POSTs. format: date-time type: string - rpcPort: - description: RpcPort is the RPC port of the node. - format: int32 - type: integer + resources: + description: Resources groups compute and storage resource metrics. + properties: + cpu: + description: CPU is the number of SPDK CPU cores allocated to + this node. + format: int32 + type: integer + devices: + description: Devices is the device summary (online/total) reported + by the backend. + type: string + memory: + description: Memory is the SPDK memory allocation reported by + the backend. + type: string + volumes: + description: Volumes is the current number of logical volumes + on this node. + format: int32 + type: integer + type: object status: description: Status is the backend-reported node status (e.g. online, suspended, offline). @@ -247,10 +289,6 @@ spec: description: UUID is the backend storage node UUID. Set once after node-add completes. type: string - volumes: - description: Volumes is the number of logical volumes on this node. - format: int32 - type: integer type: object type: object served: true diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 6670a96c..c6bbc015 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2339,43 +2339,85 @@ spec: ActiveOpsRef is the name of the currently active StorageNodeOps CR targeting this node. Empty when no operation is in progress. Used for mutual exclusion. type: string - cpu: - description: CPU is the number of CPU cores allocated to the node. - format: int32 - type: integer - devices: - description: Devices is the device list reported by the backend. - type: string health: description: Health is the backend-reported node health flag. type: boolean hostname: description: Hostname is the node hostname as reported by the backend. type: string - lvolPort: - description: LvolPort is the lvol port of the node. - format: int32 - type: integer - memory: - description: Memory is the memory allocation reported by the backend. - type: string - mgmtIp: - description: MgmtIp is the management IP address of the node. - type: string - nvmfPort: - description: NvmfPort is the NVMe-oF port of the node. - format: int32 - type: integer + latencyMetrics: + description: |- + LatencyMetrics holds the fio-measured baseline NVMe-oF latency for this node, + used by the volume rebalancer to make data-placement decisions. + properties: + baselineMeasuredAt: + description: BaselineMeasuredAt is when the baseline was established. + format: date-time + type: string + baselineP50NS: + description: BaselineP50NS is the p50 write latency (nanoseconds) + from the initial empty-cluster benchmark. + format: int64 + type: integer + baselineP99NS: + description: BaselineP99NS is the p99 write latency (nanoseconds) + from the initial empty-cluster benchmark. + format: int64 + type: integer + nodeUUID: + description: NodeUUID is the backend storage node UUID. + type: string + required: + - nodeUUID + type: object + ports: + description: Ports groups network connectivity fields (addresses and + ports). + properties: + lvol: + description: Lvol is the logical-volume subsystem port. + format: int32 + type: integer + management: + description: Management is the management IP address of the node. + type: string + nvmeof: + description: NvmeOf is the NVMe-oF fabric port. + format: int32 + type: integer + rpc: + description: Rpc is the RPC/management API port. + format: int32 + type: integer + type: object postedAt: description: |- PostedAt is the timestamp when the node-add POST was sent. Used as a provisioning guard against duplicate POSTs. format: date-time type: string - rpcPort: - description: RpcPort is the RPC port of the node. - format: int32 - type: integer + resources: + description: Resources groups compute and storage resource metrics. + properties: + cpu: + description: CPU is the number of SPDK CPU cores allocated to + this node. + format: int32 + type: integer + devices: + description: Devices is the device summary (online/total) reported + by the backend. + type: string + memory: + description: Memory is the SPDK memory allocation reported by + the backend. + type: string + volumes: + description: Volumes is the current number of logical volumes + on this node. + format: int32 + type: integer + type: object status: description: Status is the backend-reported node status (e.g. online, suspended, offline). @@ -2387,10 +2429,6 @@ spec: description: UUID is the backend storage node UUID. Set once after node-add completes. type: string - volumes: - description: Volumes is the number of logical volumes on this node. - format: int32 - type: integer type: object type: object served: true diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index b3d7e38e..55ba901d 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -180,13 +180,17 @@ func (r *StorageNodeSetReconciler) ensureStorageNodeCR( sn.Status.UUID = ns.UUID sn.Status.Status = ns.Status sn.Status.Health = ns.Health - sn.Status.MgmtIp = ns.MgmtIp sn.Status.Hostname = ns.Hostname - sn.Status.CPU = ns.CPU - sn.Status.Volumes = ns.Volumes - sn.Status.RpcPort = ns.RpcPort - sn.Status.LvolPort = ns.LvolPort - sn.Status.NvmfPort = ns.NvmfPort + sn.Status.Resources = &simplyblockv1alpha1.StorageNodeResources{ + CPU: ns.CPU, + Volumes: ns.Volumes, + } + sn.Status.Ports = &simplyblockv1alpha1.StorageNodePorts{ + Management: ns.MgmtIp, + Rpc: ns.RpcPort, + Lvol: ns.LvolPort, + NvmeOf: ns.NvmfPort, + } if patchErr := r.Status().Patch(ctx, sn, patch); patchErr != nil { log.Error(patchErr, "failed to pre-populate StorageNode status", "name", name) } else { @@ -392,13 +396,17 @@ func (r *StorageNodeSetReconciler) syncManualStorageNodeStatus( if n.Status != sn.Status.Status || n.Health != sn.Status.Health { n.Status = sn.Status.Status n.Health = sn.Status.Health - n.MgmtIp = sn.Status.MgmtIp n.Hostname = sn.Status.Hostname - n.CPU = sn.Status.CPU - n.Volumes = sn.Status.Volumes - n.RpcPort = sn.Status.RpcPort - n.LvolPort = sn.Status.LvolPort - n.NvmfPort = sn.Status.NvmfPort + if p := sn.Status.Ports; p != nil { + n.MgmtIp = p.Management + n.RpcPort = p.Rpc + n.LvolPort = p.Lvol + n.NvmfPort = p.NvmeOf + } + if r := sn.Status.Resources; r != nil { + n.CPU = r.CPU + n.Volumes = r.Volumes + } changed = true } found = true @@ -406,18 +414,23 @@ func (r *StorageNodeSetReconciler) syncManualStorageNodeStatus( } } if !found { - sns.Status.Nodes = append(sns.Status.Nodes, simplyblockv1alpha1.NodeStatus{ + entry := simplyblockv1alpha1.NodeStatus{ Hostname: sn.Spec.WorkerNode, UUID: sn.Status.UUID, Status: sn.Status.Status, Health: sn.Status.Health, - MgmtIp: sn.Status.MgmtIp, - CPU: sn.Status.CPU, - Volumes: sn.Status.Volumes, - RpcPort: sn.Status.RpcPort, - LvolPort: sn.Status.LvolPort, - NvmfPort: sn.Status.NvmfPort, - }) + } + if p := sn.Status.Ports; p != nil { + entry.MgmtIp = p.Management + entry.RpcPort = p.Rpc + entry.LvolPort = p.Lvol + entry.NvmfPort = p.NvmeOf + } + if r := sn.Status.Resources; r != nil { + entry.CPU = r.CPU + entry.Volumes = r.Volumes + } + sns.Status.Nodes = append(sns.Status.Nodes, entry) changed = true } } diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 0f575bd1..88c55e0e 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -212,13 +212,17 @@ func (r *StorageNodeReconciler) pollUUIDFromBackend( sn.Status.UUID = n.UUID sn.Status.Status = n.Status sn.Status.Health = n.Health - sn.Status.MgmtIp = n.IP sn.Status.Hostname = n.Hostname - sn.Status.CPU = &cpu - sn.Status.Volumes = &volumes - sn.Status.RpcPort = &rpcPort - sn.Status.LvolPort = &lvolPort - sn.Status.NvmfPort = &nvmfPort + sn.Status.Resources = &simplyblockv1alpha1.StorageNodeResources{ + CPU: &cpu, + Volumes: &volumes, + } + sn.Status.Ports = &simplyblockv1alpha1.StorageNodePorts{ + Management: n.IP, + NvmeOf: &nvmfPort, + Lvol: &lvolPort, + Rpc: &rpcPort, + } if err := r.Status().Patch(ctx, sn, patch); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("pollUUIDFromBackend: %w", err) } @@ -490,13 +494,17 @@ func (r *StorageNodeReconciler) syncStatus( patch := client.MergeFrom(sn.DeepCopy()) sn.Status.Status = resp.Status sn.Status.Health = resp.Health - sn.Status.CPU = &cpu - sn.Status.Volumes = &volumes - sn.Status.MgmtIp = resp.IP sn.Status.Hostname = resp.Hostname - sn.Status.RpcPort = &rpcPort - sn.Status.LvolPort = &lvolPort - sn.Status.NvmfPort = &nvmfPort + sn.Status.Resources = &simplyblockv1alpha1.StorageNodeResources{ + CPU: &cpu, + Volumes: &volumes, + } + sn.Status.Ports = &simplyblockv1alpha1.StorageNodePorts{ + Management: resp.IP, + NvmeOf: &nvmfPort, + Lvol: &lvolPort, + Rpc: &rpcPort, + } if err := r.Status().Patch(ctx, sn, patch); err != nil { log.Error(err, "failed to patch StorageNode status") From b85a981d89faed2e50b3c34890df7ab6b0541ccf Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 16 Jul 2026 09:53:00 +0100 Subject: [PATCH 40/52] docs: update design document with nested StorageNode status structure and latencyMetrics field --- .../design-storagenodeset-storagenode.md | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/operator/docs/designs/design-storagenodeset-storagenode.md b/operator/docs/designs/design-storagenodeset-storagenode.md index 09ea5611..fb114d88 100644 --- a/operator/docs/designs/design-storagenodeset-storagenode.md +++ b/operator/docs/designs/design-storagenodeset-storagenode.md @@ -162,21 +162,39 @@ type StorageNodeOverrides struct { // ... other mutable per-node tuning knobs } +// StorageNodeResources groups compute and storage resource fields. +type StorageNodeResources struct { + CPU *int32 `json:"cpu,omitempty"` + Memory string `json:"memory,omitempty"` + Volumes *int32 `json:"volumes,omitempty"` + Devices string `json:"devices,omitempty"` +} + +// StorageNodePorts groups network connectivity fields. +type StorageNodePorts struct { + Management string `json:"management,omitempty"` // management IP + NvmeOf *int32 `json:"nvmeof,omitempty"` // NVMe-oF fabric port + Lvol *int32 `json:"lvol,omitempty"` // lvol subsystem port + Rpc *int32 `json:"rpc,omitempty"` // RPC/management API port +} + type StorageNodeStatus struct { // UUID is the backend storage node UUID. Set once after node-add completes. UUID string `json:"uuid,omitempty"` Status string `json:"status,omitempty"` Health bool `json:"health,omitempty"` - CPU *int32 `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - Volumes *int32 `json:"volumes,omitempty"` - Devices string `json:"devices,omitempty"` - MgmtIp string `json:"mgmtIp,omitempty"` Hostname string `json:"hostname,omitempty"` Uptime string `json:"uptime,omitempty"` - RpcPort *int32 `json:"rpcPort,omitempty"` - LvolPort *int32 `json:"lvolPort,omitempty"` - NvmfPort *int32 `json:"nvmfPort,omitempty"` + + // Resources groups compute and storage resource metrics. + Resources *StorageNodeResources `json:"resources,omitempty"` + + // Ports groups network connectivity fields (addresses and ports). + Ports *StorageNodePorts `json:"ports,omitempty"` + + // LatencyMetrics holds fio-measured baseline NVMe-oF latency for this node, + // used by the volume rebalancer for data-placement decisions. + LatencyMetrics *NodeLatencyMetrics `json:"latencyMetrics,omitempty"` // PostedAt is when the node-add POST was sent (provisioning guard). PostedAt *metav1.Time `json:"postedAt,omitempty"` @@ -373,6 +391,12 @@ Implemented on 2026-07-15. - **Validation**: `spec.workerNodes` enforced unique via `+listType=set`; `spec.nodeConfigs` keys must match a `workerNodes` entry (CEL rule with `MaxItems=200`, `MaxProperties=200` to bound cost). +- **`StorageNodeStatus` restructured** into logical nested groups per reviewer feedback: + - `status.resources` — cpu, memory, volumes, devices + - `status.ports` — management (IP), nvmeof, lvol, rpc + - `status.latencyMetrics` — fio-measured baseline NVMe-oF latency (p50/p99) for the + volume rebalancer, mirroring the existing `NodeLatencyMetrics` type already on + `StorageNodeSet.status.latencyMetrics[]` - **42+ unit tests** covering both new reconcilers, drain state machine helpers, and StorageNodeSet status aggregation. See full test plan: [`docs/tests/test-plan-storagenode-ops.md`](../tests/test-plan-storagenode-ops.md) From f7d3e693a2dd8cb238b38b3e7125597b74dadc01 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 16 Jul 2026 10:21:02 +0100 Subject: [PATCH 41/52] operator: create StorageNode CRs per socket per nodeIndex with unambiguous s{socket}-n{nodeIdx} naming --- .../simplyblockstoragenodeset_storagenode.go | 137 ++++++++++-------- ...ockstoragenodeset_storagenode_unit_test.go | 10 +- 2 files changed, 80 insertions(+), 67 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 55ba901d..554e767d 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -51,13 +51,28 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( ) error { log := logf.FromContext(ctx) - // Build the expected set of (worker, socket) pairs. + // Compute nodesPerSocket (default 1). + nodesPerSocket := 1 + if sns.Spec.NodesPerSocket != nil && *sns.Spec.NodesPerSocket > 1 { + nodesPerSocket = int(*sns.Spec.NodesPerSocket) + } sockets := effectiveSockets(sns) - type workerSocket struct{ worker, socket string } - expected := make(map[workerSocket]struct{}, len(sns.Spec.WorkerNodes)*len(sockets)) + + // Build the expected set of (worker, globalOrdinal) pairs. + // globalOrdinal = socketPosition * nodesPerSocket + nodeIndex + // This uniquely identifies each backend storage node per worker host. + type workerOrdinal struct { + worker string + ordinal int + } + expected := make(map[workerOrdinal]struct{}, len(sns.Spec.WorkerNodes)*len(sockets)*nodesPerSocket) for _, worker := range sns.Spec.WorkerNodes { - for _, socket := range sockets { - expected[workerSocket{worker, socket}] = struct{}{} + for si, socket := range sockets { + _ = socket // socket string is embedded in the ordinal calculation + for ni := 0; ni < nodesPerSocket; ni++ { + ordinal := si*nodesPerSocket + ni + expected[workerOrdinal{worker, ordinal}] = struct{}{} + } } } @@ -70,35 +85,32 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( return fmt.Errorf("listing owned StorageNode CRs: %w", err) } - // Delete stale CRs (worker removed from spec.workerNodes or socket removed). - // Manually created StorageNode CRs (no controller OwnerReference pointing to - // this StorageNodeSet) are preserved — they can reference the fleet config - // without being listed in spec.workerNodes. + // Delete stale CRs (worker removed from spec.workerNodes, socket removed, + // or nodesPerSocket reduced). Manually created CRs (no OwnerReference) are kept. for i := range owned.Items { sn := &owned.Items[i] - key := workerSocket{sn.Spec.WorkerNode, socketLabel(sn.Spec.SocketIndex)} - if _, ok := expected[key]; ok { + ordinal := 0 + if sn.Spec.SocketIndex != nil { + ordinal = int(*sn.Spec.SocketIndex) + } + if _, ok := expected[workerOrdinal{sn.Spec.WorkerNode, ordinal}]; ok { continue } - // Skip if this CR was not created by the operator (no controller owner). owner := metav1.GetControllerOf(sn) if owner == nil || owner.Name != sns.Name { - continue + continue // manually created — preserve } if err := r.Delete(ctx, sn); err != nil && !apierrors.IsNotFound(err) { log.Error(err, "failed to delete stale StorageNode CR", "name", sn.Name) } else { log.Info("deleted stale StorageNode CR", "name", sn.Name, - "worker", sn.Spec.WorkerNode, "socket", socketLabel(sn.Spec.SocketIndex)) + "worker", sn.Spec.WorkerNode, "ordinal", ordinal) } } - // Determine which workers host FDB pods — these must be added sequentially. + // Determine which workers host FDB pods — added sequentially. fdbWorkers := r.fdbWorkerSet(ctx, sns) - // Track whether any FDB worker's StorageNode CR is still awaiting its UUID. - // If so, stop creating new FDB CRs — the StorageNodeReconciler will POST - // the current one and we wait until it comes online before creating the next. fdbInFlight := false for _, sn := range owned.Items { if !fdbWorkers[sn.Spec.WorkerNode] { @@ -110,30 +122,34 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( } } - // Create or sync each expected (worker, socket). + // Create or sync one StorageNode CR per (worker, socket, nodeIdx). + // globalOrdinal = socketPosition × nodesPerSocket + nodeIdx is stored as + // SocketIndex and used by pollUUIDFromBackend to select the correct backend node. for _, worker := range sns.Spec.WorkerNodes { - for _, socket := range sockets { - // For FDB workers: only create the next CR when no other FDB - // worker is still being provisioned (UUID not yet assigned). - if fdbWorkers[worker] && fdbInFlight { - // Check if THIS worker already has a CR (already in progress or done). - crName := storageNodeCRName(sns.Name, worker, socket) - alreadyExists := false - for _, sn := range owned.Items { - if sn.Name == crName { - alreadyExists = true - break + for si, socket := range sockets { + for ni := 0; ni < nodesPerSocket; ni++ { + globalOrdinal := si*nodesPerSocket + ni + + if fdbWorkers[worker] && fdbInFlight { + crName := storageNodeCRName(sns.Name, worker, socket, ni) + alreadyExists := false + for _, sn := range owned.Items { + if sn.Name == crName { + alreadyExists = true + break + } + } + if !alreadyExists { + log.Info("FDB worker: deferring StorageNode CR creation until previous FDB node is online", + "worker", worker, "socket", socket, "nodeIdx", ni) + continue } } - if !alreadyExists { - log.Info("FDB worker: deferring StorageNode CR creation until previous FDB node is online", - "worker", worker) - continue // skip — revisit on next reconcile + if err := r.ensureStorageNodeCR(ctx, sns, worker, socket, ni, globalOrdinal); err != nil { + log.Error(err, "failed to ensure StorageNode CR", + "worker", worker, "socket", socket, "nodeIdx", ni) } } - if err := r.ensureStorageNodeCR(ctx, sns, worker, socket); err != nil { - log.Error(err, "failed to ensure StorageNode CR", "worker", worker, "socket", socket) - } } } @@ -145,28 +161,32 @@ func (r *StorageNodeSetReconciler) reconcileStorageNodeCRs( return nil } -// ensureStorageNodeCR creates or patches a StorageNode CR for (worker, socket). +// ensureStorageNodeCR creates or patches a StorageNode CR. +// socket is the NUMA socket identifier (from socketsToUse), nodeIdx is the +// per-socket node index (0..nodesPerSocket-1), and globalOrdinal is used as +// SocketIndex for backend node lookup in pollUUIDFromBackend. func (r *StorageNodeSetReconciler) ensureStorageNodeCR( ctx context.Context, sns *simplyblockv1alpha1.StorageNodeSet, worker, socket string, + nodeIdx, globalOrdinal int, ) error { log := logf.FromContext(ctx) - name := storageNodeCRName(sns.Name, worker, socket) + name := storageNodeCRName(sns.Name, worker, socket, nodeIdx) var existing simplyblockv1alpha1.StorageNode err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: sns.Namespace}, &existing) if apierrors.IsNotFound(err) { // Create. - sn := buildStorageNodeCR(sns, name, worker, socket) + sn := buildStorageNodeCR(sns, name, worker, globalOrdinal) if setErr := controllerutil.SetControllerReference(sns, sn, r.Scheme); setErr != nil { return fmt.Errorf("setting owner reference on StorageNode %s: %w", name, setErr) } if createErr := r.Create(ctx, sn); createErr != nil { return fmt.Errorf("creating StorageNode %s: %w", name, createErr) } - log.Info("created StorageNode CR", "name", name, "worker", worker, "socket", socket) + log.Info("created StorageNode CR", "name", name, "worker", worker, "socket", socket, "nodeIdx", nodeIdx) // Backward compatibility: pre-populate UUID and status from the legacy // StorageNodeSet.status.nodes[] so nodes that were already provisioned @@ -263,18 +283,22 @@ func (r *StorageNodeSetReconciler) aggregateStorageNodeStatus( // storageNodeCRName builds a deterministic, DNS-label-safe name for a StorageNode CR. // Pattern: {sns-name}-{sanitised-worker}-{socket}, truncated to 63 chars with // an FNV-32 suffix to prevent collisions on long names. -func storageNodeCRName(snsName, worker, socket string) string { - // Sanitise worker hostname: lowercase, replace non-alnum with '-'. +// storageNodeCRName builds a deterministic, DNS-label-safe name for a StorageNode CR. +// Pattern: {sns}-{worker}-s{socket}-n{nodeIdx} +// Both the NUMA socket identifier and the per-socket node index are encoded in the +// name so it is unambiguous across different socketsToUse × nodesPerSocket combinations. +func storageNodeCRName(snsName, worker, socket string, nodeIdx int) string { + nodeIdxStr := fmt.Sprintf("%d", nodeIdx) sanitised := sanitiseDNSLabel(worker) - raw := snsName + "-" + sanitised + "-" + socket - raw = strings.ToLower(raw) + raw := strings.ToLower(snsName + "-" + sanitised + "-s" + socket + "-n" + nodeIdxStr) const maxLen = 63 if len(raw) <= maxLen { return raw } - // Append 7-char FNV hash suffix before truncation. - h := fnv32Hash(worker + socket) + // Append 7-char FNV-32 hash suffix before truncation to prevent collisions + // when two workers share a long common prefix. + h := fnv32Hash(worker + socket + nodeIdxStr) suffix := fmt.Sprintf("-%06x", h) keep := maxLen - len(suffix) if keep < 1 { @@ -300,13 +324,10 @@ func sanitiseDNSLabel(s string) string { // buildStorageNodeCR constructs a new StorageNode CR for the given worker and socket. func buildStorageNodeCR( sns *simplyblockv1alpha1.StorageNodeSet, - name, worker, socket string, + name, worker string, + ordinal int, ) *simplyblockv1alpha1.StorageNode { - var idx int32 - if socket != "" { - fmt.Sscanf(socket, "%d", &idx) //nolint:errcheck - } - socketIndex := &idx + idx := int32(ordinal) sn := &simplyblockv1alpha1.StorageNode{ ObjectMeta: metav1.ObjectMeta{ @@ -320,7 +341,7 @@ func buildStorageNodeCR( Spec: simplyblockv1alpha1.StorageNodeSpec{ StorageNodeSetRef: sns.Name, WorkerNode: worker, - SocketIndex: socketIndex, + SocketIndex: &idx, }, } @@ -493,11 +514,3 @@ func effectiveSockets(sns *simplyblockv1alpha1.StorageNodeSet) []string { return sns.Spec.SocketsToUse } -// socketLabel converts a *int32 socket index back to the string representation -// used in storageNodeCRName, for matching against expected keys. -func socketLabel(idx *int32) string { - if idx == nil { - return "0" - } - return fmt.Sprintf("%d", *idx) -} diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go index 25f0464c..13258380 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode_unit_test.go @@ -45,7 +45,7 @@ func newSNSReconciler(t *testing.T, objects ...client.Object) *StorageNodeSetRec // ── TestStorageNodeCRName ────────────────────────────────────────────────────── func TestStorageNodeCRName_SimpleCase(t *testing.T) { - name := storageNodeCRName("my-sns", "worker-a.example.com", "0") + name := storageNodeCRName("my-sns", "worker-a.example.com", "0", 0) if name == "" { t.Fatal("expected non-empty name") } @@ -60,7 +60,7 @@ func TestStorageNodeCRName_SimpleCase(t *testing.T) { func TestStorageNodeCRName_TruncatesLongNames(t *testing.T) { longWorker := "vm" + strings.Repeat("a", 60) + ".simplyblock3.localdomain" - name := storageNodeCRName("simplyblock-node", longWorker, "0") + name := storageNodeCRName("simplyblock-node", longWorker, "0", 0) if len(name) > 63 { t.Errorf("truncated name still exceeds 63 chars: len=%d", len(name)) } @@ -69,15 +69,15 @@ func TestStorageNodeCRName_TruncatesLongNames(t *testing.T) { func TestStorageNodeCRName_CollisionGuard(t *testing.T) { // Two workers sharing a long prefix must produce distinct names. base := "vm" + strings.Repeat("x", 55) + ".example.com" - name1 := storageNodeCRName("sns", base+"1", "0") - name2 := storageNodeCRName("sns", base+"2", "0") + name1 := storageNodeCRName("sns", base+"1", "0", 0) + name2 := storageNodeCRName("sns", base+"2", "0", 0) if name1 == name2 { t.Errorf("collision: both workers mapped to %q", name1) } } func TestStorageNodeCRName_IsDNSLabelSafe(t *testing.T) { - name := storageNodeCRName("my-sns", "vm01.simplyblock3.localdomain", "0") + name := storageNodeCRName("my-sns", "vm01.simplyblock3.localdomain", "0", 0) for _, c := range name { if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '.' { t.Errorf("invalid character %q in name %q", c, name) From 744e036d92bcada155cc75f9123e1ff5015c472c Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 16 Jul 2026 10:24:43 +0100 Subject: [PATCH 42/52] docs: update design document with s{socket}-n{nodeIdx} naming, nodesPerSocket support, and nested status structure --- .../design-storagenodeset-storagenode.md | 29 +++++++++++++++---- .../simplyblockstoragenodeset_storagenode.go | 1 - 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/operator/docs/designs/design-storagenodeset-storagenode.md b/operator/docs/designs/design-storagenodeset-storagenode.md index fb114d88..6913b8a9 100644 --- a/operator/docs/designs/design-storagenodeset-storagenode.md +++ b/operator/docs/designs/design-storagenodeset-storagenode.md @@ -367,10 +367,26 @@ Implemented on 2026-07-15. - **Parallel node add** enforced via `countInFlightNodes` (counts siblings with `PostedAt != nil && UUID == ""`). **FDB sequential constraint** preserved via `isFDBWorkerBlocked` — any FDB worker waits until no other FDB sibling is in-flight. -- **`pollUUIDFromBackend`** polls the backend by worker IP + socket index to retrieve the UUID +- **`pollUUIDFromBackend`** polls the backend by worker IP + global ordinal to retrieve the UUID for manually created `StorageNode` CRs (whose worker is not in `spec.workerNodes` and - therefore never appears in `StorageNodeSet.status.nodes[]`). Supports multi-socket by - sorting backend nodes by RPC port and matching by socket index position. + therefore never appears in `StorageNodeSet.status.nodes[]`). Supports multi-socket and + multi-node-per-socket by sorting backend nodes by RPC port and picking `matching[globalOrdinal]`. +- **`StorageNode` CR naming** uses the pattern `{sns}-{worker}-s{socket}-n{nodeIdx}` encoding + both the NUMA socket identifier (from `spec.socketsToUse`) and the per-socket node index + (0..`nodesPerSocket-1`). The `SocketIndex` field stores the global ordinal + `socketPosition × nodesPerSocket + nodeIdx` used by `pollUUIDFromBackend` to select the + correct backend node. This is unambiguous across all `socketsToUse × nodesPerSocket` + combinations — e.g. `s0-n1` (socket 0, node 1) and `s1-n0` (socket 1, node 0) are + always distinct regardless of the configuration. + + Examples: + + | `socketsToUse` | `nodesPerSocket` | CRs created per worker | + |---|---|---| + | `["0"]` | `1` | `s0-n0` | + | `["0","1"]` | `1` | `s0-n0`, `s1-n0` | + | `["0"]` | `2` | `s0-n0`, `s0-n1` | + | `["0","1"]` | `2` | `s0-n0`, `s0-n1`, `s1-n0`, `s1-n1` | - **Manually created `StorageNode` CRs** — users can create a `StorageNode` CR referencing an existing `StorageNodeSet` without adding the worker to `spec.workerNodes`. The operator: - Does NOT delete it (no OwnerReference = not stale) @@ -422,8 +438,11 @@ should be allowed. How many concurrent drains are permitted given FTT constraint unlimited — one per node simultaneously. FDB sequential constraint is enforced. **Q3: StorageNode naming** — Resolved ✓ -Pattern: `{storagenodeset-name}-{sanitised-worker}-{socket}`. Implemented in -`storageNodeCRName()` with FNV-32 collision guard on truncation. +Pattern: `{sns}-{worker}-s{socket}-n{nodeIdx}`. Both the NUMA socket identifier (from +`spec.socketsToUse`) and the per-socket node index (0..`nodesPerSocket-1`) are encoded so +names are unambiguous across all `socketsToUse × nodesPerSocket` combinations. FNV-32 hash +suffix prevents collision when long worker names are truncated to the 63-char Kubernetes +limit. Implemented in `storageNodeCRName()` in `simplyblockstoragenodeset_storagenode.go`. **Q4: Adoption of pre-existing backend nodes** — Resolved ✓ `syncUUIDFromNodeSet` reads existing UUIDs from `StorageNodeSet.status.nodes[]`. diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 554e767d..73201ea3 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -513,4 +513,3 @@ func effectiveSockets(sns *simplyblockv1alpha1.StorageNodeSet) []string { } return sns.Spec.SocketsToUse } - From 4084408bc009146b3cbd2add4dddd05585141418 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Thu, 16 Jul 2026 15:37:42 +0100 Subject: [PATCH 43/52] operator: add expand field to StorageNodeSet and StorageNodeOverrides for cluster expansion node adds --- operator/api/v1alpha1/storagenode_types.go | 6 ++++++ operator/api/v1alpha1/storagenodeset_types.go | 7 +++++++ .../api/v1alpha1/zz_generated.deepcopy.go | 10 ++++++++++ .../storage.simplyblock.io_storagenodes.yaml | 6 ++++++ ...torage.simplyblock.io_storagenodesets.yaml | 13 +++++++++++++ operator/dist/install.yaml | 19 +++++++++++++++++++ .../controller/storagenode_controller.go | 5 +++++ operator/internal/utils/types.go | 2 ++ 8 files changed, 68 insertions(+) diff --git a/operator/api/v1alpha1/storagenode_types.go b/operator/api/v1alpha1/storagenode_types.go index 2abb6c44..c3f7db9d 100644 --- a/operator/api/v1alpha1/storagenode_types.go +++ b/operator/api/v1alpha1/storagenode_types.go @@ -104,6 +104,12 @@ type StorageNodeOverrides struct { // +kubebuilder:validation:Minimum=1 // +optional FailureDomain *int32 `json:"failureDomain,omitempty"` + + // Expand marks this node as a cluster-expansion add. When true the backend + // node-add endpoint receives expand=true, triggering rebalancing behaviour + // appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + // +optional + Expand *bool `json:"expand,omitempty"` } // StorageNodeSpec defines the desired state of a StorageNode. diff --git a/operator/api/v1alpha1/storagenodeset_types.go b/operator/api/v1alpha1/storagenodeset_types.go index a25897aa..cae7d935 100644 --- a/operator/api/v1alpha1/storagenodeset_types.go +++ b/operator/api/v1alpha1/storagenodeset_types.go @@ -168,6 +168,13 @@ type StorageNodeSetSpec struct { // +optional NodeFailureDomains map[string]int32 `json:"nodeFailureDomains,omitempty"` + // Expand indicates that storage nodes added from this StorageNodeSet are being + // added to expand an already-active cluster. When true the backend node-add + // endpoint receives expand=true, which triggers the appropriate rebalancing + // behaviour for in-place cluster growth. + // +optional + Expand *bool `json:"expand,omitempty"` + // +kubebuilder:validation:MaxProperties=200 // NodeConfigs allows per-worker-node configuration overrides keyed by the // Kubernetes worker node name. Entries are propagated to the corresponding diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 6082ffaf..b1f8d0b6 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1801,6 +1801,11 @@ func (in *StorageNodeOverrides) DeepCopyInto(out *StorageNodeOverrides) { *out = new(int32) **out = **in } + if in.Expand != nil { + in, out := &in.Expand, &out.Expand + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageNodeOverrides. @@ -2031,6 +2036,11 @@ func (in *StorageNodeSetSpec) DeepCopyInto(out *StorageNodeSetSpec) { (*out)[key] = val } } + if in.Expand != nil { + in, out := &in.Expand, &out.Expand + *out = new(bool) + **out = **in + } if in.NodeConfigs != nil { in, out := &in.NodeConfigs, &out.NodeConfigs *out = make(map[string]StorageNodeOverrides, len(*in)) diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml index c4571803..db3db153 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml @@ -88,6 +88,12 @@ spec: description: EnableCpuTopology overrides topology-aware CPU handling for this node. type: boolean + expand: + description: |- + Expand marks this node as a cluster-expansion add. When true the backend + node-add endpoint receives expand=true, triggering rebalancing behaviour + appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + type: boolean failureDomain: description: |- FailureDomain is the failure-domain group index (≥ 1) for this node. diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml index aeba2fd9..6ad2e32b 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodesets.yaml @@ -160,6 +160,13 @@ spec: enableCpuTopology: description: EnableCpuTopology enables topology-aware CPU handling. type: boolean + expand: + description: |- + Expand indicates that storage nodes added from this StorageNodeSet are being + added to expand an already-active cluster. When true the backend node-add + endpoint receives expand=true, which triggers the appropriate rebalancing + behaviour for in-place cluster growth. + type: boolean forceFormat4K: description: ForceFormat4K forces 4K blocksize formatting of the NVMe device where supported. @@ -306,6 +313,12 @@ spec: description: EnableCpuTopology overrides topology-aware CPU handling for this node. type: boolean + expand: + description: |- + Expand marks this node as a cluster-expansion add. When true the backend + node-add endpoint receives expand=true, triggering rebalancing behaviour + appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + type: boolean failureDomain: description: |- FailureDomain is the failure-domain group index (≥ 1) for this node. diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index c6bbc015..0058b9a4 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2228,6 +2228,12 @@ spec: description: EnableCpuTopology overrides topology-aware CPU handling for this node. type: boolean + expand: + description: |- + Expand marks this node as a cluster-expansion add. When true the backend + node-add endpoint receives expand=true, triggering rebalancing behaviour + appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + type: boolean failureDomain: description: |- FailureDomain is the failure-domain group index (≥ 1) for this node. @@ -2597,6 +2603,13 @@ spec: enableCpuTopology: description: EnableCpuTopology enables topology-aware CPU handling. type: boolean + expand: + description: |- + Expand indicates that storage nodes added from this StorageNodeSet are being + added to expand an already-active cluster. When true the backend node-add + endpoint receives expand=true, which triggers the appropriate rebalancing + behaviour for in-place cluster growth. + type: boolean forceFormat4K: description: ForceFormat4K forces 4K blocksize formatting of the NVMe device where supported. @@ -2743,6 +2756,12 @@ spec: description: EnableCpuTopology overrides topology-aware CPU handling for this node. type: boolean + expand: + description: |- + Expand marks this node as a cluster-expansion add. When true the backend + node-add endpoint receives expand=true, triggering rebalancing behaviour + appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + type: boolean failureDomain: description: |- FailureDomain is the failure-domain group index (≥ 1) for this node. diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 88c55e0e..2f2a9616 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -350,6 +350,7 @@ func (r *StorageNodeReconciler) provisionNode( Format4K: utils.BoolPtrOrFalse(sns.Spec.ForceFormat4K), SpdkSystemMemory: eff.SpdkSystemMemory, FailureDomain: effectiveFailureDomain(sn, sns), + Expand: utils.BoolPtrOrFalse(eff.Expand), } endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes", clusterUUID) @@ -557,6 +558,7 @@ func effectiveNodeConfig(sn *simplyblockv1alpha1.StorageNode, sns *simplyblockv1 EnableCpuTopology: sns.Spec.EnableCpuTopology, ReservedSystemCPU: sns.Spec.ReservedSystemCPU, UbuntuHost: sns.Spec.UbuntuHost, + Expand: sns.Spec.Expand, } if sn.Spec.Overrides == nil { return eff @@ -610,6 +612,9 @@ func effectiveNodeConfig(sn *simplyblockv1alpha1.StorageNode, sns *simplyblockv1 if o.FailureDomain != nil { eff.FailureDomain = o.FailureDomain } + if o.Expand != nil { + eff.Expand = o.Expand + } return eff } diff --git a/operator/internal/utils/types.go b/operator/internal/utils/types.go index 7d05134d..7773aa55 100644 --- a/operator/internal/utils/types.go +++ b/operator/internal/utils/types.go @@ -120,4 +120,6 @@ type StorageNodeSetAddParams struct { // FailureDomain assigns this node to a failure-domain group (integer ≥ 1). // Required when the cluster has EnableFailureDomain=true; omit (zero value) otherwise. FailureDomain int `json:"failure_domain,omitempty"` + // Expand signals that this node is being added to expand an already-active cluster. + Expand bool `json:"expand,omitempty"` } From a16d77ed223530e888aa23695abc110cc867e3da Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 17 Jul 2026 09:58:39 +0100 Subject: [PATCH 44/52] added storagenodes and storagenodeops crds to helm-chart --- ...storage.simplyblock.io_storagenodeops.yaml | 163 ++++++++++ .../storage.simplyblock.io_storagenodes.yaml | 303 ++++++++++++++++++ ...torage.simplyblock.io_storagenodesets.yaml | 266 ++++++++++----- 3 files changed, 650 insertions(+), 82 deletions(-) create mode 100644 helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodeops.yaml create mode 100644 helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml diff --git a/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodeops.yaml b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodeops.yaml new file mode 100644 index 00000000..94bf3181 --- /dev/null +++ b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodeops.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: storagenodeops.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: StorageNodeOps + listKind: StorageNodeOpsList + plural: storagenodeops + shortNames: + - snops + singular: storagenodeops + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.storageNodeRef + name: Node + type: string + - jsonPath: .spec.action + name: Action + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.subPhase + name: SubPhase + type: string + - jsonPath: .status.message + name: Message + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StorageNodeOps is a one-shot operational CR targeting a single StorageNode. + Analogous to a Kubernetes Job — it drives an action (shutdown, restart, suspend, + resume, remove/drain) to completion and records the result. Only one + StorageNodeOps can be active per StorageNode at a time. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageNodeOpsSpec defines the desired state of a StorageNodeOps. + properties: + action: + description: Action is the operation to perform. Immutable. + enum: + - shutdown + - restart + - suspend + - resume + - remove + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + drain: + description: Drain configures the drain workflow. Only applicable + when action=remove. + properties: + systemVolumeFilterRegex: + description: |- + SystemVolumeFilterRegex is a Go regular expression matched against backend + volume names. Matching volumes are treated as system volumes: excluded from + drain migration and deleted inline during the Verifying phase. + Defaults to "^sb-fio-baseline-.*". + type: string + type: object + force: + description: Force enables forced execution where the backend supports + it. + type: boolean + reattachVolume: + description: |- + ReattachVolume reattaches volumes during restart. + Only applicable when action=restart. + type: boolean + storageNodeRef: + description: StorageNodeRef is the name of the target StorageNode. + Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - action + - storageNodeRef + type: object + status: + description: StorageNodeOpsStatus holds the observed state of a StorageNodeOps. + properties: + completedAt: + description: CompletedAt is when the operation finished (successfully + or not). + format: date-time + type: string + message: + description: Message is a human-readable description of the current + state or failure reason. + type: string + phase: + description: Phase is the high-level lifecycle phase. + enum: + - Pending + - Running + - Succeeded + - Failed + type: string + startedAt: + description: StartedAt is when the operation began. + format: date-time + type: string + subPhase: + description: SubPhase tracks the active drain step when action=remove + and phase=Running. + enum: + - Validating + - Suspending + - Migrating + - Verifying + - Removing + type: string + triggered: + description: |- + Triggered indicates the backend action POST has been sent (used during + Suspending to avoid duplicate POSTs across reconcile iterations). + type: boolean + volumesMigrated: + description: VolumesMigrated is the count of volumes successfully + migrated (drain only). + type: integer + volumesPending: + description: VolumesPending is the count of volumes awaiting migration + (drain only). + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml new file mode 100644 index 00000000..db3db153 --- /dev/null +++ b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml @@ -0,0 +1,303 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: storagenodes.storage.simplyblock.io +spec: + group: storage.simplyblock.io + names: + kind: StorageNode + listKind: StorageNodeList + plural: storagenodes + shortNames: + - sn + singular: storagenode + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.workerNode + name: Worker + type: string + - jsonPath: .spec.socketIndex + name: Socket + type: integer + - jsonPath: .status.uuid + name: UUID + type: string + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.health + name: Health + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StorageNode is the Schema for a single backend storage node instance. + One StorageNode CR exists per (workerNode, socketIndex) pair and is owned + by the parent StorageNodeSet. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: StorageNodeSpec defines the desired state of a StorageNode. + properties: + overrides: + description: |- + Overrides holds per-node configuration propagated from + StorageNodeSet.spec.nodeConfigs[workerNode] on every reconcile. + properties: + corePercentage: + description: CorePercentage overrides the percentage of cores + allocated to SPDK for this node (0-99). + format: int32 + type: integer + deviceNames: + description: |- + DeviceNames explicitly defines the NVMe namespace names to use on this node + (e.g. ["nvme0n1","nvme1n1"]). + items: + type: string + type: array + driveSizeRange: + description: DriveSizeRange overrides the drive size range filter + for this node. + type: string + enableCpuTopology: + description: EnableCpuTopology overrides topology-aware CPU handling + for this node. + type: boolean + expand: + description: |- + Expand marks this node as a cluster-expansion add. When true the backend + node-add endpoint receives expand=true, triggering rebalancing behaviour + appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + type: boolean + failureDomain: + description: |- + FailureDomain is the failure-domain group index (≥ 1) for this node. + Required when the parent StorageCluster has enableFailureDomains=true. + Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + format: int32 + minimum: 1 + type: integer + journalManager: + description: JournalManagerSpec overrides journal manager tuning + for this node. + properties: + count: + description: Count is the number of journal managers to configure. + format: int32 + type: integer + percentPerDevice: + description: PercentPerDevice is the journal manager capacity + percentage per device. + format: int32 + type: integer + type: object + maxLogicalVolumeCount: + description: MaxLogicalVolumeCount overrides the maximum number + of logical volumes for this node. + format: int32 + type: integer + maxSize: + description: MaxSize overrides the maximum allocatable size of + huge pages for this node. + type: string + pcieAllowList: + description: PcieAllowList overrides the list of PCI addresses + allowed for use on this node. + items: + type: string + type: array + pcieDenyList: + description: PcieDenyList overrides the list of PCI addresses + excluded from use on this node. + items: + type: string + type: array + pcieModel: + description: PcieModel overrides the PCI model filter for this + node. + type: string + reservedSystemCPU: + description: ReservedSystemCPU overrides the CPUs reserved for + system workloads on this node. + type: string + skipKubeletConfiguration: + description: |- + SkipKubeletConfiguration overrides whether kubelet configuration changes are + skipped for this node. + type: boolean + spdkImage: + description: SpdkImage overrides the SPDK image for this node + (e.g. for phased rollouts). + type: string + spdkProxyImage: + description: SpdkProxyImage overrides the SPDK proxy image for + this node. + type: string + spdkSystemMemory: + description: |- + SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + (e.g. "4G", "512M"). + pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ + type: string + ubuntuHost: + description: UbuntuHost overrides the Ubuntu host OS flag for + this node. + type: boolean + type: object + socketIndex: + description: SocketIndex is the NUMA socket index (0-based). Immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + storageNodeSetRef: + description: StorageNodeSetRef is the name of the owning StorageNodeSet. + Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + workerNode: + description: WorkerNode is the Kubernetes node hostname this StorageNode + runs on. Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf + required: + - storageNodeSetRef + - workerNode + type: object + x-kubernetes-validations: + - message: field socketIndex is immutable once set + rule: '!has(oldSelf.socketIndex) || has(self.socketIndex)' + status: + description: StorageNodeStatus holds the observed state of a StorageNode. + properties: + activeOpsRef: + description: |- + ActiveOpsRef is the name of the currently active StorageNodeOps CR targeting + this node. Empty when no operation is in progress. Used for mutual exclusion. + type: string + health: + description: Health is the backend-reported node health flag. + type: boolean + hostname: + description: Hostname is the node hostname as reported by the backend. + type: string + latencyMetrics: + description: |- + LatencyMetrics holds the fio-measured baseline NVMe-oF latency for this node, + used by the volume rebalancer to make data-placement decisions. + properties: + baselineMeasuredAt: + description: BaselineMeasuredAt is when the baseline was established. + format: date-time + type: string + baselineP50NS: + description: BaselineP50NS is the p50 write latency (nanoseconds) + from the initial empty-cluster benchmark. + format: int64 + type: integer + baselineP99NS: + description: BaselineP99NS is the p99 write latency (nanoseconds) + from the initial empty-cluster benchmark. + format: int64 + type: integer + nodeUUID: + description: NodeUUID is the backend storage node UUID. + type: string + required: + - nodeUUID + type: object + ports: + description: Ports groups network connectivity fields (addresses and + ports). + properties: + lvol: + description: Lvol is the logical-volume subsystem port. + format: int32 + type: integer + management: + description: Management is the management IP address of the node. + type: string + nvmeof: + description: NvmeOf is the NVMe-oF fabric port. + format: int32 + type: integer + rpc: + description: Rpc is the RPC/management API port. + format: int32 + type: integer + type: object + postedAt: + description: |- + PostedAt is the timestamp when the node-add POST was sent. + Used as a provisioning guard against duplicate POSTs. + format: date-time + type: string + resources: + description: Resources groups compute and storage resource metrics. + properties: + cpu: + description: CPU is the number of SPDK CPU cores allocated to + this node. + format: int32 + type: integer + devices: + description: Devices is the device summary (online/total) reported + by the backend. + type: string + memory: + description: Memory is the SPDK memory allocation reported by + the backend. + type: string + volumes: + description: Volumes is the current number of logical volumes + on this node. + format: int32 + type: integer + type: object + status: + description: Status is the backend-reported node status (e.g. online, + suspended, offline). + type: string + uptime: + description: Uptime is the node uptime as reported by the backend. + type: string + uuid: + description: UUID is the backend storage node UUID. Set once after + node-add completes. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodesets.yaml b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodesets.yaml index 297e2a29..6ad2e32b 100644 --- a/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodesets.yaml +++ b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodesets.yaml @@ -14,7 +14,33 @@ spec: singular: storagenodeset scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.totalNodes + name: Total + type: integer + - jsonPath: .status.onlineNodes + name: Online + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.offlineNodes + name: Offline + priority: 1 + type: integer + - jsonPath: .status.suspendedNodes + name: Suspended + priority: 1 + type: integer + - jsonPath: .status.creatingNodes + name: Creating + priority: 1 + type: integer + - jsonPath: .status.removedNodes + name: Removed + priority: 1 + type: integer + name: v1alpha1 schema: openAPIV3Schema: description: StorageNodeSet is the Schema for the storagenodesets API @@ -39,15 +65,6 @@ spec: spec: description: spec defines the desired state of StorageNodeSet properties: - action: - description: Action triggers an imperative node operation. - enum: - - shutdown - - restart - - suspend - - resume - - remove - type: string clusterImage: description: |- ClusterImage is the container image used for storage-node workloads. @@ -143,8 +160,12 @@ spec: enableCpuTopology: description: EnableCpuTopology enables topology-aware CPU handling. type: boolean - force: - description: Force enables forced action execution where supported. + expand: + description: |- + Expand indicates that storage nodes added from this StorageNodeSet are being + added to expand an already-active cluster. When true the backend node-add + endpoint receives expand=true, which triggers the appropriate rebalancing + behaviour for in-place cluster growth. type: boolean forceFormat4K: description: ForceFormat4K forces 4K blocksize formatting of the NVMe @@ -259,6 +280,129 @@ spec: x-kubernetes-validations: - message: field is immutable rule: self == oldSelf + nodeConfigs: + additionalProperties: + description: |- + StorageNodeOverrides holds per-node configuration that overrides the parent + StorageNodeSet fleet defaults for a specific worker node. Populated by the + StorageNodeSetReconciler from StorageNodeSet.spec.nodeConfigs[workerNode] on + every reconcile. The StorageNodeSet is the single source of truth — users + should not edit this struct directly on the StorageNode. + + Fields here mirror the configurable (non-immutable, non-infrastructure) fields + of StorageNodeSetSpec. When a field is set here it takes precedence over the + fleet default; when omitted the fleet default applies. + properties: + corePercentage: + description: CorePercentage overrides the percentage of cores + allocated to SPDK for this node (0-99). + format: int32 + type: integer + deviceNames: + description: |- + DeviceNames explicitly defines the NVMe namespace names to use on this node + (e.g. ["nvme0n1","nvme1n1"]). + items: + type: string + type: array + driveSizeRange: + description: DriveSizeRange overrides the drive size range filter + for this node. + type: string + enableCpuTopology: + description: EnableCpuTopology overrides topology-aware CPU + handling for this node. + type: boolean + expand: + description: |- + Expand marks this node as a cluster-expansion add. When true the backend + node-add endpoint receives expand=true, triggering rebalancing behaviour + appropriate for in-place cluster growth. Overrides StorageNodeSet.spec.expand. + type: boolean + failureDomain: + description: |- + FailureDomain is the failure-domain group index (≥ 1) for this node. + Required when the parent StorageCluster has enableFailureDomains=true. + Overrides StorageNodeSet.spec.nodeFailureDomains[workerNode] when both are set. + format: int32 + minimum: 1 + type: integer + journalManager: + description: JournalManagerSpec overrides journal manager tuning + for this node. + properties: + count: + description: Count is the number of journal managers to + configure. + format: int32 + type: integer + percentPerDevice: + description: PercentPerDevice is the journal manager capacity + percentage per device. + format: int32 + type: integer + type: object + maxLogicalVolumeCount: + description: MaxLogicalVolumeCount overrides the maximum number + of logical volumes for this node. + format: int32 + type: integer + maxSize: + description: MaxSize overrides the maximum allocatable size + of huge pages for this node. + type: string + pcieAllowList: + description: PcieAllowList overrides the list of PCI addresses + allowed for use on this node. + items: + type: string + type: array + pcieDenyList: + description: PcieDenyList overrides the list of PCI addresses + excluded from use on this node. + items: + type: string + type: array + pcieModel: + description: PcieModel overrides the PCI model filter for this + node. + type: string + reservedSystemCPU: + description: ReservedSystemCPU overrides the CPUs reserved for + system workloads on this node. + type: string + skipKubeletConfiguration: + description: |- + SkipKubeletConfiguration overrides whether kubelet configuration changes are + skipped for this node. + type: boolean + spdkImage: + description: SpdkImage overrides the SPDK image for this node + (e.g. for phased rollouts). + type: string + spdkProxyImage: + description: SpdkProxyImage overrides the SPDK proxy image for + this node. + type: string + spdkSystemMemory: + description: |- + SpdkSystemMemory overrides the SPDK huge-page memory allocation for this node + (e.g. "4G", "512M"). + pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ + type: string + ubuntuHost: + description: UbuntuHost overrides the Ubuntu host OS flag for + this node. + type: boolean + type: object + description: |- + NodeConfigs allows per-worker-node configuration overrides keyed by the + Kubernetes worker node name. Entries are propagated to the corresponding + StorageNode.spec.overrides by the StorageNodeReconciler on every reconcile. + The StorageNodeSet is the single source of truth for all per-node config, + including failure domain assignment via nodeConfigs[worker].failureDomain. + maxProperties: 200 + type: object nodeFailureDomains: additionalProperties: format: int32 @@ -271,9 +415,6 @@ spec: the same group index so the control plane can spread erasure-coding chunks across independent fault groups. type: object - nodeUUID: - description: NodeUUID is required when action is specified - type: string nodesPerSocket: description: NodesPerSocket defines how many storage nodes are created per NUMA socket. @@ -315,10 +456,6 @@ spec: pcieModel: description: PcieModel filters devices by PCI model. type: string - reattachVolume: - description: ReattachVolume reattaches volumes during restart where - supported by the backend. - type: boolean reservedSystemCPU: description: ReservedSystemCPU defines CPUs reserved for system workloads. type: string @@ -349,12 +486,6 @@ spec: When omitted the backend default is used. pattern: ^[0-9]+(G|GI|GB|GiB|M|MI|MB|MiB|g|gi|gb|gib|m|mi|mb|mib)?$ type: string - systemVolumeFilterRegex: - description: |- - SystemVolumeFilterRegex is a Go regular expression matched against backend - volume names. Matching volumes are excluded from drain migration and from - the final verification check. Defaults to "^sb-fio-baseline-.*". - type: string tolerations: description: Tolerations configures pod tolerations for storage-node pods. @@ -399,15 +530,14 @@ spec: ubuntuHost: description: UbuntuHost indicates the node host OS is Ubuntu. type: boolean - workerNode: - description: WorkerNode is a single worker node used by action flows. - type: string workerNodes: description: WorkerNodes is the set of Kubernetes worker nodes to manage. items: type: string + maxItems: 200 type: array + x-kubernetes-list-type: set required: - clusterName type: object @@ -423,52 +553,10 @@ spec: status: description: status defines the observed state of StorageNodeSet properties: - actionStatus: - description: ActionStatus tracks the latest action execution status. - properties: - action: - description: Action is the requested action name. - type: string - message: - description: Message is a human-readable action result or error. - type: string - nodeUUID: - description: NodeUUID is the target node UUID for the action. - type: string - observedGeneration: - description: ObservedGeneration is the resource generation observed - by this status. - format: int64 - type: integer - state: - type: string - subPhase: - description: SubPhase tracks the active drain step within the - remove action. - enum: - - Validating - - Suspending - - Migrating - - Verifying - - Removing - type: string - triggered: - description: Triggered indicates whether the underlying backend - action has been fired. - type: boolean - updatedAt: - description: UpdatedAt is the timestamp of the last status transition. - format: date-time - type: string - volumesMigrated: - description: VolumesMigrated is the count of volumes successfully - migrated so far. - type: integer - volumesPending: - description: VolumesPending is the count of volumes still awaiting - migration. - type: integer - type: object + creatingNodes: + description: CreatingNodes is the count of StorageNode CRs with status + "in_creation". + type: integer drainCoordination: description: DrainCoordination tracks the upgrade-drain state per worker node. @@ -599,6 +687,14 @@ spec: type: integer type: object type: array + offlineNodes: + description: OfflineNodes is the count of StorageNode CRs with status + "offline". + type: integer + onlineNodes: + description: OnlineNodes is the count of StorageNode CRs with status + "online". + type: integer pendingNodeAdds: additionalProperties: format: date-time @@ -610,6 +706,10 @@ spec: POSTs — it is a separate map field so patches to Status.Nodes never inadvertently delete it. type: object + removedNodes: + description: RemovedNodes is the count of StorageNode CRs with status + "removed". + type: integer schedulingFailedWorkers: additionalProperties: type: boolean @@ -618,23 +718,25 @@ spec: a FailedScheduling event during node add. Used to emit a recovery event when the node subsequently comes online. type: object + suspendedNodes: + description: SuspendedNodes is the count of StorageNode CRs with status + "suspended". + type: integer + totalNodes: + description: TotalNodes is the total number of owned StorageNode CRs. + type: integer type: object required: - spec type: object x-kubernetes-validations: - - message: nodeUUID is required when action is specified - rule: '!(has(self.spec.action) && self.spec.action != "" && (!has(self.spec.nodeUUID) - || self.spec.nodeUUID == ""))' - - message: maxLogicalVolumeCount, workerNodes, and mgmtIfname are required - when action is not specified - rule: (has(self.spec.action) && self.spec.action != "") || (has(self.spec.maxLogicalVolumeCount) - && has(self.spec.workerNodes) && size(self.spec.workerNodes) > 0 && has(self.spec.mgmtIfname) - && self.spec.mgmtIfname != "") - message: all nodeFailureDomains values must be >= 1 (failure-domain group index) rule: '!has(self.spec.nodeFailureDomains) || self.spec.nodeFailureDomains.all(k, self.spec.nodeFailureDomains[k] >= 1)' + - message: nodeConfigs keys must match a workerNode entry + rule: '!has(self.spec.nodeConfigs) || self.spec.nodeConfigs.all(k, self.spec.workerNodes.exists(w, + w == k))' served: true storage: true subresources: From a5a57883a4dd58a4497c012ebe7faef8bda06b66 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 17 Jul 2026 12:57:07 +0100 Subject: [PATCH 45/52] fix: clear stale UUID on 404 from syncStatus to trigger re-adoption of node after cluster reset --- .../templates/simplyblock-operator.yaml | 29 ++++++++++++++++++- .../controller/storagenode_controller.go | 17 +++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/helm-charts/charts/simplyblock-operator/templates/simplyblock-operator.yaml b/helm-charts/charts/simplyblock-operator/templates/simplyblock-operator.yaml index c4238ff7..869d30de 100644 --- a/helm-charts/charts/simplyblock-operator/templates/simplyblock-operator.yaml +++ b/helm-charts/charts/simplyblock-operator/templates/simplyblock-operator.yaml @@ -314,7 +314,34 @@ rules: - patch - update - watch - +- apiGroups: + - storage.simplyblock.io + resources: + - storagenodes + - storagenodes/status + - storagenodes/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.simplyblock.io + resources: + - storagenodeops + - storagenodeops/status + - storagenodeops/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 2f2a9616..18a33253 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -473,6 +473,23 @@ func (r *StorageNodeReconciler) syncStatus( endpoint := fmt.Sprintf("/api/v2/clusters/%s/storage-nodes/%s", clusterUUID, sn.Status.UUID) body, status, err := apiClient.Do(ctx, http.MethodGet, endpoint, nil) if err != nil || status >= 300 { + if status == http.StatusNotFound { + // The stored UUID no longer exists on the backend — the cluster may + // have been reset and nodes re-created with new UUIDs. Clear the UUID + // so the next reconcile calls syncUUIDFromNodeSet to adopt the new one + // from StorageNodeSet.status.nodes[]. + log.Info("backend node not found (404) — clearing stale UUID to trigger re-adoption", + "staleUUID", sn.Status.UUID) + patch := client.MergeFrom(sn.DeepCopy()) + sn.Status.UUID = "" + sn.Status.Status = "" + sn.Status.Health = false + sn.Status.PostedAt = nil + if patchErr := r.Status().Patch(ctx, sn, patch); patchErr != nil { + log.Error(patchErr, "failed to clear stale UUID from StorageNode") + } + return ctrl.Result{Requeue: true}, nil + } if err == nil { err = fmt.Errorf("status %d", status) } From cc33caea22d314af7e3080890eca85d08f17d72a Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 17 Jul 2026 15:25:22 +0100 Subject: [PATCH 46/52] fix: preserve PostedAt when clearing stale UUID on 404 to prevent duplicate node_add POSTs --- .../internal/controller/storagenode_controller.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/operator/internal/controller/storagenode_controller.go b/operator/internal/controller/storagenode_controller.go index 18a33253..8e16d675 100644 --- a/operator/internal/controller/storagenode_controller.go +++ b/operator/internal/controller/storagenode_controller.go @@ -475,16 +475,17 @@ func (r *StorageNodeReconciler) syncStatus( if err != nil || status >= 300 { if status == http.StatusNotFound { // The stored UUID no longer exists on the backend — the cluster may - // have been reset and nodes re-created with new UUIDs. Clear the UUID - // so the next reconcile calls syncUUIDFromNodeSet to adopt the new one - // from StorageNodeSet.status.nodes[]. - log.Info("backend node not found (404) — clearing stale UUID to trigger re-adoption", + // have been reset and nodes re-created with new UUIDs. + // Clear only UUID (PostedAt is intentionally preserved): provisionNode + // checks PostedAt and returns early without POSTing, while + // syncUUIDFromNodeSet and pollUUIDFromBackend find the new UUID from + // StorageNodeSet.status.nodes[] or by querying the backend by IP. + log.Info("backend node not found (404) — clearing stale UUID for re-adoption", "staleUUID", sn.Status.UUID) patch := client.MergeFrom(sn.DeepCopy()) sn.Status.UUID = "" sn.Status.Status = "" sn.Status.Health = false - sn.Status.PostedAt = nil if patchErr := r.Status().Patch(ctx, sn, patch); patchErr != nil { log.Error(patchErr, "failed to clear stale UUID from StorageNode") } From 9798e3f5c1d615ffc93a0487208d7ffecb57f701 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Fri, 17 Jul 2026 16:09:04 +0100 Subject: [PATCH 47/52] added cluster-expand service --- .../templates/controlplane_deploy.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/helm-charts/charts/simplyblock-operator/templates/controlplane_deploy.yaml b/helm-charts/charts/simplyblock-operator/templates/controlplane_deploy.yaml index fc30d741..4137a9a2 100755 --- a/helm-charts/charts/simplyblock-operator/templates/controlplane_deploy.yaml +++ b/helm-charts/charts/simplyblock-operator/templates/controlplane_deploy.yaml @@ -661,6 +661,23 @@ spec: {{ toYaml .volumeMounts | nindent 12 }} resources: {{ toYaml .resources | nindent 12 }} +{{- end }} + + - name: tasks-runner-cluster-expand + image: "{{ .Values.image.simplyblock.repository }}:{{ .Values.image.simplyblock.tag }}" + command: ["python3", "simplyblock_core/services/tasks_runner_cluster_expand.py"] + imagePullPolicy: "{{ .Values.image.simplyblock.pullPolicy }}" + env: + - name: PROMETHEUS_URL + value: "{{ .Values.prometheus.simplyblock.prometheusURL }}" + - name: PROMETHEUS_PORT + value: "{{ .Values.prometheus.simplyblock.prometheusPORT }}" +{{- with (include "simplyblock.commonContainer" . | fromYaml) }} +{{ toYaml .env | nindent 12 }} + volumeMounts: +{{ toYaml .volumeMounts | nindent 12 }} + resources: +{{ toYaml .resources | nindent 12 }} {{- end }} - name: tasks-runner-snapshot-replication From c7491b56dd0eae23da63bbc0f42ec8ac236e6fe5 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 20 Jul 2026 11:16:03 +0100 Subject: [PATCH 48/52] operator: add SocketID and NodeIndex fields to StorageNode for clear NUMA socket and per-socket node display --- .../storage.simplyblock.io_storagenodes.yaml | 29 +++++++++++++++++-- operator/api/v1alpha1/storagenode_types.go | 17 +++++++++-- .../api/v1alpha1/zz_generated.deepcopy.go | 5 ++++ .../storage.simplyblock.io_storagenodes.yaml | 29 +++++++++++++++++-- operator/dist/install.yaml | 29 +++++++++++++++++-- .../simplyblockstoragenodeset_storagenode.go | 16 ++++++---- 6 files changed, 111 insertions(+), 14 deletions(-) diff --git a/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml index db3db153..70565d1f 100644 --- a/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml +++ b/helm-charts/charts/simplyblock-operator/crds/storage.simplyblock.io_storagenodes.yaml @@ -20,8 +20,11 @@ spec: - jsonPath: .spec.workerNode name: Worker type: string - - jsonPath: .spec.socketIndex + - jsonPath: .spec.socketId name: Socket + type: string + - jsonPath: .spec.nodeIndex + name: NodeIdx type: integer - jsonPath: .status.uuid name: UUID @@ -63,6 +66,14 @@ spec: spec: description: StorageNodeSpec defines the desired state of a StorageNode. properties: + nodeIndex: + description: NodeIndex is the per-socket node index (0..nodesPerSocket-1). + Immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf overrides: description: |- Overrides holds per-node configuration propagated from @@ -169,8 +180,18 @@ spec: this node. type: boolean type: object + socketId: + description: SocketID is the NUMA socket identifier from spec.socketsToUse + (e.g. "0", "1"). Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf socketIndex: - description: SocketIndex is the NUMA socket index (0-based). Immutable. + description: |- + SocketIndex is the global ordinal (socketPosition × nodesPerSocket + nodeIndex). + Used internally by the operator to select the correct backend node from the + RPC-port-sorted list in pollUUIDFromBackend. Immutable. format: int32 type: integer x-kubernetes-validations: @@ -195,6 +216,10 @@ spec: - workerNode type: object x-kubernetes-validations: + - message: field socketId is immutable once set + rule: '!has(oldSelf.socketId) || has(self.socketId)' + - message: field nodeIndex is immutable once set + rule: '!has(oldSelf.nodeIndex) || has(self.nodeIndex)' - message: field socketIndex is immutable once set rule: '!has(oldSelf.socketIndex) || has(self.socketIndex)' status: diff --git a/operator/api/v1alpha1/storagenode_types.go b/operator/api/v1alpha1/storagenode_types.go index c3f7db9d..ef8cfe27 100644 --- a/operator/api/v1alpha1/storagenode_types.go +++ b/operator/api/v1alpha1/storagenode_types.go @@ -124,7 +124,19 @@ type StorageNodeSpec struct { // +k8s:immutable WorkerNode string `json:"workerNode"` - // SocketIndex is the NUMA socket index (0-based). Immutable. + // SocketID is the NUMA socket identifier from spec.socketsToUse (e.g. "0", "1"). Immutable. + // +k8s:immutable + // +optional + SocketID string `json:"socketId,omitempty"` + + // NodeIndex is the per-socket node index (0..nodesPerSocket-1). Immutable. + // +k8s:immutable + // +optional + NodeIndex *int32 `json:"nodeIndex,omitempty"` + + // SocketIndex is the global ordinal (socketPosition × nodesPerSocket + nodeIndex). + // Used internally by the operator to select the correct backend node from the + // RPC-port-sorted list in pollUUIDFromBackend. Immutable. // +k8s:immutable // +optional SocketIndex *int32 `json:"socketIndex,omitempty"` @@ -217,7 +229,8 @@ type StorageNodeStatus struct { // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced,shortName=sn // +kubebuilder:printcolumn:name="Worker",type=string,JSONPath=".spec.workerNode" -// +kubebuilder:printcolumn:name="Socket",type=integer,JSONPath=".spec.socketIndex" +// +kubebuilder:printcolumn:name="Socket",type=string,JSONPath=".spec.socketId" +// +kubebuilder:printcolumn:name="NodeIdx",type=integer,JSONPath=".spec.nodeIndex" // +kubebuilder:printcolumn:name="UUID",type=string,JSONPath=".status.uuid" // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=".status.status" // +kubebuilder:printcolumn:name="Health",type=boolean,JSONPath=".status.health" diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index b1f8d0b6..bde3b5a8 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -2113,6 +2113,11 @@ func (in *StorageNodeSetStatus) DeepCopy() *StorageNodeSetStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageNodeSpec) DeepCopyInto(out *StorageNodeSpec) { *out = *in + if in.NodeIndex != nil { + in, out := &in.NodeIndex, &out.NodeIndex + *out = new(int32) + **out = **in + } if in.SocketIndex != nil { in, out := &in.SocketIndex, &out.SocketIndex *out = new(int32) diff --git a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml index db3db153..70565d1f 100644 --- a/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml +++ b/operator/config/crd/bases/storage.simplyblock.io_storagenodes.yaml @@ -20,8 +20,11 @@ spec: - jsonPath: .spec.workerNode name: Worker type: string - - jsonPath: .spec.socketIndex + - jsonPath: .spec.socketId name: Socket + type: string + - jsonPath: .spec.nodeIndex + name: NodeIdx type: integer - jsonPath: .status.uuid name: UUID @@ -63,6 +66,14 @@ spec: spec: description: StorageNodeSpec defines the desired state of a StorageNode. properties: + nodeIndex: + description: NodeIndex is the per-socket node index (0..nodesPerSocket-1). + Immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf overrides: description: |- Overrides holds per-node configuration propagated from @@ -169,8 +180,18 @@ spec: this node. type: boolean type: object + socketId: + description: SocketID is the NUMA socket identifier from spec.socketsToUse + (e.g. "0", "1"). Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf socketIndex: - description: SocketIndex is the NUMA socket index (0-based). Immutable. + description: |- + SocketIndex is the global ordinal (socketPosition × nodesPerSocket + nodeIndex). + Used internally by the operator to select the correct backend node from the + RPC-port-sorted list in pollUUIDFromBackend. Immutable. format: int32 type: integer x-kubernetes-validations: @@ -195,6 +216,10 @@ spec: - workerNode type: object x-kubernetes-validations: + - message: field socketId is immutable once set + rule: '!has(oldSelf.socketId) || has(self.socketId)' + - message: field nodeIndex is immutable once set + rule: '!has(oldSelf.nodeIndex) || has(self.nodeIndex)' - message: field socketIndex is immutable once set rule: '!has(oldSelf.socketIndex) || has(self.socketIndex)' status: diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 0058b9a4..ccbaae76 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -2160,8 +2160,11 @@ spec: - jsonPath: .spec.workerNode name: Worker type: string - - jsonPath: .spec.socketIndex + - jsonPath: .spec.socketId name: Socket + type: string + - jsonPath: .spec.nodeIndex + name: NodeIdx type: integer - jsonPath: .status.uuid name: UUID @@ -2203,6 +2206,14 @@ spec: spec: description: StorageNodeSpec defines the desired state of a StorageNode. properties: + nodeIndex: + description: NodeIndex is the per-socket node index (0..nodesPerSocket-1). + Immutable. + format: int32 + type: integer + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf overrides: description: |- Overrides holds per-node configuration propagated from @@ -2309,8 +2320,18 @@ spec: this node. type: boolean type: object + socketId: + description: SocketID is the NUMA socket identifier from spec.socketsToUse + (e.g. "0", "1"). Immutable. + type: string + x-kubernetes-validations: + - message: field is immutable + rule: self == oldSelf socketIndex: - description: SocketIndex is the NUMA socket index (0-based). Immutable. + description: |- + SocketIndex is the global ordinal (socketPosition × nodesPerSocket + nodeIndex). + Used internally by the operator to select the correct backend node from the + RPC-port-sorted list in pollUUIDFromBackend. Immutable. format: int32 type: integer x-kubernetes-validations: @@ -2335,6 +2356,10 @@ spec: - workerNode type: object x-kubernetes-validations: + - message: field socketId is immutable once set + rule: '!has(oldSelf.socketId) || has(self.socketId)' + - message: field nodeIndex is immutable once set + rule: '!has(oldSelf.nodeIndex) || has(self.nodeIndex)' - message: field socketIndex is immutable once set rule: '!has(oldSelf.socketIndex) || has(self.socketIndex)' status: diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index 73201ea3..ca77a8cd 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -179,7 +179,7 @@ func (r *StorageNodeSetReconciler) ensureStorageNodeCR( if apierrors.IsNotFound(err) { // Create. - sn := buildStorageNodeCR(sns, name, worker, globalOrdinal) + sn := buildStorageNodeCR(sns, name, worker, socket, nodeIdx, globalOrdinal) if setErr := controllerutil.SetControllerReference(sns, sn, r.Scheme); setErr != nil { return fmt.Errorf("setting owner reference on StorageNode %s: %w", name, setErr) } @@ -321,13 +321,15 @@ func sanitiseDNSLabel(s string) string { return strings.Trim(b.String(), "-.") } -// buildStorageNodeCR constructs a new StorageNode CR for the given worker and socket. +// buildStorageNodeCR constructs a new StorageNode CR for the given worker, +// NUMA socket identifier, per-socket node index, and global ordinal. func buildStorageNodeCR( sns *simplyblockv1alpha1.StorageNodeSet, - name, worker string, - ordinal int, + name, worker, socketID string, + nodeIdx, ordinal int, ) *simplyblockv1alpha1.StorageNode { - idx := int32(ordinal) + globalOrdinal := int32(ordinal) + ni := int32(nodeIdx) sn := &simplyblockv1alpha1.StorageNode{ ObjectMeta: metav1.ObjectMeta{ @@ -341,7 +343,9 @@ func buildStorageNodeCR( Spec: simplyblockv1alpha1.StorageNodeSpec{ StorageNodeSetRef: sns.Name, WorkerNode: worker, - SocketIndex: &idx, + SocketID: socketID, + NodeIndex: &ni, + SocketIndex: &globalOrdinal, }, } From 95d0999d3aff4338c6e5ccdd2a47eb3ee3480c09 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 20 Jul 2026 11:26:03 +0100 Subject: [PATCH 49/52] fix: remove dead labelWorkerNode referencing deleted WorkerNode field causing lint typecheck failure --- .../simplyblockstoragenodeset_controller.go | 23 -------- ...lockstoragenodeset_controller_unit_test.go | 53 ------------------- 2 files changed, 76 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 20e7edd1..7ac1b390 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -472,29 +472,6 @@ func (r *StorageNodeSetReconciler) labelWorkerNodes(ctx context.Context, sn *sim return nil } -func (r *StorageNodeSetReconciler) labelWorkerNode(ctx context.Context, sn *simplyblockv1alpha1.StorageNodeSet) error { - var node corev1.Node - if err := r.Get(ctx, client.ObjectKey{Name: sn.Spec.WorkerNode}, &node); err != nil { - r.Recorder.Eventf(sn, corev1.EventTypeWarning, "WorkerNodeNotFound", - "worker node %q: %v", sn.Spec.WorkerNode, err) - return err - } - - if node.Labels == nil { - node.Labels = map[string]string{} - } - - key := "io.simplyblock.node-type" - value := "simplyblock-storage-plane-" + sn.Spec.ClusterName - - node.Labels[key] = value - if err := r.Update(ctx, &node); err != nil { - return err - } - - return nil -} - func (r *StorageNodeSetReconciler) reconcileDaemonSet( ctx context.Context, snCR *simplyblockv1alpha1.StorageNodeSet, diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go index b093c355..5cb6fb46 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go @@ -114,32 +114,6 @@ func TestStorageNodeSetLabelingHelpers(t *testing.T) { } }) - t.Run("labelWorkerNode labels single worker node", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-label-one", - Namespace: "default", - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: "cluster-b", - WorkerNode: "node-one", - }, - } - node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-one"}} - r := newStorageNodeSetStateTestReconciler(t, sn, node) - - if err := r.labelWorkerNode(context.Background(), sn); err != nil { - t.Fatalf("labelWorkerNode returned error: %v", err) - } - - var out corev1.Node - if err := r.Get(context.Background(), client.ObjectKey{Name: "node-one"}, &out); err != nil { - t.Fatalf("failed to fetch node: %v", err) - } - if out.Labels["io.simplyblock.node-type"] != "simplyblock-storage-plane-cluster-b" { - t.Fatalf("expected worker node label to be set") - } - }) t.Run("labelWorkerNodes records an event when a worker node is missing", func(t *testing.T) { sn := &simplyblockv1alpha1.StorageNodeSet{ @@ -169,33 +143,6 @@ func TestStorageNodeSetLabelingHelpers(t *testing.T) { } }) - t.Run("labelWorkerNode records an event when the worker node is missing", func(t *testing.T) { - sn := &simplyblockv1alpha1.StorageNodeSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sn-label-one-missing", - Namespace: "default", - }, - Spec: simplyblockv1alpha1.StorageNodeSetSpec{ - ClusterName: "cluster-b", - WorkerNode: "vm12.simplyblock4.localdomain", - }, - } - r := newStorageNodeSetStateTestReconciler(t, sn) - - if err := r.labelWorkerNode(context.Background(), sn); err == nil { - t.Fatalf("expected labelWorkerNode to return an error for a missing node") - } - - recorder := r.Recorder.(*record.FakeRecorder) - select { - case event := <-recorder.Events: - if !strings.Contains(event, "WorkerNodeNotFound") || !strings.Contains(event, "vm12.simplyblock4.localdomain") { - t.Fatalf("unexpected event content: %q", event) - } - default: - t.Fatalf("expected a WorkerNodeNotFound event to be recorded") - } - }) } func TestStorageNodeSetDaemonSetReconcileCreatesWhenMissing(t *testing.T) { From 76dd08f84017a9278dd5735ca53caf4449890bb3 Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 20 Jul 2026 11:30:19 +0100 Subject: [PATCH 50/52] formatted file internal/controller/simplyblockstoragenodeset_controller_unit_test.go --- .../controller/simplyblockstoragenodeset_controller_unit_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go index 5cb6fb46..8aece778 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller_unit_test.go @@ -114,7 +114,6 @@ func TestStorageNodeSetLabelingHelpers(t *testing.T) { } }) - t.Run("labelWorkerNodes records an event when a worker node is missing", func(t *testing.T) { sn := &simplyblockv1alpha1.StorageNodeSet{ ObjectMeta: metav1.ObjectMeta{ From 5b0a58ed8797fa9e8ed8c00944ca338def80df5b Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 20 Jul 2026 13:49:34 +0100 Subject: [PATCH 51/52] fix: gate pollNodeOnline on all StorageNode CRs having UUIDs to prevent premature timeout with multi-socket/multi-node configs --- .../simplyblockstoragenodeset_controller.go | 12 ++++++--- .../simplyblockstoragenodeset_storagenode.go | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/operator/internal/controller/simplyblockstoragenodeset_controller.go b/operator/internal/controller/simplyblockstoragenodeset_controller.go index 7ac1b390..36cd5f49 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_controller.go +++ b/operator/internal/controller/simplyblockstoragenodeset_controller.go @@ -273,11 +273,15 @@ func (r *StorageNodeSetReconciler) reconcileWorkerNode( return ctrl.Result{RequeueAfter: time.Second * 10}, nil } - // StorageNodeReconciler is now the sole owner of provisioning. Skip this - // node if its StorageNode CR already has PostedAt set — the new reconciler - // has taken over and we must not double-POST. + // StorageNodeReconciler is the sole owner of provisioning. Only call + // pollNodeOnline once ALL StorageNode CRs for this worker have their UUID set + // (all nodes confirmed online). Until then requeue — calling pollNodeOnline + // before all nodes are posted would time out waiting for expectedPerHost nodes. if r.storageNodeAlreadyPosted(ctx, snCR.Namespace, nodeName) { - return r.pollNodeOnline(ctx, apiClient, clusterUUID, ip, nodeName, expectedPerHost, snCR) + if r.allStorageNodesOnline(ctx, snCR.Namespace, nodeName, expectedPerHost) { + return r.pollNodeOnline(ctx, apiClient, clusterUUID, ip, nodeName, expectedPerHost, snCR) + } + return ctrl.Result{RequeueAfter: waitForNodeOnlineWaitInterval}, nil } // StorageNodeReconciler is the sole owner of provisioning. If it hasn't diff --git a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go index ca77a8cd..15eabfa9 100644 --- a/operator/internal/controller/simplyblockstoragenodeset_storagenode.go +++ b/operator/internal/controller/simplyblockstoragenodeset_storagenode.go @@ -487,6 +487,31 @@ func (r *StorageNodeSetReconciler) storageNodePostedAt( return nil } +// allStorageNodesOnline returns true if the number of StorageNode CRs for the +// given worker that have a non-empty UUID equals expectedPerHost. Used to gate +// pollNodeOnline so it is only called once every node has been posted AND +// received its UUID from the backend, avoiding premature timeout. +func (r *StorageNodeSetReconciler) allStorageNodesOnline( + ctx context.Context, + namespace, workerNode string, + expectedPerHost int, +) bool { + var snList simplyblockv1alpha1.StorageNodeList + if err := r.List(ctx, &snList, + client.InNamespace(namespace), + client.MatchingFields{"spec.workerNode": workerNode}, + ); err != nil { + return false + } + online := 0 + for _, sn := range snList.Items { + if sn.Status.UUID != "" { + online++ + } + } + return online >= expectedPerHost +} + // storageNodeAlreadyPosted returns true if the StorageNode CR for the given // worker node already has PostedAt set, meaning StorageNodeReconciler has taken // over provisioning and this reconciler must not duplicate the POST. From 250599c4d986e33ca9e572ca057ddc541cbd638f Mon Sep 17 00:00:00 2001 From: geoffrey1330 Date: Mon, 20 Jul 2026 14:33:26 +0100 Subject: [PATCH 52/52] added test script test_storagenode_ops.sh --- operator/test/utils/test_storagenode_ops.sh | 470 ++++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100755 operator/test/utils/test_storagenode_ops.sh diff --git a/operator/test/utils/test_storagenode_ops.sh b/operator/test/utils/test_storagenode_ops.sh new file mode 100755 index 00000000..76e59a78 --- /dev/null +++ b/operator/test/utils/test_storagenode_ops.sh @@ -0,0 +1,470 @@ +#!/bin/bash +# Regression test: StorageNode and StorageNodeOps API +# +# Usage: +# ./test_storagenode_ops.sh # run all tests +# ./test_storagenode_ops.sh 1 3 5 # run specific tests by number +# +# Tests: +# 1 — StorageNode CRs created for each workerNode in StorageNodeSet +# 2 — StorageNode fields: socketIndex, uuid, status, health, overrides +# 3 — StorageNode status aggregated in StorageNodeSet (totalNodes/onlineNodes) +# 4 — nodeConfigs override reflected in StorageNode.spec.overrides +# 5 — StorageNodeOps remove: happy path (drain + remove, node goes offline) +# 6 — StorageNodeOps remove: mutual exclusion — second ops waits for first +# 7 — StorageNodeOps remove: unknown action rejected with Failed phase +# 8 — StorageNodeOps remove: activeOpsRef cleared after completion +# 9 — StorageNode events mirrored from StorageNodeOps (NodeRemoved on StorageNode) +# 10 — StorageNodeSet status wide columns visible (-o wide shows Offline/Removed) +# 11 — Cluster expansion: add a new workerNode, StorageNode CR created and provisioned + +set -euo pipefail + +NAMESPACE="${NAMESPACE:-simplyblock}" +STORAGENODESET="${STORAGENODESET:-simplyblock-node}" + +# Pick the third worker for destructive tests (remove) +TARGET_WORKER="vm04.simplyblock3.localdomain" +TARGET_SN="simplyblock-node-vm04.simplyblock3.localdomain-0" + +TIMEOUT_ONLINE=120 +TIMEOUT_OPS=180 + +PASSED=0 +FAILED=0 + +pass() { echo "[PASS] $*"; PASSED=$((PASSED + 1)); } +fail() { echo "[FAIL] $*"; FAILED=$((FAILED + 1)); } +info() { echo "[INFO] $*"; } +section() { echo ""; echo "══════════════════════════════════════════"; echo " $*"; echo "══════════════════════════════════════════"; } + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +wait_for_sn_status() { + local name=$1 want=$2 timeout=${3:-$TIMEOUT_ONLINE} + local elapsed=0 + while [[ $elapsed -lt $timeout ]]; do + got=$(kubectl -n "$NAMESPACE" get storagenode "$name" \ + -o jsonpath='{.status.status}' 2>/dev/null || true) + [[ "$got" == "$want" ]] && return 0 + sleep 5; elapsed=$((elapsed + 5)) + done + return 1 +} + +wait_for_ops_phase() { + local name=$1 want=$2 timeout=${3:-$TIMEOUT_OPS} + local elapsed=0 + while [[ $elapsed -lt $timeout ]]; do + got=$(kubectl -n "$NAMESPACE" get storagenodeops "$name" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true) + [[ "$got" == "$want" ]] && return 0 + sleep 5; elapsed=$((elapsed + 5)) + done + return 1 +} + +delete_ops() { + kubectl -n "$NAMESPACE" delete storagenodeops "$1" --ignore-not-found &>/dev/null || true +} + +# ── Test selection ──────────────────────────────────────────────────────────── + +run_test() { + local n=$1 + [[ ${#TESTS[@]} -eq 0 ]] && return 0 + for t in "${TESTS[@]}"; do [[ "$t" == "$n" ]] && return 0; done + return 1 +} + +TESTS=() +for arg in "$@"; do TESTS+=("$arg"); done + +# ── Tests ───────────────────────────────────────────────────────────────────── + +section "Test 1 — StorageNode CRs created for each workerNode" +if run_test 1; then + workers=$(kubectl -n "$NAMESPACE" get storagenodeset "$STORAGENODESET" \ + -o jsonpath='{.spec.workerNodes[*]}' 2>/dev/null) + sns=$(kubectl -n "$NAMESPACE" get storagenode \ + -o jsonpath='{.items[*].spec.workerNode}' 2>/dev/null) + all_found=true + for w in $workers; do + if ! echo "$sns" | grep -q "$w"; then + fail "No StorageNode CR found for worker $w" + all_found=false + fi + done + $all_found && pass "StorageNode CRs exist for all workerNodes" +fi + +section "Test 2 — StorageNode fields populated" +if run_test 2; then + errors=0 + for sn in $(kubectl -n "$NAMESPACE" get storagenode -o name); do + name=$(echo "$sn" | cut -d/ -f2) + socket=$(kubectl -n "$NAMESPACE" get storagenode "$name" \ + -o jsonpath='{.spec.socketIndex}' 2>/dev/null) + uuid=$(kubectl -n "$NAMESPACE" get storagenode "$name" \ + -o jsonpath='{.status.uuid}' 2>/dev/null) + status=$(kubectl -n "$NAMESPACE" get storagenode "$name" \ + -o jsonpath='{.status.status}' 2>/dev/null) + if [[ -z "$uuid" ]]; then + fail "$name: uuid is empty" + errors=$((errors + 1)) + fi + if [[ "$status" != "online" ]]; then + fail "$name: status is '$status', expected online" + errors=$((errors + 1)) + fi + if [[ -z "$socket" ]]; then + fail "$name: socketIndex is empty" + errors=$((errors + 1)) + fi + done + [[ $errors -eq 0 ]] && pass "All StorageNode CRs have uuid, status=online, socketIndex" +fi + +section "Test 3 — StorageNodeSet status aggregation" +if run_test 3; then + total=$(kubectl -n "$NAMESPACE" get storagenodeset "$STORAGENODESET" \ + -o jsonpath='{.status.totalNodes}' 2>/dev/null) + online=$(kubectl -n "$NAMESPACE" get storagenodeset "$STORAGENODESET" \ + -o jsonpath='{.status.onlineNodes}' 2>/dev/null) + worker_count=$(kubectl -n "$NAMESPACE" get storagenodeset "$STORAGENODESET" \ + -o jsonpath='{.spec.workerNodes}' 2>/dev/null | python3 -c "import json,sys; print(len(json.load(sys.stdin)))") + if [[ "$total" == "$worker_count" ]]; then + pass "totalNodes=$total matches workerNodes count" + else + fail "totalNodes=$total, expected $worker_count" + fi + if [[ "$online" -gt 0 ]]; then + pass "onlineNodes=$online > 0" + else + fail "onlineNodes=$online — expected > 0" + fi +fi + +section "Test 4 — nodeConfigs override in StorageNode.spec.overrides" +if run_test 4; then + # Check that at least one StorageNode has overrides populated + found_overrides=false + for sn in $(kubectl -n "$NAMESPACE" get storagenode -o name); do + name=$(echo "$sn" | cut -d/ -f2) + overrides=$(kubectl -n "$NAMESPACE" get storagenode "$name" \ + -o jsonpath='{.spec.overrides}' 2>/dev/null) + if [[ -n "$overrides" && "$overrides" != "{}" ]]; then + found_overrides=true + max_lvol=$(kubectl -n "$NAMESPACE" get storagenode "$name" \ + -o jsonpath='{.spec.overrides.maxLogicalVolumeCount}' 2>/dev/null) + info "$name: maxLogicalVolumeCount=$max_lvol" + fi + done + $found_overrides && pass "StorageNode.spec.overrides populated from nodeConfigs" \ + || fail "No StorageNode has spec.overrides — check nodeConfigs in StorageNodeSet" +fi + +section "Test 5 — StorageNodeOps remove: happy path" +if run_test 5; then + info "Removing $TARGET_SN via StorageNodeOps" + delete_ops "test-remove-vm04" + kubectl apply -f - </dev/null) + [[ "$sn_status" == "removed" ]] \ + && pass "StorageNode status=removed after ops" \ + || fail "StorageNode status='$sn_status', expected 'removed'" + else + phase=$(kubectl -n "$NAMESPACE" get storagenodeops test-remove-vm04 \ + -o jsonpath='{.status.phase}' 2>/dev/null) + fail "StorageNodeOps did not reach Succeeded (phase=$phase)" + fi + delete_ops "test-remove-vm04" +fi + +section "Test 6 — StorageNodeOps mutual exclusion" +if run_test 6; then + # Need an online node — use vm02 + sn2="simplyblock-node-vm02.simplyblock3.localdomain-0" + delete_ops "test-mutex-a"; delete_ops "test-mutex-b" + + # Create two ops targeting the same node simultaneously + kubectl apply -f - </dev/null) + phase_b=$(kubectl -n "$NAMESPACE" get storagenodeops test-mutex-b \ + -o jsonpath='{.status.phase}' 2>/dev/null) + + if [[ -n "$active" ]]; then + pass "activeOpsRef set to '$active' — only one ops active at a time" + else + fail "activeOpsRef is empty — mutual exclusion may not be working" + fi + + # Clean up — delete both, resume node if needed + delete_ops "test-mutex-a"; delete_ops "test-mutex-b" + # Resume the node if it got suspended + if [[ "$(kubectl -n "$NAMESPACE" get storagenode "$sn2" -o jsonpath='{.status.status}' 2>/dev/null)" == "suspended" ]]; then + kubectl apply -f - </dev/null) + if [[ -z "$active" ]]; then + pass "activeOpsRef cleared after ops completion" + else + fail "activeOpsRef='$active' still set after ops completed" + fi + # Resume if suspended + if [[ "$(kubectl -n "$NAMESPACE" get storagenode "$sn2" -o jsonpath='{.status.status}' 2>/dev/null)" == "suspended" ]]; then + delete_ops "test-resume-cleanup" + kubectl apply -f - </dev/null | wc -l) + if [[ $events -gt 0 ]]; then + pass "StorageNode has $events event(s) — events are being mirrored" + kubectl -n "$NAMESPACE" get events \ + --field-selector "involvedObject.name=$sn2,involvedObject.kind=StorageNode" \ + --no-headers 2>/dev/null + else + fail "No events found on StorageNode $sn2" + fi + # Resume + delete_ops "test-events" + if [[ "$(kubectl -n "$NAMESPACE" get storagenode "$sn2" -o jsonpath='{.status.status}' 2>/dev/null)" == "suspended" ]]; then + kubectl apply -f - </dev/null) + if echo "$output" | grep -q "OFFLINE\|SUSPENDED\|CREATING\|REMOVED"; then + pass "Wide columns (OFFLINE, SUSPENDED, CREATING, REMOVED) visible in -o wide" + echo "$output" + else + fail "Wide columns not visible — check printcolumn markers on StorageNodeSet" + fi +fi + +section "Test 11 — Cluster expansion: manually created StorageNode referencing existing StorageNodeSet" +if run_test 11; then + # Create a StorageNode CR manually, referencing the existing StorageNodeSet. + # The operator must NOT delete it (no OwnerReference = manually created). + # Fleet defaults from the StorageNodeSet fill in any unset fields. + EXPAND_WORKER="${EXPAND_WORKER:-vm15.simplyblock3.localdomain}" + EXPAND_SN_NAME="manual-expand-${EXPAND_WORKER%%.*}" + TIMEOUT_EXPANSION="${TIMEOUT_EXPANSION:-300}" + + info "Manually creating StorageNode for $EXPAND_WORKER referencing $STORAGENODESET" + + # Clean up any leftover + kubectl -n "$NAMESPACE" delete storagenode "$EXPAND_SN_NAME" \ + --ignore-not-found &>/dev/null || true + + # Create the StorageNode CR manually — no OwnerReference. + # Only specify the overrides that differ from fleet defaults; the rest are + # inherited from the referenced StorageNodeSet. + kubectl apply -f - </dev/null; then + pass "Manually created StorageNode survived reconciler (not deleted as stale)" + else + fail "Manually created StorageNode was incorrectly deleted by the reconciler" + fi + + # 2. Per-node ConfigMap for the existing StorageNodeSet should include the new worker + elapsed=0; cm_ok=false + while [[ $elapsed -lt 60 ]]; do + keys=$(kubectl -n "$NAMESPACE" get configmap "${STORAGENODESET}-per-node-config" \ + -o jsonpath='{.data}' 2>/dev/null | \ + python3 -c "import json,sys; d=json.load(sys.stdin); print(' '.join(d.keys()))" 2>/dev/null || true) + if echo "$keys" | grep -q "$EXPAND_WORKER"; then + cm_ok=true; break + fi + sleep 5; elapsed=$((elapsed + 5)) + done + $cm_ok \ + && pass "Per-node ConfigMap updated with entry for manual worker $EXPAND_WORKER" \ + || fail "Per-node ConfigMap did not include $EXPAND_WORKER within 60s" + + # 3. Overrides merged with fleet defaults in ConfigMap entry + max_lvol=$(kubectl -n "$NAMESPACE" get configmap "${STORAGENODESET}-per-node-config" \ + -o jsonpath="{.data['$EXPAND_WORKER']}" 2>/dev/null | \ + grep "^MAX_LVOL=" | cut -d= -f2 || true) + [[ "$max_lvol" == "15" ]] \ + && pass "Override maxLogicalVolumeCount=15 reflected in ConfigMap" \ + || fail "Expected MAX_LVOL=15 in ConfigMap, got '$max_lvol'" + + # 4. StorageNodeReconciler picks it up and provisions it + info "Waiting up to ${TIMEOUT_EXPANSION}s for $EXPAND_SN_NAME to come online..." + if wait_for_sn_status "$EXPAND_SN_NAME" "online" "$TIMEOUT_EXPANSION"; then + pass "Manually created StorageNode $EXPAND_SN_NAME reached status=online" + uuid=$(kubectl -n "$NAMESPACE" get storagenode "$EXPAND_SN_NAME" \ + -o jsonpath='{.status.uuid}' 2>/dev/null) + [[ -n "$uuid" ]] && pass "StorageNode uuid=$uuid populated" \ + || fail "StorageNode uuid is empty after provisioning" + else + status=$(kubectl -n "$NAMESPACE" get storagenode "$EXPAND_SN_NAME" \ + -o jsonpath='{.status.status}' 2>/dev/null) + fail "StorageNode did not reach online within ${TIMEOUT_EXPANSION}s (status=$status)" + fi + + # Cleanup + info "Cleaning up manually created StorageNode $EXPAND_SN_NAME..." + kubectl -n "$NAMESPACE" delete storagenode "$EXPAND_SN_NAME" \ + --ignore-not-found &>/dev/null || true +fi + +# ── Summary ─────────────────────────────────────────────────────────────────── + +echo "" +echo "══════════════════════════════════════════" +echo " Results: $PASSED passed, $FAILED failed" +echo "══════════════════════════════════════════" +[[ $FAILED -eq 0 ]]