diff --git a/operator/cmd/main.go b/operator/cmd/main.go index abff1d22..6998b6ac 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -42,6 +42,7 @@ import ( simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" "github.com/simplyblock/simplyblock-operator/internal/controller" "github.com/simplyblock/simplyblock-operator/internal/utils" + "github.com/simplyblock/simplyblock-operator/internal/volumemigration/autobalancing" "github.com/simplyblock/simplyblock-operator/internal/webapi" internalwebhook "github.com/simplyblock/simplyblock-operator/internal/webhook" // +kubebuilder:scaffold:imports @@ -418,6 +419,14 @@ func main() { mgr.GetWebhookServer().Register("/mutate-v1-pod-simplyblock-rebalancer", &webhook.Admission{Handler: &internalwebhook.SimplyblockRebalancerInjector{Client: mgr.GetClient()}}) setupLog.Info("registered simplyblock-rebalancer mutating webhook") + + mgr.GetWebhookServer().Register("/mutate-v1-pvc-simplyblock-placement", + &webhook.Admission{Handler: &internalwebhook.SimplyblockVolumePlacementInjector{ + Client: mgr.GetClient(), + APIClient: webapi.NewClient(), + NodeSelector: autobalancing.NewStorageNodeSelector(mgr.GetClient()), + }}) + setupLog.Info("registered simplyblock-volume-placement mutating webhook") }() if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/operator/config/webhook/manifests.yaml b/operator/config/webhook/manifests.yaml index 27f4173e..c40c30d7 100644 --- a/operator/config/webhook/manifests.yaml +++ b/operator/config/webhook/manifests.yaml @@ -23,3 +23,22 @@ webhooks: resources: - pods sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-v1-pvc-simplyblock-placement + failurePolicy: Ignore + name: simplyblock-volume-placement-injector.simplyblock.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - persistentvolumeclaims + sideEffects: None diff --git a/operator/dist/install.yaml b/operator/dist/install.yaml index 7a25c9b4..364da056 100644 --- a/operator/dist/install.yaml +++ b/operator/dist/install.yaml @@ -3446,3 +3446,22 @@ webhooks: resources: - pods sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: simplyblock-operator-webhook-service + namespace: simplyblock-operator-system + path: /mutate-v1-pvc-simplyblock-placement + failurePolicy: Ignore + name: simplyblock-volume-placement-injector.simplyblock.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - persistentvolumeclaims + sideEffects: None diff --git a/operator/docs/designs/design-primary-node-placement.md b/operator/docs/designs/design-primary-node-placement.md new file mode 100644 index 00000000..3e16589d --- /dev/null +++ b/operator/docs/designs/design-primary-node-placement.md @@ -0,0 +1,453 @@ +# Design Document: Load-Aware Primary Node Placement at Volume Creation + +**Status:** Proposed +**Author:** Manohar Reddy +**Date:** 2026-07-14 +**Related Issues:** https://github.com/simplyblock/simplyblock-operator/issues/308 (Auto-Placement of Volumes at Creation) +--- + +## Table of Contents + +1. [Background](#1-background) +2. [Goals and Non-Goals](#2-goals-and-non-goals) +3. [Architecture Overview](#3-architecture-overview) +4. [Node Selection Algorithm](#4-node-selection-algorithm) +5. [Webhook Handler](#5-webhook-handler) +6. [Data Model Changes](#6-data-model-changes) +7. [Backend API Requirements](#7-backend-api-requirements) +8. [Failure Modes and Fallback](#8-failure-modes-and-fallback) +9. [Configuration](#9-configuration) +10. [Observability](#10-observability) +11. [Testing Strategy](#11-testing-strategy) +12. [Open Questions](#12-open-questions) + +--- + +## 1. Background + +Volumes are currently placed by `sbcli`'s `_get_next_3_nodes()` +(`simplyblock_core/controllers/lvol_controller.py`), a weighted-random pick over +online storage nodes keyed **only** on subsystem count per node +(`constants.weights = {"lvol": 100}`). It has no notion of actual I/O load: a node +hosting few, but extremely hot, volumes is just as likely to receive the next +volume as an idle one. + +The operator already computes a much better "how loaded is this node right now" +signal for a different purpose — `internal/volumemigration/autobalancing.StorageNodeSelector`, +built for the auto-rebalancer (Issue #130). It combines live Prometheus p99 write +latency with a per-node fio baseline into a per-node deviation score, and already +picks the "coolest" node as a migration target (`pickColdTarget`). + +`add_lvol_ha` already accepts an explicit `host_id_or_name` and skips its own +weighted-random pick entirely when one is supplied. `spdk-csi` already reads a +`simplyblock.io/host-id` PVC annotation and forwards it as `host_id` on +`CreateVolume` — this path is proven end-to-end today by the operator's own +benchmark-volume provisioner (`operator/internal/controller/benchmark_provisioner.go`), +which always sets `HostID` explicitly. What is missing is anything that computes +that annotation from real load data, synchronously, for user-created (PVC-driven) +volumes. + +This design closes that gap: the operator computes the best primary node using the +same node-hotness signal the rebalancer already uses, and stamps it onto the PVC as +`simplyblock.io/host-id` before the CSI provisioner ever calls `CreateVolume`. + +--- + +## 2. Goals and Non-Goals + +### Goals + +- Reuse the rebalancer's existing node-hotness signal (current Prometheus p99 + latency vs. per-node fio baseline) to rank storage nodes by load at volume + creation time. +- Compute the selected primary node **entirely inside the operator** — no new + logic in `sbcli`, no new logic in `spdk-csi`. +- Inject the decision via the existing, already-supported `simplyblock.io/host-id` + PVC annotation contract, so `spdk-csi` and the backend require zero changes to + the creation path itself. +- Never override an explicit user-supplied `host_id` annotation (manual pinning + always wins). +- Degrade silently to today's behavior (`sbcli`'s weighted-random pick) whenever + the load signal isn't available for a cluster — this feature is strictly + additive on top of clusters that already opted into latency benchmarking + (Issue #130). +- Avoid picking a node that is offline, unhealthy, a secondary node, or already at + subsystem capacity. + +### Non-Goals + +- Changing `sbcli`'s fallback placement algorithm (`_get_next_3_nodes`) itself. +- Node-affinity / topology-aware placement (pinning a volume to the consumer + pod's worker node) — tracked separately (Issue #272). +- Capacity-based placement (disk space utilization) — out of scope, same as the + rebalancer (Issue #130 §2 Non-Goals). +- Cross-cluster placement. +- Per-block-size or per-volume-QoS-aware scoring (Phase 2 of Issue #130, not + needed here). + +--- + +## 3. Architecture Overview + +``` +PVC created (user) + │ + ▼ +┌──────────────────────────────────────────────────────────────────────────┐ +│ SimplyblockVolumePlacementInjector (NEW mutating webhook) │ +│ │ +│ 1. Skip if simplyblock.io/host-id already set (explicit pin wins) │ +│ 2. Resolve StorageClass → cluster_id / pool_name params │ +│ 3. Resolve StorageCluster CR by Status.UUID == cluster_id │ +│ 4. Skip if AutoRebalancing disabled or PrometheusURL unset │ +│ 5. GET storage-nodes (webapi.Client) → filter online/healthy/ │ +│ non-secondary/under-capacity │ +│ 6. autobalancing.StorageNodeSelector.SelectBestNode(...) │ +│ → lowest current-latency-deviation eligible node │ +│ 7. Patch PVC: simplyblock.io/host-id = │ +└───────────────────────────────┬──────────────────────────────────────────┘ + │ admission response (mutating patch) + ▼ + PVC persisted with host-id annotation + │ + ▼ +┌──────────────────────────────────────────────────────────────────────────┐ +│ external-provisioner → spdk-csi CreateVolume │ +│ prepareCreateVolumeReq() re-fetches PVC live (fetchPVCAnnotations) │ +│ → reads simplyblock.io/host-id → CreateLVolData.HostID │ +└───────────────────────────────┬──────────────────────────────────────────┘ + │ HTTP +┌───────────────────────────────▼──────────────────────────────────────────┐ +│ SimplyBlock Backend: add_lvol_ha(host_id_or_name=) │ +│ host_node set → _get_next_3_nodes() is NEVER called │ +│ _resolve_lvol_subsystem() still enforces max_lvol as a hard backstop │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +**Why a mutating webhook, not a new backend/CSI call:** `spdk-csi`'s +`fetchPVCAnnotations` (`pkg/spdk/controllerserver.go:1240`) performs a **live** GET +of the PVC object at `CreateVolume` time — it does not rely on CSI request +parameters cached earlier in the provisioning pipeline. A webhook that mutates the +PVC at admission time (before the external-provisioner sidecar even notices the +PVC) is therefore guaranteed to be visible by the time `spdk-csi` reads it. This +requires **zero changes to `spdk-csi`**. + +This mirrors the existing `SimplyblockRebalancerInjector` pod-mutating webhook +(`operator/internal/webhook/simplyblock_rebalancer_injector.go`), which already +follows the same pattern: resolve the owning `StorageCluster` from context, check +whether the relevant feature is enabled for that cluster, patch if so, allow +unconditionally (`failurePolicy=Ignore`) otherwise. + +--- + +## 4. Node Selection Algorithm + +Reuses the exact signal `autobalancing.StorageNodeSelector` computes for the +rebalancer (Issue #130 §5.2): + +``` +latencyDeviationPct(node) = (currentP99NS - baselineP99NS) / baselineP99NS × 100 +``` + +Where `currentP99NS` is queried live from Prometheus +(`simplyblock_node_fio_write_latency_p99_ns`) and `baselineP99NS` is the one-time +fio baseline stored on the owning `StorageNodeSet` CR (`Status.LatencyMetrics`). + +### New entry point: `SelectBestNode` + +`pickColdTarget` (`storage_node_selector.go`) already contains the core "pick the +coolest node" loop, gated by `MinHotColdDifferencePct` (a migration-specific +"must be meaningfully cooler than a given hot source" rule that doesn't apply to +placement). This design extracts the ranking core into a new exported method: + +```go +// SelectBestNode returns the least-loaded eligible node — the one with the +// lowest current latency deviation — across the given candidate pool. Unlike +// pickColdTarget, there is no source node to compare against and no +// MinHotColdDifferencePct gate: placement always wants the single best +// candidate, however small its lead over the second-best. +func (sns *StorageNodeSelector) SelectBestNode( + ctx context.Context, + cfg RebalancingConfig, + eligible map[string]bool, // nodeUUID -> true + inputs ...StorageNodeSelectorInput, +) (nodeUUID string, ok bool, err error) +``` + +Nodes with no latency data yet (no baseline measured, deviation = 0) rank as the +best possible candidates — consistent with how the rebalancer already treats +unmeasured nodes as migration targets (Issue #130 §6 Step 5). + +### Eligibility filter (applied before ranking) + +| Filter | Source | Rationale | +|---|---|---| +| `status == "online"` | `webapi.StorageNodeInfo.Status` | Never place on an offline node | +| `health_check == true` | `webapi.StorageNodeInfo.Healthy` | Mirrors rebalancer target eligibility (Issue #130 §6 Step 5) | +| not a secondary node | `webapi.StorageNodeInfo.IsSecondary` (new field, §6) | Only primary-capable nodes host a new lvol's primary subsystem | +| `Lvols < LvolsMax` | `webapi.StorageNodeInfo.Lvols` / `.LvolsMax` (new fields, §6) | Mirrors `sbcli`'s own `max_lvol` capacity gate (`_resolve_lvol_subsystem`) so we don't hand the backend a node it will immediately reject | + +The capacity check is an approximation of `sbcli`'s `count_lvol_subsystems` (which +counts distinct subsystems, not raw lvol count, since namespaced pools share a +subsystem across lvols) — it is slightly conservative but avoids the common +"clearly full node" case. `_resolve_lvol_subsystem`'s exact check remains the +authoritative backstop server-side regardless. + +--- + +## 5. Webhook Handler + +New file: `operator/internal/webhook/simplyblock_volume_placement_injector.go`, +same shape as `simplyblock_rebalancer_injector.go`. + +```go +// +kubebuilder:webhook:path=/mutate-v1-pvc-simplyblock-placement,mutating=true,failurePolicy=ignore,sideEffects=None,groups="",resources=persistentvolumeclaims,verbs=create,versions=v1,name=simplyblock-volume-placement-injector.simplyblock.io,admissionReviewVersions=v1 + +type SimplyblockVolumePlacementInjector struct { + Client client.Client + APIClient *webapi.Client + NodeSelector *autobalancing.StorageNodeSelector +} +``` + +### Handle flow + +1. Decode the PVC. Allow unmodified if: + - `simplyblock.io/host-id` (or deprecated `simplybk/host-id`) is already set. + - `pvc.Spec.StorageClassName` is unset, or the referenced `StorageClass`'s + `Provisioner` isn't the simplyblock CSI driver, or `parameters["cluster_id"]` + is empty. +2. Resolve the `StorageCluster` CR whose `Status.UUID == cluster_id` — same lookup + pattern as `SimplyblockRebalancerInjector.resolveConfig`. Allow unmodified if + not found, or if `Spec.VolumeMigrationSettings.AutoRebalancing` is nil/disabled, + or `PrometheusURL` is unset. +3. Build `autobalancing.RebalancingConfig` via the existing + `autobalancing.ResolveRebalancingConfig(spec)`. +4. `APIClient.GetStorageNodes(ctx, clusterUUID)` — same call + `VolumeRebalancerReconciler` already makes (in-cluster service-account auth, no + per-cluster secret). +5. Apply the eligibility filter (§4). +6. `NodeSelector.SelectBestNode(...)`. Allow unmodified if no eligible node. +7. Patch the PVC: `simplyblock.io/host-id = `. + +Any error at steps 2–6 (backend unreachable, Prometheus query failure, no +StorageNodeSet baseline yet) results in `admission.Allowed(...)` with no patch — +`sbcli`'s existing weighted-random pick runs exactly as it does today. This +mirrors `failurePolicy=Ignore` on the webhook registration itself: the feature +can never block volume provisioning. + +### Registration + +`operator/cmd/main.go`, alongside the existing registration (~line 418), under the +same `webhookReady` gate: + +```go +mgr.GetWebhookServer().Register("/mutate-v1-pvc-simplyblock-placement", + &webhook.Admission{Handler: &internalwebhook.SimplyblockVolumePlacementInjector{ + Client: mgr.GetClient(), + APIClient: webapi.NewClient(), + NodeSelector: autobalancing.NewStorageNodeSelector(mgr.GetClient()), + }}) +``` + +`make manifests` regenerates `config/webhook/manifests.yaml` (and the Helm chart / +`dist/install.yaml` copies) from the new marker. + +### RBAC + +No new PVC write RBAC is needed — the mutation happens via the admission response +patch, not a client-side `Update`. New read RBAC: + +```go +// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storageclusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodesets,verbs=get;list;watch +``` + +`storageclasses` and the two `storage.simplyblock.io` reads are already granted to +other controllers (`simplyblockpool_controller.go`, `volumerebalancer_controller.go`) +and land in the same aggregated ClusterRole — verify at implementation time +whether a distinct marker is still needed for kubebuilder to pick it up for this +file, but no new *permission* is required. + +--- + +## 6. Data Model Changes + +All additions are additive; nothing existing changes shape. + +### 6.1 `operator/internal/webapi/rebalancing.go` — `StorageNodeInfo` + +```go +type StorageNodeInfo struct { + UUID string `json:"id"` + Status string `json:"status"` + Healthy bool `json:"health_check"` + TotalBytes int64 `json:"total_capacity_bytes"` + Lvols int `json:"lvols"` // NEW — already returned by the v2 API + LvolsMax int `json:"lvols_max"` // NEW — already returned by the v2 API + IsSecondary bool `json:"is_secondary"` // NEW — requires a backend field addition, §7 +} +``` + +`Lvols` / `LvolsMax` require **no backend change** — `StorageNodeDTO` in +`simplyblock_web/api/v2/_dtos.py` already serializes `lvols` and `lvols_max` +(`model.lvols`, `model.max_lvol`); the Go struct simply never mapped them because +the rebalancer never needed them. + +### 6.2 No StorageCluster/StorageNodeSet CRD changes + +This design reads existing fields only: `StorageCluster.Spec.VolumeMigrationSettings.AutoRebalancing` +(Issue #130 §4.1) and `StorageNodeSet.Status.LatencyMetrics` (Issue #130 §4.3, +implemented on `StorageNodeSet` in this codebase). + +--- + +## 7. Backend API Requirements + +One additive field, the **only** `sbcli` change in this design: + +`simplyblock_web/api/v2/_dtos.py` — `StorageNodeDTO` doesn't currently expose +whether a node is a secondary node (only `secondary_node_id`, which is a +*primary's* pointer to its own secondary; a secondary node's own record isn't +distinguishable via the v2 API today). Add a passthrough field mirroring +`model.is_secondary_node`: + +```python +class StorageNodeDTO(BaseModel): + ... + is_secondary: bool # NEW + + @staticmethod + def from_model(model: StorageNode, stat_obj: ...): + return StorageNodeDTO( + ..., + is_secondary=model.is_secondary_node, # NEW + ) +``` + +No new logic, no new endpoint — a one-field API exposure of data the model +already tracks (`simplyblock_core/models/storage_node.py:72`). + +--- + +## 8. Failure Modes and Fallback + +| Condition | Behavior | +|---|---| +| `simplyblock.io/host-id` already set on the PVC | Skip — explicit pin always wins | +| StorageClass isn't simplyblock-provisioned, or has no `cluster_id` | Skip | +| `StorageCluster` not found for `cluster_id` | Skip (log) | +| `AutoRebalancing` nil/disabled or `PrometheusURL` unset for the cluster | Skip — cluster hasn't opted into the load signal | +| Backend API (`GetStorageNodes`) unreachable | Skip (log); `failurePolicy=Ignore` also protects at the webhook-server level | +| Prometheus unreachable / query error | Skip (log) | +| No eligible node (all offline/unhealthy/secondary/at-capacity) | Skip (log) | +| Pool has `qos_host` set (`pool.has_qos()`) | Not special-cased — `add_lvol_ha` overrides any `host_id` with `pool.qos_host` regardless, so the injected annotation is harmless but ignored. Documented, not fixed, in v1. | + +In every skip case the PVC is admitted unmodified and `sbcli`'s existing +weighted-random placement (`_get_next_3_nodes`) runs exactly as it does today — +this feature can only ever make placement *better or unchanged*, never worse or +blocking. + +--- + +## 9. Configuration + +No new configuration surface. The feature activates automatically, per cluster, +whenever that cluster already has: + +```yaml +spec: + volumeMigrationSettings: + autoRebalancing: + enabled: true + latencyBenchmarkEnabled: true + prometheusURL: "http://prometheus.monitoring:9090" +``` + +(the same fields Issue #130 introduced). Clusters without latency benchmarking +enabled see no behavior change. + +--- + +## 10. Observability + +### Kubernetes Events + +| Event | Type | Reason | +|---|---|---| +| Primary node selected for new PVC | `Normal` | `PrimaryNodeSelected` | +| Selection skipped — no signal available | (none; logged only, high frequency expected) | — | + +### Prometheus Metrics + +| Metric | Labels | Description | +|---|---|---| +| `simplyblock_placement_decisions_total` | `cluster_uuid`, `result` (`selected`\|`skipped`) | Count of webhook invocations by outcome | +| `simplyblock_placement_selected_node_deviation_pct` | `cluster_uuid`, `node_uuid` | Latency deviation of the node chosen, at selection time | + +--- + +## 11. Testing Strategy + +### Unit Tests + +Mirroring `simplyblock_rebalancer_injector_test.go`, with a fake `client.Client`, +fixture `StorageCluster`/`StorageNodeSet` CRs, and a stubbed `webapi.Client` + +Prometheus response: + +- Annotation already set → PVC unmodified. +- StorageClass missing / not simplyblock-provisioned → PVC unmodified. +- `AutoRebalancing` disabled or `PrometheusURL` unset → PVC unmodified. +- Multiple eligible nodes with different deviations → lowest-deviation node chosen. +- Offline / unhealthy / secondary / at-capacity nodes excluded from candidates. +- No eligible node → PVC unmodified (no error surfaced to the CO). +- Backend or Prometheus error → PVC unmodified, error logged, request still `Allowed`. + +### Regression + +- `go build ./...` and `make test` after extracting `SelectBestNode` out of + `pickColdTarget`, to confirm the rebalancer's existing migration-target + selection behavior (still gated by `MinHotColdDifferencePct`) is unchanged. + +### Manual / E2E + +On a test cluster with `latencyBenchmarkEnabled: true` and `prometheusURL` set: +create a PVC against a StorageClass carrying `cluster_id`/`pool_name` parameters; +confirm `kubectl get pvc -o yaml` shows `simplyblock.io/host-id` stamped before +the PVC binds, and that the resulting lvol's `storage_node_id` matches the +coolest node per `rebalancer_node_latency_deviation_pct` at that moment. + +--- + +## 12. Open Questions + +**Q1: Should the webhook also weigh current IOPS/throughput, not just latency +deviation?** The rebalancer's volume-level IO score doesn't apply (no history for +a not-yet-created volume), but a node-level aggregate IOPS/throughput signal could +supplement latency deviation the same way `iopsWeight`/`throughputWeight` do for +migration ranking. Deferred — latency deviation alone is a meaningful +improvement over pure subsystem-count weighting and keeps this a small, additive +change. + +**Q2: Should capacity (disk space) factor into node selection?** Explicitly out +of scope here (matching Issue #130), but a node that's I/O-cool and about to fill +up is a poor placement choice. Worth a follow-up design. + +**Q3: What about pools with `qos_host` set?** Currently the injected annotation is +silently overridden server-side (§8). Should the webhook skip computing/injecting +for such pools entirely, as a minor optimization? Not required for correctness, +low priority. + +**Q4: Should there be a per-PVC opt-out annotation** (e.g. +`simplyblock.io/disable-smart-placement: "true"`) for workloads that want the +legacy weighted-random behavior even when the cluster has benchmarking enabled? + +**Q5: Multi-cluster StorageClasses.** `resolveClusterSelection` in `spdk-csi` +supports zone/region-mapped multi-cluster StorageClasses +(`paramZoneClusterMap`/`paramRegionClusterMap`), resolved from the CSI +`CreateVolumeRequest`'s topology requirements — information the webhook does not +have access to at PVC-admission time (topology is resolved later, during +scheduling). For such StorageClasses the webhook cannot determine `cluster_id` +up front and must skip. Confirm this is an acceptable, documented limitation +rather than a gap to close. diff --git a/operator/internal/volumemigration/autobalancing/storage_node_selector.go b/operator/internal/volumemigration/autobalancing/storage_node_selector.go index aa0f556f..bfb54d57 100644 --- a/operator/internal/volumemigration/autobalancing/storage_node_selector.go +++ b/operator/internal/volumemigration/autobalancing/storage_node_selector.go @@ -114,7 +114,7 @@ func (sns *StorageNodeSelector) SelectStorageNodes( ) ([]NodeMigrationPair, error) { // computeLatencyDeviations also emits the per-node / max deviation gauges as a side // effect; the per-cluster stats it returns are not needed for pairing here. - deviations, _, err := sns.computeLatencyDeviations(ctx, cfg.PrometheusURL, cfg.LatencyPercentile, inputs...) + deviations, err := sns.computeLatencyDeviations(ctx, cfg.PrometheusURL, cfg.LatencyPercentile, inputs...) if err != nil { return nil, err } @@ -157,6 +157,41 @@ func (sns *StorageNodeSelector) SelectStorageNodes( return pairs, nil } +// SelectBestNode returns the least-loaded eligible node — the one with the lowest current +// latency deviation — for primary volume placement at creation time. Unlike pickColdTarget +// (migration-target selection), there is no source node to compare against and no +// MinHotColdDifferencePct gate: placement always wants the single best candidate, however +// small its lead over the second-best. A node absent from the current Prometheus reading +// (e.g. freshly onboarded, not yet scraped) is treated as deviation 0 — the best possible +// score — rather than excluded, so it isn't systematically avoided. +func (sns *StorageNodeSelector) SelectBestNode( + ctx context.Context, + cfg RebalancingConfig, + eligible map[string]bool, + inputs ...StorageNodeSelectorInput, +) (nodeUUID string, ok bool, err error) { + deviations, err := sns.computeLatencyDeviations(ctx, cfg.PrometheusURL, cfg.LatencyPercentile, inputs...) + if err != nil { + return "", false, err + } + + var best string + var bestDev float64 + found := false + for uuid, isEligible := range eligible { + if !isEligible { + continue + } + dev := deviations[uuid] + if !found || dev < bestDev { + best = uuid + bestDev = dev + found = true + } + } + return best, found, nil +} + // pickColdTarget returns the coolest eligible migration target for the hot source from // the flat node pool, or ok=false when none qualifies. A candidate qualifies when it is // eligible (isMigrationTargetEligible) and at least cfg.MinHotColdDifferencePct @@ -191,17 +226,18 @@ func isMigrationTargetEligible(src, cand nodeRef, _ RebalancingConfig) bool { } // computeLatencyDeviations collects per-node latency from Prometheus and StorageNode CRs, -// computes deviation from baseline, emits Prometheus gauges, and returns the per-node -// deviation map and per-cluster aggregate statistics. +// computes deviation from baseline, and emits Prometheus gauges (per-node deviation and +// per-cluster max deviation) as a side effect. Only the per-node deviation map is returned — +// no caller currently needs the per-cluster stats beyond the gauge they're emitted into. func (sns *StorageNodeSelector) computeLatencyDeviations( ctx context.Context, prometheusURL string, percentile string, inputs ...StorageNodeSelectorInput, -) (deviations map[string]float64, statsByCluster map[string]ClusterDeviationStats, err error) { +) (deviations map[string]float64, err error) { latencyByNode, err := sns.collectLatencyState(ctx, prometheusURL, percentile, inputs...) if err != nil { - return nil, nil, err + return nil, err } deviations = make(map[string]float64, len(latencyByNode)) for nodeUUID, ld := range latencyByNode { @@ -209,11 +245,11 @@ func (sns *StorageNodeSelector) computeLatencyDeviations( rebalancerNodeLatencyDeviationPct.WithLabelValues(ld.clusterUUID, nodeUUID).Set(dev) deviations[nodeUUID] = dev } - statsByCluster = deviationStats(latencyByNode, deviations) + statsByCluster := deviationStats(latencyByNode, deviations) for clusterUUID, stats := range statsByCluster { rebalancerMaxLatencyDeviationPct.WithLabelValues(clusterUUID).Set(stats.MaxDeviationPct) } - return deviations, statsByCluster, nil + return deviations, nil } // collectLatencyState builds a nodeUUID → nodeLatencyData map by combining: diff --git a/operator/internal/webapi/rebalancing.go b/operator/internal/webapi/rebalancing.go index 22dc54cf..746129ed 100644 --- a/operator/internal/webapi/rebalancing.go +++ b/operator/internal/webapi/rebalancing.go @@ -16,6 +16,14 @@ type StorageNodeInfo struct { Status string `json:"status"` Healthy bool `json:"health_check"` TotalBytes int64 `json:"total_capacity_bytes"` + // Lvols and LvolsMax are the current and maximum lvol-subsystem counts for the + // node, used to filter out at-capacity nodes during primary node placement. + Lvols int `json:"lvols"` + LvolsMax int `json:"lvols_max"` + // IsSecondary is true when this node record is a secondary (HA failover) node + // rather than a primary-capable one. Secondary nodes are never eligible as a + // new volume's primary. + IsSecondary bool `json:"is_secondary"` } // CapacityStat holds the capacity sub-object present on VolumeDTO. diff --git a/operator/internal/webhook/simplyblock_volume_placement_injector.go b/operator/internal/webhook/simplyblock_volume_placement_injector.go new file mode 100644 index 00000000..ee0c4554 --- /dev/null +++ b/operator/internal/webhook/simplyblock_volume_placement_injector.go @@ -0,0 +1,190 @@ +package webhook + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" + "github.com/simplyblock/simplyblock-operator/internal/volumemigration" + "github.com/simplyblock/simplyblock-operator/internal/volumemigration/autobalancing" + "github.com/simplyblock/simplyblock-operator/internal/webapi" +) + +const ( + // annotationHostID and deprecatedAnnotationHostID must match the annotation names + // spdk-csi reads in prepareCreateVolumeReq (pkg/spdk/controllerserver.go) — this + // webhook and spdk-csi share the contract but not any code. + annotationHostID = "simplyblock.io/host-id" + deprecatedAnnotationHostID = "simplybk/host-id" + + // clusterIDParam is the StorageClass parameter key holding the target cluster's + // backend UUID (see upsertStorageClass in simplyblockpool_controller.go). + clusterIDParam = "cluster_id" +) + +// +kubebuilder:webhook:path=/mutate-v1-pvc-simplyblock-placement,mutating=true,failurePolicy=ignore,sideEffects=None,groups="",resources=persistentvolumeclaims,verbs=create,versions=v1,name=simplyblock-volume-placement-injector.simplyblock.io,admissionReviewVersions=v1 +// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storageclusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=storage.simplyblock.io,resources=storagenodesets,verbs=get;list;watch + +// SimplyblockVolumePlacementInjector is a mutating admission webhook that computes the +// least-loaded eligible storage node for a new PVC's primary volume — using the same +// latency-deviation signal the auto-rebalancer (Issue #130) uses — and stamps it onto the +// PVC as the simplyblock.io/host-id annotation, which spdk-csi already reads and forwards +// as host_id on CreateVolume. failurePolicy=ignore, and every skip/error path below allows +// the PVC unmodified, so this can never block volume provisioning: sbcli's own +// weighted-random pick (_get_next_3_nodes) runs as the fallback exactly as it does today. +type SimplyblockVolumePlacementInjector struct { + Client client.Client + APIClient *webapi.Client + NodeSelector *autobalancing.StorageNodeSelector +} + +func (h *SimplyblockVolumePlacementInjector) Handle( + ctx context.Context, + req admission.Request, +) admission.Response { + log := logf.FromContext(ctx).WithValues("pvc", req.Name, "namespace", req.Namespace) + + pvc := &corev1.PersistentVolumeClaim{} + if err := json.Unmarshal(req.Object.Raw, pvc); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if pvc.Annotations[annotationHostID] != "" || pvc.Annotations[deprecatedAnnotationHostID] != "" { + log.V(1).Info("Skipping: host-id already set") + return admission.Allowed("host-id already set") + } + + clusterUUID, ok := h.resolveClusterID(ctx, pvc, log) + if !ok { + return admission.Allowed("not a simplyblock-provisioned PVC") + } + + nodeUUID, ok := h.selectPrimaryNode(ctx, clusterUUID, log) + if !ok { + return admission.Allowed("no load-based placement decision available") + } + + patched := pvc.DeepCopy() + if patched.Annotations == nil { + patched.Annotations = make(map[string]string) + } + patched.Annotations[annotationHostID] = nodeUUID + + original, err := json.Marshal(pvc) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + patchedRaw, err := json.Marshal(patched) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + log.Info("Selected primary node for new volume", "nodeUUID", nodeUUID, "clusterUUID", clusterUUID) + return admission.PatchResponseFromRaw(original, patchedRaw) +} + +// resolveClusterID resolves the PVC's StorageClass to a simplyblock cluster UUID. ok=false +// means "not our PVC" (no StorageClass, not our provisioner, no cluster_id) — not an error. +func (h *SimplyblockVolumePlacementInjector) resolveClusterID( + ctx context.Context, + pvc *corev1.PersistentVolumeClaim, + log logr.Logger, +) (clusterUUID string, ok bool) { + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName == "" { + return "", false + } + + sc := &storagev1.StorageClass{} + if err := h.Client.Get(ctx, client.ObjectKey{Name: *pvc.Spec.StorageClassName}, sc); err != nil { + log.V(1).Info("Skipping: cannot get StorageClass", "storageClass", *pvc.Spec.StorageClassName, "error", err.Error()) + return "", false + } + if sc.Provisioner != utils.CSIProvisioner { + return "", false + } + clusterUUID = sc.Parameters[clusterIDParam] + return clusterUUID, clusterUUID != "" +} + +// selectPrimaryNode resolves the StorageCluster owning clusterUUID, checks that it has +// opted into latency-driven auto-rebalancing (Issue #130), fetches its storage nodes, +// filters them for placement eligibility, and ranks the survivors by current latency +// deviation. ok=false covers every "no signal available" case (feature not enabled, +// backend/Prometheus unreachable, no eligible node) — all of them fall through to the +// caller allowing the PVC unmodified. +func (h *SimplyblockVolumePlacementInjector) selectPrimaryNode( + ctx context.Context, + clusterUUID string, + log logr.Logger, +) (nodeUUID string, ok bool) { + var list simplyblockv1alpha1.StorageClusterList + if err := h.Client.List(ctx, &list); err != nil { + log.Error(err, "Failed to list StorageClusters") + return "", false + } + var cr *simplyblockv1alpha1.StorageCluster + for i := range list.Items { + if list.Items[i].Status.UUID == clusterUUID { + cr = &list.Items[i] + break + } + } + if cr == nil { + log.V(1).Info("Skipping: no matching StorageCluster found", "clusterUUID", clusterUUID) + return "", false + } + + vms := cr.Spec.VolumeMigrationSettings + if vms == nil || vms.AutoRebalancing == nil { + log.V(1).Info("Skipping: auto-rebalancing not configured", "cluster", cr.Name) + return "", false + } + spec := vms.AutoRebalancing + if spec.Enabled != nil && !*spec.Enabled { + log.V(1).Info("Skipping: auto-rebalancing disabled", "cluster", cr.Name) + return "", false + } + + cfg, err := autobalancing.ResolveRebalancingConfig(spec) + if err != nil { + log.V(1).Info("Skipping: invalid rebalancing configuration", "cluster", cr.Name, "error", err.Error()) + return "", false + } + + nodes, err := h.APIClient.GetStorageNodes(ctx, clusterUUID) + if err != nil { + log.Error(err, "Cannot list storage nodes; skipping smart placement", "cluster", cr.Name) + return "", false + } + + eligible := make(map[string]bool, len(nodes)) + storageNodes := make([]volumemigration.StorageNode, 0, len(nodes)) + for _, n := range nodes { + storageNodes = append(storageNodes, volumemigration.StorageNode{UUID: n.UUID, ClusterUUID: clusterUUID}) + eligible[n.UUID] = n.Status == "online" && n.Healthy && !n.IsSecondary && n.Lvols < n.LvolsMax + } + + nodeUUID, ok, err = h.NodeSelector.SelectBestNode(ctx, cfg, eligible, autobalancing.StorageNodeSelectorInput{ + Namespace: cr.Namespace, + StorageNodes: storageNodes, + }) + if err != nil { + log.Error(err, "Cannot select best node; skipping smart placement", "cluster", cr.Name) + return "", false + } + if !ok { + log.V(1).Info("Skipping: no eligible node found", "cluster", cr.Name) + return "", false + } + return nodeUUID, true +} diff --git a/operator/internal/webhook/simplyblock_volume_placement_injector_test.go b/operator/internal/webhook/simplyblock_volume_placement_injector_test.go new file mode 100644 index 00000000..0a9eb341 --- /dev/null +++ b/operator/internal/webhook/simplyblock_volume_placement_injector_test.go @@ -0,0 +1,377 @@ +package webhook + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + jsonpatchapply "github.com/evanphx/json-patch/v5" + jsonpatch "gomodules.xyz/jsonpatch/v2" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + simplyblockv1alpha1 "github.com/simplyblock/simplyblock-operator/api/v1alpha1" + "github.com/simplyblock/simplyblock-operator/internal/utils" + "github.com/simplyblock/simplyblock-operator/internal/volumemigration/autobalancing" + "github.com/simplyblock/simplyblock-operator/internal/webapi" +) + +const placementStorageClassName = "sb-sc" + +func pvcAdmissionRequest(t *testing.T, pvc *corev1.PersistentVolumeClaim) admission.Request { + t.Helper() + raw, err := json.Marshal(pvc) + if err != nil { + t.Fatalf("marshal pvc: %v", err) + } + return admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: runtime.RawExtension{Raw: raw}, + }, + } +} + +func makePlacementPVC(storageClassName *string, annotations map[string]string) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc1", Namespace: "default", Annotations: annotations}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: storageClassName}, + } +} + +func makePlacementStorageClass(provisioner string, params map[string]string) *storagev1.StorageClass { + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: placementStorageClassName}, + Provisioner: provisioner, + Parameters: params, + } +} + +func makePlacementCluster(autoRebalancing *simplyblockv1alpha1.VolumeRebalancingSettings) *simplyblockv1alpha1.StorageCluster { + var vms *simplyblockv1alpha1.VolumeMigrationSettings + if autoRebalancing != nil { + vms = &simplyblockv1alpha1.VolumeMigrationSettings{AutoRebalancing: autoRebalancing} + } + return &simplyblockv1alpha1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster1", Namespace: "default"}, + Spec: simplyblockv1alpha1.StorageClusterSpec{VolumeMigrationSettings: vms}, + } +} + +// applyPVCPatches applies the RFC6902 patch set produced by Handle to the original PVC +// via a real JSON-patch library, mirroring what the k8s apiserver does — avoids having to +// guess the exact path granularity the diff library chose for the annotations map. +func applyPVCPatches(t *testing.T, pvc *corev1.PersistentVolumeClaim, patches []jsonpatch.JsonPatchOperation) *corev1.PersistentVolumeClaim { + t.Helper() + original, err := json.Marshal(pvc) + if err != nil { + t.Fatalf("marshal original pvc: %v", err) + } + patchBytes, err := json.Marshal(patches) + if err != nil { + t.Fatalf("marshal patches: %v", err) + } + decoded, err := jsonpatchapply.DecodePatch(patchBytes) + if err != nil { + t.Fatalf("decode patch: %v", err) + } + resultRaw, err := decoded.Apply(original) + if err != nil { + t.Fatalf("apply patch: %v", err) + } + var result corev1.PersistentVolumeClaim + if err := json.Unmarshal(resultRaw, &result); err != nil { + t.Fatalf("unmarshal patched pvc: %v", err) + } + return &result +} + +// fakeWebapiServer serves a canned []webapi.StorageNodeInfo for any GET request, +// mirroring GetStorageNodes' expected response shape. +func fakeWebapiServer(t *testing.T, nodes []webapi.StorageNodeInfo) *httptest.Server { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(nodes); err != nil { + t.Errorf("encode fake storage nodes response: %v", err) + } + })) + t.Cleanup(ts.Close) + return ts +} + +type promSample struct { + ClusterUUID string + NodeUUID string + ValueNS int64 +} + +// fakePrometheusServer serves a canned instant-query vector result for any GET request to +// /api/v1/query, in the exact envelope prometheus/client_golang's API client expects: +// {"status":"success","data":{"resultType":"vector","result":[...]}}. +func fakePrometheusServer(t *testing.T, samples []promSample) *httptest.Server { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + type metric struct { + Cluster string `json:"cluster"` + Node string `json:"node"` + } + type result struct { + Metric metric `json:"metric"` + // Value is [, ""] per the + // Prometheus HTTP API instant-query format — the timestamp is a bare + // JSON number, only the sample value itself is a quoted string. + Value [2]any `json:"value"` + } + results := make([]result, 0, len(samples)) + for _, s := range samples { + results = append(results, result{ + Metric: metric{Cluster: s.ClusterUUID, Node: s.NodeUUID}, + Value: [2]any{1700000000, strconv.FormatInt(s.ValueNS, 10)}, + }) + } + resp := map[string]any{ + "status": "success", + "data": map[string]any{ + "resultType": "vector", + "result": results, + }, + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resp); err != nil { + t.Errorf("encode fake prometheus response: %v", err) + } + })) + t.Cleanup(ts.Close) + return ts +} + +// ── Handle: skip conditions (all resolvable without any network call) ─────────── + +func TestSimplyblockVolumePlacementInjector_Handle_SkipConditions(t *testing.T) { + enabledRebalancing := &simplyblockv1alpha1.VolumeRebalancingSettings{ + Enabled: boolRef(true), + PrometheusURL: strRef("http://unused:9090"), + } + + cases := []struct { + name string + pvc *corev1.PersistentVolumeClaim + sc *storagev1.StorageClass + cluster *simplyblockv1alpha1.StorageCluster + }{ + { + name: "host-id annotation already set — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), + map[string]string{annotationHostID: "some-node"}), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "deprecated host-id annotation already set — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), + map[string]string{deprecatedAnnotationHostID: "some-node"}), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "no StorageClassName set — skipped", + pvc: makePlacementPVC(nil, nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "StorageClass not found — skipped", + pvc: makePlacementPVC(strRef("does-not-exist"), nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "StorageClass not simplyblock-provisioned — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), nil), + sc: makePlacementStorageClass("some-other-provisioner", map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "StorageClass missing cluster_id param — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"pool_name": "pool1"}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "no matching StorageCluster — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": "00000000-0000-0000-0000-000000000000"}), + cluster: makePlacementCluster(enabledRebalancing), + }, + { + name: "VolumeMigrationSettings not configured — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(nil), + }, + { + name: "AutoRebalancing disabled — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(&simplyblockv1alpha1.VolumeRebalancingSettings{Enabled: boolRef(false)}), + }, + { + name: "invalid rebalancing config (missing prometheusURL) — skipped", + pvc: makePlacementPVC(strRef(placementStorageClassName), nil), + sc: makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}), + cluster: makePlacementCluster(&simplyblockv1alpha1.VolumeRebalancingSettings{Enabled: boolRef(true)}), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + scheme := newScheme(t) + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tc.sc, tc.cluster). + WithStatusSubresource(tc.cluster). + Build() + + tc.cluster.Status.UUID = testClusterUUID + if err := c.Status().Update(context.Background(), tc.cluster); err != nil { + t.Fatalf("set cluster status: %v", err) + } + + h := &SimplyblockVolumePlacementInjector{Client: c} + resp := h.Handle(context.Background(), pvcAdmissionRequest(t, tc.pvc)) + + if !resp.Allowed { + t.Errorf("Allowed = false, want true") + } + if len(resp.Patches) > 0 { + t.Errorf("expected no patch, got %v", resp.Patches) + } + }) + } +} + +// ── Handle: full selection path — eligibility filtering + coolest-node ranking ── + +func TestSimplyblockVolumePlacementInjector_Handle_SelectsCoolestEligibleNode(t *testing.T) { + const ( + namespace = "default" + baselineNS = int64(1_000_000) + ) + + promSrv := fakePrometheusServer(t, []promSample{ + {ClusterUUID: testClusterUUID, NodeUUID: "hot", ValueNS: 3_000_000}, // dev = 200% + {ClusterUUID: testClusterUUID, NodeUUID: "cool", ValueNS: 1_100_000}, // dev = 10% — winner + {ClusterUUID: testClusterUUID, NodeUUID: "offline", ValueNS: 500_000}, // excluded: offline + {ClusterUUID: testClusterUUID, NodeUUID: "secondary", ValueNS: 500_000}, // excluded: secondary + {ClusterUUID: testClusterUUID, NodeUUID: "atcapacity", ValueNS: 500_000}, // excluded: at capacity + }) + webSrv := fakeWebapiServer(t, []webapi.StorageNodeInfo{ + {UUID: "hot", Status: "online", Healthy: true, Lvols: 1, LvolsMax: 10}, + {UUID: "cool", Status: "online", Healthy: true, Lvols: 1, LvolsMax: 10}, + {UUID: "offline", Status: "offline", Healthy: true, Lvols: 1, LvolsMax: 10}, + {UUID: "secondary", Status: "online", Healthy: true, IsSecondary: true, Lvols: 1, LvolsMax: 10}, + {UUID: "atcapacity", Status: "online", Healthy: true, Lvols: 10, LvolsMax: 10}, + }) + + cluster := makePlacementCluster(&simplyblockv1alpha1.VolumeRebalancingSettings{ + Enabled: boolRef(true), + PrometheusURL: strRef(promSrv.URL), + }) + sc := makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}) + pvc := makePlacementPVC(strRef(placementStorageClassName), nil) + + baseline := func(uuid string) simplyblockv1alpha1.NodeLatencyMetrics { + return simplyblockv1alpha1.NodeLatencyMetrics{NodeUUID: uuid, BaselineP50NS: baselineNS} + } + nodeSet := &simplyblockv1alpha1.StorageNodeSet{ + ObjectMeta: metav1.ObjectMeta{Name: "nodeset1", Namespace: namespace}, + } + nodeSet.Status.LatencyMetrics = []simplyblockv1alpha1.NodeLatencyMetrics{ + baseline("hot"), baseline("cool"), baseline("offline"), baseline("secondary"), baseline("atcapacity"), + } + + scheme := newScheme(t) + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(sc, cluster, nodeSet). + WithStatusSubresource(cluster, nodeSet). + Build() + + cluster.Status.UUID = testClusterUUID + if err := c.Status().Update(context.Background(), cluster); err != nil { + t.Fatalf("set cluster status: %v", err) + } + if err := c.Status().Update(context.Background(), nodeSet); err != nil { + t.Fatalf("set nodeset status: %v", err) + } + + h := &SimplyblockVolumePlacementInjector{ + Client: c, + APIClient: webapi.NewClient(webSrv.URL), + NodeSelector: autobalancing.NewStorageNodeSelector(c), + } + + resp := h.Handle(context.Background(), pvcAdmissionRequest(t, pvc)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true") + } + if len(resp.Patches) == 0 { + t.Fatalf("expected a patch, got none") + } + + patched := applyPVCPatches(t, pvc, resp.Patches) + if got := patched.Annotations[annotationHostID]; got != "cool" { + t.Errorf("host-id = %q, want %q", got, "cool") + } +} + +// ── Handle: no eligible node — allowed unmodified, real network stack exercised ── + +func TestSimplyblockVolumePlacementInjector_Handle_NoEligibleNode(t *testing.T) { + promSrv := fakePrometheusServer(t, nil) + webSrv := fakeWebapiServer(t, []webapi.StorageNodeInfo{ + {UUID: "offline", Status: "offline", Healthy: true, Lvols: 1, LvolsMax: 10}, + {UUID: "unhealthy", Status: "online", Healthy: false, Lvols: 1, LvolsMax: 10}, + }) + + cluster := makePlacementCluster(&simplyblockv1alpha1.VolumeRebalancingSettings{ + Enabled: boolRef(true), + PrometheusURL: strRef(promSrv.URL), + }) + sc := makePlacementStorageClass(utils.CSIProvisioner, map[string]string{"cluster_id": testClusterUUID}) + pvc := makePlacementPVC(strRef(placementStorageClassName), nil) + + scheme := newScheme(t) + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(sc, cluster). + WithStatusSubresource(cluster). + Build() + + cluster.Status.UUID = testClusterUUID + if err := c.Status().Update(context.Background(), cluster); err != nil { + t.Fatalf("set cluster status: %v", err) + } + + h := &SimplyblockVolumePlacementInjector{ + Client: c, + APIClient: webapi.NewClient(webSrv.URL), + NodeSelector: autobalancing.NewStorageNodeSelector(c), + } + + resp := h.Handle(context.Background(), pvcAdmissionRequest(t, pvc)) + if !resp.Allowed { + t.Fatalf("Allowed = false, want true") + } + if len(resp.Patches) > 0 { + t.Errorf("expected no patch, got %v", resp.Patches) + } +}