diff --git a/CHANGELOG.md b/CHANGELOG.md index edfc31d..1198e9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,24 @@ All notable changes to this project are documented here. The format follows stalls visibly (`UpdateBlocked`) rather than evicting anything. See `docs/updates.md`. +- **An observability pack**, off by default because it needs the Prometheus Operator + CRDs: a `ServiceMonitor`, a Grafana dashboard delivered as a sidecar-discovered + `ConfigMap`, and eight warning-biased alerts. The two capacity layers get their own + alerts — `GPUCellPoolNoFreePhysicalGPU` (no whole device left in the infrastructure + cluster, and the pool wants one) versus `GPUCellPoolSharedGPUExhausted` (healthy + cells whose GPU is entirely handed out) — because they are different problems with + different remedies. Enable with `--set monitoring.enabled=true`; see + `docs/observability.md`. +- A `Service` in front of the metrics port, and a `metrics` container port. There was + no way to scrape the endpoint at all before: nothing exposed it. + ### Fixed +- **The chart claimed the metrics endpoint had authn/authz.** It does not — it is + HTTPS with a self-signed certificate and no authorization. The claim is corrected + and the real posture (and how to restrict it) documented; adding the filter pulls + `k8s.io/apiserver` into a deliberately small dependency tree, so it is tracked + separately rather than done in passing. - **A replacement cell inherited its predecessor's teardown.** Cell names are reused (index 0 is always `-0`) and status rows are keyed by name, so a freshly created cell adopted the previous incarnation's `Draining` phase and was diff --git a/Makefile b/Makefile index 204447c..acfe849 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,15 @@ manifests: controller-gen ## Generate CRDs and RBAC into config/, and sync the c # kubebuilder markers and the chart's hand-written copy drifted independently — # and the chart silently lacked the Cluster API rules it needed. sed -n '/^rules:/,$$p' config/rbac/role.yaml | tail -n +2 > charts/gpucellpool/rules.yaml + $(MAKE) dashboards-sync + +.PHONY: dashboards-sync +dashboards-sync: ## Copy the Grafana dashboards into the chart. + # config/grafana is the source of truth; the chart needs its own copy because + # Helm can only package files inside the chart directory. `verify` diffs the + # two, so an edit to one and not the other fails CI instead of shipping a + # dashboard nobody sees. + cp config/grafana/*.json charts/gpucellpool/dashboards/ .PHONY: generate generate: controller-gen ## Generate DeepCopy methods. diff --git a/charts/gpucellpool/dashboards/gpucellpool-overview.json b/charts/gpucellpool/dashboards/gpucellpool-overview.json new file mode 100644 index 0000000..bccbefb --- /dev/null +++ b/charts/gpucellpool/dashboards/gpucellpool-overview.json @@ -0,0 +1,505 @@ +{ + "title": "GPUCellPool \u2014 overview", + "uid": "gpucellpool-overview", + "tags": [ + "gpucellpool", + "kubeswift", + "hami", + "gpu" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "description": "GPUCellPool composes KubeSwift VM isolation with HAMi GPU sharing. The two capacity rows are deliberately never merged: physical GPUs are whole PCI devices in the infrastructure cluster, shared capacity is memory and compute fractions inside the cells' own cluster. No HAMi fraction is ever translated into a physical allocation.", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "label": "Data source" + }, + { + "name": "namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "label": "Namespace", + "query": "label_values(gpucell_cells_desired, namespace)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "pool", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "label": "Pool", + "query": "label_values(gpucell_cells_desired{namespace=~\"$namespace\"}, pool)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "type": "row", + "id": 1, + "title": "Pool health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "timeseries", + "id": 2, + "title": "Cells by phase", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (phase) (gpucell_cells{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{phase}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Ready is the three-way AND: the VM running, the Node registered, AND the expected GPU advertised. A cell sitting in AwaitingGPUCapacity is the failure this operator exists to make visible \u2014 a booted VM whose GPU never surfaced." + }, + { + "type": "timeseries", + "id": 3, + "title": "Desired vs Ready", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(gpucell_cells_desired{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "desired" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(gpucell_cells{pool=~\"$pool\", namespace=~\"$namespace\", phase=\"Ready\"})", + "legendFormat": "ready" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Desired follows spec.replicas, or status.desiredReplicas when autoscaling is on." + }, + { + "type": "stat", + "id": 4, + "title": "Workload cluster reachable", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(gpucell_workload_cluster_reachable{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "reachable" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "0 freezes every destructive path: nothing is removed, replaced or scaled down while it is unknown whether a cell holds work. Capacity below is then the last known value, not a current one.", + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + } + }, + { + "type": "row", + "id": 5, + "title": "Physical GPUs \u2014 the infrastructure cluster (whole devices)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + } + }, + { + "type": "timeseries", + "id": 6, + "title": "Whole GPUs by state", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (gpucell_physical_gpus{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "held = passed through to this pool's cells. free = unallocated in the infrastructure cluster. This is the OUTER layer: whole PCI devices, nothing to do with the fractions HAMi hands out inside them." + }, + { + "type": "timeseries", + "id": 7, + "title": "Cell startup (p50 / p90)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(gpucell_cell_startup_seconds_bucket{pool=~\"$pool\", namespace=~\"$namespace\"}[6h])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.9, sum by (le) (rate(gpucell_cell_startup_seconds_bucket{pool=~\"$pool\", namespace=~\"$namespace\"}[6h])))", + "legendFormat": "p90" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "description": "Creation to first Ready. Measured at 4m45s from a baked image, about three minutes of which is cloning the root disk. This number decides whether autoscaling can be reactive at all." + }, + { + "type": "row", + "id": 8, + "title": "Shared GPU capacity \u2014 the workload cluster (HAMi fractions)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + } + }, + { + "type": "timeseries", + "id": 9, + "title": "GPU memory", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (gpucell_capacity_gpu_memory_bytes{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "description": "Always valid: bytes are comparable across GPU models. available == 0 with healthy cells means the shared GPU is full \u2014 a different problem from having no free physical device." + }, + { + "type": "timeseries", + "id": 10, + "title": "GPU compute (homogeneous pools only)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (gpucell_capacity_gpu_compute_percent{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "description": "Percent OF A DEVICE, so it is only published while every advertised GPU is the same model: 100 of a GTX 1080 and 100 of an H200 are not 200 of anything. A heterogeneous pool leaves this empty on purpose." + }, + { + "type": "stat", + "id": 11, + "title": "Devices advertised", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(gpucell_capacity_gpu_devices{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "devices" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "From HAMi's node registration annotation, NOT from nvidia.com/gpu allocatable \u2014 HAMi inflates that by deviceSplitCount (default 10), so allocatable is not a device count.", + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + } + }, + { + "type": "row", + "id": 12, + "title": "Decisions and reliability", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + } + }, + { + "type": "timeseries", + "id": 13, + "title": "Scaling decisions by reason", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (increase(gpucell_scale_decisions_total{pool=~\"$pool\", namespace=~\"$namespace\"}[1h]))", + "legendFormat": "{{reason}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Records REFUSALS as well as actions, which is the point: AtMaxReplicas, NoIdleCell, DemandUnsatisfiable and CellShapeUnknown each mean a different thing and call for a different response. See docs/autoscaling.md." + }, + { + "type": "timeseries", + "id": 14, + "title": "Cell phase transitions", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (from, to) (increase(gpucell_cell_transitions_total{pool=~\"$pool\", namespace=~\"$namespace\"}[1h]))", + "legendFormat": "{{from}} \u2192 {{to}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Churn is invisible in a phase gauge: an oscillating cell looks healthy at every instant." + }, + { + "type": "timeseries", + "id": 15, + "title": "Errors", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (rate(gpucell_capacity_scrape_errors_total{pool=~\"$pool\", namespace=~\"$namespace\"}[15m]))", + "legendFormat": "scrape: {{reason}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(gpucell_reconcile_errors_total{pool=~\"$pool\", namespace=~\"$namespace\"}[15m]))", + "legendFormat": "reconcile" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "A non-zero scrape rate means every capacity figure above is stale \u2014 a failed read retains the previous values rather than reporting zero." + } + ] +} diff --git a/charts/gpucellpool/templates/_helpers.tpl b/charts/gpucellpool/templates/_helpers.tpl index 69c9f64..041207b 100644 --- a/charts/gpucellpool/templates/_helpers.tpl +++ b/charts/gpucellpool/templates/_helpers.tpl @@ -51,3 +51,12 @@ tls.key: {{ $cert.Key | b64enc }} ca.crt: {{ $ca.Cert | b64enc }} {{- end -}} {{- end -}} + +{{/* +The metrics port, taken from metrics.bindAddress (":8443" -> "8443"). Derived +rather than configured twice, so the Service, the container port and the +ServiceMonitor cannot disagree with what the manager is actually listening on. +*/}} +{{- define "gpucellpool.metricsPort" -}} +{{- regexReplaceAll ".*:" (.Values.metrics.bindAddress | toString) "" -}} +{{- end -}} diff --git a/charts/gpucellpool/templates/deployment.yaml b/charts/gpucellpool/templates/deployment.yaml index 33723ef..c9a1355 100644 --- a/charts/gpucellpool/templates/deployment.yaml +++ b/charts/gpucellpool/templates/deployment.yaml @@ -35,6 +35,10 @@ spec: ports: - name: health containerPort: 8081 + {{- if ne (.Values.metrics.bindAddress | toString) "0" }} + - name: metrics + containerPort: {{ include "gpucellpool.metricsPort" . }} + {{- end }} {{- if .Values.webhook.enabled }} - name: webhook containerPort: {{ .Values.webhook.port }} diff --git a/charts/gpucellpool/templates/monitoring/dashboards.yaml b/charts/gpucellpool/templates/monitoring/dashboards.yaml new file mode 100644 index 0000000..580b983 --- /dev/null +++ b/charts/gpucellpool/templates/monitoring/dashboards.yaml @@ -0,0 +1,31 @@ +{{- $mon := .Values.monitoring | default dict }} +{{- $db := $mon.dashboards | default dict }} +{{- if and $mon.enabled (ternary $db.enabled true (hasKey $db "enabled")) }} +{{- $ns := $db.namespace | default .Release.Namespace }} +{{- $label := $db.label | default "grafana_dashboard" }} +{{- range $path, $_ := .Files.Glob "dashboards/*.json" }} +{{- $name := base $path | trimSuffix ".json" }} +--- +# A Grafana dashboard, discovered by the Grafana sidecar via the {{ $label }} +# label. The JSONs under dashboards/ are synced verbatim from config/grafana/ by +# `make dashboards-sync`, and drift fails `make verify` in CI — a hand-edited copy +# here would silently diverge from the source of truth. +# +# Set monitoring.dashboards.namespace to your Grafana namespace if its sidecar +# only watches its own (that is the default). +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + namespace: {{ $ns }} + labels: + {{ $label }}: "1" + {{- include "gpucellpool.labels" $ | nindent 4 }} + {{- with $db.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +data: + {{ base $path }}: | +{{ $.Files.Get $path | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/gpucellpool/templates/monitoring/prometheusrule.yaml b/charts/gpucellpool/templates/monitoring/prometheusrule.yaml new file mode 100644 index 0000000..8ee0283 --- /dev/null +++ b/charts/gpucellpool/templates/monitoring/prometheusrule.yaml @@ -0,0 +1,141 @@ +{{- $mon := .Values.monitoring | default dict }} +{{- $pr := $mon.prometheusRule | default dict }} +{{- if and $mon.enabled (ternary $pr.enabled true (hasKey $pr "enabled")) }} +# A starter alert pack. Warning-biased on purpose: almost nothing here means "wake +# someone", and several conditions are states this operator reports deliberately +# rather than faults (a saturated shared GPU, a pool holding no cells). +# +# The two layers are kept apart, which is the whole reason these metrics are shaped +# the way they are: "no GPU left in the infrastructure cluster" and "the shared GPU +# inside the cells is full" are different problems with different remedies, and an +# alert that merged them would be useless. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "gpucellpool.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gpucellpool.labels" . | nindent 4 }} + {{- with $pr.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: gpucellpool.transport + rules: + - alert: GPUCellPoolWorkloadClusterUnreachable + expr: gpucell_workload_cluster_reachable == 0 + for: 10m + labels: {severity: warning} + annotations: + summary: "Pool {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} cannot reach its workload cluster" + description: >- + Nothing below this is trustworthy while it lasts, and every destructive + path is frozen by design: no cell is removed, replaced or scaled down + when it cannot be known whether it holds work. Capacity figures are the + last known values, not current ones. Check the kubeconfig Secret and the + network path from the operator to the workload apiserver. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md + + - name: gpucellpool.cells + rules: + - alert: GPUCellPoolCellsNotReady + expr: (gpucell_cells_desired - sum by (pool, namespace) (gpucell_cells{phase="Ready"})) > 0 + for: 30m + labels: {severity: warning} + annotations: + summary: "Pool {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} is short of Ready cells" + description: >- + A cell takes about five minutes to become Ready, so thirty minutes short + means it is stuck rather than starting. status.cells[].phase names the + layer: AllocatingGPU is the infrastructure cluster, Joining is the + bootstrap, AwaitingGPUCapacity is HAMi not advertising the device. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md + + - alert: GPUCellPoolCellFailed + expr: gpucell_cells{phase="Failed"} > 0 + for: 15m + labels: {severity: warning} + annotations: + summary: "Pool {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} has a Failed cell" + description: >- + A failed cell is replaced behind a backoff, so this firing repeatedly + means replacement is not fixing it — usually a bad image, an expired + join credential, or a GPU that never surfaces in the guest. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md + + - alert: GPUCellPoolCellFlapping + expr: sum by (pool, namespace) (rate(gpucell_cell_transitions_total[30m])) > 0.01 + for: 30m + labels: {severity: warning} + annotations: + summary: "Cells in {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} are churning" + description: >- + Roughly a phase change every 100 seconds, sustained. An oscillating cell + looks healthy at any single instant, which is exactly why this is measured + as a rate rather than read off a gauge. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md + + - name: gpucellpool.physical + rules: + # The OUTER layer: whole devices in the infrastructure cluster. + - alert: GPUCellPoolNoFreePhysicalGPU + expr: | + gpucell_physical_gpus{state="free"} == 0 + and on (pool, namespace) + (gpucell_cells_desired - sum by (pool, namespace) (gpucell_cells{phase="Ready"})) > 0 + for: 20m + labels: {severity: warning} + annotations: + summary: "Pool {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} wants a cell and the cluster has no free GPU" + description: >- + The pool is not broken — the infrastructure cluster is full. It + deliberately does not queue guests that could never be scheduled. Free a + device, add a GPU node, or lower the pool's size. Note the `and on` here: + a saturated cluster whose pools are all satisfied is not worth an alert. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md + + - name: gpucellpool.workload + rules: + # The INNER layer: fractions of the GPUs inside the cells. Distinct from the + # alert above, and the reason both exist. + - alert: GPUCellPoolSharedGPUExhausted + expr: | + gpucell_capacity_gpu_memory_bytes{state="available"} == 0 + and on (pool, namespace) gpucell_capacity_gpu_devices > 0 + for: 30m + labels: {severity: warning} + annotations: + summary: "Every GPU in {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} is fully allocated" + description: >- + The cells are healthy; their GPU memory is entirely handed out to HAMi + workloads, so the next pod requesting GPU memory will not schedule. This + is the inner layer — free memory by removing workloads, or add a cell. + Enabling spec.autoscaling lets pending demand add one for you. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/autoscaling.md + + - alert: GPUCellPoolCapacityStale + expr: sum by (pool, namespace) (rate(gpucell_capacity_scrape_errors_total[15m])) > 0 + for: 15m + labels: {severity: warning} + annotations: + summary: "Capacity for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}} is stale" + description: >- + Reads of the workload cluster are failing, so every capacity figure is + the last known value rather than a current one — deliberately, because + reporting zero from a failed read would be the worst possible lie. Treat + the numbers on the dashboards as untrustworthy until this clears. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md + + - name: gpucellpool.operator + rules: + - alert: GPUCellPoolReconcileErrors + expr: sum by (pool, namespace) (rate(gpucell_reconcile_errors_total[15m])) > 0 + for: 15m + labels: {severity: warning} + annotations: + summary: "Reconcile errors for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pool }}`}}" + description: >- + The operator is erroring rather than reporting state. Check the manager + logs; a persistent error usually means RBAC in one of the two clusters. + runbook_url: https://github.com/kubeswift-io/gpucellpool/blob/main/docs/runbook.md +{{- end }} diff --git a/charts/gpucellpool/templates/monitoring/service.yaml b/charts/gpucellpool/templates/monitoring/service.yaml new file mode 100644 index 0000000..b20d31d --- /dev/null +++ b/charts/gpucellpool/templates/monitoring/service.yaml @@ -0,0 +1,20 @@ +{{- if ne (.Values.metrics.bindAddress | toString) "0" }} +# A Service in front of the metrics port. Separate from the webhook Service on +# purpose: a ServiceMonitor selects by Service, and mixing the two would scrape +# the webhook port on every pod that serves it. +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gpucellpool.fullname" . }}-metrics + namespace: {{ .Release.Namespace }} + labels: {{- include "gpucellpool.labels" . | nindent 4 }} + app.kubernetes.io/component: metrics +spec: + type: ClusterIP + selector: {{- include "gpucellpool.selectorLabels" . | nindent 4 }} + ports: + - name: metrics + port: {{ include "gpucellpool.metricsPort" . }} + targetPort: metrics + protocol: TCP +{{- end }} diff --git a/charts/gpucellpool/templates/monitoring/servicemonitor.yaml b/charts/gpucellpool/templates/monitoring/servicemonitor.yaml new file mode 100644 index 0000000..cce1218 --- /dev/null +++ b/charts/gpucellpool/templates/monitoring/servicemonitor.yaml @@ -0,0 +1,42 @@ +{{- /* +Nil-safe access: with `helm upgrade --reuse-values --set monitoring.enabled=true` +the monitoring subtree from values.yaml is NOT merged, so every nested key may be +absent. `ternary ... (hasKey ...)` keeps an explicit `enabled: false` working, +where sprig's `default` would swallow the false. +*/ -}} +{{- $mon := .Values.monitoring | default dict }} +{{- $sm := $mon.serviceMonitor | default dict }} +{{- if and $mon.enabled (ternary $sm.enabled true (hasKey $sm "enabled")) }} +# Scrapes the manager's /metrics endpoint (the gpucell_* series plus +# controller-runtime's reconcile and workqueue metrics). Needs the Prometheus +# Operator CRDs; the monitoring.enabled gate keeps a default install free of +# that dependency. +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "gpucellpool.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gpucellpool.labels" . | nindent 4 }} + {{- with $sm.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: {{- include "gpucellpool.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: metrics + endpoints: + - port: metrics + path: /metrics + interval: {{ $sm.interval | default "30s" }} + {{- if .Values.metrics.secure }} + # The manager serves HTTPS with a self-signed certificate, so verification + # is skipped. It does not authenticate scrapers — see issue #11 and the + # metrics.secure comment in values.yaml. + scheme: https + tlsConfig: + insecureSkipVerify: true + {{- end }} +{{- end }} diff --git a/charts/gpucellpool/values.yaml b/charts/gpucellpool/values.yaml index ead907c..5c14a67 100644 --- a/charts/gpucellpool/values.yaml +++ b/charts/gpucellpool/values.yaml @@ -33,10 +33,42 @@ webhook: issuerRef: {} # e.g. {name: my-issuer, kind: ClusterIssuer, group: cert-manager.io} metrics: - # 0 disables the metrics endpoint. Serving is HTTPS with authn/authz. + # 0 disables the metrics endpoint. bindAddress: ":8443" + # secure serves over HTTPS with a self-signed certificate. It does NOT + # authenticate or authorize scrapers: anything able to reach the pod on this + # port can read the series, which carry pool and namespace names and capacity + # figures — no credentials, no guest data. Restrict it with a NetworkPolicy if + # that matters to you, or set bindAddress to "0" to serve nothing. secure: true +# Prometheus Operator integration and Grafana dashboards. Off by default: on it +# requires the monitoring.coreos.com CRDs, and a default install should not depend +# on them. +monitoring: + enabled: false + # ServiceMonitor scraping the manager's /metrics endpoint. + serviceMonitor: + enabled: true + interval: 30s + # Extra labels for Prometheus instances that select ServiceMonitors by label + # (e.g. release: kube-prometheus-stack). + additionalLabels: {} + # Grafana dashboards as sidecar-discovered ConfigMaps. + dashboards: + enabled: true + # The sidecar discovery label (kube-prometheus-stack's default shown). + label: grafana_dashboard + # Namespace for the ConfigMaps. Empty = the release namespace; set it to your + # Grafana namespace if the sidecar only watches its own (its default). + namespace: "" + # Extra annotations, e.g. grafana_folder: KubeSwift. + annotations: {} + # Starter alert pack — warning-biased; see docs/observability.md. + prometheusRule: + enabled: true + additionalLabels: {} + # Install the CRDs. Note that `helm upgrade` never updates files in crds/, so a # CRD change needs `kubectl apply -f charts/gpucellpool/crds/` — the same caveat # KubeSwift has. diff --git a/config/grafana/gpucellpool-overview.json b/config/grafana/gpucellpool-overview.json new file mode 100644 index 0000000..bccbefb --- /dev/null +++ b/config/grafana/gpucellpool-overview.json @@ -0,0 +1,505 @@ +{ + "title": "GPUCellPool \u2014 overview", + "uid": "gpucellpool-overview", + "tags": [ + "gpucellpool", + "kubeswift", + "hami", + "gpu" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "description": "GPUCellPool composes KubeSwift VM isolation with HAMi GPU sharing. The two capacity rows are deliberately never merged: physical GPUs are whole PCI devices in the infrastructure cluster, shared capacity is memory and compute fractions inside the cells' own cluster. No HAMi fraction is ever translated into a physical allocation.", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "label": "Data source" + }, + { + "name": "namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "label": "Namespace", + "query": "label_values(gpucell_cells_desired, namespace)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "pool", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "label": "Pool", + "query": "label_values(gpucell_cells_desired{namespace=~\"$namespace\"}, pool)", + "refresh": 1, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "type": "row", + "id": 1, + "title": "Pool health", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "timeseries", + "id": 2, + "title": "Cells by phase", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (phase) (gpucell_cells{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{phase}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Ready is the three-way AND: the VM running, the Node registered, AND the expected GPU advertised. A cell sitting in AwaitingGPUCapacity is the failure this operator exists to make visible \u2014 a booted VM whose GPU never surfaced." + }, + { + "type": "timeseries", + "id": 3, + "title": "Desired vs Ready", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(gpucell_cells_desired{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "desired" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(gpucell_cells{pool=~\"$pool\", namespace=~\"$namespace\", phase=\"Ready\"})", + "legendFormat": "ready" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Desired follows spec.replicas, or status.desiredReplicas when autoscaling is on." + }, + { + "type": "stat", + "id": 4, + "title": "Workload cluster reachable", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "min(gpucell_workload_cluster_reachable{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "reachable" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "0 freezes every destructive path: nothing is removed, replaced or scaled down while it is unknown whether a cell holds work. Capacity below is then the last known value, not a current one.", + "options": { + "colorMode": "background", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + } + }, + { + "type": "row", + "id": 5, + "title": "Physical GPUs \u2014 the infrastructure cluster (whole devices)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + } + }, + { + "type": "timeseries", + "id": 6, + "title": "Whole GPUs by state", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (gpucell_physical_gpus{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "held = passed through to this pool's cells. free = unallocated in the infrastructure cluster. This is the OUTER layer: whole PCI devices, nothing to do with the fractions HAMi hands out inside them." + }, + { + "type": "timeseries", + "id": 7, + "title": "Cell startup (p50 / p90)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(gpucell_cell_startup_seconds_bucket{pool=~\"$pool\", namespace=~\"$namespace\"}[6h])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.9, sum by (le) (rate(gpucell_cell_startup_seconds_bucket{pool=~\"$pool\", namespace=~\"$namespace\"}[6h])))", + "legendFormat": "p90" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "description": "Creation to first Ready. Measured at 4m45s from a baked image, about three minutes of which is cloning the root disk. This number decides whether autoscaling can be reactive at all." + }, + { + "type": "row", + "id": 8, + "title": "Shared GPU capacity \u2014 the workload cluster (HAMi fractions)", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + } + }, + { + "type": "timeseries", + "id": 9, + "title": "GPU memory", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (gpucell_capacity_gpu_memory_bytes{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "description": "Always valid: bytes are comparable across GPU models. available == 0 with healthy cells means the shared GPU is full \u2014 a different problem from having no free physical device." + }, + { + "type": "timeseries", + "id": 10, + "title": "GPU compute (homogeneous pools only)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (gpucell_capacity_gpu_compute_percent{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "description": "Percent OF A DEVICE, so it is only published while every advertised GPU is the same model: 100 of a GTX 1080 and 100 of an H200 are not 200 of anything. A heterogeneous pool leaves this empty on purpose." + }, + { + "type": "stat", + "id": 11, + "title": "Devices advertised", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(gpucell_capacity_gpu_devices{pool=~\"$pool\", namespace=~\"$namespace\"})", + "legendFormat": "devices" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "From HAMi's node registration annotation, NOT from nvidia.com/gpu allocatable \u2014 HAMi inflates that by deviceSplitCount (default 10), so allocatable is not a device count.", + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + } + }, + { + "type": "row", + "id": 12, + "title": "Decisions and reliability", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + } + }, + { + "type": "timeseries", + "id": 13, + "title": "Scaling decisions by reason", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (increase(gpucell_scale_decisions_total{pool=~\"$pool\", namespace=~\"$namespace\"}[1h]))", + "legendFormat": "{{reason}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Records REFUSALS as well as actions, which is the point: AtMaxReplicas, NoIdleCell, DemandUnsatisfiable and CellShapeUnknown each mean a different thing and call for a different response. See docs/autoscaling.md." + }, + { + "type": "timeseries", + "id": 14, + "title": "Cell phase transitions", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (from, to) (increase(gpucell_cell_transitions_total{pool=~\"$pool\", namespace=~\"$namespace\"}[1h]))", + "legendFormat": "{{from}} \u2192 {{to}}" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "Churn is invisible in a phase gauge: an oscillating cell looks healthy at every instant." + }, + { + "type": "timeseries", + "id": 15, + "title": "Errors", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 25 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (reason) (rate(gpucell_capacity_scrape_errors_total{pool=~\"$pool\", namespace=~\"$namespace\"}[15m]))", + "legendFormat": "scrape: {{reason}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(rate(gpucell_reconcile_errors_total{pool=~\"$pool\", namespace=~\"$namespace\"}[15m]))", + "legendFormat": "reconcile" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "description": "A non-zero scrape rate means every capacity figure above is stale \u2014 a failed read retains the previous values rather than reporting zero." + } + ] +} diff --git a/docs/README.md b/docs/README.md index 268e883..9d102bb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,7 @@ Start here if you are installing or operating a pool. | [cell-image](cell-image.md) | building and publishing a cell image with `hack/build-cell-image.sh` | | [clusterapi-cells](clusterapi-cells.md) | `provisioner: ClusterAPI` — cells as Cluster API Machines | | [limitations](limitations.md) | what is not implemented, not validated on hardware, or deliberately manual | +| [observability](observability.md) | the twelve `gpucell_*` metrics, the dashboard, and the alert pack | | [runbook](runbook.md) | what to check when a pool is not doing what you expect | `config/samples/` has ready-to-apply manifests: a `SwiftGuest`-provisioned pool, diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..44bf6fa --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,119 @@ +# Observability + +The operator exports twelve `gpucell_*` series, a Grafana dashboard and a starter +alert pack. All of it is off by default: turning it on requires the Prometheus +Operator CRDs, and a default install should not depend on them. + +```bash +helm upgrade --install gpucellpool oci://ghcr.io/kubeswift-io/charts/gpucellpool \ + --namespace gpucellpool-system --create-namespace \ + --set monitoring.enabled=true +``` + +That renders a `ServiceMonitor`, a `PrometheusRule` and the dashboard as a +sidecar-discovered `ConfigMap`. Each piece can be disabled on its own +(`monitoring.serviceMonitor.enabled`, `.prometheusRule.enabled`, +`.dashboards.enabled`). + +Two settings usually need adjusting for a given cluster: + +| Value | When | +|---|---| +| `monitoring.serviceMonitor.additionalLabels` | your Prometheus selects ServiceMonitors by label — kube-prometheus-stack wants `release: ` | +| `monitoring.dashboards.namespace` | your Grafana sidecar only watches its own namespace, which is its default. Point this at the Grafana namespace | + +## The metrics + +| Metric | Type | Labels | What it is for | +|---|---|---|---| +| `gpucell_cells_desired` | gauge | `pool`, `namespace` | the count the pool is aiming for — `spec.replicas`, or `status.desiredReplicas` when autoscaling is on | +| `gpucell_cells` | gauge | + `phase` | cells in each phase | +| `gpucell_cell_startup_seconds` | histogram | `pool`, `namespace` | creation to *first* Ready; a later Ready after a regression is not a startup | +| `gpucell_cell_transitions_total` | counter | + `from`, `to` | churn — an oscillating cell looks healthy in any instantaneous phase | +| `gpucell_physical_gpus` | gauge | + `state` (`held`, `free`) | **outer layer**: whole PCI devices | +| `gpucell_capacity_gpu_devices` | gauge | `pool`, `namespace` | **inner layer**: devices the capacity provider advertises | +| `gpucell_capacity_gpu_memory_bytes` | gauge | + `state` (`total`, `allocated`, `available`) | inner layer: GPU memory | +| `gpucell_capacity_gpu_compute_percent` | gauge | + `state` | inner layer: compute, homogeneous pools only | +| `gpucell_workload_cluster_reachable` | gauge | `pool`, `namespace` | 1 when the last reconcile reached the workload cluster | +| `gpucell_capacity_scrape_errors_total` | counter | + `reason` | capacity figures are stale | +| `gpucell_scale_decisions_total` | counter | + `reason`, `scaled_up` | every scaling decision, **including refusals** | +| `gpucell_reconcile_errors_total` | counter | `pool`, `namespace` | the operator is erroring rather than reporting | + +### Why the two capacity families are separate + +This is the one thing to understand before writing your own queries. +`gpucell_physical_gpus` counts **whole PCI devices in the infrastructure +cluster**. `gpucell_capacity_gpu_*` counts **memory and compute fractions inside +the cells' own cluster**, as HAMi accounts for them. They are never merged, +because "no GPU left in the cluster" and "the shared GPU inside the cells is +full" are different problems with different remedies: + +```promql +# the infrastructure cluster is out of devices, and this pool wants one +gpucell_physical_gpus{state="free"} == 0 + and on (pool, namespace) (gpucell_cells_desired - sum by (pool, namespace) (gpucell_cells{phase="Ready"})) > 0 + +# the cells are healthy and their GPU is fully handed out +gpucell_capacity_gpu_memory_bytes{state="available"} == 0 + and on (pool, namespace) gpucell_capacity_gpu_devices > 0 +``` + +Two traps worth naming: + +- **Device counts never come from `nvidia.com/gpu` allocatable.** HAMi inflates + it by `deviceSplitCount` (default 10), so a one-GPU node advertises 10. Use + `gpucell_capacity_gpu_devices`, which is parsed from HAMi's node registration. +- **Compute is a percentage *of a device*,** so the pool-wide aggregate is only + published while the pool is homogeneous. 100 of a GTX 1080 and 100 of an H200 + are not 200 of anything. + +### Stale, not zero + +A failed read of the workload cluster **retains** the previous capacity values +rather than reporting zero — reporting "0 GPUs, plenty free" from a failed scrape +would be the worst possible lie. `gpucell_capacity_scrape_errors_total` is +therefore not a nice-to-have: a rising rate means every capacity number on the +dashboard is untrustworthy, and `GPUCellPoolCapacityStale` alerts on exactly +that. + +## The alerts + +Eight rules, all `warning`. Nothing here means "wake someone": several of these +conditions are states the operator reports deliberately rather than faults. + +| Alert | Fires when | Layer | +|---|---|---| +| `GPUCellPoolWorkloadClusterUnreachable` | unreachable for 10m — every destructive path is frozen meanwhile | transport | +| `GPUCellPoolCellsNotReady` | short of Ready cells for 30m (a cell takes ~5 min, so this is stuck, not starting) | cells | +| `GPUCellPoolCellFailed` | a Failed cell persists for 15m — replacement is not fixing it | cells | +| `GPUCellPoolCellFlapping` | sustained phase churn | cells | +| `GPUCellPoolNoFreePhysicalGPU` | the pool wants a cell **and** no device is free | outer | +| `GPUCellPoolSharedGPUExhausted` | healthy cells whose GPU memory is entirely allocated | inner | +| `GPUCellPoolCapacityStale` | capacity reads are failing | inner | +| `GPUCellPoolReconcileErrors` | the operator is erroring | operator | + +`GPUCellPoolNoFreePhysicalGPU` is deliberately conjoined with "the pool actually +wants a cell". A saturated cluster whose pools are all satisfied is not a fault, +and alerting on it would train people to ignore the rule. + +## The dashboard + +`config/grafana/gpucellpool-overview.json` is the source of truth; the copy under +`charts/gpucellpool/dashboards/` is synced by `make manifests` (Helm can only +package files inside the chart). Drift between them fails `make verify`, so a +hand-edit to one is caught rather than shipping a dashboard nobody sees. + +Four rows, in the order you would actually read them: pool health, the physical +layer, the shared-capacity layer, then decisions and reliability. It is templated +by namespace and pool, so one dashboard covers a fleet. + +## A note on the metrics endpoint + +`metrics.secure: true` serves HTTPS with a self-signed certificate — which is why +the ServiceMonitor sets `insecureSkipVerify`. It does **not** authenticate +scrapers: anything able to reach the pod on that port can read the series. They +carry pool and namespace names, cell phases and capacity figures — no credentials +and no guest data — which is why this is tracked as an enhancement +([#11](https://github.com/kubeswift-io/gpucellpool/issues/11)) rather than a +defect. Restrict the port with a NetworkPolicy if it matters to you, or set +`metrics.bindAddress: "0"` to serve nothing at all.