From 8af330c5e9afd54267c1e8e60b893f1ff553dd99 Mon Sep 17 00:00:00 2001 From: Jagat Thakkar Date: Wed, 5 Aug 2026 11:44:33 -0500 Subject: [PATCH 1/5] feat: Add WeightsAndBiases readiness and infra-state metrics Expose per-CR readiness and per-dependency state as gauges on the operator metrics endpoint, mirroring the existing wandb_application_info pattern. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/metrics/metrics.go | 63 ++++++++++++++++++++++++++++- internal/metrics/metrics_test.go | 69 ++++++++++++++++++++++++++++---- 2 files changed, 124 insertions(+), 8 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index b7730a8b..d22747b4 100755 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -18,8 +18,69 @@ var ApplicationInfo = prometheus.NewGaugeVec( []string{"application_name", "namespace", "image", "tag", "digest"}, ) +// WeightsAndBiasesReady is 1 when a WeightsAndBiases CR reports Ready, else 0. +var WeightsAndBiasesReady = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "wandb_weightsandbiases_ready", + Help: "Whether a WeightsAndBiases custom resource is Ready (1) or not (0).", + }, + []string{"namespace", "name"}, +) + +// InfraState carries the current state of each backing dependency in the `state` +// label; value is always 1 for the active state. +var InfraState = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "wandb_infra_state", + Help: "Current state of each backing dependency of a WeightsAndBiases custom resource.", + }, + []string{"namespace", "name", "component", "instance_name", "state"}, +) + func init() { - metrics.Registry.MustRegister(ApplicationInfo) + metrics.Registry.MustRegister(ApplicationInfo, WeightsAndBiasesReady, InfraState) +} + +func SetWeightsAndBiasesReady(namespace, name string, ready bool) { + value := 0.0 + if ready { + value = 1 + } + WeightsAndBiasesReady.With(prometheus.Labels{ + "namespace": namespace, + "name": name, + }).Set(value) +} + +// SetInfraState records a dependency's current state, clearing its prior state +// series first so a transition doesn't leave the old state active. +func SetInfraState(namespace, name, component, instanceName, state string) { + InfraState.DeletePartialMatch(prometheus.Labels{ + "namespace": namespace, + "name": name, + "component": component, + "instance_name": instanceName, + }) + InfraState.With(prometheus.Labels{ + "namespace": namespace, + "name": name, + "component": component, + "instance_name": instanceName, + "state": state, + }).Set(1) +} + +// DeleteWeightsAndBiasesMetrics clears the readiness and infra-state series for +// a CR being torn down, so a deleted resource doesn't linger as Ready forever. +func DeleteWeightsAndBiasesMetrics(namespace, name string) { + WeightsAndBiasesReady.DeletePartialMatch(prometheus.Labels{ + "namespace": namespace, + "name": name, + }) + InfraState.DeletePartialMatch(prometheus.Labels{ + "namespace": namespace, + "name": name, + }) } // SetApplicationInfo records the running image for a single Application. diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index ac12f3a5..0c7108f7 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -9,10 +9,23 @@ import ( ) func gatherApplicationInfo(t *testing.T) []*dto.Metric { + t.Helper() + return gather(t, ApplicationInfo) +} + +func labelMap(m *dto.Metric) map[string]string { + out := make(map[string]string, len(m.Label)) + for _, lp := range m.Label { + out[lp.GetName()] = lp.GetValue() + } + return out +} + +func gather(t *testing.T, c prometheus.Collector) []*dto.Metric { t.Helper() out := make(chan prometheus.Metric, 64) go func() { - ApplicationInfo.Collect(out) + c.Collect(out) close(out) }() var metrics []*dto.Metric @@ -24,12 +37,54 @@ func gatherApplicationInfo(t *testing.T) []*dto.Metric { return metrics } -func labelMap(m *dto.Metric) map[string]string { - out := make(map[string]string, len(m.Label)) - for _, lp := range m.Label { - out[lp.GetName()] = lp.GetValue() - } - return out +func TestSetWeightsAndBiasesReady_FlipsValue(t *testing.T) { + t.Cleanup(WeightsAndBiasesReady.Reset) + WeightsAndBiasesReady.Reset() + + SetWeightsAndBiasesReady("wandb", "prod", true) + got := gather(t, WeightsAndBiasesReady) + assert.Len(t, got, 1) + assert.Equal(t, 1.0, got[0].Gauge.GetValue()) + + SetWeightsAndBiasesReady("wandb", "prod", false) + got = gather(t, WeightsAndBiasesReady) + assert.Len(t, got, 1) + assert.Equal(t, 0.0, got[0].Gauge.GetValue()) +} + +func TestSetInfraState_ReplacesPriorState(t *testing.T) { + t.Cleanup(InfraState.Reset) + InfraState.Reset() + + SetInfraState("wandb", "prod", "mysql", "default", "Pending") + SetInfraState("wandb", "prod", "mysql", "default", "Healthy") + + got := gather(t, InfraState) + assert.Len(t, got, 1, "a state transition must not leave the previous state as an active series") + got0 := labelMap(got[0]) + assert.Equal(t, "Healthy", got0["state"]) + assert.Equal(t, "default", got0["instance_name"]) +} + +func TestDeleteWeightsAndBiasesMetrics_ScopesToCR(t *testing.T) { + t.Cleanup(func() { WeightsAndBiasesReady.Reset(); InfraState.Reset() }) + WeightsAndBiasesReady.Reset() + InfraState.Reset() + + SetWeightsAndBiasesReady("wandb", "gone", true) + SetInfraState("wandb", "gone", "mysql", "default", "Healthy") + SetWeightsAndBiasesReady("wandb", "stays", true) + SetInfraState("wandb", "stays", "redis", "default", "Healthy") + + DeleteWeightsAndBiasesMetrics("wandb", "gone") + + ready := gather(t, WeightsAndBiasesReady) + assert.Len(t, ready, 1) + assert.Equal(t, "stays", labelMap(ready[0])["name"]) + + infra := gather(t, InfraState) + assert.Len(t, infra, 1) + assert.Equal(t, "stays", labelMap(infra[0])["name"]) } func TestSetApplicationInfo_EmitsExpectedLabels(t *testing.T) { From db33b34fb32ef967672da5c3c11a65306fdedfc4 Mon Sep 17 00:00:00 2001 From: Jagat Thakkar Date: Wed, 5 Aug 2026 11:44:33 -0500 Subject: [PATCH 2/5] feat: Emit CR readiness and infra-state metrics during reconcile Set wandb_weightsandbiases_ready wherever readiness is decided, and publish each dependency's state after infra status inference. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/controller/reconciler/readiness.go | 2 ++ .../controller/reconciler/reconcile_v2.go | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/internal/controller/reconciler/readiness.go b/internal/controller/reconciler/readiness.go index b7ef38d5..fc16e9fc 100644 --- a/internal/controller/reconciler/readiness.go +++ b/internal/controller/reconciler/readiness.go @@ -7,6 +7,7 @@ import ( "strings" apiv2 "github.com/wandb/operator/api/v2" + wmetrics "github.com/wandb/operator/internal/metrics" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,6 +24,7 @@ const ( func setReadyStatus(wandb *apiv2.WeightsAndBiases, ready bool, reason, message string) { wandb.Status.Ready = ready + wmetrics.SetWeightsAndBiasesReady(wandb.Namespace, wandb.Name, ready) status := metav1.ConditionFalse if ready { status = metav1.ConditionTrue diff --git a/internal/controller/reconciler/reconcile_v2.go b/internal/controller/reconciler/reconcile_v2.go index 57891d89..409a35d2 100644 --- a/internal/controller/reconciler/reconcile_v2.go +++ b/internal/controller/reconciler/reconcile_v2.go @@ -271,6 +271,7 @@ func Reconcile( } } + wmetrics.DeleteWeightsAndBiasesMetrics(wandb.Namespace, wandb.Name) controllerutil.RemoveFinalizer(wandb, CleanupFinalizer) if err := client.Update(ctx, wandb); err != nil { log.Error("Failed to remove finalizer '%s'", logx.ErrAttr(err)) @@ -348,6 +349,8 @@ func Reconcile( } ctrlResults = append(ctrlResults, res) + recordInfraStateMetrics(wandb) + if err = inferState(ctx, client, wandb); err != nil { errorCount++ } @@ -1585,6 +1588,27 @@ func clickHouseAllReady(wandb *apiv2.WeightsAndBiases) bool { return allInstancesReady(wandb.Spec.ClickHouse, wandb.Status.ClickHouseStatus, func(s apiv2.ClickHouseInfraStatus) bool { return s.Ready }) } +// recordInfraStateMetrics publishes the current state of each dependency as +// wandb_infra_state, keyed by component and status-map instance. +func recordInfraStateMetrics(wandb *apiv2.WeightsAndBiases) { + ns, name := wandb.Namespace, wandb.Name + for key, s := range wandb.Status.MySQLStatus { + wmetrics.SetInfraState(ns, name, "mysql", key, s.State) + } + for key, s := range wandb.Status.RedisStatus { + wmetrics.SetInfraState(ns, name, "redis", key, s.State) + } + for key, s := range wandb.Status.ObjectStoreStatus { + wmetrics.SetInfraState(ns, name, "objectstore", key, s.State) + } + for key, s := range wandb.Status.ClickHouseStatus { + wmetrics.SetInfraState(ns, name, "clickhouse", key, s.State) + } + if wandb.Spec.Kafka.ManagedKafka != nil { + wmetrics.SetInfraState(ns, name, "kafka", "", wandb.Status.KafkaStatus.State) + } +} + func inferState( ctx context.Context, client ctrlClient.Client, wandb *apiv2.WeightsAndBiases, ) error { From 2f23ba24e354b18894b55dcb8ea19b3dc2986d1e Mon Sep 17 00:00:00 2001 From: Jagat Thakkar Date: Wed, 5 Aug 2026 13:22:32 -0500 Subject: [PATCH 3/5] feat: Add W&B Operator Health dashboard A dedicated "is the operator itself healthy?" board with a description header and per-section notes explaining each panel in plain language. Covers reconcile health (errors, results, latency, workers, terminal errors, panics), workqueue backlog, Kubernetes API-client calls, admission/conversion webhooks, and Go process health, plus CR readiness, unhealthy-dependency count, per-dependency state, and image-version drift from the operator's wandb_* gauges. Queries match job=~"(.*/)?wandb-operator" so they follow the operator to any namespace and other controller-runtime operators don't bleed in. Current-state tiles use instant queries so transient startup states don't linger. Panels the operator's client-go build doesn't emit (rest-client and webhook latency histograms) are left out rather than shown empty. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../dashboards/wandb-operator-health.json | 1599 +++++++++++++++++ 1 file changed, 1599 insertions(+) create mode 100644 deploy/telemetry/dashboards/wandb-operator-health.json diff --git a/deploy/telemetry/dashboards/wandb-operator-health.json b/deploy/telemetry/dashboards/wandb-operator-health.json new file mode 100644 index 00000000..eec2d2c6 --- /dev/null +++ b/deploy/telemetry/dashboards/wandb-operator-health.json @@ -0,0 +1,1599 @@ +{ + "__inputs": [ + { + "name": "DS_VICTORIAMETRICS", + "label": "VictoriaMetrics", + "type": "datasource", + "pluginId": "victoriametrics-metrics-datasource", + "pluginName": "VictoriaMetrics" + } + ], + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "# W&B Operator Health\n\n**What this shows:** the health of the **W&B operator** \u2014 the controller that installs W&B and keeps it running by reconciling your `WeightsAndBiases` resource. It is *not* the W&B app or its databases. Open this board when an install or upgrade is stuck, a resource won't turn Ready, or the operator seems unresponsive.\n\n**Who it's for:** whoever runs the operator (SRE / platform).\n\n**How to read it:** green means healthy, red means it needs attention. Every section below opens with a note like this one that explains its panels and what a *bad* reading looks like. All data comes from the operator's own `/metrics` endpoint \u2014 the controller-runtime, Kubernetes-client and Go-runtime libraries expose most of it automatically, plus three gauges the operator publishes itself (`wandb_weightsandbiases_ready`, `wandb_infra_state`, `wandb_application_info`).\n\n**When something here looks wrong, drill into:** [Application](/d/wandb-application) for app request health \u00b7 [Managed Install](/d/wandb-managed-install-performance) for database / dependency resources \u00b7 [Telemetry Overview](/d/wandb-telemetry-overview) for the metrics pipeline itself.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "row", + "title": "W&B Resources", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Are the W&B resources healthy, and is every service on the same version?\n\n- **CR Ready** \u2014 one tile per `WeightsAndBiases` resource. Green **Ready** means the operator finished reconciling and everything it manages is up. Red **Not Ready** means it's still working or blocked \u2014 the reason shows up in the dependency panels to the right.\n- **Unhealthy dependencies** \u2014 how many backing services (MySQL, Redis, Kafka, object store, ClickHouse) are in any state other than *Healthy*. **0 (green) is what you want.** Any higher number is exactly *why* a resource above isn't Ready \u2014 check the table for which one.\n- **Dependency states** \u2014 every backing service and its current state (*Healthy / Degraded / Pending / Error / Unavailable*), so you can see *which* service is in *which* state.\n- **Distinct image versions** \u2014 how many different image tags the managed services are running. **1** means everything is on the same version; **2 or more** means **version drift** (usually a half-finished upgrade), and the *Managed service versions* table shows which service is behind.\n\n*These tiles show the current value only, so a service that was briefly Pending while starting up won't linger here.*\n" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + } + }, + { + "type": "stat", + "title": "CR Ready", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Ready" + }, + "1": { + "text": "Ready" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "wandb_weightsandbiases_ready", + "legendFormat": "{{namespace}}/{{name}}", + "refId": "A", + "instant": true + } + ], + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + } + }, + { + "type": "stat", + "title": "Unhealthy dependencies", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(wandb_infra_state{state!=\"Healthy\"} == 1) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Backing services in any state other than Healthy right now. 0 is good.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + } + }, + { + "type": "stat", + "title": "Distinct image versions", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(count by (tag) (wandb_application_info))", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Number of image tags in use across managed services. 1 = consistent; 2+ = version drift.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + } + }, + { + "type": "table", + "title": "Dependency states", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "targets": [ + { + "expr": "wandb_infra_state == 1", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "job": true, + "instance": true, + "pod": true, + "container": true, + "endpoint": true, + "service": true, + "prometheus": true + }, + "indexByName": { + "namespace": 0, + "name": 1, + "component": 2, + "instance_name": 3, + "state": 4 + }, + "renameByName": { + "namespace": "Namespace", + "name": "Name", + "component": "Component", + "instance_name": "Instance", + "state": "State" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "md" + }, + "description": "Current state of each backing dependency.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + } + }, + { + "type": "table", + "title": "Managed service versions", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "targets": [ + { + "expr": "wandb_application_info", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "job": true, + "instance": true, + "pod": true, + "container": true, + "endpoint": true, + "service": true, + "prometheus": true, + "namespace": true + }, + "indexByName": { + "application_name": 0, + "image": 1, + "tag": 2, + "digest": 3 + }, + "renameByName": { + "application_name": "Service", + "image": "Image", + "tag": "Tag", + "digest": "Digest" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "md" + }, + "description": "Image the operator wants each service to run.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + } + }, + { + "type": "row", + "title": "Reconciler", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Is the operator successfully doing its job?\n\nThe operator works in a loop called a **reconcile**: whenever your resource (or something it manages) changes, the operator runs this loop to make the cluster match what you asked for. These panels show whether that loop is healthy.\n\n- **Reconcile errors / sec** \u2014 how often a loop fails. **Should stay at zero.** A steady non-zero rate means the operator keeps trying and failing to finish setting something up (check the operator logs for the reason).\n- **Terminal reconcile errors** \u2014 failures the operator has **given up** retrying. Should be **0**; above zero means a resource is stuck and won't recover on its own.\n- **Reconcile panics** \u2014 times the operator's code crashed mid-loop (it recovers, but it shouldn't happen). Should be **0**; a non-zero value is a bug worth reporting.\n- **Reconciles / sec by result** \u2014 every loop ends as *success*, *error*, or *requeue* (asked to try again later). Lots of *requeue* usually means it's waiting on a dependency to come up.\n- **Reconcile errors by controller** \u2014 which specific controller is the one failing.\n- **Reconcile latency** \u2014 how long a loop takes (50th / 95th / 99th percentile). Rising latency means loops are getting slow.\n- **Active vs max workers** \u2014 how many loops run at once versus the configured limit. If *active* sits pinned on the *max* line, the operator is at capacity and work will start to queue.\n" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + } + }, + { + "type": "stat", + "title": "Reconcile errors / sec", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Reconcile loops ending in error, per second. Should be zero.", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + } + }, + { + "type": "stat", + "title": "Terminal reconcile errors", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(controller_runtime_terminal_reconcile_errors_total{job=~\"(.*/)?wandb-operator\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Total reconciles the operator gave up retrying since it started. Should be 0.", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + } + }, + { + "type": "stat", + "title": "Reconcile panics", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(controller_runtime_reconcile_panics_total{job=~\"(.*/)?wandb-operator\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Total times a reconcile crashed and was recovered since the operator started. Should be 0.", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + } + }, + { + "type": "timeseries", + "title": "Reconciles / sec by result", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (result) (rate(controller_runtime_reconcile_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 46 + } + }, + { + "type": "timeseries", + "title": "Reconcile errors / sec by controller", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (controller) (rate(controller_runtime_reconcile_errors_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{controller}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 46 + } + }, + { + "type": "timeseries", + "title": "Reconcile latency", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "description": "How long a reconcile loop takes.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 55 + } + }, + { + "type": "timeseries", + "title": "Active vs max workers", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (controller) (controller_runtime_active_workers{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "active {{controller}}", + "refId": "A" + }, + { + "expr": "sum(controller_runtime_max_concurrent_reconciles{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "max concurrent", + "refId": "B" + } + ], + "description": "Loops running concurrently versus the configured limit. Active pinned at max = saturated.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 55 + } + }, + { + "type": "row", + "title": "Workqueue", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Is the operator keeping up with its workload?\n\nWork the operator needs to do is placed on a **queue** (one per controller) and processed in order. These panels show whether the queue is draining or backing up.\n\n- **Queue depth** \u2014 items waiting to be processed. It should sit near zero and drain quickly. A depth that keeps **climbing** means the operator is falling behind.\n- **Adds & retries / sec** \u2014 how fast new work arrives versus how often failed work is retried. A spike in *retries* goes hand-in-hand with the reconcile errors above.\n- **Queue wait & work p95** \u2014 how long items **wait** in the queue versus how long they take to **process**. Both rising means the operator is overloaded.\n- **Oldest in-flight work** \u2014 the age of the oldest item still being worked on. A large, growing value means a single reconcile is stuck.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 65 + } + }, + { + "type": "timeseries", + "title": "Queue depth", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (name) (workqueue_depth{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "description": "Items waiting to be reconciled, per controller.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 71 + } + }, + { + "type": "timeseries", + "title": "Adds & retries / sec", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (name) (rate(workqueue_adds_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "adds {{name}}", + "refId": "A" + }, + { + "expr": "sum by (name) (rate(workqueue_retries_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "retries {{name}}", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 71 + } + }, + { + "type": "timeseries", + "title": "Queue wait & work p95", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, name) (rate(workqueue_queue_duration_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "wait {{name}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, name) (rate(workqueue_work_duration_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "work {{name}}", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 80 + } + }, + { + "type": "timeseries", + "title": "Oldest in-flight work", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "max by (name) (workqueue_unfinished_work_seconds{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "unfinished {{name}}", + "refId": "A" + }, + { + "expr": "max by (name) (workqueue_longest_running_processor_seconds{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "longest {{name}}", + "refId": "B" + } + ], + "description": "Age of the oldest item still being processed. A large, growing value means a stuck reconcile.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 80 + } + }, + { + "type": "row", + "title": "Kubernetes API client", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 89 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Are the operator's calls to Kubernetes healthy?\n\nThe operator constantly reads and writes objects in the Kubernetes API (Deployments, Secrets, your resource, and so on). If those calls fail, nothing the operator does can succeed.\n\n- **Requests / sec by code** \u2014 API responses grouped by HTTP status. The vast majority should be **2xx**. A wall of **4xx** points at permission (RBAC) problems or conflicts; **5xx** points at an unhealthy API server.\n- **Errors / sec (4xx / 5xx)** \u2014 the failing responses on their own, so a small error rate isn't hidden by all the healthy traffic.\n\n*There's no latency panel here on purpose: this operator's Kubernetes client doesn't publish a request-latency metric, so rather than show an always-empty chart we leave it out.*\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 90 + } + }, + { + "type": "timeseries", + "title": "Requests / sec by code", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(rest_client_requests_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 96 + } + }, + { + "type": "timeseries", + "title": "Errors / sec (4xx / 5xx)", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(rest_client_requests_total{job=~\"(.*/)?wandb-operator\", code=~\"[45]..\"}[$__rate_interval])) or vector(0)", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 96 + } + }, + { + "type": "row", + "title": "Admission & conversion webhooks", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 105 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Are the operator's webhooks serving?\n\nBefore Kubernetes accepts a `WeightsAndBiases` (or the operator's other resources) it calls the operator's **webhooks** to fill in defaults (*mutate*), check the object is valid (*validate*), and convert between API versions v1\u2194v2 (*convert*). If these fail, `kubectl apply` is rejected and upgrades can break.\n\n- **Webhook requests / sec by code** \u2014 should be almost entirely **2xx**. These fire on every apply and during reconciles, so on a busy cluster they're constantly active.\n- **Webhook errors / sec (5xx)** \u2014 server-side webhook failures. Should be **0**.\n- **Webhook panics** \u2014 crashes inside a webhook handler. Should be **0**.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 106 + } + }, + { + "type": "timeseries", + "title": "Webhook requests / sec by code", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(controller_runtime_webhook_requests_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 112 + } + }, + { + "type": "timeseries", + "title": "Webhook errors / sec (5xx)", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(controller_runtime_webhook_requests_total{job=~\"(.*/)?wandb-operator\", code=~\"5..\"}[$__rate_interval])) or vector(0)", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 112 + } + }, + { + "type": "stat", + "title": "Webhook panics", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "(sum(controller_runtime_webhook_panics_total{job=~\"(.*/)?wandb-operator\"}) + sum(controller_runtime_conversion_webhook_panics_total{job=~\"(.*/)?wandb-operator\"})) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Total crashes inside admission or conversion webhook handlers since start. Should be 0.", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 121 + } + }, + { + "type": "stat", + "title": "Webhook requests in flight", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(controller_runtime_webhook_requests_in_flight{job=~\"(.*/)?wandb-operator\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Webhook calls being served right now. A number that only grows suggests handlers are hanging.", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 121 + } + }, + { + "type": "row", + "title": "Process & runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 128 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Is the operator process itself healthy?\n\nStandard signals about the running program \u2014 useful for spotting crashes and memory leaks.\n\n- **Uptime** \u2014 how long the operator has been running. If this keeps **resetting to a small value**, the operator is crash-looping.\n- **Leader election** \u2014 for high availability you can run several operator replicas, but only **one** is allowed to act at a time: it holds a \"leader\" lock and the others stand by. This tile shows whether the replica you're looking at is the active **Leader** or on **Standby**. Exactly one replica should read *Leader*.\n- **Goroutines** and **Memory** \u2014 should stay stable. A line that only ever climbs suggests a leak.\n- **CPU cores used** and **Open file descriptors** \u2014 resource usage. File descriptors climbing toward the max is a warning sign.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 129 + } + }, + { + "type": "stat", + "title": "Uptime", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "time() - process_start_time_seconds{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Time since the operator process started. Resets to zero on restart.", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 135 + } + }, + { + "type": "stat", + "title": "Leader election", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Standby" + }, + "1": { + "text": "Leader" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "max(leader_election_master_status{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Whether this operator replica currently holds the leader lock. Exactly one replica should be Leader.", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 135 + } + }, + { + "type": "timeseries", + "title": "Goroutines", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "go_goroutines{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "goroutines", + "refId": "A" + } + ], + "description": "Concurrent Go routines. Steady is good; a steadily climbing line suggests a leak.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 142 + } + }, + { + "type": "timeseries", + "title": "Memory", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "resident", + "refId": "A" + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "go heap in-use", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 142 + } + }, + { + "type": "timeseries", + "title": "CPU cores used", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])", + "legendFormat": "cores", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 151 + } + }, + { + "type": "timeseries", + "title": "Open file descriptors", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "process_open_fds{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "open", + "refId": "A" + }, + { + "expr": "process_max_fds{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "max", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 151 + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "wandb", + "operator", + "observability" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "title": "W&B Operator Health", + "uid": "wandb-operator-health", + "version": 1 +} From 72b1bd2b0b937186dc9b263354c01e7e1bfa7c59 Mon Sep 17 00:00:00 2001 From: Jagat Thakkar Date: Wed, 5 Aug 2026 13:22:32 -0500 Subject: [PATCH 4/5] feat: Wire the Operator Health dashboard into the telemetry chart Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/telemetry/templates/telemetry-ui.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/deploy/telemetry/templates/telemetry-ui.yaml b/deploy/telemetry/templates/telemetry-ui.yaml index 58550846..11d95b23 100644 --- a/deploy/telemetry/templates/telemetry-ui.yaml +++ b/deploy/telemetry/templates/telemetry-ui.yaml @@ -196,4 +196,24 @@ spec: datasourceName: VictoriaLogs json: |- {{ .Files.Get "dashboards/wandb-application.json" | nindent 4 }} +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: wandb-operator-health + namespace: {{ include "telemetry.namespace" . }} + labels: + app.kubernetes.io/component: telemetry + app.kubernetes.io/part-of: wandb +spec: + uid: wandb-operator-health + folder: W&B + instanceSelector: + matchLabels: + dashboards: grafana + datasources: + - inputName: DS_VICTORIAMETRICS + datasourceName: VictoriaMetrics + json: |- +{{ .Files.Get "dashboards/wandb-operator-health.json" | nindent 4 }} {{- end }} From ce50fe2ad94ea09798829d991dc0aaca229a9d00 Mon Sep 17 00:00:00 2001 From: Jagat Thakkar Date: Wed, 5 Aug 2026 13:55:44 -0500 Subject: [PATCH 5/5] feat: Add operator liveness, restart and CPU-throttling panels Extend the operator-health board's Process & runtime section with up/liveness, uptime, last-restarted and restarts-in-range (from the operator's own process metrics) and container CPU-throttling (cAdvisor, scoped to the operator pod by name so it stays namespace-portable). Drops the "who it's for" line. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../dashboards/wandb-operator-health.json | 253 +++++++++++++++++- 1 file changed, 239 insertions(+), 14 deletions(-) diff --git a/deploy/telemetry/dashboards/wandb-operator-health.json b/deploy/telemetry/dashboards/wandb-operator-health.json index eec2d2c6..f0fc85e2 100644 --- a/deploy/telemetry/dashboards/wandb-operator-health.json +++ b/deploy/telemetry/dashboards/wandb-operator-health.json @@ -22,7 +22,7 @@ "transparent": false, "options": { "mode": "markdown", - "content": "# W&B Operator Health\n\n**What this shows:** the health of the **W&B operator** \u2014 the controller that installs W&B and keeps it running by reconciling your `WeightsAndBiases` resource. It is *not* the W&B app or its databases. Open this board when an install or upgrade is stuck, a resource won't turn Ready, or the operator seems unresponsive.\n\n**Who it's for:** whoever runs the operator (SRE / platform).\n\n**How to read it:** green means healthy, red means it needs attention. Every section below opens with a note like this one that explains its panels and what a *bad* reading looks like. All data comes from the operator's own `/metrics` endpoint \u2014 the controller-runtime, Kubernetes-client and Go-runtime libraries expose most of it automatically, plus three gauges the operator publishes itself (`wandb_weightsandbiases_ready`, `wandb_infra_state`, `wandb_application_info`).\n\n**When something here looks wrong, drill into:** [Application](/d/wandb-application) for app request health \u00b7 [Managed Install](/d/wandb-managed-install-performance) for database / dependency resources \u00b7 [Telemetry Overview](/d/wandb-telemetry-overview) for the metrics pipeline itself.\n" + "content": "# W&B Operator Health\n\n**What this shows:** the health of the **W&B operator** \u2014 the controller that installs W&B and keeps it running by reconciling your `WeightsAndBiases` resource. It is *not* the W&B app or its databases. Open this board when an install or upgrade is stuck, a resource won't turn Ready, or the operator seems unresponsive.\n\n**How to read it:** green means healthy, red means it needs attention. Every section below opens with a note like this one that explains its panels and what a *bad* reading looks like. All data comes from the operator's own `/metrics` endpoint \u2014 the controller-runtime, Kubernetes-client and Go-runtime libraries expose most of it automatically, plus three gauges the operator publishes itself (`wandb_weightsandbiases_ready`, `wandb_infra_state`, `wandb_application_info`).\n\n**When something here looks wrong, drill into:** [Application](/d/wandb-application) for app request health \u00b7 [Managed Install](/d/wandb-managed-install-performance) for database / dependency resources \u00b7 [Telemetry Overview](/d/wandb-telemetry-overview) for the metrics pipeline itself.\n" }, "gridPos": { "h": 6, @@ -1281,15 +1281,83 @@ "transparent": false, "options": { "mode": "markdown", - "content": "### Is the operator process itself healthy?\n\nStandard signals about the running program \u2014 useful for spotting crashes and memory leaks.\n\n- **Uptime** \u2014 how long the operator has been running. If this keeps **resetting to a small value**, the operator is crash-looping.\n- **Leader election** \u2014 for high availability you can run several operator replicas, but only **one** is allowed to act at a time: it holds a \"leader\" lock and the others stand by. This tile shows whether the replica you're looking at is the active **Leader** or on **Standby**. Exactly one replica should read *Leader*.\n- **Goroutines** and **Memory** \u2014 should stay stable. A line that only ever climbs suggests a leak.\n- **CPU cores used** and **Open file descriptors** \u2014 resource usage. File descriptors climbing toward the max is a warning sign.\n" + "content": "### Is the operator process itself healthy?\n\nStandard signals about the running program \u2014 useful for spotting crashes, restarts and memory leaks.\n\n- **Up** \u2014 is the operator alive and serving metrics right now? Green means yes.\n- **Uptime** and **Last restarted** \u2014 how long the operator has been running and when it last (re)started. A short, shrinking uptime means it's crash-looping.\n- **Restarts (in range)** \u2014 how many times the operator process restarted within the dashboard's selected time window. A one-off is usually just a deploy; a climbing count means crash-looping.\n- **Leader election** \u2014 for high availability you can run several operator replicas, but only **one** acts at a time: it holds a \"leader\" lock and the others stand by. This shows whether the replica you're looking at is the active **Leader** or on **Standby**. Exactly one replica should read *Leader*.\n- **CPU throttling** \u2014 how much the container is being CPU-throttled by its limit. **0% is ideal;** a high value means the CPU limit is too low.\n- **Goroutines** and **Memory** \u2014 should stay stable. A line that only ever climbs suggests a leak.\n- **CPU cores used** and **Open file descriptors** \u2014 resource usage. File descriptors climbing toward the max is a warning sign.\n" }, "gridPos": { - "h": 6, + "h": 8, "w": 24, "x": 0, "y": 129 } }, + { + "type": "stat", + "title": "Up", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Down" + }, + "1": { + "text": "Up" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "min(up{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Whether the operator is alive and serving its metrics endpoint right now.", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 137 + } + }, { "type": "stat", "title": "Uptime", @@ -1319,18 +1387,116 @@ }, "targets": [ { - "expr": "time() - process_start_time_seconds{job=~\"(.*/)?wandb-operator\"}", + "expr": "time() - max(process_start_time_seconds{job=~\"(.*/)?wandb-operator\"})", "legendFormat": "", "refId": "A", "instant": true } ], - "description": "Time since the operator process started. Resets to zero on restart.", + "description": "Time since the operator process started. Resets on restart.", "gridPos": { "h": 7, - "w": 12, + "w": 8, + "x": 8, + "y": 137 + } + }, + { + "type": "stat", + "title": "Last restarted", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "max(process_start_time_seconds{job=~\"(.*/)?wandb-operator\"}) * 1000", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "When the operator process last (re)started.", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 137 + } + }, + { + "type": "stat", + "title": "Restarts (in range)", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(changes(process_start_time_seconds{job=~\"(.*/)?wandb-operator\"}[$__range]))", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Operator process restarts within the selected time range. One is usually a deploy; a growing count means crash-looping.", + "gridPos": { + "h": 7, + "w": 8, "x": 0, - "y": 135 + "y": 144 } }, { @@ -1396,9 +1562,68 @@ "description": "Whether this operator replica currently holds the leader lock. Exactly one replica should be Leader.", "gridPos": { "h": 7, - "w": 12, - "x": 12, - "y": 135 + "w": 8, + "x": 8, + "y": 144 + } + }, + { + "type": "stat", + "title": "CPU throttling", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_periods_total{container=\"operator\", pod=~\".*wandb-operator-[^-]+-[^-]+\"}[$__rate_interval])) / sum(rate(container_cpu_cfs_periods_total{container=\"operator\", pod=~\".*wandb-operator-[^-]+-[^-]+\"}[$__rate_interval])) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Share of CPU scheduling periods the operator container was throttled. 0% is ideal.", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 144 } }, { @@ -1440,7 +1665,7 @@ "h": 9, "w": 12, "x": 0, - "y": 142 + "y": 151 } }, { @@ -1486,7 +1711,7 @@ "h": 9, "w": 12, "x": 12, - "y": 142 + "y": 151 } }, { @@ -1527,7 +1752,7 @@ "h": 9, "w": 12, "x": 0, - "y": 151 + "y": 160 } }, { @@ -1573,7 +1798,7 @@ "h": 9, "w": 12, "x": 12, - "y": 151 + "y": 160 } } ],