diff --git a/deploy/telemetry/dashboards/wandb-operator-health.json b/deploy/telemetry/dashboards/wandb-operator-health.json new file mode 100644 index 00000000..f0fc85e2 --- /dev/null +++ b/deploy/telemetry/dashboards/wandb-operator-health.json @@ -0,0 +1,1824 @@ +{ + "__inputs": [ + { + "name": "DS_VICTORIAMETRICS", + "label": "VictoriaMetrics", + "type": "datasource", + "pluginId": "victoriametrics-metrics-datasource", + "pluginName": "VictoriaMetrics" + } + ], + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "# W&B Operator Health\n\n**What this shows:** the health of the **W&B operator** \u2014 the controller that installs W&B and keeps it running by reconciling your `WeightsAndBiases` resource. It is *not* the W&B app or its databases. Open this board when an install or upgrade is stuck, a resource won't turn Ready, or the operator seems unresponsive.\n\n**How to read it:** green means healthy, red means it needs attention. Every section below opens with a note like this one that explains its panels and what a *bad* reading looks like. All data comes from the operator's own `/metrics` endpoint \u2014 the controller-runtime, Kubernetes-client and Go-runtime libraries expose most of it automatically, plus three gauges the operator publishes itself (`wandb_weightsandbiases_ready`, `wandb_infra_state`, `wandb_application_info`).\n\n**When something here looks wrong, drill into:** [Application](/d/wandb-application) for app request health \u00b7 [Managed Install](/d/wandb-managed-install-performance) for database / dependency resources \u00b7 [Telemetry Overview](/d/wandb-telemetry-overview) for the metrics pipeline itself.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "row", + "title": "W&B Resources", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Are the W&B resources healthy, and is every service on the same version?\n\n- **CR Ready** \u2014 one tile per `WeightsAndBiases` resource. Green **Ready** means the operator finished reconciling and everything it manages is up. Red **Not Ready** means it's still working or blocked \u2014 the reason shows up in the dependency panels to the right.\n- **Unhealthy dependencies** \u2014 how many backing services (MySQL, Redis, Kafka, object store, ClickHouse) are in any state other than *Healthy*. **0 (green) is what you want.** Any higher number is exactly *why* a resource above isn't Ready \u2014 check the table for which one.\n- **Dependency states** \u2014 every backing service and its current state (*Healthy / Degraded / Pending / Error / Unavailable*), so you can see *which* service is in *which* state.\n- **Distinct image versions** \u2014 how many different image tags the managed services are running. **1** means everything is on the same version; **2 or more** means **version drift** (usually a half-finished upgrade), and the *Managed service versions* table shows which service is behind.\n\n*These tiles show the current value only, so a service that was briefly Pending while starting up won't linger here.*\n" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + } + }, + { + "type": "stat", + "title": "CR Ready", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Not Ready" + }, + "1": { + "text": "Ready" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "targets": [ + { + "expr": "wandb_weightsandbiases_ready", + "legendFormat": "{{namespace}}/{{name}}", + "refId": "A", + "instant": true + } + ], + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + } + }, + { + "type": "stat", + "title": "Unhealthy dependencies", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(wandb_infra_state{state!=\"Healthy\"} == 1) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Backing services in any state other than Healthy right now. 0 is good.", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + } + }, + { + "type": "stat", + "title": "Distinct image versions", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(count by (tag) (wandb_application_info))", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Number of image tags in use across managed services. 1 = consistent; 2+ = version drift.", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + } + }, + { + "type": "table", + "title": "Dependency states", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "targets": [ + { + "expr": "wandb_infra_state == 1", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "job": true, + "instance": true, + "pod": true, + "container": true, + "endpoint": true, + "service": true, + "prometheus": true + }, + "indexByName": { + "namespace": 0, + "name": 1, + "component": 2, + "instance_name": 3, + "state": 4 + }, + "renameByName": { + "namespace": "Namespace", + "name": "Name", + "component": "Component", + "instance_name": "Instance", + "state": "State" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "md" + }, + "description": "Current state of each backing dependency.", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + } + }, + { + "type": "table", + "title": "Managed service versions", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "targets": [ + { + "expr": "wandb_application_info", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "__name__": true, + "job": true, + "instance": true, + "pod": true, + "container": true, + "endpoint": true, + "service": true, + "prometheus": true, + "namespace": true + }, + "indexByName": { + "application_name": 0, + "image": 1, + "tag": 2, + "digest": 3 + }, + "renameByName": { + "application_name": "Service", + "image": "Image", + "tag": "Tag", + "digest": "Digest" + } + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "md" + }, + "description": "Image the operator wants each service to run.", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + } + }, + { + "type": "row", + "title": "Reconciler", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Is the operator successfully doing its job?\n\nThe operator works in a loop called a **reconcile**: whenever your resource (or something it manages) changes, the operator runs this loop to make the cluster match what you asked for. These panels show whether that loop is healthy.\n\n- **Reconcile errors / sec** \u2014 how often a loop fails. **Should stay at zero.** A steady non-zero rate means the operator keeps trying and failing to finish setting something up (check the operator logs for the reason).\n- **Terminal reconcile errors** \u2014 failures the operator has **given up** retrying. Should be **0**; above zero means a resource is stuck and won't recover on its own.\n- **Reconcile panics** \u2014 times the operator's code crashed mid-loop (it recovers, but it shouldn't happen). Should be **0**; a non-zero value is a bug worth reporting.\n- **Reconciles / sec by result** \u2014 every loop ends as *success*, *error*, or *requeue* (asked to try again later). Lots of *requeue* usually means it's waiting on a dependency to come up.\n- **Reconcile errors by controller** \u2014 which specific controller is the one failing.\n- **Reconcile latency** \u2014 how long a loop takes (50th / 95th / 99th percentile). Rising latency means loops are getting slow.\n- **Active vs max workers** \u2014 how many loops run at once versus the configured limit. If *active* sits pinned on the *max* line, the operator is at capacity and work will start to queue.\n" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + } + }, + { + "type": "stat", + "title": "Reconcile errors / sec", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Reconcile loops ending in error, per second. Should be zero.", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + } + }, + { + "type": "stat", + "title": "Terminal reconcile errors", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(controller_runtime_terminal_reconcile_errors_total{job=~\"(.*/)?wandb-operator\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Total reconciles the operator gave up retrying since it started. Should be 0.", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + } + }, + { + "type": "stat", + "title": "Reconcile panics", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(controller_runtime_reconcile_panics_total{job=~\"(.*/)?wandb-operator\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Total times a reconcile crashed and was recovered since the operator started. Should be 0.", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + } + }, + { + "type": "timeseries", + "title": "Reconciles / sec by result", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (result) (rate(controller_runtime_reconcile_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 46 + } + }, + { + "type": "timeseries", + "title": "Reconcile errors / sec by controller", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (controller) (rate(controller_runtime_reconcile_errors_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{controller}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 46 + } + }, + { + "type": "timeseries", + "title": "Reconcile latency", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "description": "How long a reconcile loop takes.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 55 + } + }, + { + "type": "timeseries", + "title": "Active vs max workers", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (controller) (controller_runtime_active_workers{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "active {{controller}}", + "refId": "A" + }, + { + "expr": "sum(controller_runtime_max_concurrent_reconciles{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "max concurrent", + "refId": "B" + } + ], + "description": "Loops running concurrently versus the configured limit. Active pinned at max = saturated.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 55 + } + }, + { + "type": "row", + "title": "Workqueue", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Is the operator keeping up with its workload?\n\nWork the operator needs to do is placed on a **queue** (one per controller) and processed in order. These panels show whether the queue is draining or backing up.\n\n- **Queue depth** \u2014 items waiting to be processed. It should sit near zero and drain quickly. A depth that keeps **climbing** means the operator is falling behind.\n- **Adds & retries / sec** \u2014 how fast new work arrives versus how often failed work is retried. A spike in *retries* goes hand-in-hand with the reconcile errors above.\n- **Queue wait & work p95** \u2014 how long items **wait** in the queue versus how long they take to **process**. Both rising means the operator is overloaded.\n- **Oldest in-flight work** \u2014 the age of the oldest item still being worked on. A large, growing value means a single reconcile is stuck.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 65 + } + }, + { + "type": "timeseries", + "title": "Queue depth", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (name) (workqueue_depth{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "description": "Items waiting to be reconciled, per controller.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 71 + } + }, + { + "type": "timeseries", + "title": "Adds & retries / sec", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (name) (rate(workqueue_adds_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "adds {{name}}", + "refId": "A" + }, + { + "expr": "sum by (name) (rate(workqueue_retries_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "retries {{name}}", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 71 + } + }, + { + "type": "timeseries", + "title": "Queue wait & work p95", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, name) (rate(workqueue_queue_duration_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "wait {{name}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, name) (rate(workqueue_work_duration_seconds_bucket{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])))", + "legendFormat": "work {{name}}", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 80 + } + }, + { + "type": "timeseries", + "title": "Oldest in-flight work", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "max by (name) (workqueue_unfinished_work_seconds{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "unfinished {{name}}", + "refId": "A" + }, + { + "expr": "max by (name) (workqueue_longest_running_processor_seconds{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "longest {{name}}", + "refId": "B" + } + ], + "description": "Age of the oldest item still being processed. A large, growing value means a stuck reconcile.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 80 + } + }, + { + "type": "row", + "title": "Kubernetes API client", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 89 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Are the operator's calls to Kubernetes healthy?\n\nThe operator constantly reads and writes objects in the Kubernetes API (Deployments, Secrets, your resource, and so on). If those calls fail, nothing the operator does can succeed.\n\n- **Requests / sec by code** \u2014 API responses grouped by HTTP status. The vast majority should be **2xx**. A wall of **4xx** points at permission (RBAC) problems or conflicts; **5xx** points at an unhealthy API server.\n- **Errors / sec (4xx / 5xx)** \u2014 the failing responses on their own, so a small error rate isn't hidden by all the healthy traffic.\n\n*There's no latency panel here on purpose: this operator's Kubernetes client doesn't publish a request-latency metric, so rather than show an always-empty chart we leave it out.*\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 90 + } + }, + { + "type": "timeseries", + "title": "Requests / sec by code", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(rest_client_requests_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 96 + } + }, + { + "type": "timeseries", + "title": "Errors / sec (4xx / 5xx)", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(rest_client_requests_total{job=~\"(.*/)?wandb-operator\", code=~\"[45]..\"}[$__rate_interval])) or vector(0)", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 96 + } + }, + { + "type": "row", + "title": "Admission & conversion webhooks", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 105 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Are the operator's webhooks serving?\n\nBefore Kubernetes accepts a `WeightsAndBiases` (or the operator's other resources) it calls the operator's **webhooks** to fill in defaults (*mutate*), check the object is valid (*validate*), and convert between API versions v1\u2194v2 (*convert*). If these fail, `kubectl apply` is rejected and upgrades can break.\n\n- **Webhook requests / sec by code** \u2014 should be almost entirely **2xx**. These fire on every apply and during reconciles, so on a busy cluster they're constantly active.\n- **Webhook errors / sec (5xx)** \u2014 server-side webhook failures. Should be **0**.\n- **Webhook panics** \u2014 crashes inside a webhook handler. Should be **0**.\n" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 106 + } + }, + { + "type": "timeseries", + "title": "Webhook requests / sec by code", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(controller_runtime_webhook_requests_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 112 + } + }, + { + "type": "timeseries", + "title": "Webhook errors / sec (5xx)", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "sum by (code) (rate(controller_runtime_webhook_requests_total{job=~\"(.*/)?wandb-operator\", code=~\"5..\"}[$__rate_interval])) or vector(0)", + "legendFormat": "{{code}}", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 112 + } + }, + { + "type": "stat", + "title": "Webhook panics", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "(sum(controller_runtime_webhook_panics_total{job=~\"(.*/)?wandb-operator\"}) + sum(controller_runtime_conversion_webhook_panics_total{job=~\"(.*/)?wandb-operator\"})) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Total crashes inside admission or conversion webhook handlers since start. Should be 0.", + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 121 + } + }, + { + "type": "stat", + "title": "Webhook requests in flight", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(controller_runtime_webhook_requests_in_flight{job=~\"(.*/)?wandb-operator\"}) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Webhook calls being served right now. A number that only grows suggests handlers are hanging.", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 121 + } + }, + { + "type": "row", + "title": "Process & runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 128 + } + }, + { + "type": "text", + "title": "", + "transparent": false, + "options": { + "mode": "markdown", + "content": "### Is the operator process itself healthy?\n\nStandard signals about the running program \u2014 useful for spotting crashes, restarts and memory leaks.\n\n- **Up** \u2014 is the operator alive and serving metrics right now? Green means yes.\n- **Uptime** and **Last restarted** \u2014 how long the operator has been running and when it last (re)started. A short, shrinking uptime means it's crash-looping.\n- **Restarts (in range)** \u2014 how many times the operator process restarted within the dashboard's selected time window. A one-off is usually just a deploy; a climbing count means crash-looping.\n- **Leader election** \u2014 for high availability you can run several operator replicas, but only **one** acts at a time: it holds a \"leader\" lock and the others stand by. This shows whether the replica you're looking at is the active **Leader** or on **Standby**. Exactly one replica should read *Leader*.\n- **CPU throttling** \u2014 how much the container is being CPU-throttled by its limit. **0% is ideal;** a high value means the CPU limit is too low.\n- **Goroutines** and **Memory** \u2014 should stay stable. A line that only ever climbs suggests a leak.\n- **CPU cores used** and **Open file descriptors** \u2014 resource usage. File descriptors climbing toward the max is a warning sign.\n" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 129 + } + }, + { + "type": "stat", + "title": "Up", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Down" + }, + "1": { + "text": "Up" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "min(up{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Whether the operator is alive and serving its metrics endpoint right now.", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 137 + } + }, + { + "type": "stat", + "title": "Uptime", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "time() - max(process_start_time_seconds{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Time since the operator process started. Resets on restart.", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 137 + } + }, + { + "type": "stat", + "title": "Last restarted", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "dateTimeFromNow" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "max(process_start_time_seconds{job=~\"(.*/)?wandb-operator\"}) * 1000", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "When the operator process last (re)started.", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 137 + } + }, + { + "type": "stat", + "title": "Restarts (in range)", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(changes(process_start_time_seconds{job=~\"(.*/)?wandb-operator\"}[$__range]))", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Operator process restarts within the selected time range. One is usually a deploy; a growing count means crash-looping.", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 144 + } + }, + { + "type": "stat", + "title": "Leader election", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Standby" + }, + "1": { + "text": "Leader" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "max(leader_election_master_status{job=~\"(.*/)?wandb-operator\"})", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Whether this operator replica currently holds the leader lock. Exactly one replica should be Leader.", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 144 + } + }, + { + "type": "stat", + "title": "CPU throttling", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(container_cpu_cfs_throttled_periods_total{container=\"operator\", pod=~\".*wandb-operator-[^-]+-[^-]+\"}[$__rate_interval])) / sum(rate(container_cpu_cfs_periods_total{container=\"operator\", pod=~\".*wandb-operator-[^-]+-[^-]+\"}[$__rate_interval])) or vector(0)", + "legendFormat": "", + "refId": "A", + "instant": true + } + ], + "description": "Share of CPU scheduling periods the operator container was throttled. 0% is ideal.", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 144 + } + }, + { + "type": "timeseries", + "title": "Goroutines", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "go_goroutines{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "goroutines", + "refId": "A" + } + ], + "description": "Concurrent Go routines. Steady is good; a steadily climbing line suggests a leak.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 151 + } + }, + { + "type": "timeseries", + "title": "Memory", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "resident", + "refId": "A" + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "go heap in-use", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 151 + } + }, + { + "type": "timeseries", + "title": "CPU cores used", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=~\"(.*/)?wandb-operator\"}[$__rate_interval])", + "legendFormat": "cores", + "refId": "A" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 160 + } + }, + { + "type": "timeseries", + "title": "Open file descriptors", + "datasource": { + "type": "victoriametrics-metrics-datasource", + "uid": "${DS_VICTORIAMETRICS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "expr": "process_open_fds{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "open", + "refId": "A" + }, + { + "expr": "process_max_fds{job=~\"(.*/)?wandb-operator\"}", + "legendFormat": "max", + "refId": "B" + } + ], + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 160 + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "wandb", + "operator", + "observability" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "title": "W&B Operator Health", + "uid": "wandb-operator-health", + "version": 1 +} diff --git a/deploy/telemetry/templates/telemetry-ui.yaml b/deploy/telemetry/templates/telemetry-ui.yaml index 58550846..11d95b23 100644 --- a/deploy/telemetry/templates/telemetry-ui.yaml +++ b/deploy/telemetry/templates/telemetry-ui.yaml @@ -196,4 +196,24 @@ spec: datasourceName: VictoriaLogs json: |- {{ .Files.Get "dashboards/wandb-application.json" | nindent 4 }} +--- +apiVersion: grafana.integreatly.org/v1beta1 +kind: GrafanaDashboard +metadata: + name: wandb-operator-health + namespace: {{ include "telemetry.namespace" . }} + labels: + app.kubernetes.io/component: telemetry + app.kubernetes.io/part-of: wandb +spec: + uid: wandb-operator-health + folder: W&B + instanceSelector: + matchLabels: + dashboards: grafana + datasources: + - inputName: DS_VICTORIAMETRICS + datasourceName: VictoriaMetrics + json: |- +{{ .Files.Get "dashboards/wandb-operator-health.json" | nindent 4 }} {{- end }} diff --git a/internal/controller/reconciler/readiness.go b/internal/controller/reconciler/readiness.go index b7ef38d5..fc16e9fc 100644 --- a/internal/controller/reconciler/readiness.go +++ b/internal/controller/reconciler/readiness.go @@ -7,6 +7,7 @@ import ( "strings" apiv2 "github.com/wandb/operator/api/v2" + wmetrics "github.com/wandb/operator/internal/metrics" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -23,6 +24,7 @@ const ( func setReadyStatus(wandb *apiv2.WeightsAndBiases, ready bool, reason, message string) { wandb.Status.Ready = ready + wmetrics.SetWeightsAndBiasesReady(wandb.Namespace, wandb.Name, ready) status := metav1.ConditionFalse if ready { status = metav1.ConditionTrue diff --git a/internal/controller/reconciler/reconcile_v2.go b/internal/controller/reconciler/reconcile_v2.go index 57891d89..409a35d2 100644 --- a/internal/controller/reconciler/reconcile_v2.go +++ b/internal/controller/reconciler/reconcile_v2.go @@ -271,6 +271,7 @@ func Reconcile( } } + wmetrics.DeleteWeightsAndBiasesMetrics(wandb.Namespace, wandb.Name) controllerutil.RemoveFinalizer(wandb, CleanupFinalizer) if err := client.Update(ctx, wandb); err != nil { log.Error("Failed to remove finalizer '%s'", logx.ErrAttr(err)) @@ -348,6 +349,8 @@ func Reconcile( } ctrlResults = append(ctrlResults, res) + recordInfraStateMetrics(wandb) + if err = inferState(ctx, client, wandb); err != nil { errorCount++ } @@ -1585,6 +1588,27 @@ func clickHouseAllReady(wandb *apiv2.WeightsAndBiases) bool { return allInstancesReady(wandb.Spec.ClickHouse, wandb.Status.ClickHouseStatus, func(s apiv2.ClickHouseInfraStatus) bool { return s.Ready }) } +// recordInfraStateMetrics publishes the current state of each dependency as +// wandb_infra_state, keyed by component and status-map instance. +func recordInfraStateMetrics(wandb *apiv2.WeightsAndBiases) { + ns, name := wandb.Namespace, wandb.Name + for key, s := range wandb.Status.MySQLStatus { + wmetrics.SetInfraState(ns, name, "mysql", key, s.State) + } + for key, s := range wandb.Status.RedisStatus { + wmetrics.SetInfraState(ns, name, "redis", key, s.State) + } + for key, s := range wandb.Status.ObjectStoreStatus { + wmetrics.SetInfraState(ns, name, "objectstore", key, s.State) + } + for key, s := range wandb.Status.ClickHouseStatus { + wmetrics.SetInfraState(ns, name, "clickhouse", key, s.State) + } + if wandb.Spec.Kafka.ManagedKafka != nil { + wmetrics.SetInfraState(ns, name, "kafka", "", wandb.Status.KafkaStatus.State) + } +} + func inferState( ctx context.Context, client ctrlClient.Client, wandb *apiv2.WeightsAndBiases, ) error { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index b7730a8b..d22747b4 100755 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -18,8 +18,69 @@ var ApplicationInfo = prometheus.NewGaugeVec( []string{"application_name", "namespace", "image", "tag", "digest"}, ) +// WeightsAndBiasesReady is 1 when a WeightsAndBiases CR reports Ready, else 0. +var WeightsAndBiasesReady = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "wandb_weightsandbiases_ready", + Help: "Whether a WeightsAndBiases custom resource is Ready (1) or not (0).", + }, + []string{"namespace", "name"}, +) + +// InfraState carries the current state of each backing dependency in the `state` +// label; value is always 1 for the active state. +var InfraState = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "wandb_infra_state", + Help: "Current state of each backing dependency of a WeightsAndBiases custom resource.", + }, + []string{"namespace", "name", "component", "instance_name", "state"}, +) + func init() { - metrics.Registry.MustRegister(ApplicationInfo) + metrics.Registry.MustRegister(ApplicationInfo, WeightsAndBiasesReady, InfraState) +} + +func SetWeightsAndBiasesReady(namespace, name string, ready bool) { + value := 0.0 + if ready { + value = 1 + } + WeightsAndBiasesReady.With(prometheus.Labels{ + "namespace": namespace, + "name": name, + }).Set(value) +} + +// SetInfraState records a dependency's current state, clearing its prior state +// series first so a transition doesn't leave the old state active. +func SetInfraState(namespace, name, component, instanceName, state string) { + InfraState.DeletePartialMatch(prometheus.Labels{ + "namespace": namespace, + "name": name, + "component": component, + "instance_name": instanceName, + }) + InfraState.With(prometheus.Labels{ + "namespace": namespace, + "name": name, + "component": component, + "instance_name": instanceName, + "state": state, + }).Set(1) +} + +// DeleteWeightsAndBiasesMetrics clears the readiness and infra-state series for +// a CR being torn down, so a deleted resource doesn't linger as Ready forever. +func DeleteWeightsAndBiasesMetrics(namespace, name string) { + WeightsAndBiasesReady.DeletePartialMatch(prometheus.Labels{ + "namespace": namespace, + "name": name, + }) + InfraState.DeletePartialMatch(prometheus.Labels{ + "namespace": namespace, + "name": name, + }) } // SetApplicationInfo records the running image for a single Application. diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index ac12f3a5..0c7108f7 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -9,10 +9,23 @@ import ( ) func gatherApplicationInfo(t *testing.T) []*dto.Metric { + t.Helper() + return gather(t, ApplicationInfo) +} + +func labelMap(m *dto.Metric) map[string]string { + out := make(map[string]string, len(m.Label)) + for _, lp := range m.Label { + out[lp.GetName()] = lp.GetValue() + } + return out +} + +func gather(t *testing.T, c prometheus.Collector) []*dto.Metric { t.Helper() out := make(chan prometheus.Metric, 64) go func() { - ApplicationInfo.Collect(out) + c.Collect(out) close(out) }() var metrics []*dto.Metric @@ -24,12 +37,54 @@ func gatherApplicationInfo(t *testing.T) []*dto.Metric { return metrics } -func labelMap(m *dto.Metric) map[string]string { - out := make(map[string]string, len(m.Label)) - for _, lp := range m.Label { - out[lp.GetName()] = lp.GetValue() - } - return out +func TestSetWeightsAndBiasesReady_FlipsValue(t *testing.T) { + t.Cleanup(WeightsAndBiasesReady.Reset) + WeightsAndBiasesReady.Reset() + + SetWeightsAndBiasesReady("wandb", "prod", true) + got := gather(t, WeightsAndBiasesReady) + assert.Len(t, got, 1) + assert.Equal(t, 1.0, got[0].Gauge.GetValue()) + + SetWeightsAndBiasesReady("wandb", "prod", false) + got = gather(t, WeightsAndBiasesReady) + assert.Len(t, got, 1) + assert.Equal(t, 0.0, got[0].Gauge.GetValue()) +} + +func TestSetInfraState_ReplacesPriorState(t *testing.T) { + t.Cleanup(InfraState.Reset) + InfraState.Reset() + + SetInfraState("wandb", "prod", "mysql", "default", "Pending") + SetInfraState("wandb", "prod", "mysql", "default", "Healthy") + + got := gather(t, InfraState) + assert.Len(t, got, 1, "a state transition must not leave the previous state as an active series") + got0 := labelMap(got[0]) + assert.Equal(t, "Healthy", got0["state"]) + assert.Equal(t, "default", got0["instance_name"]) +} + +func TestDeleteWeightsAndBiasesMetrics_ScopesToCR(t *testing.T) { + t.Cleanup(func() { WeightsAndBiasesReady.Reset(); InfraState.Reset() }) + WeightsAndBiasesReady.Reset() + InfraState.Reset() + + SetWeightsAndBiasesReady("wandb", "gone", true) + SetInfraState("wandb", "gone", "mysql", "default", "Healthy") + SetWeightsAndBiasesReady("wandb", "stays", true) + SetInfraState("wandb", "stays", "redis", "default", "Healthy") + + DeleteWeightsAndBiasesMetrics("wandb", "gone") + + ready := gather(t, WeightsAndBiasesReady) + assert.Len(t, ready, 1) + assert.Equal(t, "stays", labelMap(ready[0])["name"]) + + infra := gather(t, InfraState) + assert.Len(t, infra, 1) + assert.Equal(t, "stays", labelMap(infra[0])["name"]) } func TestSetApplicationInfo_EmitsExpectedLabels(t *testing.T) {