From 8f69a32744af73efd264aa7a334b9ed4a210c810 Mon Sep 17 00:00:00 2001 From: David Flores Date: Mon, 15 Jun 2026 21:37:41 +0100 Subject: [PATCH 1/2] chore(autocon5): bump sonda to 1.16.0 and move aggregate scrape to /scenarios/metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sonda 1.16.0 (released 2026-06-15) renamed the aggregate scenario-metrics route from `/metrics` to `/scenarios/metrics` and reclaimed `/metrics` for the server-self endpoint (Prometheus convention). The workshop's Telegraf scrape configs, both hands-on guides, the docker-compose comment block, the data-pipelines doc, and the srl1 catalog comment all needed the URL flip. Also bumps the pinned `SONDA_IMAGE` from 1.14.0 to 1.16.0 in docker-compose.yml and refreshes the .env.example commented example. End-to-end verified: a fresh `nobs autocon5 up` against this branch produces 6 hits for `interface_oper_state{intf_role="peer"}` in Prometheus (3 per device, 2 devices) — same shape Part 1 promises. --- workshops/autocon5/.env.example | 2 +- workshops/autocon5/docker-compose.yml | 10 +++++----- workshops/autocon5/docs/data-pipelines.md | 4 ++-- .../autocon5/guides/part-1-telemetry-and-queries.md | 12 ++++++------ workshops/autocon5/guides/part-2-dashboards.md | 4 ++-- workshops/autocon5/sonda/catalog/srl1-metrics.yaml | 2 +- workshops/autocon5/telegraf/telegraf-srl1.conf.toml | 4 ++-- workshops/autocon5/telegraf/telegraf-srl2.conf.toml | 4 ++-- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/workshops/autocon5/.env.example b/workshops/autocon5/.env.example index 3db3511..ea96ded 100644 --- a/workshops/autocon5/.env.example +++ b/workshops/autocon5/.env.example @@ -8,7 +8,7 @@ # Image pins (optional). Leave commented to use the defaults baked into the # compose file. # --------------------------------------------------------------------------- -# SONDA_IMAGE=ghcr.io/davidban77/sonda:1.3.0 +# SONDA_IMAGE=ghcr.io/davidban77/sonda:1.16.0 # PROMETHEUS_IMAGE=docker.io/prom/prometheus:v2.52.0 # LOKI_IMAGE=docker.io/grafana/loki:3.1.1 # GRAFANA_IMAGE=docker.io/grafana/grafana:13.0.1 diff --git a/workshops/autocon5/docker-compose.yml b/workshops/autocon5/docker-compose.yml index 2a0f428..f112fb6 100644 --- a/workshops/autocon5/docker-compose.yml +++ b/workshops/autocon5/docker-compose.yml @@ -34,11 +34,11 @@ services: # ========================================================================= # Both devices register their baseline scenarios on sonda-server. Each - # telegraf scrapes the aggregate /metrics endpoint, filtered by a device + # telegraf scrapes the aggregate /scenarios/metrics endpoint, filtered by a device # label (`source` for srl1's gNMI shape, `agent_host` for srl2's SNMP shape). sonda-server: container_name: sonda-server - image: ${SONDA_IMAGE:-ghcr.io/davidban77/sonda:1.14.0} + image: ${SONDA_IMAGE:-ghcr.io/davidban77/sonda:1.16.0} command: ["--port", "8080", "--bind", "0.0.0.0", "--catalog", "/catalog"] volumes: - ./sonda/catalog:/catalog:ro @@ -68,7 +68,7 @@ services: # to Loki; srl2 streams emit RFC 5424 syslog UDP to Vector (vector:1514). sonda-logs: container_name: sonda-logs - image: ${SONDA_IMAGE:-ghcr.io/davidban77/sonda:1.14.0} + image: ${SONDA_IMAGE:-ghcr.io/davidban77/sonda:1.16.0} entrypoint: ["/sonda"] command: ["run", "/catalog/all-logs.yaml"] volumes: @@ -87,7 +87,7 @@ services: # canonical `interface_*` / `bgp_*` series in Prometheus. # ========================================================================= - # telegraf-srl1: scrapes the aggregate /metrics endpoint filtered by + # telegraf-srl1: scrapes the aggregate /scenarios/metrics endpoint filtered by # `source:srl1` (gNMI-raw `srl_*`), renames to canonical (`interface_*`, # `bgp_*`), exposes a Prom scrape endpoint on :9005. telegraf-srl1: @@ -111,7 +111,7 @@ services: sonda-setup: condition: service_completed_successfully - # telegraf-srl2: scrapes the aggregate /metrics endpoint filtered by + # telegraf-srl2: scrapes the aggregate /scenarios/metrics endpoint filtered by # `agent_host:srl2` (SNMP-raw `ifHC*`, `bgpPeer*`), renames to canonical, # exposes a Prom scrape endpoint on :9005. telegraf-srl2: diff --git a/workshops/autocon5/docs/data-pipelines.md b/workshops/autocon5/docs/data-pipelines.md index d40e13d..ad438a8 100644 --- a/workshops/autocon5/docs/data-pipelines.md +++ b/workshops/autocon5/docs/data-pipelines.md @@ -213,7 +213,7 @@ Workshop CLI (host machine) Prefect flows (in-container) │ http://localhost:8085) │ http://sonda-server:8080) ▼ ▼ ┌──────────────────────────────────────────────────┐ - │ sonda-server v1.3.0 │ + │ sonda-server v1.16.0 │ │ POST /events handler │ │ • validates payload │ │ • forwards to sink config inline │ @@ -318,7 +318,7 @@ docker compose --project-name autocon5 restart sonda-setup ## Why `incident.py` uses remote_write -Every other workshop metric flows through the pull path — **sonda emits → Telegraf scrapes `/metrics` → Prometheus scrapes Telegraf**. `nobs autocon5 incident` is the exception: its cascade body uses `encoder/sink: remote_write` and pushes directly to Prometheus, bypassing Telegraf. This is intentional, not legacy. +Every other workshop metric flows through the pull path — **sonda emits → Telegraf scrapes `/scenarios/metrics` → Prometheus scrapes Telegraf**. `nobs autocon5 incident` is the exception: its cascade body uses `encoder/sink: remote_write` and pushes directly to Prometheus, bypassing Telegraf. This is intentional, not legacy. ### Why the exception exists diff --git a/workshops/autocon5/guides/part-1-telemetry-and-queries.md b/workshops/autocon5/guides/part-1-telemetry-and-queries.md index 54dab6c..916867d 100644 --- a/workshops/autocon5/guides/part-1-telemetry-and-queries.md +++ b/workshops/autocon5/guides/part-1-telemetry-and-queries.md @@ -40,7 +40,7 @@ The two devices speak different protocols at the source: - **`srl1` emits gNMI** — that's the telemetry shape SR Linux puts on the wire natively. Field names like `srl_interface_oper_state`, tags like `source`. **Telegraf-srl1** scrapes this raw shape, renames `srl_*` to canonical (`interface_*`, `bgp_*`) and `source` to `device`. Out the other side: the shared schema this workshop's dashboards and alerts speak. - **`srl2` emits SNMP** — the classic shape from IF-MIB / BGP4-MIB. Field names like `ifOperStatus`, `ifHCInOctets`; tags like `agent_host`, `ifDescr`. **Telegraf-srl2** scrapes the raw SNMP shape and renames every field and every tag to the same canonical schema. Out the other side: byte-for-byte identical to what srl1 produces. -Both raw shapes live on `sonda-server` (the lab's synthetic-telemetry runtime). Each Telegraf scrapes its device's per-scenario `/metrics` endpoints on a 10-second cadence — same scrape pattern Prometheus would use against real exporters in production. +Both raw shapes live on `sonda-server` (the lab's synthetic-telemetry runtime). Each Telegraf scrapes its device's per-scenario `/scenarios/metrics` endpoints on a 10-second cadence — same scrape pattern Prometheus would use against real exporters in production. #### See the raw shape, before Telegraf touches it @@ -48,7 +48,7 @@ Both raw shapes live on `sonda-server` (the lab's synthetic-telemetry runtime). The pipeline has three layers you can inspect directly from your browser: -1. **Raw gNMI from srl1** (sonda-server, before Telegraf): +1. **Raw gNMI from srl1** (sonda-server, before Telegraf): Look for `srl_*` metric names and the `source="srl1"` tag. This is what an SR Linux device emits on its gNMI stream. For example: @@ -58,7 +58,7 @@ The pipeline has three layers you can inspect directly from your browser: The pack `workshops/autocon5/sonda/catalog/srlinux-gnmi-interfaces-raw.yaml` lists every metric in this shape. -2. **Raw SNMP from srl2** (sonda-server, before Telegraf): +2. **Raw SNMP from srl2** (sonda-server, before Telegraf): Different shape entirely — IF-MIB names (`ifOperStatus`, `ifHCInOctets`) and the `agent_host="srl2"` tag. For example: @@ -80,9 +80,9 @@ The pipeline has three layers you can inspect directly from your browser: A single PromQL query for `interface_oper_state` returns rows from both devices in the same shape. The vendor difference is invisible at this layer. -??? info "Why the sonda `/metrics` endpoint is safe for two readers at once" +??? info "Why the sonda `/scenarios/metrics` endpoint is safe for two readers at once" - `sonda-server` exposes two shapes of metric endpoint: the **aggregate** `/metrics?label=key:value` you just used, and a **per-scenario** `/scenarios/{id}/metrics` for a single scenario by ID. + `sonda-server` exposes two shapes of metric endpoint: the **aggregate** `/scenarios/metrics?label=key:value` you just used, and a **per-scenario** `/scenarios/{id}/metrics` for a single scenario by ID. - The aggregate endpoint is **snapshot-style**: each scrape gets a consistent picture without consuming anything. Telegraf reads it every 10 seconds; you can read it concurrently from your browser; both see the same bytes. - The per-scenario endpoint is **drain-on-read**: each read consumes the scenario's emission buffer. Telegraf doesn't use this endpoint precisely because two consumers can't share a drain-on-read buffer without racing. @@ -646,7 +646,7 @@ UPDOWN log lines start appearing in the live stream within seconds of the first Click each URL and grep for one specific metric: - - — raw SNMP (pre-Telegraf): `bgpPeerState`, `ifHCInOctets`, `agent_host=srl2` + - — raw SNMP (pre-Telegraf): `bgpPeerState`, `ifHCInOctets`, `agent_host=srl2` - — telegraf-srl2's normalized output: `bgp_oper_state`, `interface_in_octets`, `device=srl2` - — Prometheus stores the same data after one more scrape hop diff --git a/workshops/autocon5/guides/part-2-dashboards.md b/workshops/autocon5/guides/part-2-dashboards.md index ae3ce65..9bba800 100644 --- a/workshops/autocon5/guides/part-2-dashboards.md +++ b/workshops/autocon5/guides/part-2-dashboards.md @@ -232,8 +232,8 @@ Worth noting: `srl1` and `srl2` arrive through different upstream pipelines (gNM **See it yourself — five URLs walk the three layers of each pipeline:** - 1. **Raw gNMI from srl1** (sonda-server, before Telegraf): . Look for `srl_*` metric names (`srl_interface_oper_state`, `srl_bgp_oper_state`) and the `source="srl1"` tag — what an SR Linux device emits on its gNMI stream. - 2. **Raw SNMP from srl2** (sonda-server, before Telegraf): . Look for the IF-MIB / BGP4-MIB names (`ifHCInOctets`, `bgpPeerState`, `cbgpPeerOperStatus`) and the `agent_host="srl2"` tag — the classic SNMP shape. + 1. **Raw gNMI from srl1** (sonda-server, before Telegraf): . Look for `srl_*` metric names (`srl_interface_oper_state`, `srl_bgp_oper_state`) and the `source="srl1"` tag — what an SR Linux device emits on its gNMI stream. + 2. **Raw SNMP from srl2** (sonda-server, before Telegraf): . Look for the IF-MIB / BGP4-MIB names (`ifHCInOctets`, `bgpPeerState`, `cbgpPeerOperStatus`) and the `agent_host="srl2"` tag — the classic SNMP shape. 3. **Telegraf-srl1's normalized output**: . The `srl_*` names are now plain `interface_*` / `bgp_*`, and the `source` tag has been renamed to `device`. Same data, canonical shape. 4. **Telegraf-srl2's normalized output**: . The SNMP names (`ifHCInOctets`, etc.) are now also `interface_*` / `bgp_*`, and `agent_host` is now `device`. Identical to telegraf-srl1's output above — except for one label we keep on purpose: `collection_type=gnmi` vs `collection_type=snmp`, so you can debug which pipeline a sample came from. 5. **Final view in Prometheus**: . A single query for `interface_oper_state{device=~"srl1|srl2"}` returns rows from both devices in the same shape — the vendor difference is invisible at this layer. diff --git a/workshops/autocon5/sonda/catalog/srl1-metrics.yaml b/workshops/autocon5/sonda/catalog/srl1-metrics.yaml index 6b8e005..baf5489 100644 --- a/workshops/autocon5/sonda/catalog/srl1-metrics.yaml +++ b/workshops/autocon5/sonda/catalog/srl1-metrics.yaml @@ -1,5 +1,5 @@ # srl1 telemetry — RAW gNMI shape (`srl_*` names + `source` tag). -# Registered on sonda-server; telegraf-srl1 scrapes the aggregate /metrics +# Registered on sonda-server; telegraf-srl1 scrapes the aggregate /scenarios/metrics # endpoint and normalizes the shapes to the canonical schema. # Broken-on-purpose: BGP peer 10.1.99.2 and interface ethernet-1/11. # diff --git a/workshops/autocon5/telegraf/telegraf-srl1.conf.toml b/workshops/autocon5/telegraf/telegraf-srl1.conf.toml index c832950..78857ab 100644 --- a/workshops/autocon5/telegraf/telegraf-srl1.conf.toml +++ b/workshops/autocon5/telegraf/telegraf-srl1.conf.toml @@ -5,14 +5,14 @@ ############################################################################### # DATA COLLECTION # ############################################################################### -# Scrape sonda's aggregate /metrics, filtered to srl1 (gNMI shape). +# Scrape sonda's aggregate /scenarios/metrics, filtered to srl1 (gNMI shape). # `include_state=running,unresolved,held` is the inversion-pattern filter: # baselines sit in `unresolved` (with `if_unresolved: open`) at rest, the # cascade signal flips them to `paused` while a cascade is running, and # `held` admits snap_to-equipped scenarios that have frozen at a recovery # value (Pattern C, sonda PR #440 / 1.13.2+). [[inputs.prometheus]] - urls = ["http://sonda-server:8080/metrics?label=source:srl1&include_state=running,unresolved,held"] + urls = ["http://sonda-server:8080/scenarios/metrics?label=source:srl1&include_state=running,unresolved,held"] metric_version = 2 tagexclude = ["url", "host"] diff --git a/workshops/autocon5/telegraf/telegraf-srl2.conf.toml b/workshops/autocon5/telegraf/telegraf-srl2.conf.toml index 18e236a..e0bc8ed 100644 --- a/workshops/autocon5/telegraf/telegraf-srl2.conf.toml +++ b/workshops/autocon5/telegraf/telegraf-srl2.conf.toml @@ -5,14 +5,14 @@ ############################################################################### # DATA COLLECTION # ############################################################################### -# Scrape sonda's aggregate /metrics, filtered to srl2 (SNMP-raw shape). +# Scrape sonda's aggregate /scenarios/metrics, filtered to srl2 (SNMP-raw shape). # `include_state=running,unresolved,held` is the inversion-pattern filter: # baselines sit in `unresolved` (with `if_unresolved: open`) at rest, the # cascade signal flips them to `paused` while a cascade is running, and # `held` admits snap_to-equipped scenarios that have frozen at a recovery # value (Pattern C, sonda PR #440 / 1.13.2+). [[inputs.prometheus]] - urls = ["http://sonda-server:8080/metrics?label=agent_host:srl2&include_state=running,unresolved,held"] + urls = ["http://sonda-server:8080/scenarios/metrics?label=agent_host:srl2&include_state=running,unresolved,held"] metric_version = 2 tagexclude = ["url", "host"] From 9ed93eb9a4a878d28a3d5f8f72ba730cf129b8cd Mon Sep 17 00:00:00 2001 From: David Flores Date: Mon, 15 Jun 2026 22:49:17 +0100 Subject: [PATCH 2/2] =?UTF-8?q?chore(autocon5,docs):=20catch=20remaining?= =?UTF-8?q?=20/metrics=20=E2=86=92=20/scenarios/metrics=20sites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the previous commit. Missed two surfaces: - `docs/workshop/tour.md`: the rendered docs-site Tour page (separate from the autocon5 in-repo guides) had its API table row, the pre-curl prose, and three curl examples still pointing at the legacy aggregate `/metrics` URL. Updated all five sites. - `workshops/autocon5/sonda/scripts/sonda-setup.sh`: header comment describing what Telegraf scrapes still said `/metrics`. The `/metrics` mention in tour.md's "what is Prometheus" paragraph stays — that's a generic reference to the Prometheus exposition format, not the sonda endpoint. --- docs/workshop/tour.md | 10 +++++----- workshops/autocon5/sonda/scripts/sonda-setup.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/workshop/tour.md b/docs/workshop/tour.md index 9374413..a60a962 100644 --- a/docs/workshop/tour.md +++ b/docs/workshop/tour.md @@ -47,7 +47,7 @@ A single workshop boot registers **76 scenarios** — sonda-server fans each `ki | `GET /scenarios/{uuid}` | One scenario's live handle: identity (`id`, `name`), `state`, `elapsed_secs`, plus an embedded `stats` block. | | `GET /scenarios/{uuid}/stats` | Per-scenario emission counters: `total_events`, `current_rate`, `target_rate`, `bytes_emitted`, `errors`, `consecutive_failures`, gap/burst state, and `last_successful_write_at`. | | `GET /scenarios/{uuid}/metrics` | The single Prometheus-text sample this scenario is emitting right now. Drained on read. | -| `GET /metrics[?label=k:v]` | Aggregate Prometheus-text snapshot across every running scenario. Optional `label=key:value` filter narrows to one device. Snapshot semantics — multiple consumers (Telegraf and a curl) can read the same bytes concurrently. | +| `GET /scenarios/metrics[?label=k:v]` | Aggregate Prometheus-text snapshot across every running scenario. Optional `label=key:value` filter narrows to one device. Snapshot semantics — multiple consumers (Telegraf and a curl) can read the same bytes concurrently. | | `POST /scenarios` | Register a new scenario. The cascade flap (`nobs autocon5 flap-interface`) is one of these. | | `DELETE /scenarios/{uuid}` | Stop and unregister a scenario. | @@ -99,10 +99,10 @@ curl -s http://localhost:8085/scenarios | jq '.scenarios[0]' } ``` -To see the actual Prometheus-text samples Telegraf reads — the byte-for-byte input, before any rename or normalization — curl the aggregate `/metrics` endpoint and filter to one device with `label=key:value`. The endpoint is snapshot-style: Telegraf scrapes it every 10 seconds and you can read it concurrently without stealing samples from anyone. +To see the actual Prometheus-text samples Telegraf reads — the byte-for-byte input, before any rename or normalization — curl the aggregate `/scenarios/metrics` endpoint and filter to one device with `label=key:value`. The endpoint is snapshot-style: Telegraf scrapes it every 10 seconds and you can read it concurrently without stealing samples from anyone. ```bash -curl -s 'http://localhost:8085/metrics?label=source:srl1' | grep '^srl_bgp_oper_state{' | grep '10.1.99.2' | head -1 +curl -s 'http://localhost:8085/scenarios/metrics?label=source:srl1' | grep '^srl_bgp_oper_state{' | grep '10.1.99.2' | head -1 ``` ```text @@ -114,7 +114,7 @@ Note the metric name (`srl_bgp_oper_state`) and the `source="srl1"` tag — that srl2 uses the SNMP shape, where the device tag is `agent_host`: ```bash -curl -s 'http://localhost:8085/metrics?label=agent_host:srl2' | grep '^bgpPeerState{' | grep '10.1.2.1' | head -1 +curl -s 'http://localhost:8085/scenarios/metrics?label=agent_host:srl2' | grep '^bgpPeerState{' | grep '10.1.2.1' | head -1 ``` ```text @@ -145,7 +145,7 @@ Drop `--kind composable` to also list runnable scenarios in the catalog, or swap ### Where you'll see this in the workshop -- **Part 1** — When you query Prometheus and see `bgp_oper_state{device="srl1"}` return three rows with one stuck at `5` instead of `1`, that's a value Sonda is generating right now. `curl 'http://localhost:8085/metrics?label=source:srl1'` shows you the raw side of that same number. +- **Part 1** — When you query Prometheus and see `bgp_oper_state{device="srl1"}` return three rows with one stuck at `5` instead of `1`, that's a value Sonda is generating right now. `curl 'http://localhost:8085/scenarios/metrics?label=source:srl1'` shows you the raw side of that same number. - **Part 3** — `nobs autocon5 flap-interface` and `nobs autocon5 incident` both POST cascade scenarios to this server. - **Advanced** — When you write your own scenario, you'll POST it here. diff --git a/workshops/autocon5/sonda/scripts/sonda-setup.sh b/workshops/autocon5/sonda/scripts/sonda-setup.sh index 6910440..28bc95a 100755 --- a/workshops/autocon5/sonda/scripts/sonda-setup.sh +++ b/workshops/autocon5/sonda/scripts/sonda-setup.sh @@ -3,7 +3,7 @@ # # Runs as an init container. Waits for sonda-server, POSTs each `*.yaml` in # `$SCENARIOS_DIR` (server resolves any `pack:` refs via its own `--catalog`), -# and exits. Telegraf scrapes the aggregate /metrics endpoint filtered by +# and exits. Telegraf scrapes the aggregate /scenarios/metrics endpoint filtered by # device label, so no per-device ID file is needed. set -e