From 04e4e14f6825f9e648c6b4e76016fcd1574d06e7 Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:26:45 -0400 Subject: [PATCH] skills: Add monitoring skills (prometheus, monitoring-ops) Add prometheus skill for querying cluster metrics via promtool, and monitoring-ops skill for troubleshooting the OpenShift monitoring stack. Co-Authored-By: Claude Sonnet 4.6 --- monitoring/OWNERS | 12 + monitoring/README.md | 5 + monitoring/monitoring-ops/SKILL.md | 113 +++++++++ monitoring/prometheus/SKILL.md | 143 +++++++++++ .../prometheus/references/cluster-access.md | 223 +++++++++++++++++ monitoring/prometheus/references/querying.md | 234 ++++++++++++++++++ monitoring/prometheus/references/tsdb.md | 213 ++++++++++++++++ .../prometheus/references/validation.md | 219 ++++++++++++++++ 8 files changed, 1162 insertions(+) create mode 100644 monitoring/OWNERS create mode 100644 monitoring/README.md create mode 100644 monitoring/monitoring-ops/SKILL.md create mode 100644 monitoring/prometheus/SKILL.md create mode 100644 monitoring/prometheus/references/cluster-access.md create mode 100644 monitoring/prometheus/references/querying.md create mode 100644 monitoring/prometheus/references/tsdb.md create mode 100644 monitoring/prometheus/references/validation.md diff --git a/monitoring/OWNERS b/monitoring/OWNERS new file mode 100644 index 0000000..76fe018 --- /dev/null +++ b/monitoring/OWNERS @@ -0,0 +1,12 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +approvers: + - harche + - mrunalp + - wking +reviewers: + - harche + - mrunalp + - wking + +component: "Monitoring Lightspeed Skills" diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..8ccf56f --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,5 @@ +# Monitoring skills + +This directory contains skills which are designed to help agents query and analyze metrics from monitoring systems such as Prometheus. + +> **Note:** The skills in this directory are initial drafts. They will evolve as we test and refine them based on real-world usage. diff --git a/monitoring/monitoring-ops/SKILL.md b/monitoring/monitoring-ops/SKILL.md new file mode 100644 index 0000000..f1a508c --- /dev/null +++ b/monitoring/monitoring-ops/SKILL.md @@ -0,0 +1,113 @@ +--- +name: monitoring-ops +description: OpenShift Cluster Monitoring Operator troubleshooting and tuning +--- + +# Monitoring Operations + +Troubleshoot and tune the OpenShift monitoring stack managed by the Cluster Monitoring Operator. + +## Components + +| Component | Namespace | Key Resources | +|-----------|-----------|---------------| +| Prometheus | openshift-monitoring | StatefulSet/prometheus-k8s, ConfigMap/prometheus-k8s-rulefiles-0 | +| Alertmanager | openshift-monitoring | StatefulSet/alertmanager-main, Secret/alertmanager-main | +| Thanos Querier | openshift-monitoring | Deployment/thanos-querier | +| node-exporter | openshift-monitoring | DaemonSet/node-exporter | +| kube-state-metrics | openshift-monitoring | Deployment/kube-state-metrics | +| prometheus-operator | openshift-monitoring | Deployment/prometheus-operator | +| Metrics Server | openshift-monitoring | Deployment/metrics-server | +| User Workload Prometheus | openshift-user-workload-monitoring | StatefulSet/prometheus-user-workload | +| User Workload Alertmanager | openshift-user-workload-monitoring | StatefulSet/alertmanager-user-workload | + +## Configuration + +CMO is configured via ConfigMap `cluster-monitoring-config` in `openshift-monitoring`: + +```bash +oc -n openshift-monitoring get configmap cluster-monitoring-config -o yaml +``` + +User workload monitoring is configured via `user-workload-monitoring-config` in `openshift-user-workload-monitoring`. + +## Common Investigations + +### Prometheus Health +```bash +# Pod status +oc -n openshift-monitoring get pods -l app.kubernetes.io/name=prometheus + +# Config reload status +oc -n openshift-monitoring exec prometheus-k8s-0 -c prometheus -- promtool check config /etc/prometheus/config_out/prometheus.env.yaml + +# TSDB status +oc -n openshift-monitoring exec prometheus-k8s-0 -c prometheus -- promtool tsdb analyze /prometheus + +# Memory usage +oc -n openshift-monitoring top pod -l app.kubernetes.io/name=prometheus +``` + +### Alertmanager Health +```bash +# Pod status and cluster membership +oc -n openshift-monitoring get pods -l app.kubernetes.io/name=alertmanager + +# Config validation +oc -n openshift-monitoring get secret alertmanager-main -o jsonpath='{.data.alertmanager\.yaml}' | base64 -d | amtool check-config - + +# Cluster status +oc -n openshift-monitoring exec alertmanager-main-0 -c alertmanager -- amtool cluster show --alertmanager.url=http://localhost:9093 +``` + +### Target Scrape Issues +```bash +# Down targets +oc -n openshift-monitoring exec prometheus-k8s-0 -c prometheus -- curl -s http://localhost:9090/api/v1/targets | python3 -c "import sys,json; [print(t['scrapeUrl'],t['lastError']) for t in json.load(sys.stdin)['data']['activeTargets'] if t['health']!='up']" + +# ServiceMonitor/PodMonitor validation +oc get servicemonitors -A +oc get podmonitors -A +``` + +### Storage Issues +```bash +# PV usage for monitoring +oc -n openshift-monitoring get pvc +oc -n openshift-monitoring exec prometheus-k8s-0 -c prometheus -- df -h /prometheus + +# Retention settings +oc -n openshift-monitoring get prometheus k8s -o jsonpath='{.spec.retention}' +``` + +### Thanos Querier Issues +```bash +# Store API endpoints +oc -n openshift-monitoring exec deploy/thanos-querier -- thanos query stores + +# Query health +oc -n openshift-monitoring logs deploy/thanos-querier -c thanos-query --tail=50 +``` + +## Tuning Recommendations + +### High Memory / Cardinality +- Reduce scrape targets via ServiceMonitor label selectors +- Lower retention: `prometheusK8s.retention: 12h` in cluster-monitoring-config +- Add metric relabeling to drop high-cardinality labels +- Check for label explosion in custom ServiceMonitors + +### Slow Rule Evaluation +- Split large rule groups into smaller ones +- Increase evaluation interval for non-critical rules +- Check for expensive PromQL in recording rules (joins, high-cardinality aggregations) + +### Storage Pressure +- Reduce retention period +- Increase PVC size: `prometheusK8s.volumeClaimTemplate.spec.resources.requests.storage` +- Enable compaction tuning + +### Alertmanager Notification Failures +- Check webhook endpoints are reachable from the cluster +- Verify TLS certificates for notification receivers +- Check network policies blocking egress from openshift-monitoring diff --git a/monitoring/prometheus/SKILL.md b/monitoring/prometheus/SKILL.md new file mode 100644 index 0000000..8c42ca2 --- /dev/null +++ b/monitoring/prometheus/SKILL.md @@ -0,0 +1,143 @@ +--- +name: prometheus +description: Query and analyze Prometheus metrics on Kubernetes and OpenShift clusters using promtool. Use when the user asks about Prometheus metrics, PromQL queries, metric analysis, alerting rules, recording rules, TSDB cardinality, or anything related to Prometheus monitoring — even if they just say "check metrics" or "why is CPU high on the cluster". Also trigger when the user mentions promtool, Thanos, or wants to validate Prometheus configuration or rules. +--- + +# Prometheus Metrics Analysis via promtool + +Query, analyze, and validate Prometheus metrics on Kubernetes and OpenShift clusters using `promtool`. + +## Prerequisites + +- `promtool` must be installed (`brew install prometheus` includes it) +- `kubectl` or `oc` CLI with a valid kubeconfig pointing to the target cluster + +## Critical Rules + +These rules exist because they caused real failures during testing. Follow them exactly. + +1. **Run setup + queries in a single bash call.** Shell variables (`$PROM_URL`, `$HTTP_CONFIG`, `$TOKEN`) do not persist across separate bash invocations. Combine setup and queries into one command using `&&`. + +2. **Never use `!=` in PromQL.** Zsh mangles `!=` into `\!=` via history expansion, even inside single quotes. Bash does not have this issue, but avoid `!=` for portability. Use `=~".+"` instead of `!=""`, and `=~"^((?!value).)*$"` or a negated regex instead of `!=`: + ```bash + # WRONG — zsh corrupts this + '{container!=""}' + # CORRECT + '{container=~".+"}' + ``` + +3. **JSON output is a raw array.** `promtool -o json` outputs `[{metric:{...}, value:[ts, val]}, ...]` — NOT `{data:{result:...}}`. Parse with `jq '.[]'`, not `jq '.data.result[]'`. + +4. **Token acquisition priority.** Inside a pod, use the mounted SA token first: `TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null || true)`. Only fall back to `oc whoami -t` when running outside a pod. The SA must have `cluster-monitoring-view` ClusterRole bound for Prometheus/Thanos access on OpenShift. + +5. **`promtool check healthy/ready` returns 503 on Thanos Querier.** This is expected — Thanos doesn't expose `/-/healthy`. Verify connectivity with `promtool query instant ... 'up'` instead. + +6. **Clean up temp files when done.** Always `rm -f "$HTTP_CONFIG"` and `kill $PF_PID 2>/dev/null` (if port-forwarding) after queries complete. + +## Setup + Query (Single Bash Call) + +Every promtool session should follow this pattern in one bash command. Adapt the query section as needed. + +### OpenShift + +```bash +TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null || true) && \ +if [ -z "$TOKEN" ]; then TOKEN=$(oc whoami -t 2>/dev/null || true); fi && \ +if [ -z "$TOKEN" ]; then echo "ERROR: No token available"; exit 1; fi && \ +HOST=$(oc -n openshift-monitoring get route thanos-querier -o jsonpath='{.status.ingress[].host}') && \ +PROM_URL="https://$HOST" && \ +HTTP_CONFIG=$(mktemp /tmp/prom-http-XXXXXX.yaml) && \ +cat > "$HTTP_CONFIG" < && \ +PROM_NS="monitoring" && \ +PROM_SVC=$(kubectl get svc -n "$PROM_NS" -o jsonpath='{.items[?(@.spec.ports[*].port==9090)].metadata.name}') && \ +kubectl port-forward -n "$PROM_NS" "svc/$PROM_SVC" 9090:9090 & +PF_PID=$! && sleep 2 && \ +PROM_URL="http://localhost:9090" && \ +HTTP_CONFIG=$(mktemp /tmp/prom-http-XXXXXX.yaml) && \ +cat > "$HTTP_CONFIG" </dev/null +``` + +## Query Examples + +All examples below assume `$HTTP_CONFIG` and `$PROM_URL` are set (from the setup block above). Chain them in the same bash call. + +```bash +# Instant query (text output) +promtool query instant --http.config.file="$HTTP_CONFIG" "$PROM_URL" 'up' + +# Instant query (JSON, parsed with jq) +promtool query instant --http.config.file="$HTTP_CONFIG" -o json "$PROM_URL" \ + 'sum(rate(container_cpu_usage_seconds_total{container=~".+"}[5m])) by (namespace)' | \ + jq -r '.[] | "\(.metric.namespace): \(.value[1] | tonumber | . * 1000 | round / 1000) cores"' + +# Range query (last hour, 1-minute steps) +# macOS: date -u -v-1H Linux: date -u -d '1 hour ago' +promtool query range --http.config.file="$HTTP_CONFIG" \ + --start="$(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1H +%Y-%m-%dT%H:%M:%SZ)" \ + --end="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --step=1m \ + "$PROM_URL" 'node_memory_MemAvailable_bytes' + +# Discover all metric names +promtool query labels --http.config.file="$HTTP_CONFIG" "$PROM_URL" __name__ + +# Find series matching a selector +promtool query series --http.config.file="$HTTP_CONFIG" \ + --match='container_cpu_usage_seconds_total{namespace="default"}' \ + "$PROM_URL" +``` + +## References + +Detailed command references — read on demand when you need specifics: + +|references/cluster-access.md — Discovery, auth, and port-forward setup for OpenShift and Kubernetes +|references/querying.md — Instant, range, series, labels, analyze, and PromQL formatting +|references/validation.md — Check config, check rules, check metrics, test rules (unit testing) +|references/tsdb.md — Analyze cardinality, list blocks, dump data, create blocks from rules + +## Common PromQL Patterns + +Useful starting points when the user asks broad questions: + +| Question | PromQL | +|---|---| +| Which targets are down? | `up == 0` | +| CPU usage by namespace | `sum(rate(container_cpu_usage_seconds_total[5m])) by (namespace)` | +| Memory usage by pod | `container_memory_working_set_bytes{container=~".+"}` | +| Disk pressure | `node_filesystem_avail_bytes / node_filesystem_size_bytes < 0.1` | +| API server request rate | `sum(rate(apiserver_request_total[5m])) by (verb, resource)` | +| API server error rate | `sum(rate(apiserver_request_total{code=~"5.."}[5m])) by (resource)` | +| Pod restart count | `increase(kube_pod_container_status_restarts_total[1h]) > 0` | +| Node CPU saturation | `1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)` | +| etcd leader changes | `increase(etcd_server_leader_changes_seen_total[1h])` | +| Scrape duration | `scrape_duration_seconds` | + +## Important + +- For OpenShift, always query the Thanos Querier route — it aggregates data from all Prometheus instances. +- Use `-o json` with `jq` when you need to parse or filter results programmatically. +- **Cross-platform date**: Use `date -u -d '1 hour ago' +FMT 2>/dev/null || date -u -v-1H +FMT` to work on both Linux (GNU date) and macOS (BSD date). diff --git a/monitoring/prometheus/references/cluster-access.md b/monitoring/prometheus/references/cluster-access.md new file mode 100644 index 0000000..492ce5c --- /dev/null +++ b/monitoring/prometheus/references/cluster-access.md @@ -0,0 +1,223 @@ +# Cluster Access Reference + +How to discover, authenticate to, and connect to Prometheus on Kubernetes and OpenShift clusters. + +## Table of Contents + +1. [OpenShift Clusters](#openshift-clusters) +2. [Kubernetes Clusters](#kubernetes-clusters) +3. [Authentication Patterns](#authentication-patterns) +4. [Troubleshooting](#troubleshooting) + +--- + +## OpenShift Clusters + +OpenShift ships a managed monitoring stack with Prometheus behind a Thanos Querier front-end. + +### Discover the Thanos Querier Route + +```bash +# List monitoring routes +oc get route -n openshift-monitoring + +# Extract the Thanos Querier host +HOST=$(oc -n openshift-monitoring get route thanos-querier -o jsonpath='{.status.ingress[].host}') +PROM_URL="https://$HOST" +``` + +### Get Bearer Token + +**Current user token (simplest):** +```bash +TOKEN=$(oc whoami -t 2>/dev/null) +``` + +If this returns empty, the kubeconfig uses client certificate auth instead of a session token. Create a service account token: + +**Service account token (for client cert kubeconfigs or automation):** +```bash +oc -n openshift-monitoring create sa prometheus-reader 2>/dev/null +oc adm policy add-cluster-role-to-user cluster-monitoring-view -z prometheus-reader -n openshift-monitoring 2>/dev/null + +# OCP 4.11+ / Kubernetes 1.24+ (TokenRequest API) +TOKEN=$(oc create token prometheus-reader -n openshift-monitoring --duration=1h) +``` + +### Create HTTP Config + +```bash +HTTP_CONFIG=$(mktemp /tmp/promtool-http-XXXXXX.yaml) +cat > "$HTTP_CONFIG" </dev/null +``` + +Common service names (varies by Helm release): + +| Component | Typical Service Name | Port | +|---|---|---| +| Prometheus | `prometheus-kube-prometheus-prometheus` | 9090 | +| Alertmanager | `prometheus-kube-prometheus-alertmanager` | 9093 | +| Thanos Sidecar | `prometheus-kube-prometheus-thanos-discovery` | 10901 | + +### Port-Forward + +```bash +# Port-forward to the Prometheus service +PROM_NS="monitoring" # adjust to actual namespace +PROM_SVC="prometheus-kube-prometheus-prometheus" # adjust to actual service name + +kubectl port-forward -n "$PROM_NS" "svc/$PROM_SVC" 9090:9090 & +PF_PID=$! +sleep 2 # wait for port-forward to establish + +PROM_URL="http://localhost:9090" +``` + +If the service name isn't obvious, find the pod by label: +```bash +kubectl port-forward -n "$PROM_NS" \ + $(kubectl get pods -n "$PROM_NS" -l app.kubernetes.io/name=prometheus -o jsonpath='{.items[0].metadata.name}') \ + 9090:9090 & +PF_PID=$! +``` + +### HTTP Config for Port-Forward + +Port-forwarded connections usually don't require auth: +```bash +HTTP_CONFIG=$(mktemp /tmp/promtool-http-XXXXXX.yaml) +cat > "$HTTP_CONFIG" < "$HTTP_CONFIG" </dev/null +``` + +--- + +## Authentication Patterns + +### HTTP Config File Schema + +The `--http.config.file` YAML supports: + +```yaml +# Bearer token (most common for K8s/OCP) +authorization: + type: Bearer + credentials: + # OR read from file: + credentials_file: /path/to/token + +# Basic auth +basic_auth: + username: + password: + +# TLS client certificates (mTLS) +tls_config: + ca_file: /path/to/ca.crt + cert_file: /path/to/client.crt + key_file: /path/to/client.key + insecure_skip_verify: false +``` + +### Token Extraction from Kubeconfig + +```bash +# Direct token in kubeconfig +kubectl config view --minify --raw -o jsonpath='{.users[0].user.token}' + +# OpenShift +oc whoami -t + +# Create a short-lived token (K8s 1.24+) +kubectl create token -n --duration=1h +``` + +--- + +## Troubleshooting + +### "connection refused" on port-forward +The port-forward process may have died. Check `jobs` and restart it. + +### "401 Unauthorized" on OpenShift route +Token may have expired. Refresh with `oc whoami -t` (requires `oc login` session) or create a new SA token. + +### "certificate signed by unknown authority" +Add `insecure_skip_verify: true` to the TLS config, or provide the CA cert via `ca_file`. + +### "could not find Prometheus service" +Try broader searches: +```bash +kubectl get svc -A | grep -iE '9090|prom|thanos|monitor' +kubectl get pods -A | grep -iE 'prom|thanos' +``` diff --git a/monitoring/prometheus/references/querying.md b/monitoring/prometheus/references/querying.md new file mode 100644 index 0000000..2398b4f --- /dev/null +++ b/monitoring/prometheus/references/querying.md @@ -0,0 +1,234 @@ +# Querying Reference + +All `promtool query` subcommands for querying a Prometheus server. + +Every command below requires: +- `--http.config.file="$HTTP_CONFIG"` for auth (see cluster-access.md) +- The Prometheus server URL as `$PROM_URL` + +## Table of Contents + +1. [Instant Query](#instant-query) +2. [Range Query](#range-query) +3. [Series Discovery](#series-discovery) +4. [Label Discovery](#label-discovery) +5. [Metric Analysis](#metric-analysis) +6. [PromQL Formatting](#promql-formatting) + +--- + +## Instant Query + +Evaluate a PromQL expression at a single point in time. + +```bash +promtool query instant [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--time` | now | Evaluation time (RFC3339 or Unix timestamp) | +| `-o` / `--format` | `promql` | Output format: `promql` or `json` | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# Basic query +promtool query instant --http.config.file="$HTTP_CONFIG" "$PROM_URL" 'up' + +# JSON output — promtool outputs a raw JSON array: [{metric:{...}, value:[ts, val]}, ...] +promtool query instant --http.config.file="$HTTP_CONFIG" -o json "$PROM_URL" 'up' | jq . + +# Query at a specific time +promtool query instant --http.config.file="$HTTP_CONFIG" \ + --time="2024-01-15T10:00:00Z" \ + "$PROM_URL" 'up' + +# Aggregated query — extract namespace and value +promtool query instant --http.config.file="$HTTP_CONFIG" -o json \ + "$PROM_URL" 'sum(rate(container_cpu_usage_seconds_total[5m])) by (namespace)' \ + | jq '.[] | {namespace: .metric.namespace, value: .value[1]}' + +# Top 10 memory consumers +promtool query instant --http.config.file="$HTTP_CONFIG" -o json \ + "$PROM_URL" 'topk(10, container_memory_working_set_bytes{container!=""})' \ + | jq '.[] | {pod: .metric.pod, namespace: .metric.namespace, bytes: .value[1]}' +``` + +--- + +## Range Query + +Evaluate a PromQL expression over a time range. + +```bash +promtool query range [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--start` | | Start time (RFC3339 or Unix timestamp, required) | +| `--end` | | End time (RFC3339 or Unix timestamp, required) | +| `--step` | | Step size (duration like `1m`, `5m`, `1h`, required) | +| `-o` / `--format` | `promql` | Output format: `promql` or `json` | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# Last hour, 1-minute resolution (cross-platform: tries GNU date first, falls back to BSD) +promtool query range --http.config.file="$HTTP_CONFIG" \ + --start="$(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1H +%Y-%m-%dT%H:%M:%SZ)" \ + --end="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --step=1m \ + "$PROM_URL" 'node_memory_MemAvailable_bytes' + +# Last 24 hours, 5-minute resolution +promtool query range --http.config.file="$HTTP_CONFIG" \ + --start="$(date -u -d '1 day ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1d +%Y-%m-%dT%H:%M:%SZ)" \ + --end="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --step=5m -o json \ + "$PROM_URL" 'avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)' | jq . +``` + +### Choosing Step Size + +- **1m** — fine-grained, last 1-2 hours +- **5m** — standard, last 6-24 hours +- **15m** — daily overview +- **1h** — weekly/monthly trends + +Rule of thumb: aim for 100-500 data points per series. + +--- + +## Series Discovery + +Find time series matching label selectors. + +```bash +promtool query series [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--match` | | Series selector (required, repeatable) | +| `--start` | | Start time | +| `--end` | | End time | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# All series for a metric +promtool query series --http.config.file="$HTTP_CONFIG" \ + --match='container_cpu_usage_seconds_total' \ + "$PROM_URL" + +# Series in a specific namespace +promtool query series --http.config.file="$HTTP_CONFIG" \ + --match='container_cpu_usage_seconds_total{namespace="kube-system"}' \ + "$PROM_URL" + +# Multiple selectors (OR) +promtool query series --http.config.file="$HTTP_CONFIG" \ + --match='up' --match='scrape_duration_seconds' \ + "$PROM_URL" +``` + +--- + +## Label Discovery + +List label names or values. + +```bash +promtool query labels [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--start` | | Start time | +| `--end` | | End time | +| `--match` | | Restrict to series matching selector (repeatable) | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# List all metric names +promtool query labels --http.config.file="$HTTP_CONFIG" "$PROM_URL" __name__ + +# List all namespaces that have metrics +promtool query labels --http.config.file="$HTTP_CONFIG" "$PROM_URL" namespace + +# List pods for a specific metric +promtool query labels --http.config.file="$HTTP_CONFIG" \ + --match='container_cpu_usage_seconds_total' \ + "$PROM_URL" pod + +# List all label names (pass empty string) +promtool query labels --http.config.file="$HTTP_CONFIG" "$PROM_URL" "" +``` + +--- + +## Metric Analysis + +Analyze metric usage patterns (e.g., histogram bucket boundaries). + +```bash +promtool query analyze [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--server` | | Prometheus server URL (required) | +| `--type` | `histogram` | Metric type to analyze | +| `--duration` | `1h` | Time frame to analyze | +| `--time` | now | Query time | +| `--match` | | Series selector (required, repeatable) | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# Analyze histogram bucket distribution +promtool query analyze \ + --server="$PROM_URL" \ + --http.config.file="$HTTP_CONFIG" \ + --match='apiserver_request_duration_seconds_bucket' \ + --duration=1h + +# Analyze a specific histogram +promtool query analyze \ + --server="$PROM_URL" \ + --http.config.file="$HTTP_CONFIG" \ + --match='http_request_duration_seconds_bucket{handler="/api/v1/query"}' \ + --duration=6h +``` + +--- + +## PromQL Formatting + +Pretty-print and manipulate PromQL expressions (requires `--experimental` flag). + +```bash +# Format / pretty-print a query +promtool --experimental promql format 'sum(rate(container_cpu_usage_seconds_total{namespace!=""}[5m])) by (namespace, pod)' + +# Add a label matcher to a query +promtool --experimental promql label-matchers set \ + 'sum(rate(http_requests_total[5m])) by (code)' \ + namespace myapp + +# Set a regex label matcher +promtool --experimental promql label-matchers set -t '=~' \ + 'up' job 'kube.*' + +# Remove a label matcher +promtool --experimental promql label-matchers delete \ + 'up{job="prometheus"}' job +``` diff --git a/monitoring/prometheus/references/tsdb.md b/monitoring/prometheus/references/tsdb.md new file mode 100644 index 0000000..e8f7b8c --- /dev/null +++ b/monitoring/prometheus/references/tsdb.md @@ -0,0 +1,213 @@ +# TSDB Operations Reference + +Analyze, inspect, and manage Prometheus TSDB data. + +These commands operate on local TSDB data directories. They are useful for cardinality analysis, debugging storage issues, and backfilling data. + +## Table of Contents + +1. [Analyze](#analyze) +2. [List Blocks](#list-blocks) +3. [Dump Data](#dump-data) +4. [Create Blocks from Rules](#create-blocks-from-rules) +5. [Benchmarking](#benchmarking) +6. [Debug](#debug) + +--- + +## Analyze + +Analyze TSDB block(s) for cardinality, label statistics, and storage efficiency. + +```bash +promtool tsdb analyze [flags] [db-path] [block-id] +``` + +| Flag | Default | Description | +|---|---|---| +| `--limit` | 20 | Number of items to show per list | +| `--extended` | false | Run extended analysis | +| `--match` | | Series selector to filter analysis | + +Defaults: `db-path` = `data/`, `block-id` = last block. + +### Examples + +```bash +# Analyze local TSDB (e.g., from a Prometheus data dir) +promtool tsdb analyze /path/to/prometheus/data + +# Show top 50 metrics by cardinality +promtool tsdb analyze --limit=50 /path/to/prometheus/data + +# Extended analysis +promtool tsdb analyze --extended /path/to/prometheus/data + +# Analyze only specific metrics +promtool tsdb analyze --match='container_cpu_usage_seconds_total' /path/to/prometheus/data + +# Analyze a specific block +promtool tsdb analyze /path/to/prometheus/data 01ABCDEF12345678 +``` + +### Output Includes + +- **Block metadata**: min/max time, duration, number of series/samples/chunks +- **Label pair cardinality**: which label pairs have the most series +- **Highest cardinality labels**: labels with the most unique values +- **Highest cardinality metric names**: metrics with the most series + +This is the go-to command for diagnosing cardinality explosions. + +--- + +## List Blocks + +List all TSDB blocks with metadata. + +```bash +promtool tsdb list [flags] [db-path] +``` + +| Flag | Default | Description | +|---|---|---| +| `-r` / `--human-readable` | false | Print sizes in human-readable format | + +### Examples + +```bash +# List blocks +promtool tsdb list /path/to/prometheus/data + +# Human-readable sizes +promtool tsdb list -r /path/to/prometheus/data +``` + +--- + +## Dump Data + +Dump raw time series data from TSDB blocks. + +```bash +promtool tsdb dump [flags] [db-path] +``` + +| Flag | Default | Description | +|---|---|---| +| `--min-time` | MinInt64 | Minimum timestamp in milliseconds | +| `--max-time` | MaxInt64 | Maximum timestamp in milliseconds | +| `--match` | `{__name__=~'(?s:.*)'}` | Series selector (repeatable) | +| `--format` | `prom` | Output: `prom` or `seriesjson` | + +### Examples + +```bash +# Dump all data in Prometheus exposition format +promtool tsdb dump /path/to/prometheus/data + +# Dump specific metrics +promtool tsdb dump --match='up' /path/to/prometheus/data + +# Dump as JSON +promtool tsdb dump --format=seriesjson /path/to/prometheus/data + +# Dump a time range (timestamps in milliseconds) +promtool tsdb dump \ + --min-time=1700000000000 \ + --max-time=1700003600000 \ + /path/to/prometheus/data +``` + +### OpenMetrics Format + +```bash +promtool tsdb dump-openmetrics [flags] [db-path] +``` + +Same flags as `dump` except no `--format` (always OpenMetrics). Requires `--experimental`. + +--- + +## Create Blocks from Rules + +Backfill recording rules by querying historical data from a running Prometheus and producing TSDB blocks. + +```bash +promtool tsdb create-blocks-from rules [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--url` | `http://localhost:9090` | Prometheus API URL | +| `--start` | | Start time (required) | +| `--end` | 3 hours ago | End time | +| `--output-dir` | `data/` | Output directory | +| `--eval-interval` | `60s` | Evaluation interval | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# Backfill a recording rule for the last 7 days +promtool tsdb create-blocks-from rules \ + --url="$PROM_URL" \ + --http.config.file="$HTTP_CONFIG" \ + --start="$(date -u -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-7d +%Y-%m-%dT%H:%M:%SZ)" \ + --output-dir=/tmp/backfill-blocks \ + recording-rules.yml +``` + +### Import OpenMetrics Data + +```bash +promtool tsdb create-blocks-from openmetrics [flags] [output-dir] +``` + +| Flag | Default | Description | +|---|---|---| +| `-r` | false | Human-readable output | +| `-q` | false | Quiet mode | +| `--label` | | Labels to attach (repeatable) | + +--- + +## Benchmarking + +Benchmark TSDB write performance. + +```bash +promtool tsdb bench write [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--out` | `benchout` | Output path | +| `--metrics` | 10000 | Number of metrics | +| `--scrapes` | 3000 | Number of scrapes | + +--- + +## Debug + +Fetch debug information from a running Prometheus server. + +```bash +promtool debug pprof # Profiling data +promtool debug metrics # Current metrics +promtool debug all # Everything +``` + +All accept `--http.config.file` for authentication. + +### Examples + +```bash +# Fetch all debug info +promtool debug all "$PROM_URL" --http.config.file="$HTTP_CONFIG" + +# Profiling data only +promtool debug pprof "$PROM_URL" --http.config.file="$HTTP_CONFIG" +``` + +Output is saved to `debug.tar.gz` in the current directory. diff --git a/monitoring/prometheus/references/validation.md b/monitoring/prometheus/references/validation.md new file mode 100644 index 0000000..9ba83d5 --- /dev/null +++ b/monitoring/prometheus/references/validation.md @@ -0,0 +1,219 @@ +# Validation & Testing Reference + +Validate Prometheus configuration, lint rules, check metrics naming, and run unit tests. + +## Table of Contents + +1. [Check Config](#check-config) +2. [Check Rules](#check-rules) +3. [Check Metrics](#check-metrics) +4. [Check Health & Readiness](#check-health--readiness) +5. [Unit Testing Rules](#unit-testing-rules) + +--- + +## Check Config + +Validate Prometheus configuration files for syntax and semantic correctness. + +```bash +promtool check config [config-file...] +``` + +| Flag | Default | Description | +|---|---|---| +| `--syntax-only` | false | Only check YAML syntax, skip semantic validation | +| `--lint` | `duplicate-rules` | Linting: `all`, `duplicate-rules`, `too-long-scrape-interval`, `none` | +| `--lint-fatal` | false | Make lint errors exit with code 3 | +| `--agent` | false | Check config for Prometheus Agent mode | + +### Examples + +```bash +# Validate a config file +promtool check config prometheus.yml + +# Syntax check only (faster, no network) +promtool check config --syntax-only prometheus.yml + +# Full linting +promtool check config --lint=all --lint-fatal prometheus.yml +``` + +--- + +## Check Rules + +Validate Prometheus rule files (alerting and recording rules). + +```bash +promtool check rules [rule-file...] # reads stdin if no files +``` + +| Flag | Default | Description | +|---|---|---| +| `--lint` | `duplicate-rules` | Linting: `all`, `duplicate-rules`, `none` | +| `--lint-fatal` | false | Make lint errors exit with code 3 | + +### Examples + +```bash +# Validate rule files +promtool check rules alerts.yml recording-rules.yml + +# Validate all rule files in a directory +promtool check rules rules/*.yml + +# Read from stdin +cat my-rules.yml | promtool check rules + +# Strict linting +promtool check rules --lint=all --lint-fatal alerts.yml +``` + +--- + +## Check Metrics + +Lint metrics exposition for naming conventions and best practices. + +```bash +promtool check metrics [flags] +``` + +Reads metrics in Prometheus exposition format from stdin. + +| Flag | Default | Description | +|---|---|---| +| `--extended` | false | Print extended cardinality information | +| `--lint` | `all` | Linting: `all`, `none` | + +### Examples + +```bash +# Lint metrics from a running Prometheus +curl -s http://localhost:9090/metrics | promtool check metrics + +# Lint metrics from any instrumented endpoint +curl -s http://localhost:8080/metrics | promtool check metrics --extended + +# From a cluster pod via port-forward +kubectl port-forward -n 8080:8080 & +curl -s http://localhost:8080/metrics | promtool check metrics --extended +kill %1 +``` + +Common warnings: +- `counter should have _total suffix` +- `histogram should have _bucket/_count/_sum` +- `metric name should match [a-zA-Z_:][a-zA-Z0-9_:]*` +- `help string missing` + +--- + +## Check Health & Readiness + +Check if a Prometheus server is healthy or ready. + +```bash +promtool check healthy [flags] +promtool check ready [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--url` | `http://localhost:9090` | Prometheus server URL | +| `--http.config.file` | | HTTP client config file | + +### Examples + +```bash +# Check health +promtool check healthy --url="$PROM_URL" --http.config.file="$HTTP_CONFIG" + +# Check readiness +promtool check ready --url="$PROM_URL" --http.config.file="$HTTP_CONFIG" +``` + +--- + +## Unit Testing Rules + +Test alerting and recording rules against synthetic time series data without a running Prometheus server. + +```bash +promtool test rules [flags] +``` + +| Flag | Default | Description | +|---|---|---| +| `--run` | | Only run test groups matching regex (repeatable) | +| `--debug` | false | Print debug output | +| `--diff` | false | Print colored diff output | +| `--junit` | | File path for JUnit XML results | + +### Test File Format + +```yaml +# test-alerts.yml +rule_files: + - alerts.yml + - recording-rules.yml + +evaluation_interval: 1m + +tests: + - interval: 1m + input_series: + - series: 'up{job="prometheus", instance="localhost:9090"}' + values: '1 1 1 0 0 0 0 0 0 0' + # Expanding notation: start+incrementxcount + - series: 'http_requests_total{method="GET"}' + values: '0+10x10' # 0, 10, 20, 30, ..., 100 + + # Test alerting rules + alert_rule_test: + - eval_time: 5m + alertname: InstanceDown + exp_alerts: + - exp_labels: + severity: critical + job: prometheus + instance: 'localhost:9090' + exp_annotations: + summary: 'Instance localhost:9090 down' + + # Test PromQL expressions / recording rules + promql_expr_test: + - expr: 'http_requests_total' + eval_time: 5m + exp_samples: + - labels: 'http_requests_total{method="GET"}' + value: 50 +``` + +### Examples + +```bash +# Run all tests +promtool test rules test-alerts.yml + +# Run specific test groups +promtool test rules --run="InstanceDown" test-alerts.yml + +# Debug output +promtool test rules --debug --diff test-alerts.yml + +# JUnit output (CI integration) +promtool test rules --junit=results.xml test-alerts.yml +``` + +### Expanding Notation for Input Series + +| Pattern | Expansion | +|---|---| +| `1 2 3` | Literal values at each interval | +| `0+1x5` | 0, 1, 2, 3, 4, 5 | +| `10-2x3` | 10, 8, 6, 4 | +| `_` | Stale marker | +| `1x3 _ 5x2` | 1, 1, 1, stale, 5, 5 |