diff --git a/.github/workflows/prometheus-rules.yml b/.github/workflows/prometheus-rules.yml new file mode 100644 index 0000000..26658f2 --- /dev/null +++ b/.github/workflows/prometheus-rules.yml @@ -0,0 +1,43 @@ +name: Prometheus Rules + +on: + pull_request: + paths: + - "deploy/observability/**" + - ".github/workflows/prometheus-rules.yml" + push: + branches: ["main"] + paths: + - "deploy/observability/**" + - ".github/workflows/prometheus-rules.yml" + +concurrency: + group: promtool-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + name: promtool check + runs-on: ubuntu-latest + + env: + PROMETHEUS_IMAGE: prom/prometheus:v2.55.1 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: promtool check config + run: | + docker run --rm --entrypoint promtool \ + -v "$PWD/deploy/observability:/cfg" "$PROMETHEUS_IMAGE" \ + check config /cfg/prometheus.yml + + - name: promtool check rules + run: | + docker run --rm --entrypoint promtool \ + -v "$PWD/deploy/observability:/cfg" "$PROMETHEUS_IMAGE" \ + check rules /cfg/alerts.yml diff --git a/deploy/observability/alerts.yml b/deploy/observability/alerts.yml new file mode 100644 index 0000000..573c67f --- /dev/null +++ b/deploy/observability/alerts.yml @@ -0,0 +1,64 @@ +# Prometheus alerting rules for the local observability profile. +# Availability SLO 99% (1% error budget); multi-window multi-burn-rate on the 5xx ratio. +groups: + - name: fincore-availability + rules: + - alert: InstanceDown + expr: up == 0 + for: 2m + labels: + severity: P0 + annotations: + summary: "Target {{ $labels.job }} is down" + description: "Prometheus cannot scrape {{ $labels.job }} ({{ $labels.instance }}) for 2m." + + - alert: ErrorBudgetBurnFast + expr: | + sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[5m])) by (job) + / sum(rate(http_server_requests_seconds_count[5m])) by (job) > 0.144 + and + sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[1h])) by (job) + / sum(rate(http_server_requests_seconds_count[1h])) by (job) > 0.144 + for: 2m + labels: + severity: P1 + annotations: + summary: "Fast error-budget burn on {{ $labels.job }}" + description: "{{ $labels.job }} is burning the 99% availability budget at >14.4x (5m and 1h windows)." + + - alert: ErrorBudgetBurnSlow + expr: | + sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[30m])) by (job) + / sum(rate(http_server_requests_seconds_count[30m])) by (job) > 0.06 + and + sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[6h])) by (job) + / sum(rate(http_server_requests_seconds_count[6h])) by (job) > 0.06 + for: 15m + labels: + severity: P2 + annotations: + summary: "Slow error-budget burn on {{ $labels.job }}" + description: "{{ $labels.job }} is burning the 99% availability budget at >6x (30m and 6h windows)." + + - name: fincore-latency + rules: + - alert: HighRequestLatencyP95 + expr: | + histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket[5m])) by (le, job)) > 0.5 + for: 10m + labels: + severity: P2 + annotations: + summary: "High p95 request latency on {{ $labels.job }}" + description: "p95 request latency on {{ $labels.job }} is {{ $value | humanizeDuration }} (> 0.5s) over 10m." + + - name: fincore-health + rules: + - alert: TargetFlapping + expr: changes(up[15m]) >= 4 + for: 0m + labels: + severity: P3 + annotations: + summary: "Target {{ $labels.job }} is flapping" + description: "{{ $labels.job }} ({{ $labels.instance }}) changed up/down state >= 4 times in 15m." diff --git a/deploy/observability/prometheus.yml b/deploy/observability/prometheus.yml index a76009b..0237931 100644 --- a/deploy/observability/prometheus.yml +++ b/deploy/observability/prometheus.yml @@ -3,6 +3,9 @@ global: scrape_interval: 15s +rule_files: + - alerts.yml + scrape_configs: - job_name: ledger metrics_path: /actuator/prometheus diff --git a/docker-compose.yml b/docker-compose.yml index e8459ec..7e77458 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,6 +79,7 @@ services: profiles: [observability] volumes: - ./deploy/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./deploy/observability/alerts.yml:/etc/prometheus/alerts.yml:ro ports: - "9090:9090"