From e24de0281a2c5d9b90ce2101cf2cd0418df1d629 Mon Sep 17 00:00:00 2001
From: "@tanya_r" <tiana.rogozina@gmail.com>
Date: Fri, 19 Jun 2026 02:47:44 -0300
Subject: [PATCH] feat(observability): add prometheus alert rules with slo
 burn-rate

Add P0-P3 alerting rules loaded by the observability-profile
Prometheus: an instance-down page, multi-window multi-burn-rate alerts
on the 5xx error budget (fast and slow), a p95 request-latency alert,
and a target-flapping warning. The rules reference the metrics the
services already emit and are validated in CI by promtool (check config
and check rules) against the pinned Prometheus image.

Closes #252
---
 .github/workflows/prometheus-rules.yml | 43 +++++++++++++++++
 deploy/observability/alerts.yml        | 64 ++++++++++++++++++++++++++
 deploy/observability/prometheus.yml    |  3 ++
 docker-compose.yml                     |  1 +
 4 files changed, 111 insertions(+)
 create mode 100644 .github/workflows/prometheus-rules.yml
 create mode 100644 deploy/observability/alerts.yml

diff --git a/.github/workflows/prometheus-rules.yml b/.github/workflows/prometheus-rules.yml
new file mode 100644
index 0000000..26658f2
--- /dev/null
+++ b/.github/workflows/prometheus-rules.yml
@@ -0,0 +1,43 @@
+name: Prometheus Rules
+
+on:
+  pull_request:
+    paths:
+      - "deploy/observability/**"
+      - ".github/workflows/prometheus-rules.yml"
+  push:
+    branches: ["main"]
+    paths:
+      - "deploy/observability/**"
+      - ".github/workflows/prometheus-rules.yml"
+
+concurrency:
+  group: promtool-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: promtool check
+    runs-on: ubuntu-latest
+
+    env:
+      PROMETHEUS_IMAGE: prom/prometheus:v2.55.1
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: promtool check config
+        run: |
+          docker run --rm --entrypoint promtool \
+            -v "$PWD/deploy/observability:/cfg" "$PROMETHEUS_IMAGE" \
+            check config /cfg/prometheus.yml
+
+      - name: promtool check rules
+        run: |
+          docker run --rm --entrypoint promtool \
+            -v "$PWD/deploy/observability:/cfg" "$PROMETHEUS_IMAGE" \
+            check rules /cfg/alerts.yml
diff --git a/deploy/observability/alerts.yml b/deploy/observability/alerts.yml
new file mode 100644
index 0000000..573c67f
--- /dev/null
+++ b/deploy/observability/alerts.yml
@@ -0,0 +1,64 @@
+# Prometheus alerting rules for the local observability profile.
+# Availability SLO 99% (1% error budget); multi-window multi-burn-rate on the 5xx ratio.
+groups:
+  - name: fincore-availability
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 2m
+        labels:
+          severity: P0
+        annotations:
+          summary: "Target {{ $labels.job }} is down"
+          description: "Prometheus cannot scrape {{ $labels.job }} ({{ $labels.instance }}) for 2m."
+
+      - alert: ErrorBudgetBurnFast
+        expr: |
+          sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[5m])) by (job)
+            / sum(rate(http_server_requests_seconds_count[5m])) by (job) > 0.144
+          and
+          sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[1h])) by (job)
+            / sum(rate(http_server_requests_seconds_count[1h])) by (job) > 0.144
+        for: 2m
+        labels:
+          severity: P1
+        annotations:
+          summary: "Fast error-budget burn on {{ $labels.job }}"
+          description: "{{ $labels.job }} is burning the 99% availability budget at >14.4x (5m and 1h windows)."
+
+      - alert: ErrorBudgetBurnSlow
+        expr: |
+          sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[30m])) by (job)
+            / sum(rate(http_server_requests_seconds_count[30m])) by (job) > 0.06
+          and
+          sum(rate(http_server_requests_seconds_count{outcome="SERVER_ERROR"}[6h])) by (job)
+            / sum(rate(http_server_requests_seconds_count[6h])) by (job) > 0.06
+        for: 15m
+        labels:
+          severity: P2
+        annotations:
+          summary: "Slow error-budget burn on {{ $labels.job }}"
+          description: "{{ $labels.job }} is burning the 99% availability budget at >6x (30m and 6h windows)."
+
+  - name: fincore-latency
+    rules:
+      - alert: HighRequestLatencyP95
+        expr: |
+          histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket[5m])) by (le, job)) > 0.5
+        for: 10m
+        labels:
+          severity: P2
+        annotations:
+          summary: "High p95 request latency on {{ $labels.job }}"
+          description: "p95 request latency on {{ $labels.job }} is {{ $value | humanizeDuration }} (> 0.5s) over 10m."
+
+  - name: fincore-health
+    rules:
+      - alert: TargetFlapping
+        expr: changes(up[15m]) >= 4
+        for: 0m
+        labels:
+          severity: P3
+        annotations:
+          summary: "Target {{ $labels.job }} is flapping"
+          description: "{{ $labels.job }} ({{ $labels.instance }}) changed up/down state >= 4 times in 15m."
diff --git a/deploy/observability/prometheus.yml b/deploy/observability/prometheus.yml
index a76009b..0237931 100644
--- a/deploy/observability/prometheus.yml
+++ b/deploy/observability/prometheus.yml
@@ -3,6 +3,9 @@
 global:
   scrape_interval: 15s
 
+rule_files:
+  - alerts.yml
+
 scrape_configs:
   - job_name: ledger
     metrics_path: /actuator/prometheus
diff --git a/docker-compose.yml b/docker-compose.yml
index e8459ec..7e77458 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -79,6 +79,7 @@ services:
     profiles: [observability]
     volumes:
       - ./deploy/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./deploy/observability/alerts.yml:/etc/prometheus/alerts.yml:ro
     ports:
       - "9090:9090"