deployment: add alerting rules

README.md

@@ -257,6 +257,40 @@ Values in between indicate the fraction of maximum possible churn that is occurr
 
 This helps identify jobs that create the most indexing pressure on storage, even when their current active cardinality appears moderate.
 
+### Alerting
+
+Pre-built alert rules for cardinality monitoring are available in
+[deployment/docker/rules/alerts-cardinality.yml](https://github.com/VictoriaMetrics/vmestimator/blob/main/deployment/docker/rules/use -cardinality.yml).
+
+They require two streams with the same `group_by` but different intervals to also support churn detection:
+```yaml
+# streams.yaml
+# or use example config:
+# https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml
+
+- interval: '15m'
+  group_by: ['job']
+
+- interval: '30m'
+  group_by: ['job']
+```
+
+The included alerts are:
+
+- **JobTooHighCardinality** — fires when any job exceeds 20,000 estimated active series over the last 30 minutes.
+  The threshold is a starting point and should be calibrated to reflect the expected cardinality of your largest jobs.
+
+- **JobTooHighChurnRate** — fires when more than 10% of a job's series churned between the 15m and 30m windows.
+  Catches jobs that generate continuous indexing pressure even when their active series count looks moderate.
+
+- **CardinalityGroupLimitNearlyReached** — fires when the number of tracked groups exceeds 80% of the configured `group_limit`.
+  Acts as an early warning that some label value combinations may soon be dropped from individual tracking.
+
+- **CardinalityGroupLimitReached** — fires when groups are actively rejected because `group_limit` is full.
+  At this point, some label combinations are being counted in a shared "rejected" sketch rather than tracked individually.
+
+All alerts link to the [Cardinality Explorer dashboard](https://play-grafana.victoriametrics.com/d/mktd5h8/).
+
 ## Alternative solutions
 
 ### PromQL

deployment/docker/rules/alerts-cardinality.yml

@@ -0,0 +1,69 @@
+# Default alerts for vmestimator. These are recommendations and may require
+# threshold calibration for your specific setup. They are based on the example
+# configuration at https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml
+groups:
+  - name: 'cardinality'
+    rules:
+      - alert: 'JobTooHighCardinality'
+        expr: 'max (cardinality_estimate{group_by_keys="job",interval="30m0s"}) without(instance) > 20000'
+        for: '15m'
+        labels:
+          severity: 'warning'
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
+          summary: "High cardinality for job {{ $labels.group_by_values }}"
+          description: "Job {{ $labels.group_by_values }} has {{ $value }} estimated series over the last {{ $labels.interval }}, exceeding the 20000 series threshold."
+
+      - alert: 'JobTooHighChurnRate'
+        expr: |
+          (
+              max(cardinality_estimate{group_by_keys="job",interval="30m0s"}) without (instance)
+              -
+              max(cardinality_estimate{group_by_keys="job",interval="15m0s"}) without (instance)
+          )
+          /
+          max(cardinality_estimate{group_by_keys="job",interval="30m0s"}) without (instance) * 100 > 10
+        for: '15m'
+        labels:
+          severity: 'warning'
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-churn_short=15m0s&var-churn_long=30m0s"
+          summary: "High churn rate for job {{ $labels.group_by_values }}"
+          description: 'Job {{ $labels.group_by_values }} has {{ printf "%.0f%%" $value }} of series churned between the 15m and 30m cardinality windows, exceeding the 10% threshold.'
+
+      - alert: 'CardinalityGroupLimitNearlyReached'
+        expr: |
+          (
+            sum(vmestimator_estimator_group_size) by (job, group_by_keys, interval)
+            /
+            sum(vmestimator_estimator_group_limit) by (job, group_by_keys, interval)
+            * 100
+          ) > 80
+        for: '15m'
+        labels:
+          severity: 'warning'
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys={{ $labels.group_by_keys }}&var-interval={{ $labels.interval }}"
+          summary: "Cardinality estimate group limit nearly reached for {{ $labels.group_by_keys }}"
+          description: >
+            The number of tracked groups for group_by_keys={{ $labels.group_by_keys }}
+            has reached {{ printf "%.0f%%" $value }} of the configured group_limit. 
+            New groups may soon be rejected.
+
+            Consider investigating the source of the cardinality growth and either
+            reducing it or increasing group_limit in the stream configuration.
+
+      - alert: 'CardinalityGroupLimitReached'
+        expr: 'sum(vmestimator_estimator_group_rejected_size) by (job, group_by_keys, interval) > 0'
+        for: '5m'
+        labels:
+          severity: 'critical'
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys={{ $labels.group_by_keys }}&var-interval={{ $labels.interval }}"
+          summary: "Group limit reached for {{ $labels.group_by_keys }}"
+          description: >
+            {{ printf "%.0f" $value }} groups for group_by_keys={{ $labels.group_by_keys }}
+            cannot be tracked individually because group_limit has been reached.
+
+            Consider investigating the source of the cardinality growth and either
+            reducing it or increasing group_limit in the stream configuration.