From c085d098f60764b61be70b3883dc837e94405024 Mon Sep 17 00:00:00 2001
From: Max Kotliar <mkotlyar@victoriametrics.com>
Date: Sun, 28 Jun 2026 10:58:02 +0300
Subject: [PATCH 1/4] deployment: add alerting rules

---
 .../docker/rules/alerts-cardinality.yml       | 87 +++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 deployment/docker/rules/alerts-cardinality.yml

diff --git a/deployment/docker/rules/alerts-cardinality.yml b/deployment/docker/rules/alerts-cardinality.yml
new file mode 100644
index 0000000..a2ba6de
--- /dev/null
+++ b/deployment/docker/rules/alerts-cardinality.yml
@@ -0,0 +1,87 @@
+# File contains default list of alerts for vmestimator service.
+# The alerts below are just recommendations and may require some updates
+# and threshold calibration according to every specific setup.
+groups:
+  - name: 'cardinality'
+    rules:
+      - alert: 'JobCardinalityThresholdExceeded'
+        expr: 'max without(instance) (cardinality_estimate{group_by_keys="job"}) > 5000'
+        for: '5m'
+        labels:
+          severity: 'warning'
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
+          summary: "High cardinality for job {{ $labels.by_job }}"
+          description: "Job {{ $labels.by_job }} has {{ $value }} cardinality over the last {{ $labels.interval }}, exceeding the 5000 series threshold."
+
+      - alert: 'Cardinality10xWithin1h'
+        expr: |
+          (
+            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"})
+            /
+            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"} offset 1h)
+          ) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
+          summary: "Cardinality spike for {{ $labels.group_by_keys }}={{ $labels.group_by_values }}"
+          description: >
+            {{ $labels.group_by_keys }}={{ $labels.group_by_values }} cardinality over the last {{ $labels.interval }} is {{ printf "%.1fx" $value }}, which is 10x higher than 1 hour ago.
+            Check for new high-cardinality labels recently introduced.
+
+      - alert: 'Cardinality10xWithin24h'
+        expr: |
+          (
+            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"})
+            /
+            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"} offset 24h)
+          ) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
+          summary: "Cardinality spike for {{ $labels.group_by_keys }}={{ $labels.group_by_values }}"
+          description: >
+            {{ $labels.group_by_keys }}={{ $labels.group_by_values }} cardinality over the last {{ $labels.interval }} is {{ printf "%.1fx" $value }}, which is 10x higher than 24 hours ago.
+            Check for new high-cardinality labels recently introduced.
+
+      - alert: 'CardinalityGroupLimitNearlyReached'
+        expr: |
+          (
+            sum(vmestimator_estimator_group_size) by (job, group_by_keys, interval)
+            /
+            sum(vmestimator_estimator_group_limit) by (job, group_by_keys, interval)
+          ) > 0.8
+        for: '5m'
+        labels:
+          severity: 'warning'
+        annotations:
+          summary: "Cardinality estimate group limit nearly reached for {{ $labels.group_by_keys }}"
+          description: >
+            The number of tracked groups for group_by_keys={{ $labels.group_by_keys }}
+            has reached {{ printf "%.0f%%" (mulf $value 100) }} of the configured
+            group_limit.
+            
+            New groups may soon be rejected.
+            
+            Consider investigating the source of the cardinality growth and either
+            reducing it or increasing group_limit in the stream configuration.
+
+      - alert: 'CardinalityGroupLimitReached'
+        expr: 'sum(vmestimator_estimator_group_rejected_size) by (job, group_by_keys, interval) > 0'
+        for: '5m'
+        labels:
+          severity: 'critical'
+        annotations:
+          summary: "Group limit reached for {{ $labels.group_by_keys }}"
+          description: >
+            {{ printf "%.0f" $value }} groups for group_by_keys={{ $labels.group_by_keys }}
+            cannot be tracked individually because group_limit has been reached.
+            
+            These groups are being counted in the shared overflow sketch.
+            
+            Consider investigating the source of the cardinality growth and either
+            reducing it or increasing group_limit in the stream configuration.
\ No newline at end of file

From 4d7a19a34908bdabb0e663aa3e176796f0514732 Mon Sep 17 00:00:00 2001
From: Max Kotliar <mkotlyar@victoriametrics.com>
Date: Mon, 29 Jun 2026 17:30:04 +0300
Subject: [PATCH 2/4] deployment: upd alerts

---
 .../docker/rules/alerts-cardinality.yml       | 51 ++++---------------
 1 file changed, 9 insertions(+), 42 deletions(-)

diff --git a/deployment/docker/rules/alerts-cardinality.yml b/deployment/docker/rules/alerts-cardinality.yml
index a2ba6de..4d84a78 100644
--- a/deployment/docker/rules/alerts-cardinality.yml
+++ b/deployment/docker/rules/alerts-cardinality.yml
@@ -5,48 +5,14 @@ groups:
   - name: 'cardinality'
     rules:
       - alert: 'JobCardinalityThresholdExceeded'
-        expr: 'max without(instance) (cardinality_estimate{group_by_keys="job"}) > 5000'
-        for: '5m'
+        expr: 'max without(instance) (cardinality_estimate{group_by_keys="job"}) > 20000'
+        for: '15m'
         labels:
           severity: 'warning'
         annotations:
           dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
           summary: "High cardinality for job {{ $labels.by_job }}"
-          description: "Job {{ $labels.by_job }} has {{ $value }} cardinality over the last {{ $labels.interval }}, exceeding the 5000 series threshold."
-
-      - alert: 'Cardinality10xWithin1h'
-        expr: |
-          (
-            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"})
-            /
-            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"} offset 1h)
-          ) > 10
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
-          summary: "Cardinality spike for {{ $labels.group_by_keys }}={{ $labels.group_by_values }}"
-          description: >
-            {{ $labels.group_by_keys }}={{ $labels.group_by_values }} cardinality over the last {{ $labels.interval }} is {{ printf "%.1fx" $value }}, which is 10x higher than 1 hour ago.
-            Check for new high-cardinality labels recently introduced.
-
-      - alert: 'Cardinality10xWithin24h'
-        expr: |
-          (
-            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"})
-            /
-            max without(job, instance) (cardinality_estimate{group_by_keys!="__global__",group_by_keys!="__group__"} offset 24h)
-          ) > 10
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
-          summary: "Cardinality spike for {{ $labels.group_by_keys }}={{ $labels.group_by_values }}"
-          description: >
-            {{ $labels.group_by_keys }}={{ $labels.group_by_values }} cardinality over the last {{ $labels.interval }} is {{ printf "%.1fx" $value }}, which is 10x higher than 24 hours ago.
-            Check for new high-cardinality labels recently introduced.
+          description: "Job {{ $labels.by_job }} has {{ $value }} cardinality over the last {{ $labels.interval }}, exceeding the 20000 series threshold."
 
       - alert: 'CardinalityGroupLimitNearlyReached'
         expr: |
@@ -54,15 +20,17 @@ groups:
             sum(vmestimator_estimator_group_size) by (job, group_by_keys, interval)
             /
             sum(vmestimator_estimator_group_limit) by (job, group_by_keys, interval)
-          ) > 0.8
-        for: '5m'
+            * 100
+          ) > 80
+        for: '15m'
         labels:
           severity: 'warning'
         annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys{{ $labels.group_by_keys }}=job&var-interval={{ $labels.interval }}"
           summary: "Cardinality estimate group limit nearly reached for {{ $labels.group_by_keys }}"
           description: >
             The number of tracked groups for group_by_keys={{ $labels.group_by_keys }}
-            has reached {{ printf "%.0f%%" (mulf $value 100) }} of the configured
+            has reached {{ printf "%.0f%%" $value }} of the configured
             group_limit.
             
             New groups may soon be rejected.
@@ -76,12 +44,11 @@ groups:
         labels:
           severity: 'critical'
         annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys{{ $labels.group_by_keys }}=job&var-interval={{ $labels.interval }}"
           summary: "Group limit reached for {{ $labels.group_by_keys }}"
           description: >
             {{ printf "%.0f" $value }} groups for group_by_keys={{ $labels.group_by_keys }}
             cannot be tracked individually because group_limit has been reached.
             
-            These groups are being counted in the shared overflow sketch.
-            
             Consider investigating the source of the cardinality growth and either
             reducing it or increasing group_limit in the stream configuration.
\ No newline at end of file

From 1cadf2faa926f8d73be0a0d5d8728f29d96ab434 Mon Sep 17 00:00:00 2001
From: Max Kotliar <mkotlyar@victoriametrics.com>
Date: Mon, 29 Jun 2026 18:58:19 +0300
Subject: [PATCH 3/4] deployment: refine alerts

---
 .../docker/rules/alerts-cardinality.yml       | 39 +++++++++++++------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/deployment/docker/rules/alerts-cardinality.yml b/deployment/docker/rules/alerts-cardinality.yml
index 4d84a78..4a0b770 100644
--- a/deployment/docker/rules/alerts-cardinality.yml
+++ b/deployment/docker/rules/alerts-cardinality.yml
@@ -1,18 +1,35 @@
-# File contains default list of alerts for vmestimator service.
-# The alerts below are just recommendations and may require some updates
-# and threshold calibration according to every specific setup.
+# Default alerts for vmestimator. These are recommendations and may require
+# threshold calibration for your specific setup. They are based on the example
+# configuration at https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml
 groups:
   - name: 'cardinality'
     rules:
-      - alert: 'JobCardinalityThresholdExceeded'
-        expr: 'max without(instance) (cardinality_estimate{group_by_keys="job"}) > 20000'
+      - alert: 'JobTooHighCardinality'
+        expr: 'max (cardinality_estimate{group_by_keys="job",interval="30m0s"}) without(instance) > 20000'
         for: '15m'
         labels:
           severity: 'warning'
         annotations:
           dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-interval={{ $labels.interval }}"
-          summary: "High cardinality for job {{ $labels.by_job }}"
-          description: "Job {{ $labels.by_job }} has {{ $value }} cardinality over the last {{ $labels.interval }}, exceeding the 20000 series threshold."
+          summary: "High cardinality for job {{ $labels.group_by_values }}"
+          description: "Job {{ $labels.group_by_values }} has {{ $value }} estimated series over the last {{ $labels.interval }}, exceeding the 20000 series threshold."
+
+      - alert: 'JobTooHighChurnRate'
+        expr: |
+          (
+              max(cardinality_estimate{group_by_keys="job",interval="30m0s"}) without (instance)
+              -
+              max(cardinality_estimate{group_by_keys="job",interval="15m0s"}) without (instance)
+          )
+          /
+          max(cardinality_estimate{group_by_keys="job",interval="30m0s"}) without (instance) * 100 > 10
+        for: '15m'
+        labels:
+          severity: 'warning'
+        annotations:
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys=job&var-churn_short=15m0s&var-churn_long=30m0s"
+          summary: "High churn rate for job {{ $labels.group_by_values }}"
+          description: 'Job {{ $labels.group_by_values }} has {{ printf "%.0f%%" $value }} of series churned between the 15m and 30m cardinality windows, exceeding the 10% threshold.'
 
       - alert: 'CardinalityGroupLimitNearlyReached'
         expr: |
@@ -26,13 +43,11 @@ groups:
         labels:
           severity: 'warning'
         annotations:
-          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys{{ $labels.group_by_keys }}=job&var-interval={{ $labels.interval }}"
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys={{ $labels.group_by_keys }}&var-interval={{ $labels.interval }}"
           summary: "Cardinality estimate group limit nearly reached for {{ $labels.group_by_keys }}"
           description: >
             The number of tracked groups for group_by_keys={{ $labels.group_by_keys }}
-            has reached {{ printf "%.0f%%" $value }} of the configured
-            group_limit.
-            
+            has reached {{ printf "%.0f%%" $value }} of the configured group_limit. 
             New groups may soon be rejected.
             
             Consider investigating the source of the cardinality growth and either
@@ -44,7 +59,7 @@ groups:
         labels:
           severity: 'critical'
         annotations:
-          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys{{ $labels.group_by_keys }}=job&var-interval={{ $labels.interval }}"
+          dashboard: "{{ $externalURL }}/d/mktd5h8?var-group_by_keys={{ $labels.group_by_keys }}&var-interval={{ $labels.interval }}"
           summary: "Group limit reached for {{ $labels.group_by_keys }}"
           description: >
             {{ printf "%.0f" $value }} groups for group_by_keys={{ $labels.group_by_keys }}

From f47353af8f0f4737f1f6b85f64b290bee25a3913 Mon Sep 17 00:00:00 2001
From: Max Kotliar <mkotlyar@victoriametrics.com>
Date: Mon, 29 Jun 2026 19:03:49 +0300
Subject: [PATCH 4/4] readme: add alerting section to use cases

---
 README.md | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/README.md b/README.md
index a3a55f4..d33b6e6 100644
--- a/README.md
+++ b/README.md
@@ -257,6 +257,40 @@ Values in between indicate the fraction of maximum possible churn that is occurr
 
 This helps identify jobs that create the most indexing pressure on storage, even when their current active cardinality appears moderate.
 
+### Alerting
+
+Pre-built alert rules for cardinality monitoring are available in
+[deployment/docker/rules/alerts-cardinality.yml](https://github.com/VictoriaMetrics/vmestimator/blob/main/deployment/docker/rules/use -cardinality.yml).
+
+They require two streams with the same `group_by` but different intervals to also support churn detection:
+```yaml
+# streams.yaml
+# or use example config:
+# https://github.com/VictoriaMetrics/vmestimator/blob/main/streams.yaml
+
+- interval: '15m'
+  group_by: ['job']
+
+- interval: '30m'
+  group_by: ['job']
+```
+
+The included alerts are:
+
+- **JobTooHighCardinality** — fires when any job exceeds 20,000 estimated active series over the last 30 minutes.
+  The threshold is a starting point and should be calibrated to reflect the expected cardinality of your largest jobs.
+
+- **JobTooHighChurnRate** — fires when more than 10% of a job's series churned between the 15m and 30m windows.
+  Catches jobs that generate continuous indexing pressure even when their active series count looks moderate.
+
+- **CardinalityGroupLimitNearlyReached** — fires when the number of tracked groups exceeds 80% of the configured `group_limit`.
+  Acts as an early warning that some label value combinations may soon be dropped from individual tracking.
+
+- **CardinalityGroupLimitReached** — fires when groups are actively rejected because `group_limit` is full.
+  At this point, some label combinations are being counted in a shared "rejected" sketch rather than tracked individually.
+
+All alerts link to the [Cardinality Explorer dashboard](https://play-grafana.victoriametrics.com/d/mktd5h8/).
+
 ## Alternative solutions
 
 ### PromQL