From f729454b04f288a17b72b77fada6fc803ad88e8b Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:46:59 +0000 Subject: [PATCH 1/7] test: cover rollup watermark selection --- .github/workflows/test.yml | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c829022..3c2cfaf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7081,6 +7081,126 @@ jobs: run: | psql -h localhost -U postgres -d postgres -v ON_ERROR_STOP=1 -f devel/tests/query_attribution_assert.sql + - name: "Test 2.0: stale and rounded-up rollup source selection" + env: + PGPASSWORD: postgres + run: | + psql -h localhost -U postgres -d postgres -v ON_ERROR_STOP=1 << 'EOF' + -- A wide aggregate read may prefer rollup_1m only when that rollup + -- covers the requested end. If raw covers the whole window while the + -- rollup worker lags, choosing rollup silently drops the recent load. + -- Conversely, a healthy rollup must keep the fast path when a + -- dashboard rounds until up to the next minute boundary. + do $$ + declare + v_wait smallint; + v_start timestamptz := date_trunc('minute', now()) - interval '2 hours'; + v_end timestamptz := date_trunc('minute', now()); + v_start_ts int4; + v_recent_ts int4; + v_config record; + a record; + begin + select + current_slot, + num_partitions, + sample_interval, + rotation_period, + rotated_at, + last_rollup_1m_ts, + last_rollup_1h_ts + into v_config + from ash.config + where singleton; + + truncate ash.sample_0, ash.sample_1, ash.sample_2; + truncate ash.rollup_1m, ash.rollup_1h; + update ash.config + set current_slot = 0, + num_partitions = 3, + sample_interval = interval '1 second', + rotation_period = interval '1 day', + rotated_at = v_end, + last_rollup_1m_ts = ash.ts_from_timestamptz( + v_start + interval '1 minute' + ), + last_rollup_1h_ts = null + where singleton; + + select ash._register_wait('active', 'IO', 'StaleRollupProof') + into v_wait; + v_start_ts := ash.ts_from_timestamptz(v_start); + v_recent_ts := ash.ts_from_timestamptz(v_end - interval '1 minute'); + + insert into ash.rollup_1m ( + ts, datid, samples, peak_backends, wait_counts, query_counts + ) values ( + v_start_ts, 0::oid, 1, 1, + array[v_wait, 1]::int4[], '{}'::int8[] + ); + insert into ash.sample ( + sample_ts, datid, active_count, data, slot + ) values + (v_start_ts, 0::oid, 1, array[-v_wait, 1, 0]::int4[], 0), + (v_recent_ts, 0::oid, 1, array[-v_wait, 1, 0]::int4[], 0); + + select * into a from ash.aas(v_start, v_end); + assert a.source = 'raw', + 'stale rollup must fall back to raw, got ' || a.source; + assert a.buckets_with_data = 2, + 'raw fallback should retain both covered minutes, got ' + || a.buckets_with_data; + assert a.backend_seconds = 2.00, + 'raw fallback should retain both backend-seconds, got ' + || a.backend_seconds; + + truncate ash.rollup_1m; + insert into ash.rollup_1m ( + ts, datid, samples, peak_backends, wait_counts, query_counts + ) + select + ash.ts_from_timestamptz(minute_ts), + 0::oid, + 1, + 1, + array[v_wait, 1]::int4[], + '{}'::int8[] + from generate_series( + v_start, + v_end - interval '1 minute', + interval '1 minute' + ) as minute_ts; + update ash.config + set last_rollup_1m_ts = ash.ts_from_timestamptz(v_end) + where singleton; + + select * into a + from ash.aas(v_start, v_end + interval '1 minute'); + assert a.source = 'rollup_1m', + 'healthy rollup must keep the fast path for a rounded-up until, ' + 'got ' || a.source; + assert a.buckets_with_data = 120, + 'healthy rollup should cover 120 complete minutes, got ' + || a.buckets_with_data; + assert a.backend_seconds = 120.00, + 'healthy rollup should retain 120 backend-seconds, got ' + || a.backend_seconds; + + truncate ash.sample_0, ash.sample_1, ash.sample_2; + truncate ash.rollup_1m, ash.rollup_1h; + update ash.config + set current_slot = v_config.current_slot, + num_partitions = v_config.num_partitions, + sample_interval = v_config.sample_interval, + rotation_period = v_config.rotation_period, + rotated_at = v_config.rotated_at, + last_rollup_1m_ts = v_config.last_rollup_1m_ts, + last_rollup_1h_ts = v_config.last_rollup_1h_ts + where singleton; + raise notice 'stale/rounded-up rollup source selection PASSED'; + end $$; + EOF + - name: "Test 2.0: minute-grain peak/p99 survive the rollup_1h seam" env: PGPASSWORD: postgres From 8376c0ff5c99947e08dc84a404665a47fb1e2b41 Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:47:59 +0000 Subject: [PATCH 2/7] fix: avoid stale rollup reads --- sql/ash-install.sql | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/sql/ash-install.sql b/sql/ash-install.sql index b5317c8..e884df7 100644 --- a/sql/ash-install.sql +++ b/sql/ash-install.sql @@ -4041,12 +4041,16 @@ $$; * grain, so for anything wider than ~1 hour whose requested start is within * rollup_1m retention we prefer rollup_1m (a raw decode of a wide window spills * hundreds of MB — the last-24h read cost ~4.5s and ~500MB before this). - * #122 tracks the separate end-watermark/completeness gap. Narrow windows + * The rollup watermark must + * also reach the latest complete minute whenever raw covers the window start; + * otherwise a stalled rollup worker would hide newer raw load. Narrow windows * still fall through to _pick_source (raw preferred) so the freshest partial - * minute is captured, and windows rollup can't cover (or where rollup is - * disabled/lagging) still fall to raw / rollup_1h. Exact-query drills and - * samples bypass this and force raw. The source column stays honest — it names - * whatever was actually read. + * minute is captured. When raw covers the window start, a rollup that is + * disabled, cannot cover it, or lags the requested end falls back through + * _pick_source. Completeness for a stalled rollup when the window starts + * before physical raw coverage remains tracked by #122. Exact-query drills + * and samples bypass this and force raw. The source column stays honest — it + * names whatever was actually read. */ create or replace function ash._pick_source_agg( since timestamptz, @@ -4060,7 +4064,23 @@ as $$ select case when extract(epoch from (until - since)) > 3600 and ash._rollup_1m_retention_start() is not null - and since >= ash._rollup_1m_retention_start() then 'rollup_1m' + and since >= ash._rollup_1m_retention_start() + and ( + ash._raw_retention_start() is null + or since < ash._raw_retention_start() + or coalesce( + ( + select ash.ts_to_timestamptz(last_rollup_1m_ts) + >= least( + until, + date_trunc('minute', now()) + ) + from ash.config + where singleton + ), + false + ) + ) then 'rollup_1m' else ash._pick_source(since) end $$; From 0420a3ce05b154a1c331df97c6f80b9090325106 Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:53:34 +0000 Subject: [PATCH 3/7] test: isolate rollup selector fixture --- .github/workflows/test.yml | 45 +++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c2cfaf..e101263 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7086,6 +7086,8 @@ jobs: PGPASSWORD: postgres run: | psql -h localhost -U postgres -d postgres -v ON_ERROR_STOP=1 << 'EOF' + begin; + -- A wide aggregate read may prefer rollup_1m only when that rollup -- covers the requested end. If raw covers the whole window while the -- rollup worker lags, choosing rollup silently drops the recent load. @@ -7096,23 +7098,11 @@ jobs: v_wait smallint; v_start timestamptz := date_trunc('minute', now()) - interval '2 hours'; v_end timestamptz := date_trunc('minute', now()); + v_historical_end timestamptz := v_end - interval '30 minutes'; v_start_ts int4; v_recent_ts int4; - v_config record; a record; begin - select - current_slot, - num_partitions, - sample_interval, - rotation_period, - rotated_at, - last_rollup_1m_ts, - last_rollup_1h_ts - into v_config - from ash.config - where singleton; - truncate ash.sample_0, ash.sample_1, ash.sample_2; truncate ash.rollup_1m, ash.rollup_1h; update ash.config @@ -7171,9 +7161,23 @@ jobs: interval '1 minute' ) as minute_ts; update ash.config - set last_rollup_1m_ts = ash.ts_from_timestamptz(v_end) + set last_rollup_1m_ts = ash.ts_from_timestamptz(v_historical_end) where singleton; + select * into a from ash.aas(v_start, v_historical_end); + assert a.source = 'rollup_1m', + 'healthy historical rollup must use its covered watermark, got ' + || a.source; + assert a.buckets_with_data = 90, + 'historical rollup should cover 90 complete minutes, got ' + || a.buckets_with_data; + assert a.backend_seconds = 90.00, + 'historical rollup should retain 90 backend-seconds, got ' + || a.backend_seconds; + + update ash.config + set last_rollup_1m_ts = ash.ts_from_timestamptz(v_end) + where singleton; select * into a from ash.aas(v_start, v_end + interval '1 minute'); assert a.source = 'rollup_1m', @@ -7186,19 +7190,10 @@ jobs: 'healthy rollup should retain 120 backend-seconds, got ' || a.backend_seconds; - truncate ash.sample_0, ash.sample_1, ash.sample_2; - truncate ash.rollup_1m, ash.rollup_1h; - update ash.config - set current_slot = v_config.current_slot, - num_partitions = v_config.num_partitions, - sample_interval = v_config.sample_interval, - rotation_period = v_config.rotation_period, - rotated_at = v_config.rotated_at, - last_rollup_1m_ts = v_config.last_rollup_1m_ts, - last_rollup_1h_ts = v_config.last_rollup_1h_ts - where singleton; raise notice 'stale/rounded-up rollup source selection PASSED'; end $$; + + rollback; EOF - name: "Test 2.0: minute-grain peak/p99 survive the rollup_1h seam" From be184fc5911fd260d30f007ca805523a035298e8 Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:54:45 +0000 Subject: [PATCH 4/7] fix: follow picker raw coverage --- sql/ash-install.sql | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/sql/ash-install.sql b/sql/ash-install.sql index e884df7..3ead861 100644 --- a/sql/ash-install.sql +++ b/sql/ash-install.sql @@ -4041,16 +4041,15 @@ $$; * grain, so for anything wider than ~1 hour whose requested start is within * rollup_1m retention we prefer rollup_1m (a raw decode of a wide window spills * hundreds of MB — the last-24h read cost ~4.5s and ~500MB before this). - * The rollup watermark must - * also reach the latest complete minute whenever raw covers the window start; - * otherwise a stalled rollup worker would hide newer raw load. Narrow windows - * still fall through to _pick_source (raw preferred) so the freshest partial - * minute is captured. When raw covers the window start, a rollup that is - * disabled, cannot cover it, or lags the requested end falls back through - * _pick_source. Completeness for a stalled rollup when the window starts - * before physical raw coverage remains tracked by #122. Exact-query drills - * and samples bypass this and force raw. The source column stays honest — it - * names whatever was actually read. + * also reach the latest complete minute whenever raw is the preferred source + * at the window start; otherwise a stalled rollup worker would hide newer raw + * load. Narrow windows still fall through to _pick_source (raw preferred) so + * the freshest partial minute is captured. When raw covers the window start, + * a rollup that is disabled, cannot cover it, or lags the requested end falls + * back through _pick_source. Completeness for a stalled rollup when the window + * starts before physical raw coverage remains tracked by #122. Exact-query + * drills and samples bypass this and force raw. The source column stays honest + * — it names whatever was actually read. */ create or replace function ash._pick_source_agg( since timestamptz, @@ -4061,13 +4060,15 @@ language sql stable set search_path = pg_catalog, ash as $$ + with fallback as ( + select ash._pick_source(since) as source + ) select case when extract(epoch from (until - since)) > 3600 and ash._rollup_1m_retention_start() is not null and since >= ash._rollup_1m_retention_start() and ( - ash._raw_retention_start() is null - or since < ash._raw_retention_start() + fallback.source <> 'raw' or coalesce( ( select ash.ts_to_timestamptz(last_rollup_1m_ts) @@ -4081,8 +4082,9 @@ as $$ false ) ) then 'rollup_1m' - else ash._pick_source(since) + else fallback.source end + from fallback $$; /* From 8288af51a82d746b8aa428d63f3bfa3a138f4344 Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:48:31 +0000 Subject: [PATCH 5/7] test: pin valid rollup retention --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e101263..cfc0a3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7110,6 +7110,7 @@ jobs: num_partitions = 3, sample_interval = interval '1 second', rotation_period = interval '1 day', + rollup_1m_retention_days = 30, rotated_at = v_end, last_rollup_1m_ts = ash.ts_from_timestamptz( v_start + interval '1 minute' From 1c76907791ae4256f807249a7b820c10fcf6fa7b Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 22:17:58 +0000 Subject: [PATCH 6/7] docs: bound rollup coverage claim --- .github/workflows/test.yml | 7 ++++--- sql/ash-install.sql | 18 +++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cfc0a3a..a57147c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7088,9 +7088,10 @@ jobs: psql -h localhost -U postgres -d postgres -v ON_ERROR_STOP=1 << 'EOF' begin; - -- A wide aggregate read may prefer rollup_1m only when that rollup - -- covers the requested end. If raw covers the whole window while the - -- rollup worker lags, choosing rollup silently drops the recent load. + -- When raw covers a wide aggregate window, rollup_1m may replace it + -- only if that rollup covers the requested end. Otherwise a lagging + -- worker would silently drop recent raw load. Completeness for a + -- stalled rollup older than physical raw coverage remains #122. -- Conversely, a healthy rollup must keep the fast path when a -- dashboard rounds until up to the next minute boundary. do $$ diff --git a/sql/ash-install.sql b/sql/ash-install.sql index 3ead861..84da148 100644 --- a/sql/ash-install.sql +++ b/sql/ash-install.sql @@ -4041,15 +4041,15 @@ $$; * grain, so for anything wider than ~1 hour whose requested start is within * rollup_1m retention we prefer rollup_1m (a raw decode of a wide window spills * hundreds of MB — the last-24h read cost ~4.5s and ~500MB before this). - * also reach the latest complete minute whenever raw is the preferred source - * at the window start; otherwise a stalled rollup worker would hide newer raw - * load. Narrow windows still fall through to _pick_source (raw preferred) so - * the freshest partial minute is captured. When raw covers the window start, - * a rollup that is disabled, cannot cover it, or lags the requested end falls - * back through _pick_source. Completeness for a stalled rollup when the window - * starts before physical raw coverage remains tracked by #122. Exact-query - * drills and samples bypass this and force raw. The source column stays honest - * — it names whatever was actually read. + * The rollup watermark must also reach the latest complete minute whenever raw + * is the preferred source at the window start; otherwise a stalled rollup + * worker would hide newer raw load. Narrow windows still fall through to + * _pick_source (raw preferred) so the freshest partial minute is captured. + * When raw covers the window start, a rollup that is disabled, cannot cover it, + * or lags the requested end falls back through _pick_source. Completeness for a + * stalled rollup when the window starts before physical raw coverage remains + * tracked by #122. Exact-query drills and samples bypass this and force raw. + * The source column stays honest — it names whatever was actually read. */ create or replace function ash._pick_source_agg( since timestamptz, From de169f46533d54ad312694bb760b28f1998df73c Mon Sep 17 00:00:00 2001 From: samo-agent <280144521+samo-agent@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:39:34 +0000 Subject: [PATCH 7/7] docs: note stale-rollup source correction Record the deliberate reader behavior change: wide windows fall back to raw when the canonical raw source is newer than the rollup watermark, while a dashboard end rounded into the next minute keeps a healthy rollup fast path. Keep the older-than-physical-raw completeness case explicitly open under #122. --- RELEASE_NOTES.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 3ebdb9a..fbcfa1b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -70,6 +70,13 @@ AAS-oriented 2.0 API: `ash.rollup_hour()` return processed minutes and hours rather than per-database rows upserted, so activity from multiple databases no longer inflates scheduler-visible results. (issue #191) +- **Wide aggregate readers now reject stale rollup coverage without losing the + fast path.** When raw is the canonical source for a wide window, readers fall + back to it if `rollup_1m` has not reached the requested end, clamped at the + latest complete-minute boundary; a dashboard `until` rounded up to the next + minute no longer makes a healthy rollup fall back to raw. Completeness when a + window starts before physical raw coverage remains tracked by issue #122. + ([PR #199](https://github.com/NikolayS/pg_ash/pull/199)) ## Retired tooling