fix: keep compacted query attribution honest

.github/workflows/test.yml

@@ -3712,6 +3712,159 @@ jobs:
           end $$;
           EOF
 
+      - name: "Issue #136: compacted query attribution stays honest"
+        env:
+          PGPASSWORD: postgres
+        run: |
+          psql -h localhost -U postgres -d postgres -v ON_ERROR_STOP=1 << 'EOF'
+          do $$
+          declare
+            v_ts int4 := (
+              ash.ts_from_timestamptz(
+                date_trunc('minute', now() - interval '20 minutes')
+              ) / 60
+            ) * 60;
+            v_from timestamptz;
+            v_to timestamptz;
+            v_datid oid;
+            v_wait_id smallint;
+            v_q1 int4;
+            v_q2 int4;
+            v_q3 int4;
+            v_result int;
+            v_aas record;
+            v_named record;
+            v_residual record;
+            v_rows int;
+            v_total numeric;
+            v_pct numeric;
+          begin
+            truncate ash.sample_0, ash.sample_1, ash.sample_2;
+            truncate ash.query_map_0, ash.query_map_1, ash.query_map_2
+              restart identity;
+            truncate ash.rollup_1m, ash.rollup_1h;
+            update ash.config
+            set current_slot = 0,
+                sample_interval = interval '1 second',
+                rollup_min_backend_seconds = 3,
+                last_rollup_1m_ts = null,
+                last_rollup_1h_ts = null
+            where singleton;
+
+            select oid into v_datid
+            from pg_database where datname = current_database();
+            select ash._register_wait('active', 'CPU*', 'CPU*')
+              into v_wait_id;
+
+            insert into ash.query_map_0 (query_id)
+            values (111), (222), (333);
+            select id into v_q1 from ash.query_map_0 where query_id = 111;
+            select id into v_q2 from ash.query_map_0 where query_id = 222;
+            select id into v_q3 from ash.query_map_0 where query_id = 333;
+
+            -- One real packed sample: total 26 appearances. Only q111 reaches
+            -- the minute rollup's threshold of three; q222/q333 and the 20
+            -- uncaptured appearances must survive as a NULL residual.
+            insert into ash.sample (
+              sample_ts, datid, active_count, data, slot
+            ) values (
+              v_ts,
+              v_datid,
+              26,
+              array[-v_wait_id, 26, v_q1, v_q1, v_q1, v_q2, v_q2, v_q3]
+                || array_fill(0, array[20]),
+              0
+            );
+
+            select ash.rollup_minute() into v_result;
+            assert v_result = 1,
+              'rollup_minute should create exactly one database row, got '
+              || v_result;
+            assert (select query_counts from ash.rollup_1m where ts = v_ts)
+                     = array[111, 3]::int8[],
+              'fixture must exercise compacted query_counts';
+
+            v_from := ash.ts_to_timestamptz(v_ts);
+            v_to := ash.ts_to_timestamptz(v_ts + 60);
+            truncate ash.sample_0, ash.sample_1, ash.sample_2;
+
+            select * into v_aas from ash.aas(v_from, v_to);
+            assert v_aas.source = 'rollup_1m'
+                   and v_aas.backend_seconds = 26.00,
+              format('fixture AAS mismatch: source=%s backend_seconds=%s',
+                     v_aas.source, v_aas.backend_seconds);
+
+            select * into v_named
+            from ash.top('query_id', v_from, v_to, n => 100)
+            where key = '111';
+            select * into v_residual
+            from ash.top('query_id', v_from, v_to, n => 100)
+            where key is null;
+            select count(*), sum(backend_seconds), sum(pct)
+              into v_rows, v_total, v_pct
+            from ash.top('query_id', v_from, v_to, n => 100);
+
+            assert v_named.backend_seconds = 3.00 and v_named.pct = 11.54,
+              format('named compacted query mismatch: seconds=%s pct=%s',
+                     v_named.backend_seconds, v_named.pct);
+            assert v_residual.backend_seconds = 23.00
+                   and v_residual.pct = 88.46,
+              format('NULL residual mismatch: seconds=%s pct=%s',
+                     v_residual.backend_seconds, v_residual.pct);
+            assert v_rows = 2 and v_total = v_aas.backend_seconds
+                   and v_pct = 100.00,
+              format('query breakdown does not reconcile: rows=%s total=%s pct=%s',
+                     v_rows, v_total, v_pct);
+
+            -- A concrete query filter cannot use compacted rollups: absence
+            -- there is not proof of zero. Every public aggregate path must
+            -- require exact raw attribution and compare must inherit the rule.
+            begin
+              perform * from ash.aas(v_from, v_to, query_id => 222);
+              raise exception 'ash.aas query_id filter should require raw';
+            exception when others then
+              assert sqlerrm like '%exact raw query attribution%'
+                     and sqlerrm like '%entirely outside raw retention%',
+                'ash.aas query_id retention error mismatch: ' || sqlerrm;
+            end;
+            begin
+              perform * from ash.timeline(v_from, v_to, query_id => 222);
+              raise exception 'ash.timeline query_id filter should require raw';
+            exception when others then
+              assert sqlerrm like '%exact raw query attribution%'
+                     and sqlerrm like '%entirely outside raw retention%',
+                'ash.timeline query_id retention error mismatch: ' || sqlerrm;
+            end;
+            begin
+              perform * from ash.top(
+                'database', v_from, v_to, query_id => 222
+              );
+              raise exception 'ash.top query_id filter should require raw';
+            exception when others then
+              assert sqlerrm like '%exact raw query attribution%'
+                     and sqlerrm like '%entirely outside raw retention%',
+                'ash.top query_id retention error mismatch: ' || sqlerrm;
+            end;
+            begin
+              perform * from ash.compare(
+                v_from, v_to, v_from, v_to, query_id => 222
+              );
+              raise exception 'ash.compare query_id filter should require raw';
+            exception when others then
+              assert sqlerrm like '%exact raw query attribution%'
+                     and sqlerrm like '%entirely outside raw retention%',
+                'ash.compare query_id retention error mismatch: ' || sqlerrm;
+            end;
+
+            truncate ash.rollup_1m, ash.rollup_1h;
+            truncate ash.query_map_0, ash.query_map_1, ash.query_map_2;
+            update ash.config
+            set last_rollup_1m_ts = null, last_rollup_1h_ts = null
+            where singleton;
+            raise notice 'issue #136 compacted query attribution PASSED';
+          end $$;
+          EOF
+
       - name: "Test 2.0: minute-grain peak/p99 survive the rollup_1h seam"
         env:
           PGPASSWORD: postgres

README.md

@@ -211,9 +211,16 @@ from ash.top(
 );
 ```
 
-Combining a query filter with a wait filter needs the raw wait-query tie. If
-the window is past raw retention, pg_ash raises with the raw-retention boundary
-instead of returning a fake empty result.
+Every explicit `query_id` filter needs exact raw attribution, even without a
+wait filter. A query breakdown combined with a wait filter also needs the raw
+wait-query tie. If the window is past raw retention, pg_ash raises with the
+raw-retention boundary instead of treating a compacted-away query as zero.
+
+An unfiltered `ash.top('query_id')` can use rollups efficiently. Rollup query
+IDs are compacted (low-volume IDs may be omitted, and hourly rows retain a top
+set), so named rows describe only preserved attribution. A `NULL` row carries
+everything not preserved—including uncaptured query IDs—and makes
+`backend_seconds` and `pct` reconcile to the window's total AAS.
 
 ### 5. Pull raw evidence
 

blueprints/AAS_API.md

@@ -36,9 +36,10 @@ removed (see §8). Sampling, storage, rollups, and admin/lifecycle functions
    with no coverage yields NULL columns plus a NOTICE (never a fake zero
    baseline, see §2.5); `report()` returns `jsonb`, so its source and coverage
    live inside a `coverage` object (`coverage.source`, always `rollup_1m` — see
-   §4). When a request *cannot* be answered (the event↔query tie needs raw
-   samples but the window exceeds raw retention), the reader raises a clear
-   exception naming the boundary — never a silent empty result.
+   §4). When a request *cannot* be answered exactly (an explicit `query_id`
+   filter, or an event↔query tie, needs raw samples but the window exceeds raw
+   retention), the reader raises a clear exception naming the boundary — never
+   a silent empty result or a compacted-away query reported as zero.
 6. **Self-describing.** Every function carries a catalog `comment` stating the
    unit, the column contract, and the recommended next call, so an AI agent
    can navigate via `\df+` / `obj_description()` alone.
@@ -138,7 +139,7 @@ avg_aas numeric, peak_aas numeric, p99_aas numeric)`
 `peak_aas` and `p99_aas` are **per-minute even on `rollup_1h`-backed windows**:
 `rollup_hour()` preserves per-minute totals in `rollup_1h.minute_counts`, so a
 long-window read keeps minute-grain extremes across the hourly seam (US-6). The
-only case that returns null `p99_aas` is a **wait/query-filtered**
+only case that returns null `p99_aas` is a **wait-filtered**
 `rollup_1h`-backed bucket, where the surviving grain is the hour.
 
 ### 2.4 `ash.top(dimension text, since, until, filters…, n int default 10, bucket, order_by text default 'avg')`
@@ -174,27 +175,30 @@ backend_seconds numeric, pct numeric)`
   membership). A plain `grant_reader` role without it sees null even with
   pgss installed, because pgss restricts query text to the owning role.
   Degrades to null, never errors.
-- For `dimension = 'query_id'`, the unattributed bucket is a **NULL `key`**
-  (not the literal string `'unknown'`): activity whose `query_id` was not
-  captured (client not reporting a queryid, truncated, or utility /
-  idle-in-transaction work). It is real load, not an error; `compare()` pairs
-  NULL keys and `summary()` renders it as `(unattributed)`.
-- Combinations requiring the event↔query association
-  (`'query_id'` + `wait_event`/`wait_event_type`, or `'wait_event'` +
-  `query_id`) are answerable **only from raw samples**. Past raw retention the
-  function raises, and the message now splits by how the window sits against the
-  boundary:
-  - **Window entirely past raw retention** — the tie is unrecoverable and
-    narrowing cannot help; the message says so and points at the untied
-    aggregate readers: *"…is entirely outside raw retention (raw retention
-    starts at `<ts>`). The tie is unrecoverable for that window — narrowing it
-    will not help. Use the untied aggregate readers instead: drop either the
-    wait filter or query_id (e.g. ash.aas(), ash.timeline(), ash.top() with
-    one of the two)."*
+- For `dimension = 'query_id'`, unattributed load has a **NULL `key`** (not the
+  literal string `'unknown'`). On raw reads this is activity whose `query_id`
+  was not captured (client not reporting a queryid, truncated, or utility /
+  idle-in-transaction work). On rollup reads, `query_counts` is deliberately
+  compacted: low-volume query IDs may be omitted and hourly rows retain only a
+  top set. The NULL row is therefore the exact residual between total load and
+  preserved named-query attribution. It combines uncaptured and
+  compacted-away attribution so `backend_seconds` and `pct` reconcile to total
+  AAS. `compare()` pairs NULL keys and `summary()` renders the row as
+  `(unattributed)`.
+- Every explicit `query_id` filter is answerable **only from raw samples**;
+  compacted rollups cannot prove either its exact count or its absence. A
+  `query_id` breakdown combined with `wait_event`/`wait_event_type` likewise
+  needs the raw event↔query association. Past raw retention the function
+  raises, and the message splits by how the window sits against the boundary:
+  - **Window entirely past raw retention** — exact attribution is
+    unrecoverable and narrowing cannot help; the message points at an
+    unfiltered aggregate read: *"…needs exact raw query attribution … is
+    entirely outside raw retention … narrowing it will not help. Use the
+    untied aggregate readers instead: remove query_id …"*
   - **Partial overlap** (window end still inside retention) — the message names
     the exact boundary to move to: *"…raw retention starts at `<ts>` but the
     requested window starts at `<ts>`. Narrow the window to start at or after
-    `<ts>` … or drill without the query/event tie."*
+    `<ts>` … or remove query_id."*
 - On a `rollup_1h`-backed window (`source = rollup_1h`), each row's `peak_aas`
   and `p99_aas` collapse to **hour** grain — the per-dimension arrays are stored
   per hour, so a one-minute spike is averaged into its hour. For a minute-grain
@@ -381,18 +385,19 @@ function does any of that.
 
 ## 6. Source selection & retention metadata
 
-- **Aggregate readers** (`aas`, `timeline`, `periods`, and non-tie `top` /
+- **Aggregate readers** (`aas`, `timeline`, `periods`, and unfiltered `top` /
   `chart`) prefer `rollup_1m` for any window wider than ~1 hour that
   `rollup_1m` fully covers — even when that window is still within raw
   retention. Rollups are far cheaper for wide windows and just as accurate at
   minute grain, so triage never pays the cost of decoding raw samples. `raw`
   is used only for narrow (≤ ~1 h) windows, where it is both cheap and
   freshest, and for windows rollups cannot cover.
-- **Leaf tie-drills** — any drill that needs the `wait_event ↔ query_id`
-  association (`top('query_id', wait_event/​wait_event_type => …)`,
-  `top('wait_event', query_id => …)`) and `samples` — force `raw`, because
-  rollups don't preserve that association; past raw retention they raise (§1
-  rule 5) rather than return empty.
+- **Exact query drills** — every explicit `query_id` filter, any query
+  breakdown that also filters `wait_event`/`wait_event_type`, and `samples`
+  force `raw`. Rollups compact query attribution and do not preserve the
+  event↔query association; past raw retention these calls raise (§1 rule 5)
+  rather than return a false zero. Unfiltered `top('query_id')` remains a fast
+  rollup read and reports compacted-away attribution in its NULL residual.
 - Each reader reports the source it used in the `source` column
   (`raw` | `rollup_1m` | `rollup_1h` | `none`). Scalar readers and `top` pick a
   single source per result (never mixing — no double-counting); `timeline`

blueprints/AAS_EXAMPLES.md

@@ -202,6 +202,13 @@ select * from ash.top('query_id', since => '2026-07-04 14:30', until => '2026-07
 (pg_stat_statements execution metrics no longer ride along — join on
 `query_id` yourself when you want them.)
 
+For rollup-backed windows, named rows are compacted query attribution: minute
+thresholds can omit low-volume IDs and hourly rows retain a top set. A NULL-key
+row carries the uncaptured plus compacted-away residual, so the query breakdown
+still reconciles to total AAS. Filtering for one concrete `query_id` always
+uses raw samples, because a missing rollup entry cannot distinguish zero from
+an omitted ID.
+
 ## 4. Leaf (the drill v1.x could only half-do)
 
 ### `query_waits(query_id, …)` → `ash.top('wait_event', query_id => …)`
@@ -262,29 +269,28 @@ And if you ask for a window whose raw samples are gone, you get an error, not
 an empty table — and the message differs by how the window sits against the
 raw-retention boundary.
 
-**Window entirely past raw retention** — the tie is gone for good; narrowing
-cannot recover it, so the error says so and redirects to the untied aggregate
-readers:
+**Window entirely past raw retention** — exact query attribution is gone for
+good; narrowing cannot recover it, so the error says so and redirects to an
+unfiltered aggregate read:
 
 ```
-ERROR:  pg_ash: this drill needs the raw wait<->query tie, but the requested
+ERROR:  pg_ash: this drill needs exact raw query attribution, but the requested
         window (2026-06-28 00:00:00+00 to 2026-06-28 01:00:00+00) is entirely
         outside raw retention (raw retention starts at 2026-07-03 00:00:00+00).
-        The tie is unrecoverable for that window — narrowing it will not help.
-        Use the untied aggregate readers instead: drop either the wait filter
-        or query_id (e.g. ash.aas(), ash.timeline(), ash.top() with one of
-        the two).
+        Exact query attribution is unrecoverable for that window — narrowing
+        it will not help. Use the untied aggregate readers instead: remove
+        query_id (or drop the wait filter when drilling queries by wait).
 ```
 
 **Partial overlap** — the window end is still inside retention, so the error
 names the exact boundary to move the start to:
 
 ```
-ERROR:  pg_ash: this drill needs raw samples; raw retention starts at
-        2026-07-03 00:00:00+00 but the requested window starts at
+ERROR:  pg_ash: this drill needs exact raw query attribution; raw retention
+        starts at 2026-07-03 00:00:00+00 but the requested window starts at
         2026-07-02 23:00:00+00. Narrow the window to start at or after
         2026-07-03 00:00:00+00 (the window end is still inside raw retention),
-        or drill without the query/event tie.
+        or remove query_id.
 ```
 
 ## 5. Long windows (rollup readers)

blueprints/AAS_USER_STORIES.md

@@ -154,7 +154,7 @@ and US-8 (machine load-report ingest) are cross-cutting or extension stories.
   2. Per-bucket `peak_aas` (and ideally `p99_aas`) preserved at hour/day grain.
   3. Works to the limit of `rollup_1h` retention and signals that horizon.
 - **Primary API:** `ash.timeline` over long spans.
-- **Coverage:** ✅ Covered — auto `rollup_1h` selection; `rollup_1h.minute_counts` preserves per-minute totals, so unfiltered/database-filtered long windows report exact minute-grain `peak_aas` and non-null `p99_aas` (seam-consistent with `rollup_1m`); `p99_aas` is null only for wait/query-filtered `rollup_1h`-backed buckets (hour grain).
+- **Coverage:** ✅ Covered — auto `rollup_1h` selection; `rollup_1h.minute_counts` preserves per-minute totals, so unfiltered/database-filtered long windows report exact minute-grain `peak_aas` and non-null `p99_aas` (seam-consistent with `rollup_1m`); `p99_aas` is null only for wait-filtered `rollup_1h`-backed buckets (hour grain). Explicit query filters require raw samples for exact attribution.
 
 ### US-7 — Before/after comparison