From 86d6d2bab6c24c4def5f1f5ab10b4c855621a72b Mon Sep 17 00:00:00 2001 From: ISRAEL JOEL NGBEDE Date: Tue, 5 May 2026 17:42:37 +0100 Subject: [PATCH] perf(store): use json_each CTE to make select_notes_since_block_by_tag cacheable Previously, diesel generated `tag IN (?, ?, ...)` with a variable number of placeholders, producing a different SQL string for each unique tag count. SQLite cannot cache prepared statements across different SQL texts, so every sync call paid a parse and compile cost. Fix by serializing the tag list as a single JSON integer array and expanding it with `json_each(?1)` inside a CTE. The SQL text is now constant regardless of how many tags are requested, allowing SQLite to cache the prepared statement across calls. Two diesel queries (find first block, then fetch notes) are also merged into one via the CTE, eliminating a round-trip. The remaining eq_any call sites in the store follow the same pattern and can be addressed with this same technique in follow-up PRs. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 1 + crates/store/src/db/models/queries/notes.rs | 112 +++++++++++--------- 2 files changed, 60 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0c95207b..50d7044c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Added `StoreReplica` gRPC service with endpoints for streaming blocks and proofs ([#1987](https://github.com/0xMiden/node/pull/1987)). - Replaced the network monitor's JavaScript dashboard with a server-rendered Maud + HTMX frontend ([#2024](https://github.com/0xMiden/node/pull/2024)). - [BREAKING] Removed `CheckNullifiers` endpoint ([#2049](https://github.com/0xMiden/node/pull/2049)). +- Rewrote `select_notes_since_block_by_tag` to pass the tag list as a JSON array via `json_each(?1)` CTE, making the prepared statement shape stable and SQLite statement cache-friendly regardless of tag count ([#2051](https://github.com/0xMiden/node/pull/2051)). ## v0.14.10 (2026-05-29) diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index 5b1eba78a..9d8c862ab 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -107,72 +107,78 @@ impl From for i32 { /// /// # Raw SQL /// +/// Tags are passed as a single JSON integer array so the prepared statement text is stable +/// regardless of how many tags are supplied. `json_each(?1)` expands the array inside SQLite. +/// /// ```sql -/// SELECT -/// committed_at, -/// batch_index, -/// note_index, -/// note_id, -/// note_type, -/// sender, -/// tag, -/// attachment, -/// inclusion_path -/// FROM -/// notes -/// WHERE -/// committed_at = ( -/// SELECT -/// committed_at -/// FROM -/// notes -/// WHERE -/// tag IN (?1) AND -/// committed_at >= ?2 AND -/// committed_at <= ?3 -/// ORDER BY -/// committed_at ASC -/// LIMIT 1 -/// ) AND -/// tag IN (?1) -/// ORDER BY -/// committed_at ASC, batch_index ASC, note_index ASC +/// WITH +/// tags(v) AS (SELECT value FROM json_each(?1)), +/// first_block(committed_at) AS ( +/// SELECT MIN(committed_at) FROM notes +/// WHERE tag IN (SELECT v FROM tags) +/// AND committed_at >= ?2 AND committed_at <= ?3 +/// ) +/// SELECT n.committed_at, n.batch_index, n.note_index, n.note_id, +/// n.note_type, n.sender, n.tag, n.attachment, n.inclusion_path +/// FROM notes n +/// CROSS JOIN first_block fb +/// WHERE n.committed_at = fb.committed_at +/// AND n.tag IN (SELECT v FROM tags) +/// ORDER BY n.committed_at ASC, n.batch_index ASC, n.note_index ASC /// ``` pub(crate) fn select_notes_since_block_by_tag( conn: &mut SqliteConnection, note_tags: &[u32], block_range: RangeInclusive, ) -> Result, DatabaseError> { + // Pass tags as a JSON integer array so the SQL text is the same for any number of tags. + const SQL: &str = "\ + WITH \ + tags(v) AS (SELECT value FROM json_each(?1)), \ + first_block(committed_at) AS ( \ + SELECT MIN(committed_at) FROM notes \ + WHERE tag IN (SELECT v FROM tags) \ + AND committed_at >= ?2 AND committed_at <= ?3 \ + ) \ + SELECT n.committed_at, n.batch_index, n.note_index, n.note_id, \ + n.note_type, n.sender, n.tag, n.attachment, n.inclusion_path \ + FROM notes n \ + CROSS JOIN first_block fb \ + WHERE n.committed_at = fb.committed_at \ + AND n.tag IN (SELECT v FROM tags) \ + ORDER BY n.committed_at ASC, n.batch_index ASC, n.note_index ASC"; + QueryParamNoteTagLimit::check(note_tags.len())?; - let desired_note_tags: Vec = note_tags.iter().map(|tag| *tag as i32).collect(); + let tags_json = build_json_int_array(note_tags.iter().map(|t| i64::from(*t))); let start_block_num = block_range.start().to_raw_sql(); let end_block_num = block_range.end().to_raw_sql(); - let Some(desired_block_num): Option = - SelectDsl::select(schema::notes::table, schema::notes::committed_at) - .filter(schema::notes::tag.eq_any(&desired_note_tags)) - .filter(schema::notes::committed_at.ge(start_block_num)) - .filter(schema::notes::committed_at.le(end_block_num)) - .order_by(schema::notes::committed_at.asc()) - .limit(1) - .get_result(conn) - .optional()? - else { - return Ok(Vec::new()); - }; - - let notes = SelectDsl::select(schema::notes::table, NoteSyncRecordRawRow::as_select()) - .filter(schema::notes::committed_at.eq(desired_block_num)) - .filter(schema::notes::tag.eq_any(&desired_note_tags)) - .order_by(( - schema::notes::committed_at.asc(), - schema::notes::batch_index.asc(), - schema::notes::note_index.asc(), - )) - .get_results::(conn) + let raw_notes = diesel::sql_query(SQL) + .bind::(&tags_json) + .bind::(start_block_num) + .bind::(end_block_num) + .load::(conn) .map_err(DatabaseError::from)?; - vec_raw_try_into(notes) + vec_raw_try_into(raw_notes) +} + +/// Builds a JSON integer array from an iterator, e.g. `[1,2,3]`. +/// +/// Used to pass integer lists through `json_each(?)` in SQLite, keeping the prepared statement +/// text stable regardless of how many values are supplied. +fn build_json_int_array(values: impl Iterator) -> String { + let mut s = String::from("["); + let mut first = true; + for v in values { + if !first { + s.push(','); + } + s.push_str(&v.to_string()); + first = false; + } + s.push(']'); + s } /// Select all notes matching the given set of identifiers