From 8c6f88913de96457572179c31de4b43c22448004 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 16 Jun 2026 08:12:32 +0100 Subject: [PATCH 1/2] perf(core): cut redundant queries on content pages - Cache getWidgetArea per request (it was the only content helper not request-cached). - Fetch taxonomy term usage-counts once per request via a shared request-cached aggregate, instead of re-running the full content_taxonomies GROUP BY for every taxonomy widget (Categories + Tags each ran it). - Make getTermsForEntries cache-aware: reuse already-hydrated per-entry terms from the request cache and only query the misses. Returns private copies so callers mutating the result can't poison the cache, and orders the miss path by label to match the hydration primer. All same-shape, all backends. Adds regression tests for cache reuse and mutation-safety. Updates the sqlite query-count snapshot (post route -1). --- .changeset/reduce-warm-queries.md | 5 + packages/core/src/taxonomies/index.ts | 90 ++++++++++++-- packages/core/src/widgets/index.ts | 111 +++++++++--------- .../get-all-terms-for-entries.test.ts | 70 +++++++++++ scripts/query-counts.snapshot.sqlite.json | 4 +- 5 files changed, 212 insertions(+), 68 deletions(-) create mode 100644 .changeset/reduce-warm-queries.md diff --git a/.changeset/reduce-warm-queries.md b/.changeset/reduce-warm-queries.md new file mode 100644 index 0000000000..4d79ab278f --- /dev/null +++ b/.changeset/reduce-warm-queries.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Reduces redundant database queries when rendering content pages: widget areas are now request-cached, taxonomy term usage-counts are fetched once per request instead of once per taxonomy widget, and `getTermsForEntries` reuses already-hydrated terms instead of re-querying. Fewer round trips per page on every backend. diff --git a/packages/core/src/taxonomies/index.ts b/packages/core/src/taxonomies/index.ts index 6d3ee776ab..fe70d52380 100644 --- a/packages/core/src/taxonomies/index.ts +++ b/packages/core/src/taxonomies/index.ts @@ -120,15 +120,10 @@ export async function getTaxonomyTerms( if (locale !== undefined) termsQuery = termsQuery.where("locale", "=", locale); const rows = await termsQuery.execute(); - // Counts are keyed by translation_group (what the pivot stores). - const countsResult = await db - .selectFrom("content_taxonomies") - .select(["taxonomy_id"]) - .select((eb) => eb.fn.count("entry_id").as("count")) - .groupBy("taxonomy_id") - .execute(); - const counts = new Map(); - for (const row of countsResult) counts.set(row.taxonomy_id, row.count); + // Counts are keyed by translation_group (what the pivot stores) and are + // locale-independent, so the aggregate is shared across every taxonomy + // rendered in this request (Categories + Tags widgets, etc.). + const counts = await getTaxonomyTermCounts(); const flatTerms: TaxonomyTermRow[] = rows.map((row) => ({ id: row.id, @@ -157,6 +152,27 @@ export async function getTaxonomyTerms( }); } +/** + * Per-translation-group usage counts across all taxonomies, in one aggregate + * scan of `content_taxonomies`. Counts are locale-independent (the pivot stores + * translation_group), so a single request-cached entry serves every taxonomy + * that renders during the request. + */ +function getTaxonomyTermCounts(): Promise> { + return requestCached("taxonomy-term-counts", async () => { + const db = await getDb(); + const countsResult = await db + .selectFrom("content_taxonomies") + .select(["taxonomy_id"]) + .select((eb) => eb.fn.count("entry_id").as("count")) + .groupBy("taxonomy_id") + .execute(); + const counts = new Map(); + for (const row of countsResult) counts.set(row.taxonomy_id, row.count); + return counts; + }); +} + /** * Get a single term by (taxonomy, slug). Honours the fallback chain — if the * slug exists in a fallback locale, we return that row (useful for deep-linking @@ -290,10 +306,57 @@ export async function getTermsForEntries( for (const id of uniqueIds) result.set(id, []); if (uniqueIds.length === 0) return result; - const db = await getDb(); const locale = resolveLocale(options.locale); + const localeKey = locale ?? "*"; - for (const chunk of chunks(uniqueIds, SQL_BATCH_SIZE)) { + // Entry-term hydration (getAllTermsForEntries -> primeEntryTermsCache) + // seeds the per-entry cache under the same key getEntryTerms uses: + // `terms:${collection}:${entryId}:${taxonomyName}:${localeKey}`, storing a + // TaxonomyTerm[] (including `[]` for entries with no terms). Satisfy those + // from cache and run the batched query only for the ids that missed. + const missedIds: string[] = []; + type CacheRead = { id: string; terms: TaxonomyTerm[] } | { id: string; miss: true }; + const cacheReads: Array> = []; + for (const id of uniqueIds) { + const cached = peekRequestCache( + `terms:${collection}:${id}:${taxonomyName}:${localeKey}`, + ); + if (cached) { + // A peeked promise can reject (e.g. a sibling getEntryTerms hit a + // missing table). Treat a rejection as a cache miss so the batched + // query path -- and its isMissingTableError guard below -- still runs, + // rather than propagating an uncaught error. + cacheReads.push( + cached.then( + (terms): CacheRead => ({ id, terms }), + (): CacheRead => ({ id, miss: true }), + ), + ); + } else { + missedIds.push(id); + } + } + for (const read of await Promise.all(cacheReads)) { + if ("miss" in read) { + missedIds.push(read.id); + continue; + } + // Return a private copy. The cached array and its term objects are shared + // with getEntryTerms/getAllTermsForEntries (primeEntryTermsCache stores + // the same references), so a caller that mutates the result -- sorting in + // place, pushing into `children` -- must not poison the cache. The + // pre-cache implementation always returned freshly built arrays. + result.set( + read.id, + read.terms.map((t) => ({ ...t, children: [...t.children] })), + ); + } + + if (missedIds.length === 0) return result; + + const db = await getDb(); + + for (const chunk of chunks(missedIds, SQL_BATCH_SIZE)) { let rows; try { let query = db @@ -311,7 +374,10 @@ export async function getTermsForEntries( ]) .where("content_taxonomies.collection", "=", collection) .where("content_taxonomies.entry_id", "in", chunk) - .where("taxonomies.name", "=", taxonomyName); + .where("taxonomies.name", "=", taxonomyName) + // Match the order getAllTermsForEntries (the cache primer) uses, so + // cache-hit and DB-miss entries in one result are ordered consistently. + .orderBy("taxonomies.label", "asc"); if (locale !== undefined) query = query.where("taxonomies.locale", "=", locale); rows = await query.execute(); } catch (error) { diff --git a/packages/core/src/widgets/index.ts b/packages/core/src/widgets/index.ts index 48aae86efe..c305ef12ee 100644 --- a/packages/core/src/widgets/index.ts +++ b/packages/core/src/widgets/index.ts @@ -1,4 +1,5 @@ import { getDb } from "../loader.js"; +import { requestCached } from "../request-cache.js"; import { getWidgetComponents as getComponentRegistry } from "./components.js"; import type { Widget, WidgetArea, WidgetRow, WidgetComponentDef } from "./types.js"; @@ -22,62 +23,64 @@ export type { * row with null widget columns, which we skip when mapping. */ export async function getWidgetArea(name: string): Promise { - const db = await getDb(); - const rows = await db - .selectFrom("_emdash_widget_areas as a") - .leftJoin("_emdash_widgets as w", "w.area_id", "a.id") - .select([ - "a.id as a_id", - "a.name as a_name", - "a.label as a_label", - "a.description as a_description", - "w.id as w_id", - "w.type as w_type", - "w.title as w_title", - "w.content as w_content", - "w.menu_name as w_menu_name", - "w.component_id as w_component_id", - "w.component_props as w_component_props", - "w.area_id as w_area_id", - "w.sort_order as w_sort_order", - "w.created_at as w_created_at", - ]) - .where("a.name", "=", name) - .orderBy("w.sort_order", "asc") - .execute(); + return requestCached(`widget-area:${name}`, async () => { + const db = await getDb(); + const rows = await db + .selectFrom("_emdash_widget_areas as a") + .leftJoin("_emdash_widgets as w", "w.area_id", "a.id") + .select([ + "a.id as a_id", + "a.name as a_name", + "a.label as a_label", + "a.description as a_description", + "w.id as w_id", + "w.type as w_type", + "w.title as w_title", + "w.content as w_content", + "w.menu_name as w_menu_name", + "w.component_id as w_component_id", + "w.component_props as w_component_props", + "w.area_id as w_area_id", + "w.sort_order as w_sort_order", + "w.created_at as w_created_at", + ]) + .where("a.name", "=", name) + .orderBy("w.sort_order", "asc") + .execute(); - const first = rows[0]; - if (!first) return null; - const widgets: Widget[] = []; - for (const row of rows) { - if (row.w_id === null) continue; // area has no widgets (left-join null row) - // Left-join makes every w_* column nullable in the type; at runtime - // they're all non-null once w_id is (we match on widgets.area_id, so - // a widget row always has the not-null columns filled). Cast is the - // price of that structural fact. - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- left-join row is non-null when w_id is set; see above - const widgetRow = { - id: row.w_id, - type: row.w_type, - title: row.w_title, - content: row.w_content, - menu_name: row.w_menu_name, - component_id: row.w_component_id, - component_props: row.w_component_props, - area_id: row.w_area_id, - sort_order: row.w_sort_order, - created_at: row.w_created_at, - } as WidgetRow; - widgets.push(rowToWidget(widgetRow)); - } + const first = rows[0]; + if (!first) return null; + const widgets: Widget[] = []; + for (const row of rows) { + if (row.w_id === null) continue; // area has no widgets (left-join null row) + // Left-join makes every w_* column nullable in the type; at runtime + // they're all non-null once w_id is (we match on widgets.area_id, so + // a widget row always has the not-null columns filled). Cast is the + // price of that structural fact. + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- left-join row is non-null when w_id is set; see above + const widgetRow = { + id: row.w_id, + type: row.w_type, + title: row.w_title, + content: row.w_content, + menu_name: row.w_menu_name, + component_id: row.w_component_id, + component_props: row.w_component_props, + area_id: row.w_area_id, + sort_order: row.w_sort_order, + created_at: row.w_created_at, + } as WidgetRow; + widgets.push(rowToWidget(widgetRow)); + } - return { - id: first.a_id, - name: first.a_name, - label: first.a_label, - description: first.a_description ?? undefined, - widgets, - }; + return { + id: first.a_id, + name: first.a_name, + label: first.a_label, + description: first.a_description ?? undefined, + widgets, + }; + }); } /** diff --git a/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts b/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts index 71194d2851..ef02b5efb9 100644 --- a/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts +++ b/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts @@ -16,6 +16,7 @@ import { runWithContext } from "../../../src/request-context.js"; import { getAllTermsForEntries, getEntryTerms, + getTermsForEntries, invalidateTermCache, } from "../../../src/taxonomies/index.js"; @@ -225,4 +226,73 @@ describe("getAllTermsForEntries", () => { // At least one DB call should have happened in the second request. expect(getDbSpy.mock.calls.length).toBeGreaterThan(callsBeforeSecondRequest); }); + + it("getTermsForEntries serves primed entries from cache without re-querying", async () => { + await db + .updateTable("_emdash_taxonomy_defs") + .set({ collections: JSON.stringify(["posts", "post"]) }) + .where("name", "=", "tag") + .execute(); + + const tag = await taxRepo.create({ name: "tag", slug: "web", label: "Web" }); + const p1 = await contentRepo.create({ type: "post", slug: "p1", data: { title: "P1" } }); + const p2 = await contentRepo.create({ type: "post", slug: "p2", data: { title: "P2" } }); + await taxRepo.attachToEntry("post", p1.id, tag.id); + + invalidateTermCache(); + const getDbSpy = vi.mocked(getDb); + + await runWithContext({ editMode: false }, async () => { + await getAllTermsForEntries("post", [p1.id, p2.id]); // primes the per-entry cache + const callsAfterBatch = getDbSpy.mock.calls.length; + + const map = await getTermsForEntries("post", [p1.id, p2.id], "tag"); + + // All entries were primed, so no further DB calls. + expect(getDbSpy.mock.calls.length).toBe(callsAfterBatch); + expect(map.get(p1.id)!.map((t) => t.slug)).toEqual(["web"]); + expect(map.get(p2.id)).toEqual([]); + }); + }); + + it("getTermsForEntries returns private copies that can't poison the cache", async () => { + await db + .updateTable("_emdash_taxonomy_defs") + .set({ collections: JSON.stringify(["posts", "post"]) }) + .where("name", "=", "tag") + .execute(); + + const tagA = await taxRepo.create({ name: "tag", slug: "a", label: "A" }); + const tagB = await taxRepo.create({ name: "tag", slug: "b", label: "B" }); + const p1 = await contentRepo.create({ type: "post", slug: "p1", data: { title: "P1" } }); + await taxRepo.attachToEntry("post", p1.id, tagA.id); + await taxRepo.attachToEntry("post", p1.id, tagB.id); + + invalidateTermCache(); + + await runWithContext({ editMode: false }, async () => { + await getAllTermsForEntries("post", [p1.id]); // prime + + const first = await getTermsForEntries("post", [p1.id], "tag"); + const arr = first.get(p1.id)!; + expect(arr.map((t) => t.slug).toSorted()).toEqual(["a", "b"]); + + // Mutate the returned array in place (truncate + push junk) and mutate + // a term's children — none of this must leak into the shared cache. + arr.length = 0; + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- deliberate junk for the mutation test + arr.push({ slug: "junk" } as unknown as (typeof arr)[number]); + + const viaEntry = await getEntryTerms("post", p1.id, "tag"); + expect(viaEntry.map((t) => t.slug).toSorted()).toEqual(["a", "b"]); + + const second = await getTermsForEntries("post", [p1.id], "tag"); + expect( + second + .get(p1.id)! + .map((t) => t.slug) + .toSorted(), + ).toEqual(["a", "b"]); + }); + }); }); diff --git a/scripts/query-counts.snapshot.sqlite.json b/scripts/query-counts.snapshot.sqlite.json index ef4072d240..814f0dc9f4 100644 --- a/scripts/query-counts.snapshot.sqlite.json +++ b/scripts/query-counts.snapshot.sqlite.json @@ -11,8 +11,8 @@ "GET /pages/about (warm)": 5, "GET /posts (cold)": 7, "GET /posts (warm)": 7, - "GET /posts/building-for-the-long-term (cold)": 18, - "GET /posts/building-for-the-long-term (warm)": 18, + "GET /posts/building-for-the-long-term (cold)": 17, + "GET /posts/building-for-the-long-term (warm)": 17, "GET /rss.xml (cold)": 5, "GET /rss.xml (warm)": 5, "GET /search (cold)": 6, From da62330cccce0f714745c09e77f5665377909665 Mon Sep 17 00:00:00 2001 From: "emdashbot[bot]" Date: Tue, 16 Jun 2026 07:32:41 +0000 Subject: [PATCH 2/2] ci: update query-count snapshots --- scripts/query-counts.snapshot.d1.json | 20 ++++++++++---------- scripts/query-counts.snapshot.sqlite.json | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/query-counts.snapshot.d1.json b/scripts/query-counts.snapshot.d1.json index 9753041a97..627aa09be5 100644 --- a/scripts/query-counts.snapshot.d1.json +++ b/scripts/query-counts.snapshot.d1.json @@ -1,22 +1,22 @@ { - "GET / (cold)": 17, - "GET / (warm)": 6, - "GET /category/development (cold)": 20, - "GET /category/development (warm)": 9, + "GET / (cold)": 16, + "GET / (warm)": 5, + "GET /category/development (cold)": 19, + "GET /category/development (warm)": 8, "GET /contributors (cold)": 15, "GET /contributors (warm)": 5, "GET /contributors-naive (cold)": 22, "GET /contributors-naive (warm)": 12, "GET /pages/about (cold)": 13, "GET /pages/about (warm)": 4, - "GET /posts (cold)": 16, - "GET /posts (warm)": 6, - "GET /posts/building-for-the-long-term (cold)": 28, - "GET /posts/building-for-the-long-term (warm)": 17, + "GET /posts (cold)": 15, + "GET /posts (warm)": 5, + "GET /posts/building-for-the-long-term (cold)": 26, + "GET /posts/building-for-the-long-term (warm)": 15, "GET /rss.xml (cold)": 15, "GET /rss.xml (warm)": 5, "GET /search (cold)": 14, "GET /search (warm)": 5, - "GET /tag/webdev (cold)": 20, - "GET /tag/webdev (warm)": 9 + "GET /tag/webdev (cold)": 19, + "GET /tag/webdev (warm)": 8 } diff --git a/scripts/query-counts.snapshot.sqlite.json b/scripts/query-counts.snapshot.sqlite.json index 814f0dc9f4..ee273a0a9d 100644 --- a/scripts/query-counts.snapshot.sqlite.json +++ b/scripts/query-counts.snapshot.sqlite.json @@ -1,22 +1,22 @@ { - "GET / (cold)": 7, - "GET / (warm)": 7, - "GET /category/development (cold)": 11, - "GET /category/development (warm)": 10, + "GET / (cold)": 6, + "GET / (warm)": 6, + "GET /category/development (cold)": 10, + "GET /category/development (warm)": 9, "GET /contributors (cold)": 6, "GET /contributors (warm)": 6, "GET /contributors-naive (cold)": 13, "GET /contributors-naive (warm)": 13, "GET /pages/about (cold)": 5, "GET /pages/about (warm)": 5, - "GET /posts (cold)": 7, - "GET /posts (warm)": 7, - "GET /posts/building-for-the-long-term (cold)": 17, - "GET /posts/building-for-the-long-term (warm)": 17, + "GET /posts (cold)": 6, + "GET /posts (warm)": 6, + "GET /posts/building-for-the-long-term (cold)": 16, + "GET /posts/building-for-the-long-term (warm)": 16, "GET /rss.xml (cold)": 5, "GET /rss.xml (warm)": 5, "GET /search (cold)": 6, "GET /search (warm)": 6, - "GET /tag/webdev (cold)": 10, - "GET /tag/webdev (warm)": 10 + "GET /tag/webdev (cold)": 9, + "GET /tag/webdev (warm)": 9 }