diff --git a/.changeset/reduce-warm-queries.md b/.changeset/reduce-warm-queries.md new file mode 100644 index 0000000000..4d79ab278f --- /dev/null +++ b/.changeset/reduce-warm-queries.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Reduces redundant database queries when rendering content pages: widget areas are now request-cached, taxonomy term usage-counts are fetched once per request instead of once per taxonomy widget, and `getTermsForEntries` reuses already-hydrated terms instead of re-querying. Fewer round trips per page on every backend. diff --git a/packages/core/src/taxonomies/index.ts b/packages/core/src/taxonomies/index.ts index 6d3ee776ab..fe70d52380 100644 --- a/packages/core/src/taxonomies/index.ts +++ b/packages/core/src/taxonomies/index.ts @@ -120,15 +120,10 @@ export async function getTaxonomyTerms( if (locale !== undefined) termsQuery = termsQuery.where("locale", "=", locale); const rows = await termsQuery.execute(); - // Counts are keyed by translation_group (what the pivot stores). - const countsResult = await db - .selectFrom("content_taxonomies") - .select(["taxonomy_id"]) - .select((eb) => eb.fn.count("entry_id").as("count")) - .groupBy("taxonomy_id") - .execute(); - const counts = new Map(); - for (const row of countsResult) counts.set(row.taxonomy_id, row.count); + // Counts are keyed by translation_group (what the pivot stores) and are + // locale-independent, so the aggregate is shared across every taxonomy + // rendered in this request (Categories + Tags widgets, etc.). + const counts = await getTaxonomyTermCounts(); const flatTerms: TaxonomyTermRow[] = rows.map((row) => ({ id: row.id, @@ -157,6 +152,27 @@ export async function getTaxonomyTerms( }); } +/** + * Per-translation-group usage counts across all taxonomies, in one aggregate + * scan of `content_taxonomies`. Counts are locale-independent (the pivot stores + * translation_group), so a single request-cached entry serves every taxonomy + * that renders during the request. + */ +function getTaxonomyTermCounts(): Promise> { + return requestCached("taxonomy-term-counts", async () => { + const db = await getDb(); + const countsResult = await db + .selectFrom("content_taxonomies") + .select(["taxonomy_id"]) + .select((eb) => eb.fn.count("entry_id").as("count")) + .groupBy("taxonomy_id") + .execute(); + const counts = new Map(); + for (const row of countsResult) counts.set(row.taxonomy_id, row.count); + return counts; + }); +} + /** * Get a single term by (taxonomy, slug). Honours the fallback chain — if the * slug exists in a fallback locale, we return that row (useful for deep-linking @@ -290,10 +306,57 @@ export async function getTermsForEntries( for (const id of uniqueIds) result.set(id, []); if (uniqueIds.length === 0) return result; - const db = await getDb(); const locale = resolveLocale(options.locale); + const localeKey = locale ?? "*"; - for (const chunk of chunks(uniqueIds, SQL_BATCH_SIZE)) { + // Entry-term hydration (getAllTermsForEntries -> primeEntryTermsCache) + // seeds the per-entry cache under the same key getEntryTerms uses: + // `terms:${collection}:${entryId}:${taxonomyName}:${localeKey}`, storing a + // TaxonomyTerm[] (including `[]` for entries with no terms). Satisfy those + // from cache and run the batched query only for the ids that missed. + const missedIds: string[] = []; + type CacheRead = { id: string; terms: TaxonomyTerm[] } | { id: string; miss: true }; + const cacheReads: Array> = []; + for (const id of uniqueIds) { + const cached = peekRequestCache( + `terms:${collection}:${id}:${taxonomyName}:${localeKey}`, + ); + if (cached) { + // A peeked promise can reject (e.g. a sibling getEntryTerms hit a + // missing table). Treat a rejection as a cache miss so the batched + // query path -- and its isMissingTableError guard below -- still runs, + // rather than propagating an uncaught error. + cacheReads.push( + cached.then( + (terms): CacheRead => ({ id, terms }), + (): CacheRead => ({ id, miss: true }), + ), + ); + } else { + missedIds.push(id); + } + } + for (const read of await Promise.all(cacheReads)) { + if ("miss" in read) { + missedIds.push(read.id); + continue; + } + // Return a private copy. The cached array and its term objects are shared + // with getEntryTerms/getAllTermsForEntries (primeEntryTermsCache stores + // the same references), so a caller that mutates the result -- sorting in + // place, pushing into `children` -- must not poison the cache. The + // pre-cache implementation always returned freshly built arrays. + result.set( + read.id, + read.terms.map((t) => ({ ...t, children: [...t.children] })), + ); + } + + if (missedIds.length === 0) return result; + + const db = await getDb(); + + for (const chunk of chunks(missedIds, SQL_BATCH_SIZE)) { let rows; try { let query = db @@ -311,7 +374,10 @@ export async function getTermsForEntries( ]) .where("content_taxonomies.collection", "=", collection) .where("content_taxonomies.entry_id", "in", chunk) - .where("taxonomies.name", "=", taxonomyName); + .where("taxonomies.name", "=", taxonomyName) + // Match the order getAllTermsForEntries (the cache primer) uses, so + // cache-hit and DB-miss entries in one result are ordered consistently. + .orderBy("taxonomies.label", "asc"); if (locale !== undefined) query = query.where("taxonomies.locale", "=", locale); rows = await query.execute(); } catch (error) { diff --git a/packages/core/src/widgets/index.ts b/packages/core/src/widgets/index.ts index 48aae86efe..c305ef12ee 100644 --- a/packages/core/src/widgets/index.ts +++ b/packages/core/src/widgets/index.ts @@ -1,4 +1,5 @@ import { getDb } from "../loader.js"; +import { requestCached } from "../request-cache.js"; import { getWidgetComponents as getComponentRegistry } from "./components.js"; import type { Widget, WidgetArea, WidgetRow, WidgetComponentDef } from "./types.js"; @@ -22,62 +23,64 @@ export type { * row with null widget columns, which we skip when mapping. */ export async function getWidgetArea(name: string): Promise { - const db = await getDb(); - const rows = await db - .selectFrom("_emdash_widget_areas as a") - .leftJoin("_emdash_widgets as w", "w.area_id", "a.id") - .select([ - "a.id as a_id", - "a.name as a_name", - "a.label as a_label", - "a.description as a_description", - "w.id as w_id", - "w.type as w_type", - "w.title as w_title", - "w.content as w_content", - "w.menu_name as w_menu_name", - "w.component_id as w_component_id", - "w.component_props as w_component_props", - "w.area_id as w_area_id", - "w.sort_order as w_sort_order", - "w.created_at as w_created_at", - ]) - .where("a.name", "=", name) - .orderBy("w.sort_order", "asc") - .execute(); + return requestCached(`widget-area:${name}`, async () => { + const db = await getDb(); + const rows = await db + .selectFrom("_emdash_widget_areas as a") + .leftJoin("_emdash_widgets as w", "w.area_id", "a.id") + .select([ + "a.id as a_id", + "a.name as a_name", + "a.label as a_label", + "a.description as a_description", + "w.id as w_id", + "w.type as w_type", + "w.title as w_title", + "w.content as w_content", + "w.menu_name as w_menu_name", + "w.component_id as w_component_id", + "w.component_props as w_component_props", + "w.area_id as w_area_id", + "w.sort_order as w_sort_order", + "w.created_at as w_created_at", + ]) + .where("a.name", "=", name) + .orderBy("w.sort_order", "asc") + .execute(); - const first = rows[0]; - if (!first) return null; - const widgets: Widget[] = []; - for (const row of rows) { - if (row.w_id === null) continue; // area has no widgets (left-join null row) - // Left-join makes every w_* column nullable in the type; at runtime - // they're all non-null once w_id is (we match on widgets.area_id, so - // a widget row always has the not-null columns filled). Cast is the - // price of that structural fact. - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- left-join row is non-null when w_id is set; see above - const widgetRow = { - id: row.w_id, - type: row.w_type, - title: row.w_title, - content: row.w_content, - menu_name: row.w_menu_name, - component_id: row.w_component_id, - component_props: row.w_component_props, - area_id: row.w_area_id, - sort_order: row.w_sort_order, - created_at: row.w_created_at, - } as WidgetRow; - widgets.push(rowToWidget(widgetRow)); - } + const first = rows[0]; + if (!first) return null; + const widgets: Widget[] = []; + for (const row of rows) { + if (row.w_id === null) continue; // area has no widgets (left-join null row) + // Left-join makes every w_* column nullable in the type; at runtime + // they're all non-null once w_id is (we match on widgets.area_id, so + // a widget row always has the not-null columns filled). Cast is the + // price of that structural fact. + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- left-join row is non-null when w_id is set; see above + const widgetRow = { + id: row.w_id, + type: row.w_type, + title: row.w_title, + content: row.w_content, + menu_name: row.w_menu_name, + component_id: row.w_component_id, + component_props: row.w_component_props, + area_id: row.w_area_id, + sort_order: row.w_sort_order, + created_at: row.w_created_at, + } as WidgetRow; + widgets.push(rowToWidget(widgetRow)); + } - return { - id: first.a_id, - name: first.a_name, - label: first.a_label, - description: first.a_description ?? undefined, - widgets, - }; + return { + id: first.a_id, + name: first.a_name, + label: first.a_label, + description: first.a_description ?? undefined, + widgets, + }; + }); } /** diff --git a/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts b/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts index 71194d2851..ef02b5efb9 100644 --- a/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts +++ b/packages/core/tests/unit/taxonomies/get-all-terms-for-entries.test.ts @@ -16,6 +16,7 @@ import { runWithContext } from "../../../src/request-context.js"; import { getAllTermsForEntries, getEntryTerms, + getTermsForEntries, invalidateTermCache, } from "../../../src/taxonomies/index.js"; @@ -225,4 +226,73 @@ describe("getAllTermsForEntries", () => { // At least one DB call should have happened in the second request. expect(getDbSpy.mock.calls.length).toBeGreaterThan(callsBeforeSecondRequest); }); + + it("getTermsForEntries serves primed entries from cache without re-querying", async () => { + await db + .updateTable("_emdash_taxonomy_defs") + .set({ collections: JSON.stringify(["posts", "post"]) }) + .where("name", "=", "tag") + .execute(); + + const tag = await taxRepo.create({ name: "tag", slug: "web", label: "Web" }); + const p1 = await contentRepo.create({ type: "post", slug: "p1", data: { title: "P1" } }); + const p2 = await contentRepo.create({ type: "post", slug: "p2", data: { title: "P2" } }); + await taxRepo.attachToEntry("post", p1.id, tag.id); + + invalidateTermCache(); + const getDbSpy = vi.mocked(getDb); + + await runWithContext({ editMode: false }, async () => { + await getAllTermsForEntries("post", [p1.id, p2.id]); // primes the per-entry cache + const callsAfterBatch = getDbSpy.mock.calls.length; + + const map = await getTermsForEntries("post", [p1.id, p2.id], "tag"); + + // All entries were primed, so no further DB calls. + expect(getDbSpy.mock.calls.length).toBe(callsAfterBatch); + expect(map.get(p1.id)!.map((t) => t.slug)).toEqual(["web"]); + expect(map.get(p2.id)).toEqual([]); + }); + }); + + it("getTermsForEntries returns private copies that can't poison the cache", async () => { + await db + .updateTable("_emdash_taxonomy_defs") + .set({ collections: JSON.stringify(["posts", "post"]) }) + .where("name", "=", "tag") + .execute(); + + const tagA = await taxRepo.create({ name: "tag", slug: "a", label: "A" }); + const tagB = await taxRepo.create({ name: "tag", slug: "b", label: "B" }); + const p1 = await contentRepo.create({ type: "post", slug: "p1", data: { title: "P1" } }); + await taxRepo.attachToEntry("post", p1.id, tagA.id); + await taxRepo.attachToEntry("post", p1.id, tagB.id); + + invalidateTermCache(); + + await runWithContext({ editMode: false }, async () => { + await getAllTermsForEntries("post", [p1.id]); // prime + + const first = await getTermsForEntries("post", [p1.id], "tag"); + const arr = first.get(p1.id)!; + expect(arr.map((t) => t.slug).toSorted()).toEqual(["a", "b"]); + + // Mutate the returned array in place (truncate + push junk) and mutate + // a term's children — none of this must leak into the shared cache. + arr.length = 0; + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- deliberate junk for the mutation test + arr.push({ slug: "junk" } as unknown as (typeof arr)[number]); + + const viaEntry = await getEntryTerms("post", p1.id, "tag"); + expect(viaEntry.map((t) => t.slug).toSorted()).toEqual(["a", "b"]); + + const second = await getTermsForEntries("post", [p1.id], "tag"); + expect( + second + .get(p1.id)! + .map((t) => t.slug) + .toSorted(), + ).toEqual(["a", "b"]); + }); + }); }); diff --git a/scripts/query-counts.snapshot.d1.json b/scripts/query-counts.snapshot.d1.json index 9753041a97..627aa09be5 100644 --- a/scripts/query-counts.snapshot.d1.json +++ b/scripts/query-counts.snapshot.d1.json @@ -1,22 +1,22 @@ { - "GET / (cold)": 17, - "GET / (warm)": 6, - "GET /category/development (cold)": 20, - "GET /category/development (warm)": 9, + "GET / (cold)": 16, + "GET / (warm)": 5, + "GET /category/development (cold)": 19, + "GET /category/development (warm)": 8, "GET /contributors (cold)": 15, "GET /contributors (warm)": 5, "GET /contributors-naive (cold)": 22, "GET /contributors-naive (warm)": 12, "GET /pages/about (cold)": 13, "GET /pages/about (warm)": 4, - "GET /posts (cold)": 16, - "GET /posts (warm)": 6, - "GET /posts/building-for-the-long-term (cold)": 28, - "GET /posts/building-for-the-long-term (warm)": 17, + "GET /posts (cold)": 15, + "GET /posts (warm)": 5, + "GET /posts/building-for-the-long-term (cold)": 26, + "GET /posts/building-for-the-long-term (warm)": 15, "GET /rss.xml (cold)": 15, "GET /rss.xml (warm)": 5, "GET /search (cold)": 14, "GET /search (warm)": 5, - "GET /tag/webdev (cold)": 20, - "GET /tag/webdev (warm)": 9 + "GET /tag/webdev (cold)": 19, + "GET /tag/webdev (warm)": 8 } diff --git a/scripts/query-counts.snapshot.sqlite.json b/scripts/query-counts.snapshot.sqlite.json index ef4072d240..ee273a0a9d 100644 --- a/scripts/query-counts.snapshot.sqlite.json +++ b/scripts/query-counts.snapshot.sqlite.json @@ -1,22 +1,22 @@ { - "GET / (cold)": 7, - "GET / (warm)": 7, - "GET /category/development (cold)": 11, - "GET /category/development (warm)": 10, + "GET / (cold)": 6, + "GET / (warm)": 6, + "GET /category/development (cold)": 10, + "GET /category/development (warm)": 9, "GET /contributors (cold)": 6, "GET /contributors (warm)": 6, "GET /contributors-naive (cold)": 13, "GET /contributors-naive (warm)": 13, "GET /pages/about (cold)": 5, "GET /pages/about (warm)": 5, - "GET /posts (cold)": 7, - "GET /posts (warm)": 7, - "GET /posts/building-for-the-long-term (cold)": 18, - "GET /posts/building-for-the-long-term (warm)": 18, + "GET /posts (cold)": 6, + "GET /posts (warm)": 6, + "GET /posts/building-for-the-long-term (cold)": 16, + "GET /posts/building-for-the-long-term (warm)": 16, "GET /rss.xml (cold)": 5, "GET /rss.xml (warm)": 5, "GET /search (cold)": 6, "GET /search (warm)": 6, - "GET /tag/webdev (cold)": 10, - "GET /tag/webdev (warm)": 10 + "GET /tag/webdev (cold)": 9, + "GET /tag/webdev (warm)": 9 }