fix(core): compute taxonomy counts on demand

.changeset/nervous-jars-shave.md

@@ -0,0 +1,5 @@
+---
+"emdash": patch
+---
+
+Fixes taxonomy term counts being recomputed on every page render even when nothing displays them. Counting term usage aggregates the whole content–term assignment table for each taxonomy, and the layout prefetch ran it for every taxonomy on every HTML response — on Cloudflare D1 this could read millions of rows per page view. Counts are now computed only when a consumer asks for them: the prefetch never does, and the Tags and Categories widgets only when their `showCount` prop is on. `getTaxonomyTerms()` takes a new `includeCounts` option (default `true`) to opt out explicitly, and terms are cached separately from their counts so both callers share one term lookup.

docs/src/content/docs/guides/taxonomies.mdx

@@ -126,6 +126,14 @@ interface TaxonomyTerm {
 }
 ```
 
+Computing `count` aggregates every content–term assignment in the taxonomy's
+collections, which is the most expensive part of the call. If you only need
+labels and slugs, skip it — `count` is then omitted from the returned terms:
+
+```ts
+const tags = await getTaxonomyTerms("tag", { includeCounts: false });
+```
+
 ### Get a Single Term
 
 The following example fetches one term by taxonomy and slug:

packages/core/src/astro/prefetch.ts

@@ -40,10 +40,15 @@ async function prefetchWidgetAreas(): Promise<void> {
 	}
 }
 
-/** Warm every taxonomy's term list via the real helper (primes per-name keys). */
+/**
+ * Warm every taxonomy's term list via the real helper (primes per-name keys).
+ * Counts are left out: they cost an aggregate over the whole assignment pivot
+ * per taxonomy, and only a consumer that renders one can say it's needed. A
+ * consumer that does asks for it and reuses the term list warmed here.
+ */
 async function prefetchTaxonomyTerms(): Promise<void> {
 	const defs = await getTaxonomyDefs();
-	await Promise.allSettled(defs.map((def) => getTaxonomyTerms(def.name)));
+	await Promise.allSettled(defs.map((def) => getTaxonomyTerms(def.name, { includeCounts: false })));
 }
 
 /** Warm every menu via the real helper (primes `menu:${name}:${locale}`). */

packages/core/src/components/widgets/Categories.astro

@@ -12,7 +12,7 @@ interface Props {
 
 const { showCount = true, hierarchical = true } = Astro.props;
 
-const categories = await getTaxonomyTerms("category");
+const categories = await getTaxonomyTerms("category", { includeCounts: showCount });
 ---
 
 <ul class="widget-categories">

packages/core/src/components/widgets/Tags.astro

@@ -12,7 +12,7 @@ interface Props {
 
 const { showCount = false, limit = 20 } = Astro.props;
 
-const allTags = await getTaxonomyTerms("tag");
+const allTags = await getTaxonomyTerms("tag", { includeCounts: showCount });
 const tags = allTags.slice(0, limit);
 ---
 

packages/core/src/taxonomies/index.ts

@@ -31,6 +31,15 @@ export interface TaxonomyQueryOptions {
 	locale?: string;
 }
 
+export interface TaxonomyTermsOptions extends TaxonomyQueryOptions {
+	/**
+	 * Populate each term's `count`. Counts aggregate the whole
+	 * `content_taxonomies` pivot against every declared collection, so callers
+	 * that don't render a count should opt out. Defaults to `true`.
+	 */
+	includeCounts?: boolean;
+}
+
 /** Invalidate cached taxonomy term data and any content that hydrates terms. */
 export function invalidateTermCache(): void {
 	invalidateTaxonomyObjectCache();
@@ -230,21 +239,54 @@ function termCountNamespaces(collections: string[]): string[] {
 /**
  * All terms of a taxonomy in a specific locale (flat for non-hierarchical,
  * tree for hierarchical).
+ *
+ * The term list and the visible-entry counts are loaded and cached separately:
+ * the list depends only on the taxonomy epoch, while the counts additionally
+ * depend on every counted collection's content epoch and cost an aggregate over
+ * the whole assignment pivot. Callers that don't render counts pass
+ * `includeCounts: false` and skip that aggregate entirely, while still sharing
+ * the term list with callers that do.
  */
 export async function getTaxonomyTerms(
 	taxonomyName: string,
-	options: TaxonomyQueryOptions = {},
+	options: TaxonomyTermsOptions = {},
 ): Promise<TaxonomyTerm[]> {
 	const locale = resolveLocale(options.locale);
-	return requestCached(`taxonomy-terms:${taxonomyName}:${locale ?? "*"}`, async () => {
-		const def = await getTaxonomyDef(taxonomyName, options);
-		if (!def) return [];
-		return cachedQuery({
-			namespace: termCountNamespaces(def.collections),
-			key: `terms:${taxonomyName}:${locale ?? "*"}`,
+	const def = await getTaxonomyDef(taxonomyName, options);
+	if (!def) return [];
+	if (options.includeCounts === false) return getTermList(def, locale);
+
+	// The two are independent, so run them concurrently to save a round trip.
+	const [terms, counts] = await Promise.all([
+		getTermList(def, locale),
+		getVisibleTermCounts(def.name, def.collections),
+	]);
+	return withCounts(terms, counts);
+}
+
+/** Terms without counts, under the cache keys the layout prefetch warms. */
+function getTermList(def: TaxonomyDef, locale: string | undefined): Promise<TaxonomyTerm[]> {
+	const localeKey = locale ?? "*";
+	return requestCached(`taxonomy-terms:${def.name}:${localeKey}`, () =>
+		cachedQuery({
+			namespace: CacheNamespace.TAXONOMIES,
+			key: `termList:${def.name}:${localeKey}`,
 			load: () => loadTaxonomyTerms(def, locale),
-		});
-	});
+		}),
+	);
+}
+
+/**
+ * Copy a term list with counts attached. Counts are keyed by translation_group
+ * (what the pivot stores) and are locale-independent. Rebuilds every node so
+ * the shared, cached count-free list is never mutated.
+ */
+function withCounts(terms: TaxonomyTerm[], counts: Map<string, number>): TaxonomyTerm[] {
+	return terms.map((term) => ({
+		...term,
+		count: counts.get(term.translationGroup ?? term.id) ?? 0,
+		children: withCounts(term.children, counts),
+	}));
 }
 
 async function loadTaxonomyTerms(
@@ -260,14 +302,7 @@ async function loadTaxonomyTerms(
 		.orderBy("label", "asc");
 	if (locale !== undefined) termsQuery = termsQuery.where("locale", "=", locale);
 
-	// Counts are keyed by translation_group (what the pivot stores) and are
-	// locale-independent. Only publicly visible entries are counted (#581);
-	// the map is request-cached so a term detail rendered on the same page
-	// reuses it. Independent of the terms query, so run both concurrently.
-	const [rows, counts] = await Promise.all([
-		termsQuery.execute(),
-		getVisibleTermCounts(def.name, def.collections),
-	]);
+	const rows = await termsQuery.execute();
 
 	const flatTerms: TaxonomyTermRow[] = rows.map((row) => ({
 		id: row.id,
@@ -280,7 +315,7 @@ async function loadTaxonomyTerms(
 		translation_group: row.translation_group,
 	}));
 
-	if (def.hierarchical) return buildTree(flatTerms, counts);
+	if (def.hierarchical) return buildTree(flatTerms);
 
 	return flatTerms.map((term) => ({
 		id: term.id,
@@ -289,7 +324,6 @@ async function loadTaxonomyTerms(
 		label: term.label,
 		description: term.data ? JSON.parse(term.data).description : undefined,
 		children: [],
-		count: counts.get(term.translation_group ?? term.id) ?? 0,
 		locale: term.locale,
 		translationGroup: term.translation_group,
 	}));
@@ -312,8 +346,16 @@ function getVisibleTermCounts(
 	// entry.
 	const scope = [...new Set(collections)].toSorted().join(",");
 	return requestCached(`taxonomy-term-counts:${taxonomyName}:${scope}`, async () => {
-		const db = await getDb();
-		return fetchVisibleTermCounts(db, taxonomyName, collections);
+		// A Map is not JSON-representable — cache the entries, rebuild on read.
+		const entries = await cachedQuery({
+			namespace: termCountNamespaces(collections),
+			key: `termCounts:${taxonomyName}:${scope}`,
+			load: async (): Promise<Array<[string, number]>> => {
+				const db = await getDb();
+				return [...(await fetchVisibleTermCounts(db, taxonomyName, collections))];
+			},
+		});
+		return new Map(entries);
 	});
 }
 
@@ -796,7 +838,7 @@ function rowToTaxonomyDef(row: {
 /**
  * Build tree structure from flat terms
  */
-function buildTree(flatTerms: TaxonomyTermRow[], counts: Map<string, number>): TaxonomyTerm[] {
+function buildTree(flatTerms: TaxonomyTermRow[]): TaxonomyTerm[] {
 	// parent_id holds the parent's translation_group, so link children by it.
 	// Key by (locale, group): a child's parent lives in the same locale, and an
 	// unfiltered set mixes locales whose translated siblings share a group —
@@ -814,7 +856,6 @@ function buildTree(flatTerms: TaxonomyTermRow[], counts: Map<string, number>): T
 			parentId: term.parent_id ?? undefined,
 			description: term.data ? JSON.parse(term.data).description : undefined,
 			children: [],
-			count: counts.get(term.translation_group ?? term.id) ?? 0,
 			locale: term.locale,
 			translationGroup: term.translation_group,
 		};

packages/core/tests/unit/taxonomies/term-count-demand.test.ts

@@ -0,0 +1,168 @@
+/**
+ * Visible term counts are demand-driven: the aggregate over the assignment
+ * pivot runs only for a caller that asked for counts. Assertions are on the SQL
+ * actually executed, because with no object-cache backend `cachedQuery` is a
+ * passthrough and every render pays the aggregate again.
+ */
+
+import Database from "better-sqlite3";
+import { Kysely, SqliteDialect } from "kysely";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+import { runMigrations } from "../../../src/database/migrations/runner.js";
+import { ContentRepository } from "../../../src/database/repositories/content.js";
+import { TaxonomyRepository } from "../../../src/database/repositories/taxonomy.js";
+import type { Database as DatabaseSchema } from "../../../src/database/types.js";
+import { runWithContext } from "../../../src/request-context.js";
+import { SchemaRegistry } from "../../../src/schema/registry.js";
+
+// Mock loader.getDb so the runtime taxonomy functions read from our test db.
+vi.mock("../../../src/loader.js", () => ({
+	getDb: vi.fn(),
+}));
+
+import { prefetchLayoutData } from "../../../src/astro/prefetch.js";
+import { getDb } from "../../../src/loader.js";
+import {
+	getTaxonomyTerms,
+	invalidateTermCache,
+	resetTaxonomyDefsCacheForTests,
+} from "../../../src/taxonomies/index.js";
+
+/** SQL of every query executed against the test database. */
+let queries: string[] = [];
+
+/** `per_collection` is the visible-count aggregate's subquery alias. */
+function countAggregateQueries(): string[] {
+	return queries.filter((q) => q.includes("per_collection"));
+}
+
+function termListQueries(): string[] {
+	return queries.filter((q) => q.includes('from "taxonomies"'));
+}
+
+describe("visible term counts are only computed on demand", () => {
+	let db: Kysely<DatabaseSchema>;
+
+	beforeEach(async () => {
+		queries = [];
+		db = new Kysely<DatabaseSchema>({
+			dialect: new SqliteDialect({ database: new Database(":memory:") }),
+			log(event) {
+				if (event.level === "query") queries.push(event.query.sql);
+			},
+		});
+		await runMigrations(db);
+		vi.mocked(getDb).mockResolvedValue(db);
+		resetTaxonomyDefsCacheForTests();
+		invalidateTermCache();
+
+		// Migrations seed the `category` (hierarchical) and `tag` defs declaring
+		// a `posts` collection; point them at the collection this test creates so
+		// the aggregate runs against a real table.
+		await new SchemaRegistry(db).createCollection({
+			slug: "post",
+			label: "Posts",
+			labelSingular: "Post",
+		});
+		await db
+			.updateTable("_emdash_taxonomy_defs")
+			.set({ collections: JSON.stringify(["post"]) })
+			.where("name", "in", ["category", "tag"])
+			.execute();
+
+		const taxRepo = new TaxonomyRepository(db);
+		const contentRepo = new ContentRepository(db);
+		const parent = await taxRepo.create({
+			name: "category",
+			slug: "tech",
+			label: "Technology",
+			data: { description: "All things tech" },
+		});
+		const child = await taxRepo.create({
+			name: "category",
+			slug: "web",
+			label: "Web",
+			parentId: parent.translationGroup ?? parent.id,
+		});
+		const tag = await taxRepo.create({ name: "tag", slug: "webdev", label: "WebDev" });
+
+		for (const [slug, term] of [
+			["published-one", parent],
+			["published-two", parent],
+			["published-three", child],
+		] as const) {
+			const entry = await contentRepo.create({
+				type: "post",
+				slug,
+				status: "published",
+				data: {},
+			});
+			await taxRepo.attachToEntry("post", entry.id, term.id);
+			await taxRepo.attachToEntry("post", entry.id, tag.id);
+		}
+	});
+
+	afterEach(async () => {
+		resetTaxonomyDefsCacheForTests();
+		invalidateTermCache();
+		await db.destroy();
+		vi.restoreAllMocks();
+	});
+
+	it("does not aggregate counts during the layout prefetch", async () => {
+		await runWithContext({ editMode: false }, async () => {
+			queries = [];
+			await prefetchLayoutData();
+
+			expect(countAggregateQueries()).toEqual([]);
+			// The term lists themselves are still warmed, one query per taxonomy.
+			expect(termListQueries()).toHaveLength(2);
+		});
+	});
+
+	it("reuses the prefetched term list and aggregates once for a caller that wants counts", async () => {
+		await runWithContext({ editMode: false }, async () => {
+			await prefetchLayoutData();
+			queries = [];
+
+			const tags = await getTaxonomyTerms("tag", { includeCounts: false });
+			expect(tags.map((t) => t.slug)).toEqual(["webdev"]);
+			expect(tags[0]).not.toHaveProperty("count");
+			expect(queries).toEqual([]);
+
+			const categories = await getTaxonomyTerms("category");
+			expect(categories[0]!.count).toBe(2);
+			expect(categories[0]!.children[0]!.count).toBe(1);
+			// One aggregate, and no second read of the warmed term list.
+			expect(countAggregateQueries()).toHaveLength(1);
+			expect(termListQueries()).toEqual([]);
+		});
+	});
+
+	it("keeps hierarchy, description and locale on the count-free list", async () => {
+		const [root] = await runWithContext({ editMode: false }, () =>
+			getTaxonomyTerms("category", { includeCounts: false }),
+		);
+
+		expect(root!.slug).toBe("tech");
+		expect(root!.description).toBe("All things tech");
+		expect(root!.locale).toBe("en");
+		expect(root!.children.map((c) => c.slug)).toEqual(["web"]);
+		expect(root!.children[0]).not.toHaveProperty("count");
+	});
+
+	it("recomputes counts per request with no object-cache backend, and never without", async () => {
+		for (const run of [1, 2]) {
+			await runWithContext({ editMode: false }, async () => {
+				queries = [];
+				await prefetchLayoutData();
+				expect(countAggregateQueries(), `prefetch run ${run}`).toEqual([]);
+
+				const categories = await getTaxonomyTerms("category");
+				expect(categories[0]!.count, `counted run ${run}`).toBe(2);
+				expect(countAggregateQueries(), `aggregate run ${run}`).toHaveLength(1);
+			});
+		}
+	});
+});

scripts/query-counts.queries.d1.json

@@ -184,7 +184,7 @@
 		"select count(\"id\") as \"count\" from \"_emdash_comments\" where \"collection\" = ? and \"content_id\" = ? and \"status\" = ?": 1,
 		"select count(*) as \"count\" from \"_emdash_collections\"": 1,
 		"SELECT COUNT(*) as count FROM \"_emdash_migrations\"": 1,
-		"SELECT taxonomy_id, SUM(count) AS count FROM ( SELECT ct.taxonomy_id AS taxonomy_id, COUNT(*) AS count FROM content_taxonomies AS ct INNER JOIN \"ec_posts\" AS e ON e.id = ct.entry_id WHERE ct.collection = ? AND ct.taxonomy_id IN (SELECT translation_group FROM taxonomies WHERE name = ?) AND (\"e\".\"status\" = 'published' OR (\"e\".\"status\" = 'scheduled' AND \"e\".\"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) AND e.deleted_at IS NULL GROUP BY ct.taxonomy_id) AS per_collection GROUP BY taxonomy_id": 2,
+		"SELECT taxonomy_id, SUM(count) AS count FROM ( SELECT ct.taxonomy_id AS taxonomy_id, COUNT(*) AS count FROM content_taxonomies AS ct INNER JOIN \"ec_posts\" AS e ON e.id = ct.entry_id WHERE ct.collection = ? AND ct.taxonomy_id IN (SELECT translation_group FROM taxonomies WHERE name = ?) AND (\"e\".\"status\" = 'published' OR (\"e\".\"status\" = 'scheduled' AND \"e\".\"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) AND e.deleted_at IS NULL GROUP BY ct.taxonomy_id) AS per_collection GROUP BY taxonomy_id": 1,
 		"UPDATE _emdash_cron_tasks SET status = 'idle', locked_at = NULL WHERE status = 'running' AND locked_at < ?": 1
 	},
 	"GET /posts/building-for-the-long-term (warm)": {
@@ -201,7 +201,7 @@
 		"SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct CROSS JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb CROSS JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC LIMIT ?": 1,
 		"SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct CROSS JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb CROSS JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1,
 		"select count(\"id\") as \"count\" from \"_emdash_comments\" where \"collection\" = ? and \"content_id\" = ? and \"status\" = ?": 1,
-		"SELECT taxonomy_id, SUM(count) AS count FROM ( SELECT ct.taxonomy_id AS taxonomy_id, COUNT(*) AS count FROM content_taxonomies AS ct INNER JOIN \"ec_posts\" AS e ON e.id = ct.entry_id WHERE ct.collection = ? AND ct.taxonomy_id IN (SELECT translation_group FROM taxonomies WHERE name = ?) AND (\"e\".\"status\" = 'published' OR (\"e\".\"status\" = 'scheduled' AND \"e\".\"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) AND e.deleted_at IS NULL GROUP BY ct.taxonomy_id) AS per_collection GROUP BY taxonomy_id": 2
+		"SELECT taxonomy_id, SUM(count) AS count FROM ( SELECT ct.taxonomy_id AS taxonomy_id, COUNT(*) AS count FROM content_taxonomies AS ct INNER JOIN \"ec_posts\" AS e ON e.id = ct.entry_id WHERE ct.collection = ? AND ct.taxonomy_id IN (SELECT translation_group FROM taxonomies WHERE name = ?) AND (\"e\".\"status\" = 'published' OR (\"e\".\"status\" = 'scheduled' AND \"e\".\"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) AND e.deleted_at IS NULL GROUP BY ct.taxonomy_id) AS per_collection GROUP BY taxonomy_id": 1
 	},
 	"GET /rss.xml (cold)": {
 		"select \"name\", \"value\" from \"options\" where \"name\" in (...)": 2,