diff --git a/.changeset/fix-d1-result-set-column-limit.md b/.changeset/fix-d1-result-set-column-limit.md new file mode 100644 index 0000000000..b7e551bafb --- /dev/null +++ b/.changeset/fix-d1-result-set-column-limit.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Fixes silent `null` entries on wide-schema collections under Cloudflare D1. The content loader's single-query `LEFT JOIN _emdash_seo` added 5 alias columns to every result set, which pushed collections with ~95+ flat user fields past D1's per-query column limit (~100). The query failed with `D1_ERROR: too many columns in result set`, the error was wrapped as a generic `Failed to load entry`, and the call site surfaced `null`. SEO is now folded into a single aggregated JSON column (mirroring how byline and taxonomy hydration already work), keeping the result-set width bounded regardless of how wide the collection schema gets while preserving the single round trip. diff --git a/packages/core/src/loader.ts b/packages/core/src/loader.ts index afab5970aa..8535777d4a 100644 --- a/packages/core/src/loader.ts +++ b/packages/core/src/loader.ts @@ -25,15 +25,23 @@ import { isMissingColumnError, isMissingTableError } from "./utils/db-errors.js" const FIELD_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; /** - * SEO columns joined in from `_emdash_seo` on the single-entry path, mapped to - * aliased result keys. SEO lives in a side table, so a LEFT JOIN folds it into - * the entry load at zero extra query cost; the result is surfaced as a nested - * `data.seo` object (see extractSeo) rather than flat fields. + * SEO columns folded into the single-entry query as a single JSON column + * (`_emdash_seo` in the result set), then expanded onto the row under these + * aliases for `extractSeo()`. Surfacing SEO as one aggregated column keeps the + * result-set width bounded regardless of how many fields the collection has, + * which matters for D1: a flat `LEFT JOIN _emdash_seo` adds 5 alias columns to + * every row and pushes wide collections (common after WordPress / ACF imports) + * past D1's per-result-set column limit, surfacing as a silent null entry. + * One JSON column is one column, so the join stays safe at any schema width. + * + * The aliases mirror the strategy used by `foldedHydrationSelects` for byline + * and taxonomy hydration: aggregate in SQL, expand in JS. SEO is 1:1 with + * content, so the subquery uses `json_object` (not the array aggregator). * * The `_emdash_` prefix on the aliases guarantees they can never collide with * a content field. Field slugs must match `/^[a-z][a-z0-9_]*$/`, so a user can - * legitimately define a `seo_title` field; selecting the joined column under - * its bare name would shadow that field in the result set and drop the user's + * legitimately define a `seo_title` field; surfacing the SEO column under its + * bare name would shadow that field in the result set and drop the user's * value. The prefix (illegal as a leading slug char) sidesteps this entirely. */ const SEO_COLUMN_ALIASES: Record = { @@ -47,6 +55,9 @@ const SEO_COLUMN_ALIASES: Record = { /** Aliased SEO result keys — excluded from generic field mapping. */ const SEO_ALIAS_COLUMNS = Object.values(SEO_COLUMN_ALIASES); +/** Folded SEO JSON column name in the result set (expanded onto aliases in JS). */ +const SEO_FOLDED_COLUMN = "_emdash_seo"; + /** * System columns excluded from entry.data * Note: slug is intentionally NOT excluded - it's useful as data.slug in templates @@ -67,15 +78,17 @@ const SYSTEM_COLUMNS = new Set([ "draft_revision_id", "locale", "translation_group", - // Aliased SEO columns joined from _emdash_seo on the single-entry path. - // Surfaced as a nested data.seo object (see extractSeo), never as flat - // fields. The aliases are _emdash_-prefixed so they can't shadow a user - // field named e.g. `seo_title`. + // Aliased SEO columns expanded from the folded _emdash_seo JSON column on + // the single-entry path. Surfaced as a nested data.seo object (see + // extractSeo), never as flat fields. The aliases are _emdash_-prefixed so + // they can't shadow a user field named e.g. `seo_title`. ...SEO_ALIAS_COLUMNS, - // Folded hydration JSON columns (see foldedHydrationSelects) — surfaced via - // the FOLDED_* markers, never as flat fields. + // Folded hydration JSON columns (see foldedHydrationSelects and + // foldedSeoSelect) — surfaced via the FOLDED_* markers or expanded onto + // SEO_ALIAS_COLUMNS, never as flat fields. "_emdash_terms", "_emdash_bylines", + SEO_FOLDED_COLUMN, ]); /** Markers for byline/taxonomy hydration folded into the content query. */ @@ -123,6 +136,62 @@ function foldedHydrationSelects(db: Kysely, type: string, outer: string) { return { terms, bylines }; } +/** + * Correlated JSON-object subquery that folds per-entry SEO into the content + * query without widening the result set: 1 row of `_emdash_seo` becomes 1 JSON + * column rather than 5 flat columns. The JSON column is expanded onto the row + * via {@link expandFoldedSeo} after the query runs, preserving the alias keys + * that {@link extractSeo} reads. Missing SEO row (no entry in `_emdash_seo`) + * yields NULL, which {@link expandFoldedSeo} treats as "no SEO" - identical to + * the prior LEFT JOIN miss behavior. + * + * Dialect-specific aggregation mirrors {@link foldedHydrationSelects}: SQLite + * `json_object` returns a JSON *string*, Postgres `json_build_object` returns + * parsed JSON; both branches are handled in expansion. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any Kysely instance +function foldedSeoSelect(db: Kysely, type: string, outer: string) { + const o = sql.ref(outer); + const pg = isPostgres(db); + // Use raw column names (not aliases) as JSON keys: the JSON is expanded back + // onto SEO_COLUMN_ALIASES in JS, and keeping the keys matched to the + // underlying columns makes the SQL readable and the expansion 1-to-1. + const pairs = + "'seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index"; + const obj = pg ? sql.raw(`json_build_object(${pairs})`) : sql.raw(`json_object(${pairs})`); + return sql`(SELECT ${obj} FROM ${sql.ref("_emdash_seo")} AS s WHERE s.collection = ${type} AND s.content_id = ${o}.id LIMIT 1) AS ${sql.ref(SEO_FOLDED_COLUMN)}`; +} + +/** + * Expand the folded `_emdash_seo` JSON column onto the row using SEO_COLUMN_ALIASES, + * so {@link extractSeo} reads it transparently. SQLite returns a JSON string + * (parse it); Postgres returns already-parsed JSON. Missing/malformed/null is + * a no-op: {@link extractSeo} returns null when the aliases are absent. + */ +function isPlainObject(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function expandFoldedSeo(row: Record): void { + const raw = row[SEO_FOLDED_COLUMN]; + delete row[SEO_FOLDED_COLUMN]; + let parsed: Record | null = null; + if (typeof raw === "string") { + try { + const candidate: unknown = JSON.parse(raw); + if (isPlainObject(candidate)) parsed = candidate; + } catch { + return; // malformed JSON: leave the row without SEO aliases (extractSeo returns null) + } + } else if (isPlainObject(raw)) { + parsed = raw; + } + if (!parsed) return; + for (const [col, alias] of Object.entries(SEO_COLUMN_ALIASES)) { + row[alias] = parsed[col] ?? null; + } +} + /** * Stash folded hydration JSON (non-enumerable) for the query.ts fast paths. * SQLite returns a JSON string (parse it); Postgres returns already-parsed JSON. @@ -1050,31 +1119,25 @@ export function emdashLoader(): LiveLoader sql`${sql.ref(`s.${col}`)} AS ${sql.ref(alias)}`, - ), - ); - // Fold byline + taxonomy hydration into the content query (see - // foldedHydrationSelects), removing the two separate hydration - // round trips per fetch. + // Byline + taxonomy hydration (foldedHydrationSelects) and per-entry + // SEO (foldedSeoSelect) are each surfaced as a single aggregated JSON + // column rather than flat columns. This keeps the result-set width + // bounded at any collection schema width: a flat `LEFT JOIN _emdash_seo` + // adds 5 alias columns to every row and pushes wide flat-schema + // collections (common after WordPress / ACF imports) past D1's + // per-result-set column limit, surfacing as a silent null entry. One + // JSON column is one column, so the join stays safe at any width and + // we keep the single round trip. const { terms: termsSelect, bylines: bylinesSelect } = foldedHydrationSelects( db, type, "c", ); + const seoSelect = foldedSeoSelect(db, type, "c"); const result = locale ? await sql>` SELECT c.*, ${seoSelect}, ${termsSelect}, ${bylinesSelect} FROM ${sql.ref(tableName)} AS c - LEFT JOIN ${sql.ref("_emdash_seo")} AS s - ON s.collection = ${type} AND s.content_id = c.id WHERE c.deleted_at IS NULL AND ((c.slug = ${id} AND c.locale = ${locale}) OR c.id = ${id}) LIMIT 1 @@ -1082,8 +1145,6 @@ export function emdashLoader(): LiveLoader>` SELECT c.*, ${seoSelect}, ${termsSelect}, ${bylinesSelect} FROM ${sql.ref(tableName)} AS c - LEFT JOIN ${sql.ref("_emdash_seo")} AS s - ON s.collection = ${type} AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ${id} OR c.id = ${id}) LIMIT 1 @@ -1094,6 +1155,11 @@ export function emdashLoader(): LiveLoader 1; const entrySlug = rowStr(row, "slug") || rowStr(row, "id"); diff --git a/packages/core/tests/unit/loader-seo.test.ts b/packages/core/tests/unit/loader-seo.test.ts index 710f496f32..5093ab57ac 100644 --- a/packages/core/tests/unit/loader-seo.test.ts +++ b/packages/core/tests/unit/loader-seo.test.ts @@ -14,10 +14,12 @@ import { /** * Regression test for #1270: SEO fields (noindex toggle, canonical URL) set in * the admin had no effect on rendered pages because the content loader never - * surfaced the `_emdash_seo` row. The loader now LEFT JOINs that table and - * attaches the result to `entry.data.seo`, which `getSeoMeta()` reads. + * surfaced the `_emdash_seo` row. The loader now folds that row into the + * single-entry query as an aggregated JSON column and attaches the expanded + * result to `entry.data.seo`, which `getSeoMeta()` reads. * - * Run on both dialects — the LEFT JOIN SQL is dialect-sensitive. + * Run on both dialects, since the JSON aggregation SQL is dialect-sensitive + * (`json_object` on SQLite, `json_build_object` on Postgres). */ describeEachDialect("Loader SEO hydration (#1270)", (dialect) => { let ctx: DialectTestContext; diff --git a/packages/core/tests/unit/loader-wide-collection.test.ts b/packages/core/tests/unit/loader-wide-collection.test.ts new file mode 100644 index 0000000000..5e2f22000e --- /dev/null +++ b/packages/core/tests/unit/loader-wide-collection.test.ts @@ -0,0 +1,157 @@ +import { it, expect, beforeEach, afterEach } from "vitest"; + +import { handleContentCreate } from "../../src/api/index.js"; +import { SeoRepository } from "../../src/database/repositories/seo.js"; +import { emdashLoader } from "../../src/loader.js"; +import { runWithContext } from "../../src/request-context.js"; +import { SchemaRegistry } from "../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../utils/test-db.js"; + +/** + * Regression test for #1600: loadEntry's SELECT shape on wide collections. + * + * When a per-collection `ec_*` table has many flat scalar columns (common when + * porting from WordPress / ACF or other builders where every section is a + * top-level field), the previous implementation did: + * + * SELECT c.*, <5 SEO alias columns> FROM ec_table c LEFT JOIN _emdash_seo s + * + * On Cloudflare D1 the per-query result-set column limit (~100) made this + * fail with `D1_ERROR: too many columns in result set` for collections + * around 95+ user columns. The loader's try/catch wrapped it as a generic + * `Failed to load entry` error and the call site returned a silent `null`. + * + * The fix folds SEO into a single aggregated JSON column (`_emdash_seo`) + * mirroring how byline and taxonomy hydration already work: aggregate in SQL, + * expand in JS. One JSON column is one column, so the result-set width stays + * bounded at any collection schema width and the single round trip is + * preserved. + * + * Run on both dialects to keep parity with loader-seo.test.ts. + */ +describeEachDialect("Loader on wide-schema collections (#1600)", (dialect) => { + let ctx: DialectTestContext; + let seoRepo: SeoRepository; + const COLLECTION = "wide_collection"; + const USER_FIELD_COUNT = 95; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + const registry = new SchemaRegistry(ctx.db); + + // Create a collection with SEO enabled and a large number of flat + // scalar fields. 95 user fields + 14 system columns + 5 SEO aliases + // would have been ~114 result-set columns under the old LEFT JOIN + // shape, well past D1's per-query limit. + await registry.createCollection({ + slug: COLLECTION, + label: "Wide Collection", + labelSingular: "Wide Entry", + }); + await registry.createField(COLLECTION, { + slug: "title", + label: "Title", + type: "string", + }); + for (let i = 1; i <= USER_FIELD_COUNT; i++) { + await registry.createField(COLLECTION, { + slug: `field_${i}`, + label: `Field ${i}`, + type: "string", + }); + } + // Enable SEO so extractSeo() has somewhere to read from. + await ctx.db + .updateTable("_emdash_collections") + .set({ has_seo: 1 }) + .where("slug", "=", COLLECTION) + .execute(); + + seoRepo = new SeoRepository(ctx.db); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + function load(idOrSlug: string) { + const loader = emdashLoader(); + return runWithContext({ db: ctx.db }, () => + loader.loadEntry!({ filter: { type: COLLECTION, id: idOrSlug } }), + ); + } + + it("loads an entry from a collection with 95+ flat user columns", async () => { + const data: Record = { title: "Wide Entry" }; + for (let i = 1; i <= USER_FIELD_COUNT; i++) { + data[`field_${i}`] = `value-${i}`; + } + const result = await handleContentCreate(ctx.db, COLLECTION, { + data, + status: "published", + }); + if (!result.success) throw new Error("Failed to create entry"); + const slug = result.data!.item.slug!; + + const loaded = await load(slug); + + expect(loaded).toBeDefined(); + expect((loaded as { data: Record }).data.title).toBe("Wide Entry"); + // Spot-check a handful of user fields across the range. + const loadedData = (loaded as { data: Record }).data; + expect(loadedData.field_1).toBe("value-1"); + expect(loadedData.field_50).toBe("value-50"); + expect(loadedData.field_95).toBe("value-95"); + }); + + it("still attaches data.seo on wide collections (folded JSON column)", async () => { + const data: Record = { title: "Wide With SEO" }; + for (let i = 1; i <= USER_FIELD_COUNT; i++) { + data[`field_${i}`] = `value-${i}`; + } + const result = await handleContentCreate(ctx.db, COLLECTION, { + data, + status: "published", + }); + if (!result.success) throw new Error("Failed to create entry"); + const item = result.data!.item; + + await seoRepo.upsert(COLLECTION, item.id, { + noIndex: true, + canonical: "https://example.com/wide", + title: "Wide SEO Title", + }); + + const loaded = await load(item.slug!); + const loadedData = (loaded as { data: Record }).data; + const seo = loadedData.seo as Record | undefined; + + expect(seo).toBeDefined(); + expect(seo!.noIndex).toBe(true); + expect(seo!.canonical).toBe("https://example.com/wide"); + expect(seo!.title).toBe("Wide SEO Title"); + }); + + it("omits data.seo when no SEO row exists, even on wide collections", async () => { + const data: Record = { title: "No SEO" }; + for (let i = 1; i <= USER_FIELD_COUNT; i++) { + data[`field_${i}`] = `value-${i}`; + } + const result = await handleContentCreate(ctx.db, COLLECTION, { + data, + status: "published", + }); + if (!result.success) throw new Error("Failed to create entry"); + const slug = result.data!.item.slug!; + + const loaded = await load(slug); + const loadedData = (loaded as { data: Record }).data; + + expect(loadedData.seo).toBeUndefined(); + }); +});