From d1bc115f27ec4b4b78cece7f47b037b4a26d16b5 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 00:40:58 +0100 Subject: [PATCH 01/29] feat(media): add usage index tables --- .../migrations/046_media_usage_index.ts | 115 ++++++++++++++ .../core/src/database/migrations/runner.ts | 2 + packages/core/src/database/types.ts | 39 +++++ .../database/media-usage-migration.test.ts | 143 ++++++++++++++++++ .../integration/database/migrations.test.ts | 3 + 5 files changed, 302 insertions(+) create mode 100644 packages/core/src/database/migrations/046_media_usage_index.ts create mode 100644 packages/core/tests/integration/database/media-usage-migration.test.ts diff --git a/packages/core/src/database/migrations/046_media_usage_index.ts b/packages/core/src/database/migrations/046_media_usage_index.ts new file mode 100644 index 0000000000..dce7d65424 --- /dev/null +++ b/packages/core/src/database/migrations/046_media_usage_index.ts @@ -0,0 +1,115 @@ +import type { Kysely } from "kysely"; + +import { currentTimestamp } from "../dialect-helpers.js"; + +/** + * Internal media usage projection tables. + * + * This migration is DDL-only by design: no backfill, no runtime media/indexing + * imports, and no content-table scans. Production rows are introduced by later + * phases once the central snapshot/indexer path is reviewed. + */ +export async function up(db: Kysely): Promise { + await db.schema + .createTable("_emdash_media_usage_sources") + .ifNotExists() + .addColumn("source_key", "text", (c) => c.primaryKey()) + .addColumn("source_type", "text", (c) => c.notNull()) + .addColumn("collection_slug", "text") + .addColumn("content_id", "text") + .addColumn("source_variant", "text", (c) => c.notNull()) + .addColumn("locale", "text") + .addColumn("translation_group", "text") + .addColumn("content_slug", "text") + .addColumn("content_title", "text") + .addColumn("content_status", "text") + .addColumn("content_scheduled_at", "text") + .addColumn("content_deleted_at", "text") + .addColumn("revision_id", "text") + .addColumn("current_generation", "text", (c) => c.notNull()) + .addColumn("schema_version", "integer", (c) => c.notNull().defaultTo(1)) + .addColumn("indexed_at", "text", (c) => c.notNull().defaultTo(currentTimestamp(db))) + .addColumn("created_at", "text", (c) => c.defaultTo(currentTimestamp(db))) + .addColumn("updated_at", "text", (c) => c.defaultTo(currentTimestamp(db))) + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_sources_content") + .ifNotExists() + .on("_emdash_media_usage_sources") + .columns(["source_type", "collection_slug", "content_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_sources_variant") + .ifNotExists() + .on("_emdash_media_usage_sources") + .columns(["source_type", "source_variant"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_sources_locale") + .ifNotExists() + .on("_emdash_media_usage_sources") + .columns(["collection_slug", "locale"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_sources_deleted") + .ifNotExists() + .on("_emdash_media_usage_sources") + .column("content_deleted_at") + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_sources_translation_group") + .ifNotExists() + .on("_emdash_media_usage_sources") + .columns(["collection_slug", "translation_group"]) + .execute(); + + await db.schema + .createTable("_emdash_media_usage") + .ifNotExists() + .addColumn("id", "text", (c) => c.primaryKey()) + .addColumn("source_key", "text", (c) => c.notNull()) + .addColumn("generation", "text", (c) => c.notNull()) + .addColumn("field_slug", "text", (c) => c.notNull()) + .addColumn("field_path", "text", (c) => c.notNull()) + .addColumn("occurrence_index", "integer", (c) => c.notNull().defaultTo(0)) + .addColumn("reference_type", "text", (c) => c.notNull()) + .addColumn("media_id", "text") + .addColumn("provider", "text", (c) => c.notNull().defaultTo("local")) + .addColumn("provider_asset_id", "text", (c) => c.notNull()) + .addColumn("media_kind", "text") + .addColumn("mime_type", "text") + .addColumn("created_at", "text", (c) => c.defaultTo(currentTimestamp(db))) + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_unique_occurrence") + .ifNotExists() + .unique() + .on("_emdash_media_usage") + .columns(["source_key", "generation", "field_path", "occurrence_index"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_media_id") + .ifNotExists() + .on("_emdash_media_usage") + .column("media_id") + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_provider_asset") + .ifNotExists() + .on("_emdash_media_usage") + .columns(["provider", "provider_asset_id"]) + .execute(); + await db.schema + .createIndex("idx__emdash_media_usage_source_generation") + .ifNotExists() + .on("_emdash_media_usage") + .columns(["source_key", "generation"]) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("_emdash_media_usage").ifExists().execute(); + await db.schema.dropTable("_emdash_media_usage_sources").ifExists().execute(); +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 0d45422a14..0c651816b0 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -47,6 +47,7 @@ import * as m042 from "./042_byline_fields.js"; import * as m043 from "./043_content_references.js"; import * as m044 from "./044_comment_reactions.js"; import * as m045 from "./045_taxonomy_parent_group.js"; +import * as m046 from "./046_media_usage_index.js"; const MIGRATIONS: Readonly> = Object.freeze({ "001_initial": m001, @@ -93,6 +94,7 @@ const MIGRATIONS: Readonly> = Object.freeze({ "043_content_references": m043, "044_comment_reactions": m044, "045_taxonomy_parent_group": m045, + "046_media_usage_index": m046, }); /** Total number of registered migrations. Exported for use in tests. */ diff --git a/packages/core/src/database/types.ts b/packages/core/src/database/types.ts index 7b437cd7f1..f8278fb34b 100644 --- a/packages/core/src/database/types.ts +++ b/packages/core/src/database/types.ts @@ -60,6 +60,43 @@ export interface MediaTable { author_id: string | null; } +export interface MediaUsageSourceTable { + source_key: string; + source_type: string; + collection_slug: string | null; + content_id: string | null; + source_variant: string; + locale: string | null; + translation_group: string | null; + content_slug: string | null; + content_title: string | null; + content_status: string | null; + content_scheduled_at: string | null; + content_deleted_at: string | null; + revision_id: string | null; + current_generation: string; + schema_version: Generated; + indexed_at: Generated; + created_at: Generated; + updated_at: Generated; +} + +export interface MediaUsageTable { + id: string; + source_key: string; + generation: string; + field_slug: string; + field_path: string; + occurrence_index: Generated; + reference_type: string; + media_id: string | null; + provider: Generated; + provider_asset_id: string; + media_kind: string | null; + mime_type: string | null; + created_at: Generated; +} + export interface UserTable { id: string; email: string; @@ -414,6 +451,8 @@ export interface Database { content_taxonomies: ContentTaxonomyTable; _emdash_taxonomy_defs: TaxonomyDefTable; media: MediaTable; + _emdash_media_usage_sources: MediaUsageSourceTable; + _emdash_media_usage: MediaUsageTable; users: UserTable; credentials: CredentialTable; auth_tokens: AuthTokenTable; diff --git a/packages/core/tests/integration/database/media-usage-migration.test.ts b/packages/core/tests/integration/database/media-usage-migration.test.ts new file mode 100644 index 0000000000..ce9797ffb2 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-migration.test.ts @@ -0,0 +1,143 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +const EXPECTED_INDEXES = [ + "idx__emdash_media_usage_sources_content", + "idx__emdash_media_usage_sources_variant", + "idx__emdash_media_usage_sources_locale", + "idx__emdash_media_usage_sources_deleted", + "idx__emdash_media_usage_sources_translation_group", + "idx__emdash_media_usage_media_id", + "idx__emdash_media_usage_provider_asset", + "idx__emdash_media_usage_source_generation", + "idx__emdash_media_usage_unique_occurrence", +] as const; + +describeEachDialect("media usage index migration", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("creates usage source and occurrence tables through registered migrations", async () => { + const sources = await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .execute(); + const usage = await ctx.db.selectFrom("_emdash_media_usage").select("id").execute(); + + expect(Array.isArray(sources)).toBe(true); + expect(Array.isArray(usage)).toBe(true); + }); + + it("accepts a content usage source and one occurrence", async () => { + const sourceKey = "content:posts:entry1:live"; + const generation = "gen1"; + + await ctx.db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: sourceKey, + source_type: "content", + collection_slug: "posts", + content_id: "entry1", + source_variant: "live", + content_slug: "hello-world", + content_title: "Hello World", + locale: "en", + translation_group: "tg1", + content_status: "published", + content_scheduled_at: null, + content_deleted_at: null, + revision_id: "rev1", + current_generation: generation, + schema_version: 1, + }) + .execute(); + + await ctx.db + .insertInto("_emdash_media_usage") + .values({ + id: "usage1", + source_key: sourceKey, + generation, + field_slug: "hero", + field_path: "hero", + occurrence_index: 0, + reference_type: "image_field", + media_id: "media1", + provider: "local", + provider_asset_id: "media1", + media_kind: "image", + mime_type: "image/jpeg", + }) + .execute(); + + const rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select(["source_key", "media_id", "provider", "provider_asset_id", "field_path"]) + .where("media_id", "=", "media1") + .execute(); + + expect(rows).toEqual([ + { + source_key: sourceKey, + media_id: "media1", + provider: "local", + provider_asset_id: "media1", + field_path: "hero", + }, + ]); + }); + + it("creates expected indexes", async () => { + const indexNames = await listIndexNames(ctx); + + for (const indexName of EXPECTED_INDEXES) { + expect(indexNames.has(indexName), `missing index ${indexName}`).toBe(true); + } + }); + + it("down() drops tables and up() recreates them", async () => { + const migration = await import("../../../src/database/migrations/046_media_usage_index.js"); + + await migration.down(ctx.db); + + await expect(sql`SELECT 1 FROM _emdash_media_usage`.execute(ctx.db)).rejects.toThrow(); + await expect(sql`SELECT 1 FROM _emdash_media_usage_sources`.execute(ctx.db)).rejects.toThrow(); + + await migration.up(ctx.db); + + const sources = await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .execute(); + expect(Array.isArray(sources)).toBe(true); + }); +}); + +async function listIndexNames(ctx: DialectTestContext): Promise> { + if (ctx.dialect === "sqlite") { + const result = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'index' + `.execute(ctx.db); + return new Set(result.rows.map((row) => row.name)); + } + + const result = await sql<{ name: string }>` + SELECT indexname AS name FROM pg_indexes WHERE schemaname = current_schema() + `.execute(ctx.db); + return new Set(result.rows.map((row) => row.name)); +} diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index 91105ac8ff..469ba12441 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -48,6 +48,8 @@ describe("Database Migrations (Integration)", () => { "_emdash_byline_fields", "_emdash_byline_field_values", "_emdash_byline_field_group_values", + "_emdash_media_usage_sources", + "_emdash_media_usage", ]; for (const table of tables) { @@ -128,6 +130,7 @@ describe("Database Migrations (Integration)", () => { "043_content_references", "044_comment_reactions", "045_taxonomy_parent_group", + "046_media_usage_index", ]; await db.deleteFrom("_emdash_migrations").where("name", "in", trailing).execute(); From 72b3253c68476853e6b4666ef65c6031a4d0a848 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 00:53:15 +0100 Subject: [PATCH 02/29] feat(media): add usage extractor --- packages/core/src/media/usage/extractor.ts | 287 ++++++++++++++ packages/core/src/media/usage/types.ts | 46 +++ .../tests/unit/media/usage-extractor.test.ts | 360 ++++++++++++++++++ 3 files changed, 693 insertions(+) create mode 100644 packages/core/src/media/usage/extractor.ts create mode 100644 packages/core/src/media/usage/types.ts create mode 100644 packages/core/tests/unit/media/usage-extractor.test.ts diff --git a/packages/core/src/media/usage/extractor.ts b/packages/core/src/media/usage/extractor.ts new file mode 100644 index 0000000000..7273cf1be1 --- /dev/null +++ b/packages/core/src/media/usage/extractor.ts @@ -0,0 +1,287 @@ +import { normalizeMime } from "../mime.js"; +import type { + ExtractedMediaUsageOccurrence, + ExtractMediaUsageOccurrencesInput, + MediaKind, + MediaUsageExtractionSubField, + MediaUsageReferenceType, +} from "./types.js"; + +const INTERNAL_MEDIA_PREFIX = "/_emdash/api/media/file/"; +const URL_LIKE_RE = /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i; + +interface MediaRef { + mediaId: string | null; + provider: string; + providerAssetId: string; + mediaKind: MediaKind | null; + mimeType: string | null; +} + +interface AddOccurrenceInput { + fieldSlug: string; + fieldPath: string; + referenceType: MediaUsageReferenceType; + value: unknown; + fallbackKind: MediaKind | null; +} + +export function extractMediaUsageOccurrences({ + fields, + data, +}: ExtractMediaUsageOccurrencesInput): ExtractedMediaUsageOccurrence[] { + const occurrences: ExtractedMediaUsageOccurrence[] = []; + const seen = new Set(); + + for (const field of fields) { + const value = data[field.slug]; + + if (field.type === "image") { + addOccurrence(occurrences, seen, { + fieldSlug: field.slug, + fieldPath: field.slug, + referenceType: "image_field", + value, + fallbackKind: "image", + }); + continue; + } + + if (field.type === "file") { + addOccurrence(occurrences, seen, { + fieldSlug: field.slug, + fieldPath: field.slug, + referenceType: "file_field", + value, + fallbackKind: null, + }); + continue; + } + + if (field.type === "repeater") { + extractRepeaterOccurrences(occurrences, seen, field.slug, value, field.validation?.subFields); + continue; + } + + if (field.type === "portableText") { + extractPortableTextOccurrences(occurrences, seen, field.slug, value); + } + } + + return occurrences; +} + +function extractRepeaterOccurrences( + occurrences: ExtractedMediaUsageOccurrence[], + seen: Set, + fieldSlug: string, + value: unknown, + subFields: readonly MediaUsageExtractionSubField[] | undefined, +): void { + if (!Array.isArray(value) || !Array.isArray(subFields)) return; + + for (const [itemIndex, item] of value.entries()) { + if (!isRecord(item)) continue; + + for (const subField of subFields) { + if (subField.type !== "image" && subField.type !== "file") continue; + + addOccurrence(occurrences, seen, { + fieldSlug, + fieldPath: `${fieldSlug}[${itemIndex}].${subField.slug}`, + referenceType: subField.type === "image" ? "image_field" : "file_field", + value: item[subField.slug], + fallbackKind: subField.type === "image" ? "image" : null, + }); + } + } +} + +function extractPortableTextOccurrences( + occurrences: ExtractedMediaUsageOccurrence[], + seen: Set, + fieldSlug: string, + value: unknown, +): void { + if (!Array.isArray(value)) return; + + for (const [blockIndex, block] of value.entries()) { + if (!isRecord(block) || block._type !== "image" || !isRecord(block.asset)) continue; + + const ref = readPortableTextAssetRef(block.asset); + if (!ref) continue; + + addRefOccurrence(occurrences, seen, { + fieldSlug, + fieldPath: `${fieldSlug}[${blockIndex}].asset.${ref.key}`, + referenceType: "portable_text_image", + ref: buildMediaRef({ + id: ref.id, + provider: readString(block.asset.provider) ?? "local", + mimeType: normalizeMimeValue(block.asset.mimeType), + fallbackKind: "image", + }), + }); + } +} + +function addOccurrence( + occurrences: ExtractedMediaUsageOccurrence[], + seen: Set, + input: AddOccurrenceInput, +): void { + const ref = readMediaRef(input.value, input.fallbackKind); + if (!ref) return; + + addRefOccurrence(occurrences, seen, { + fieldSlug: input.fieldSlug, + fieldPath: input.fieldPath, + referenceType: input.referenceType, + ref, + }); +} + +function addRefOccurrence( + occurrences: ExtractedMediaUsageOccurrence[], + seen: Set, + input: { + fieldSlug: string; + fieldPath: string; + referenceType: MediaUsageReferenceType; + ref: MediaRef | null; + }, +): void { + if (!input.ref) return; + + const occurrence: ExtractedMediaUsageOccurrence = { + fieldSlug: input.fieldSlug, + fieldPath: input.fieldPath, + occurrenceIndex: 0, + referenceType: input.referenceType, + mediaId: input.ref.mediaId, + provider: input.ref.provider, + providerAssetId: input.ref.providerAssetId, + mediaKind: input.ref.mediaKind, + mimeType: input.ref.mimeType, + }; + + const key = [ + occurrence.fieldSlug, + occurrence.fieldPath, + occurrence.occurrenceIndex, + occurrence.referenceType, + occurrence.provider, + occurrence.providerAssetId, + occurrence.mediaId ?? "", + ].join("\0"); + + if (seen.has(key)) return; + seen.add(key); + occurrences.push(occurrence); +} + +function readMediaRef(value: unknown, fallbackKind: MediaKind | null): MediaRef | null { + if (typeof value === "string") { + const id = normalizeStableId(value); + return id ? buildMediaRef({ id, provider: "local", mimeType: null, fallbackKind }) : null; + } + + if (!isRecord(value)) return null; + + const id = normalizeStableId(value.id); + if (!id) return null; + + return buildMediaRef({ + id, + provider: readString(value.provider) ?? "local", + mimeType: normalizeMimeValue(value.mimeType), + fallbackKind, + }); +} + +function buildMediaRef(input: { + id: string; + provider: string; + mimeType: string | null; + fallbackKind: MediaKind | null; +}): MediaRef | null { + const provider = input.provider.trim() || "local"; + if (provider === "external") return null; + + return { + mediaId: provider === "local" ? input.id : null, + provider, + providerAssetId: input.id, + mediaKind: mediaKindFromMime(input.mimeType) ?? input.fallbackKind, + mimeType: input.mimeType, + }; +} + +function readPortableTextAssetRef( + asset: Record, +): { key: "_ref" | "id"; id: string } | null { + const ref = normalizeStableId(asset._ref); + if (ref) return { key: "_ref", id: ref }; + + const id = normalizeStableId(asset.id); + if (id) return { key: "id", id }; + + return null; +} + +function normalizeStableId(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!trimmed) return null; + if (URL_LIKE_RE.test(trimmed)) return null; + if (trimmed.startsWith(INTERNAL_MEDIA_PREFIX)) return null; + return trimmed; +} + +function normalizeMimeValue(value: unknown): string | null { + if (typeof value !== "string") return null; + const normalized = normalizeMime(value); + return normalized.includes("/") ? normalized : null; +} + +function mediaKindFromMime(mimeType: string | null): MediaKind | null { + if (!mimeType) return null; + if (mimeType.startsWith("image/")) return "image"; + if (mimeType.startsWith("video/")) return "video"; + if (mimeType.startsWith("audio/")) return "audio"; + if (mimeType.startsWith("font/") || mimeType.startsWith("application/font-")) return "font"; + if (mimeType.startsWith("text/")) return "text"; + if (isDocumentMime(mimeType)) return "document"; + if (isArchiveMime(mimeType)) return "archive"; + return "other"; +} + +function isDocumentMime(mimeType: string): boolean { + return ( + mimeType === "application/pdf" || + mimeType === "application/msword" || + mimeType === "application/rtf" || + mimeType === "application/vnd.ms-excel" || + mimeType === "application/vnd.ms-powerpoint" || + mimeType.startsWith("application/vnd.openxmlformats-officedocument.") + ); +} + +function isArchiveMime(mimeType: string): boolean { + return ( + mimeType === "application/zip" || + mimeType === "application/gzip" || + mimeType === "application/x-tar" || + mimeType === "application/x-7z-compressed" || + mimeType === "application/x-rar-compressed" || + mimeType === "application/vnd.rar" + ); +} + +function readString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/core/src/media/usage/types.ts b/packages/core/src/media/usage/types.ts new file mode 100644 index 0000000000..a76281c7db --- /dev/null +++ b/packages/core/src/media/usage/types.ts @@ -0,0 +1,46 @@ +import type { FieldType } from "../../schema/types.js"; + +export type MediaKind = + | "image" + | "video" + | "audio" + | "document" + | "archive" + | "font" + | "text" + | "other"; + +export type MediaUsageReferenceType = "image_field" | "file_field" | "portable_text_image"; + +export interface MediaUsageExtractionSubField { + slug: string; + type: FieldType; + label?: string; +} + +export interface MediaUsageExtractionValidation { + subFields?: readonly MediaUsageExtractionSubField[]; +} + +export interface MediaUsageExtractionField { + slug: string; + type: FieldType; + validation?: MediaUsageExtractionValidation | null; +} + +export interface ExtractMediaUsageOccurrencesInput { + fields: readonly MediaUsageExtractionField[]; + data: Record; +} + +export interface ExtractedMediaUsageOccurrence { + fieldSlug: string; + fieldPath: string; + occurrenceIndex: number; + referenceType: MediaUsageReferenceType; + mediaId: string | null; + provider: string; + providerAssetId: string; + mediaKind: MediaKind | null; + mimeType: string | null; +} diff --git a/packages/core/tests/unit/media/usage-extractor.test.ts b/packages/core/tests/unit/media/usage-extractor.test.ts new file mode 100644 index 0000000000..265546788e --- /dev/null +++ b/packages/core/tests/unit/media/usage-extractor.test.ts @@ -0,0 +1,360 @@ +import { describe, expect, it } from "vitest"; + +import { extractMediaUsageOccurrences } from "../../../src/media/usage/extractor.js"; +import type { MediaUsageExtractionField } from "../../../src/media/usage/types.js"; + +function field( + slug: string, + type: MediaUsageExtractionField["type"], + validation?: MediaUsageExtractionField["validation"], +): MediaUsageExtractionField { + return { slug, type, validation }; +} + +describe("extractMediaUsageOccurrences", () => { + it("extracts top-level image and file field references", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [field("hero", "image"), field("attachment", "file"), field("title", "string")], + data: { + hero: { + id: "media-hero", + provider: "local", + mimeType: "Image/JPEG; charset=utf-8", + }, + attachment: { + id: "media-file", + provider: "local", + mimeType: "application/pdf", + }, + title: "media-title", + }, + }); + + expect(occurrences).toEqual([ + { + fieldSlug: "hero", + fieldPath: "hero", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: "media-hero", + provider: "local", + providerAssetId: "media-hero", + mediaKind: "image", + mimeType: "image/jpeg", + }, + { + fieldSlug: "attachment", + fieldPath: "attachment", + occurrenceIndex: 0, + referenceType: "file_field", + mediaId: "media-file", + provider: "local", + providerAssetId: "media-file", + mediaKind: "document", + mimeType: "application/pdf", + }, + ]); + }); + + it("extracts legacy bare local IDs and skips URLs or internal file routes", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [ + field("hero", "image"), + field("attachment", "file"), + field("external", "image"), + field("internal", "image"), + field("blank", "image"), + ], + data: { + hero: "media-hero", + attachment: "media-file", + external: "https://example.com/photo.jpg", + internal: "/_emdash/api/media/file/uploads/photo.jpg", + blank: " ", + }, + }); + + expect(occurrences).toEqual([ + { + fieldSlug: "hero", + fieldPath: "hero", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: "media-hero", + provider: "local", + providerAssetId: "media-hero", + mediaKind: "image", + mimeType: null, + }, + { + fieldSlug: "attachment", + fieldPath: "attachment", + occurrenceIndex: 0, + referenceType: "file_field", + mediaId: "media-file", + provider: "local", + providerAssetId: "media-file", + mediaKind: null, + mimeType: null, + }, + ]); + }); + + it("extracts structured external provider references without local media IDs", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [field("hero", "image"), field("video", "file")], + data: { + hero: { + id: "cf-image-1", + provider: "cloudflare-images", + mimeType: "image/png", + }, + video: { + id: "mux-video-1", + provider: "mux", + mimeType: "video/mp4", + }, + }, + }); + + expect(occurrences).toEqual([ + { + fieldSlug: "hero", + fieldPath: "hero", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: null, + provider: "cloudflare-images", + providerAssetId: "cf-image-1", + mediaKind: "image", + mimeType: "image/png", + }, + { + fieldSlug: "video", + fieldPath: "video", + occurrenceIndex: 0, + referenceType: "file_field", + mediaId: null, + provider: "mux", + providerAssetId: "mux-video-1", + mediaKind: "video", + mimeType: "video/mp4", + }, + ]); + }); + + it("extracts repeater image and defensive file subfields with stable paths", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [ + field("sections", "repeater", { + subFields: [ + { slug: "image", type: "image", label: "Image" }, + { slug: "download", type: "file", label: "Download" }, + ], + }), + ], + data: { + sections: [ + { + image: { id: "image-1", mimeType: "image/webp" }, + download: { id: "file-1", mimeType: "application/zip" }, + }, + { + image: "image-2", + download: { + id: "video-1", + provider: "mux", + mimeType: "video/mp4", + }, + }, + ], + }, + }); + + expect(occurrences).toEqual([ + { + fieldSlug: "sections", + fieldPath: "sections[0].image", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: "image-1", + provider: "local", + providerAssetId: "image-1", + mediaKind: "image", + mimeType: "image/webp", + }, + { + fieldSlug: "sections", + fieldPath: "sections[0].download", + occurrenceIndex: 0, + referenceType: "file_field", + mediaId: "file-1", + provider: "local", + providerAssetId: "file-1", + mediaKind: "archive", + mimeType: "application/zip", + }, + { + fieldSlug: "sections", + fieldPath: "sections[1].image", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: "image-2", + provider: "local", + providerAssetId: "image-2", + mediaKind: "image", + mimeType: null, + }, + { + fieldSlug: "sections", + fieldPath: "sections[1].download", + occurrenceIndex: 0, + referenceType: "file_field", + mediaId: null, + provider: "mux", + providerAssetId: "video-1", + mediaKind: "video", + mimeType: "video/mp4", + }, + ]); + }); + + it("extracts Portable Text image block asset refs", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [field("body", "portableText")], + data: { + body: [ + { _type: "block", _key: "p1", children: [] }, + { + _type: "image", + _key: "img1", + asset: { + _ref: "local-image", + url: "/_emdash/api/media/file/local-image.jpg", + }, + }, + { + _type: "image", + _key: "img2", + asset: { + id: "cf-image", + provider: "cloudflare-images", + mimeType: "image/avif", + }, + }, + { _type: "image", _key: "img3", asset: { url: "https://example.com/cat.jpg" } }, + { _type: "image", _key: "img4" }, + ], + }, + }); + + expect(occurrences).toEqual([ + { + fieldSlug: "body", + fieldPath: "body[1].asset._ref", + occurrenceIndex: 0, + referenceType: "portable_text_image", + mediaId: "local-image", + provider: "local", + providerAssetId: "local-image", + mediaKind: "image", + mimeType: null, + }, + { + fieldSlug: "body", + fieldPath: "body[2].asset.id", + occurrenceIndex: 0, + referenceType: "portable_text_image", + mediaId: null, + provider: "cloudflare-images", + providerAssetId: "cf-image", + mediaKind: "image", + mimeType: "image/avif", + }, + ]); + }); + + it("skips URL-only and malformed media values", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [ + field("hero", "image"), + field("srcOnly", "image"), + field("externalProvider", "image"), + field("badId", "file"), + field("pt", "portableText"), + ], + data: { + hero: { id: "https://example.com/photo.jpg", provider: "local" }, + srcOnly: { src: "https://example.com/photo.jpg" }, + externalProvider: { + provider: "external", + id: "", + src: "https://example.com/photo.jpg", + }, + badId: { id: 123, provider: "local" }, + pt: [ + { + _type: "image", + asset: { _ref: "/_emdash/api/media/file/uploads/photo.jpg" }, + }, + ], + }, + }); + + expect(occurrences).toEqual([]); + }); + + it("dedupes exact duplicate occurrence identities without collapsing repeated media uses", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [ + field("hero", "image"), + field("hero", "image"), + field("sections", "repeater", { + subFields: [{ slug: "image", type: "image", label: "Image" }], + }), + field("body", "portableText"), + ], + data: { + hero: { id: "shared-media" }, + sections: [{ image: { id: "shared-media" } }], + body: [{ _type: "image", asset: { _ref: "shared-media" } }], + }, + }); + + expect(occurrences).toEqual([ + { + fieldSlug: "hero", + fieldPath: "hero", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: "shared-media", + provider: "local", + providerAssetId: "shared-media", + mediaKind: "image", + mimeType: null, + }, + { + fieldSlug: "sections", + fieldPath: "sections[0].image", + occurrenceIndex: 0, + referenceType: "image_field", + mediaId: "shared-media", + provider: "local", + providerAssetId: "shared-media", + mediaKind: "image", + mimeType: null, + }, + { + fieldSlug: "body", + fieldPath: "body[0].asset._ref", + occurrenceIndex: 0, + referenceType: "portable_text_image", + mediaId: "shared-media", + provider: "local", + providerAssetId: "shared-media", + mediaKind: "image", + mimeType: null, + }, + ]); + }); +}); From 8ef7d0eed30a986548d2312adea6395dfa0a988c Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 00:59:09 +0100 Subject: [PATCH 03/29] feat(media): add usage repository --- .../src/database/repositories/media-usage.ts | 444 ++++++++++++++++++ .../database/media-usage-repository.test.ts | 225 +++++++++ 2 files changed, 669 insertions(+) create mode 100644 packages/core/src/database/repositories/media-usage.ts create mode 100644 packages/core/tests/integration/database/media-usage-repository.test.ts diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts new file mode 100644 index 0000000000..5b7ea18ad0 --- /dev/null +++ b/packages/core/src/database/repositories/media-usage.ts @@ -0,0 +1,444 @@ +import type { Kysely, Selectable, Transaction } from "kysely"; +import { ulid } from "ulidx"; + +import type { MediaKind, MediaUsageReferenceType } from "../../media/usage/types.js"; +import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js"; +import { withTransaction } from "../transaction.js"; +import type { Database, MediaUsageSourceTable, MediaUsageTable } from "../types.js"; + +type DatabaseExecutor = Kysely | Transaction; + +const OCCURRENCE_BIND_COLUMNS = 12; +const OCCURRENCE_INSERT_BATCH_SIZE = Math.max( + 1, + Math.floor(SQL_BATCH_SIZE / OCCURRENCE_BIND_COLUMNS), +); + +export interface MediaUsageSourceInput { + sourceKey: string; + sourceType: string; + collectionSlug?: string | null; + contentId?: string | null; + sourceVariant: string; + locale?: string | null; + translationGroup?: string | null; + contentSlug?: string | null; + contentTitle?: string | null; + contentStatus?: string | null; + contentScheduledAt?: string | null; + contentDeletedAt?: string | null; + revisionId?: string | null; + schemaVersion?: number; +} + +export interface MediaUsageOccurrenceInput { + fieldSlug: string; + fieldPath: string; + occurrenceIndex?: number; + referenceType: MediaUsageReferenceType; + mediaId: string | null; + provider: string; + providerAssetId: string; + mediaKind?: MediaKind | null; + mimeType?: string | null; +} + +export interface MediaUsageSource { + sourceKey: string; + sourceType: string; + collectionSlug: string | null; + contentId: string | null; + sourceVariant: string; + locale: string | null; + translationGroup: string | null; + contentSlug: string | null; + contentTitle: string | null; + contentStatus: string | null; + contentScheduledAt: string | null; + contentDeletedAt: string | null; + revisionId: string | null; + currentGeneration: string; + schemaVersion: number; + indexedAt: string; + createdAt: string; + updatedAt: string; +} + +export interface MediaUsageOccurrence { + id: string; + sourceKey: string; + generation: string; + fieldSlug: string; + fieldPath: string; + occurrenceIndex: number; + referenceType: string; + mediaId: string | null; + provider: string; + providerAssetId: string; + mediaKind: string | null; + mimeType: string | null; + createdAt: string; +} + +export interface MediaUsageRecord { + source: MediaUsageSource; + occurrence: MediaUsageOccurrence; +} + +interface JoinedUsageRow { + source_key: string; + source_type: string; + collection_slug: string | null; + content_id: string | null; + source_variant: string; + locale: string | null; + translation_group: string | null; + content_slug: string | null; + content_title: string | null; + content_status: string | null; + content_scheduled_at: string | null; + content_deleted_at: string | null; + revision_id: string | null; + current_generation: string; + schema_version: number; + indexed_at: string; + source_created_at: string; + source_updated_at: string; + occurrence_id: string; + generation: string; + field_slug: string; + field_path: string; + occurrence_index: number; + reference_type: string; + media_id: string | null; + provider: string; + provider_asset_id: string; + media_kind: string | null; + mime_type: string | null; + occurrence_created_at: string; +} + +/** Persistence-only repository for the internal media usage projection tables. */ +export class MediaUsageRepository { + constructor(private db: Kysely) {} + + async replaceSource( + source: MediaUsageSourceInput, + occurrences: readonly MediaUsageOccurrenceInput[], + ): Promise { + const generation = ulid(); + const now = new Date().toISOString(); + + await withTransaction(this.db, async (trx) => { + await this.insertOccurrences(trx, source.sourceKey, generation, occurrences); + await this.upsertSource(trx, source, generation, now); + + try { + await this.deleteStaleGenerations(trx, source.sourceKey, generation); + } catch (error) { + console.error("[media-usage] failed to delete stale generations:", error); + } + }); + + const replaced = await this.findSource(source.sourceKey); + if (!replaced) { + throw new Error(`Media usage source ${source.sourceKey} was not persisted`); + } + return replaced; + } + + async findSource(sourceKey: string): Promise { + const row = await this.db + .selectFrom("_emdash_media_usage_sources") + .selectAll() + .where("source_key", "=", sourceKey) + .executeTakeFirst(); + + return row ? rowToSource(row) : null; + } + + async findCurrentUsageByMediaId(mediaId: string): Promise { + const rows = await this.db + .selectFrom("_emdash_media_usage_sources as s") + .innerJoin("_emdash_media_usage as u", (join) => + join + .onRef("u.source_key", "=", "s.source_key") + .onRef("u.generation", "=", "s.current_generation"), + ) + .select(currentUsageSelect) + .where("u.media_id", "=", mediaId) + .orderBy("s.source_key", "asc") + .orderBy("u.field_path", "asc") + .orderBy("u.occurrence_index", "asc") + .execute(); + + return rows.map(rowToUsageRecord); + } + + async findCurrentUsageByProviderAsset( + provider: string, + providerAssetId: string, + ): Promise { + const rows = await this.db + .selectFrom("_emdash_media_usage_sources as s") + .innerJoin("_emdash_media_usage as u", (join) => + join + .onRef("u.source_key", "=", "s.source_key") + .onRef("u.generation", "=", "s.current_generation"), + ) + .select(currentUsageSelect) + .where("u.provider", "=", provider) + .where("u.provider_asset_id", "=", providerAssetId) + .orderBy("s.source_key", "asc") + .orderBy("u.field_path", "asc") + .orderBy("u.occurrence_index", "asc") + .execute(); + + return rows.map(rowToUsageRecord); + } + + async deleteSource(sourceKey: string): Promise { + return withTransaction(this.db, async (trx) => { + await trx.deleteFrom("_emdash_media_usage").where("source_key", "=", sourceKey).execute(); + const result = await trx + .deleteFrom("_emdash_media_usage_sources") + .where("source_key", "=", sourceKey) + .executeTakeFirst(); + return Number(result.numDeletedRows ?? 0); + }); + } + + async deleteContentSources(collectionSlug: string, contentId: string): Promise { + const sourceRows = await this.db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .where("source_type", "=", "content") + .where("collection_slug", "=", collectionSlug) + .where("content_id", "=", contentId) + .execute(); + const sourceKeys = sourceRows.map((row) => row.source_key); + if (sourceKeys.length === 0) return 0; + + return withTransaction(this.db, async (trx) => { + let deleted = 0; + for (const sourceKeyBatch of chunks(sourceKeys, SQL_BATCH_SIZE)) { + await trx + .deleteFrom("_emdash_media_usage") + .where("source_key", "in", sourceKeyBatch) + .execute(); + const result = await trx + .deleteFrom("_emdash_media_usage_sources") + .where("source_key", "in", sourceKeyBatch) + .executeTakeFirst(); + deleted += Number(result.numDeletedRows ?? 0); + } + return deleted; + }); + } + + private async insertOccurrences( + db: DatabaseExecutor, + sourceKey: string, + generation: string, + occurrences: readonly MediaUsageOccurrenceInput[], + ): Promise { + if (occurrences.length === 0) return; + + const rows = occurrences.map((occurrence) => ({ + id: ulid(), + source_key: sourceKey, + generation, + field_slug: occurrence.fieldSlug, + field_path: occurrence.fieldPath, + occurrence_index: occurrence.occurrenceIndex ?? 0, + reference_type: occurrence.referenceType, + media_id: occurrence.mediaId, + provider: occurrence.provider, + provider_asset_id: occurrence.providerAssetId, + media_kind: occurrence.mediaKind ?? null, + mime_type: occurrence.mimeType ?? null, + })); + + for (const rowBatch of chunks(rows, OCCURRENCE_INSERT_BATCH_SIZE)) { + await db.insertInto("_emdash_media_usage").values(rowBatch).execute(); + } + } + + private async upsertSource( + db: DatabaseExecutor, + source: MediaUsageSourceInput, + generation: string, + now: string, + ): Promise { + const row = { + source_key: source.sourceKey, + source_type: source.sourceType, + collection_slug: source.collectionSlug ?? null, + content_id: source.contentId ?? null, + source_variant: source.sourceVariant, + locale: source.locale ?? null, + translation_group: source.translationGroup ?? null, + content_slug: source.contentSlug ?? null, + content_title: source.contentTitle ?? null, + content_status: source.contentStatus ?? null, + content_scheduled_at: source.contentScheduledAt ?? null, + content_deleted_at: source.contentDeletedAt ?? null, + revision_id: source.revisionId ?? null, + current_generation: generation, + schema_version: source.schemaVersion ?? 1, + indexed_at: now, + updated_at: now, + }; + + await db + .insertInto("_emdash_media_usage_sources") + .values(row) + .onConflict((oc) => + oc.column("source_key").doUpdateSet({ + source_type: row.source_type, + collection_slug: row.collection_slug, + content_id: row.content_id, + source_variant: row.source_variant, + locale: row.locale, + translation_group: row.translation_group, + content_slug: row.content_slug, + content_title: row.content_title, + content_status: row.content_status, + content_scheduled_at: row.content_scheduled_at, + content_deleted_at: row.content_deleted_at, + revision_id: row.revision_id, + current_generation: row.current_generation, + schema_version: row.schema_version, + indexed_at: row.indexed_at, + updated_at: row.updated_at, + }), + ) + .execute(); + } + + private async deleteStaleGenerations( + db: DatabaseExecutor, + sourceKey: string, + currentGeneration: string, + ): Promise { + await db + .deleteFrom("_emdash_media_usage") + .where("source_key", "=", sourceKey) + .where("generation", "!=", currentGeneration) + .execute(); + } +} + +const currentUsageSelect = [ + "s.source_key as source_key", + "s.source_type as source_type", + "s.collection_slug as collection_slug", + "s.content_id as content_id", + "s.source_variant as source_variant", + "s.locale as locale", + "s.translation_group as translation_group", + "s.content_slug as content_slug", + "s.content_title as content_title", + "s.content_status as content_status", + "s.content_scheduled_at as content_scheduled_at", + "s.content_deleted_at as content_deleted_at", + "s.revision_id as revision_id", + "s.current_generation as current_generation", + "s.schema_version as schema_version", + "s.indexed_at as indexed_at", + "s.created_at as source_created_at", + "s.updated_at as source_updated_at", + "u.id as occurrence_id", + "u.generation as generation", + "u.field_slug as field_slug", + "u.field_path as field_path", + "u.occurrence_index as occurrence_index", + "u.reference_type as reference_type", + "u.media_id as media_id", + "u.provider as provider", + "u.provider_asset_id as provider_asset_id", + "u.media_kind as media_kind", + "u.mime_type as mime_type", + "u.created_at as occurrence_created_at", +] as const; + +function rowToSource(row: Selectable): MediaUsageSource { + return { + sourceKey: row.source_key, + sourceType: row.source_type, + collectionSlug: row.collection_slug, + contentId: row.content_id, + sourceVariant: row.source_variant, + locale: row.locale, + translationGroup: row.translation_group, + contentSlug: row.content_slug, + contentTitle: row.content_title, + contentStatus: row.content_status, + contentScheduledAt: row.content_scheduled_at, + contentDeletedAt: row.content_deleted_at, + revisionId: row.revision_id, + currentGeneration: row.current_generation, + schemaVersion: Number(row.schema_version), + indexedAt: row.indexed_at, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + +function rowToOccurrence(row: Selectable): MediaUsageOccurrence { + return { + id: row.id, + sourceKey: row.source_key, + generation: row.generation, + fieldSlug: row.field_slug, + fieldPath: row.field_path, + occurrenceIndex: Number(row.occurrence_index), + referenceType: row.reference_type, + mediaId: row.media_id, + provider: row.provider, + providerAssetId: row.provider_asset_id, + mediaKind: row.media_kind, + mimeType: row.mime_type, + createdAt: row.created_at, + }; +} + +function rowToUsageRecord(row: JoinedUsageRow): MediaUsageRecord { + return { + source: rowToSource({ + source_key: row.source_key, + source_type: row.source_type, + collection_slug: row.collection_slug, + content_id: row.content_id, + source_variant: row.source_variant, + locale: row.locale, + translation_group: row.translation_group, + content_slug: row.content_slug, + content_title: row.content_title, + content_status: row.content_status, + content_scheduled_at: row.content_scheduled_at, + content_deleted_at: row.content_deleted_at, + revision_id: row.revision_id, + current_generation: row.current_generation, + schema_version: row.schema_version, + indexed_at: row.indexed_at, + created_at: row.source_created_at, + updated_at: row.source_updated_at, + }), + occurrence: rowToOccurrence({ + id: row.occurrence_id, + source_key: row.source_key, + generation: row.generation, + field_slug: row.field_slug, + field_path: row.field_path, + occurrence_index: row.occurrence_index, + reference_type: row.reference_type, + media_id: row.media_id, + provider: row.provider, + provider_asset_id: row.provider_asset_id, + media_kind: row.media_kind, + mime_type: row.mime_type, + created_at: row.occurrence_created_at, + }), + }; +} diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts new file mode 100644 index 0000000000..b7a791991a --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -0,0 +1,225 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { SQL_BATCH_SIZE } from "../../../src/utils/chunks.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("MediaUsageRepository", (dialect) => { + let ctx: DialectTestContext; + let repo: MediaUsageRepository; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + repo = new MediaUsageRepository(ctx.db); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("replaces a source with a current generation of occurrences", async () => { + const source = await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-hero", { mimeType: "image/jpeg", mediaKind: "image" }), + occurrence("attachment", "media-file", { + referenceType: "file_field", + mimeType: "application/pdf", + mediaKind: "document", + }), + ]); + + expect(source.currentGeneration).toEqual(expect.any(String)); + expect(source.sourceKey).toBe("content:posts:entry1:live"); + expect(source.sourceVariant).toBe("live"); + + const usage = await repo.findCurrentUsageByMediaId("media-hero"); + expect(usage).toEqual([ + { + source: expect.objectContaining({ + sourceKey: "content:posts:entry1:live", + collectionSlug: "posts", + contentId: "entry1", + contentSlug: "hello-world", + contentTitle: "Hello World", + currentGeneration: source.currentGeneration, + }), + occurrence: expect.objectContaining({ + fieldSlug: "hero", + fieldPath: "hero", + mediaId: "media-hero", + provider: "local", + providerAssetId: "media-hero", + mediaKind: "image", + mimeType: "image/jpeg", + generation: source.currentGeneration, + }), + }, + ]); + }); + + it("flips generations and removes stale occurrence rows", async () => { + const first = await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-old"), + ]); + const second = await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-new"), + ]); + + expect(second.currentGeneration).not.toBe(first.currentGeneration); + expect(await repo.findCurrentUsageByMediaId("media-old")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-new")).toHaveLength(1); + + const rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select(["generation", "media_id"]) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + + expect(rows).toEqual([{ generation: second.currentGeneration, media_id: "media-new" }]); + }); + + it("supports empty replacement while preserving the source row", async () => { + const first = await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-old"), + ]); + const second = await repo.replaceSource(contentSource("entry1", "live"), []); + + expect(second.currentGeneration).not.toBe(first.currentGeneration); + expect(await repo.findSource("content:posts:entry1:live")).toEqual( + expect.objectContaining({ currentGeneration: second.currentGeneration }), + ); + expect(await repo.findCurrentUsageByMediaId("media-old")).toEqual([]); + }); + + it("deletes a single source and its occurrences", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "draft"), [occurrence("hero", "media-draft")]); + + expect(await repo.deleteSource("content:posts:entry1:live")).toBe(1); + expect(await repo.findSource("content:posts:entry1:live")).toBeNull(); + expect(await repo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-draft")).toHaveLength(1); + }); + + it("deletes all content sources for one collection and content id", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "draft"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry2", "live"), [occurrence("hero", "media-other")]); + await repo.replaceSource(contentSource("entry1", "live", { collectionSlug: "pages" }), [ + occurrence("hero", "media-page"), + ]); + + expect(await repo.deleteContentSources("posts", "entry1")).toBe(2); + expect(await repo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-draft")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-other")).toHaveLength(1); + expect(await repo.findCurrentUsageByMediaId("media-page")).toHaveLength(1); + }); + + it("finds current usage by provider asset", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("video", "mux-video-1", { + referenceType: "file_field", + provider: "mux", + mediaId: null, + providerAssetId: "mux-video-1", + mediaKind: "video", + mimeType: "video/mp4", + }), + ]); + + expect(await repo.findCurrentUsageByProviderAsset("mux", "mux-video-1")).toEqual([ + { + source: expect.objectContaining({ sourceKey: "content:posts:entry1:live" }), + occurrence: expect.objectContaining({ + mediaId: null, + provider: "mux", + providerAssetId: "mux-video-1", + mediaKind: "video", + mimeType: "video/mp4", + }), + }, + ]); + }); + + it("keeps live and draft source keys separate for the same content", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-shared")]); + await repo.replaceSource(contentSource("entry1", "draft"), [ + occurrence("draftHero", "media-shared", { fieldPath: "draftHero" }), + ]); + + const usage = await repo.findCurrentUsageByMediaId("media-shared"); + + expect(usage.map((row) => row.source.sourceKey)).toEqual([ + "content:posts:entry1:draft", + "content:posts:entry1:live", + ]); + expect(usage.map((row) => row.source.sourceVariant)).toEqual(["draft", "live"]); + }); + + it("replaces more occurrences than one D1 insert batch", async () => { + const occurrences = Array.from({ length: SQL_BATCH_SIZE + 7 }, (_, index) => + occurrence(`gallery-${index}`, `media-${index}`, { + fieldPath: `gallery[${index}].image`, + }), + ); + + const source = await repo.replaceSource(contentSource("entry1", "draft"), occurrences); + const rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select(["generation", "media_id"]) + .where("source_key", "=", source.sourceKey) + .orderBy("field_path", "asc") + .execute(); + + expect(rows).toHaveLength(SQL_BATCH_SIZE + 7); + expect(rows.every((row) => row.generation === source.currentGeneration)).toBe(true); + }); +}); + +function contentSource( + contentId: string, + variant: "live" | "draft", + overrides: Partial[0]> = {}, +): Parameters[0] { + const collectionSlug = overrides.collectionSlug ?? "posts"; + return { + sourceKey: `content:${collectionSlug}:${contentId}:${variant}`, + sourceType: "content", + collectionSlug, + contentId, + sourceVariant: variant, + locale: "en", + translationGroup: `tg-${contentId}`, + contentSlug: "hello-world", + contentTitle: "Hello World", + contentStatus: variant === "live" ? "published" : "draft", + contentScheduledAt: null, + contentDeletedAt: null, + revisionId: `rev-${contentId}-${variant}`, + ...overrides, + }; +} + +function occurrence( + fieldSlug: string, + mediaId: string, + overrides: Partial[1][number]> = {}, +): Parameters[1][number] { + return { + fieldSlug, + fieldPath: fieldSlug, + occurrenceIndex: 0, + referenceType: "image_field", + mediaId, + provider: "local", + providerAssetId: mediaId, + mediaKind: "image", + mimeType: null, + ...overrides, + }; +} From fa9c6b10a078405ef8961282beafb53dd4e7d965 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 01:10:18 +0100 Subject: [PATCH 04/29] chore(media): add usage foundation changeset --- .changeset/media-usage-index-foundation.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/media-usage-index-foundation.md diff --git a/.changeset/media-usage-index-foundation.md b/.changeset/media-usage-index-foundation.md new file mode 100644 index 0000000000..e9ebfc9991 --- /dev/null +++ b/.changeset/media-usage-index-foundation.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Adds the internal media usage index foundation for upcoming usage-aware media workflows. This creates the usage index schema during migrations but does not change Media Library behavior yet. From a97e17ee5e9b7d2dd7f3bc5417642d110f5f3dc1 Mon Sep 17 00:00:00 2001 From: "emdashbot[bot]" Date: Wed, 1 Jul 2026 00:15:26 +0000 Subject: [PATCH 05/29] ci: update query-count snapshots --- scripts/query-counts.queries.d1.json | 8 ++++---- scripts/query-counts.queries.sqlite.json | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/query-counts.queries.d1.json b/scripts/query-counts.queries.d1.json index 297e616ca0..e0fd40c6fd 100644 --- a/scripts/query-counts.queries.d1.json +++ b/scripts/query-counts.queries.d1.json @@ -120,7 +120,7 @@ "select * from \"_emdash_migrations\" limit ?": 1, "select * from \"_emdash_redirects\" where \"enabled\" = ?": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, "select count(*) as \"count\" from \"_emdash_collections\"": 1, "SELECT COUNT(*) as count FROM \"_emdash_migrations\"": 1, "UPDATE _emdash_cron_tasks SET status = 'idle', locked_at = NULL WHERE status = 'running' AND locked_at < ?": 1 @@ -131,7 +131,7 @@ "select * from \"_emdash_menu_items\" where \"menu_id\" = ? order by \"sort_order\" asc": 1, "select * from \"_emdash_menus\" where \"name\" = ? order by \"locale\" asc": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1 + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1 }, "GET /posts (cold)": { "select \"a\".\"id\" as \"a_id\", \"a\".\"name\" as \"a_name\", \"a\".\"label\" as \"a_label\", \"a\".\"description\" as \"a_description\", \"w\".\"id\" as \"w_id\", \"w\".\"type\" as \"w_type\", \"w\".\"title\" as \"w_title\", \"w\".\"content\" as \"w_content\", \"w\".\"menu_name\" as \"w_menu_name\", \"w\".\"component_id\" as \"w_component_id\", \"w\".\"component_props\" as \"w_component_props\", \"w\".\"area_id\" as \"w_area_id\", \"w\".\"sort_order\" as \"w_sort_order\", \"w\".\"created_at\" as \"w_created_at\" from \"_emdash_widget_areas\" as \"a\" left join \"_emdash_widgets\" as \"w\" on \"w\".\"area_id\" = \"a\".\"id\" where \"a\".\"name\" = ? order by \"w\".\"sort_order\" asc": 1, @@ -177,7 +177,7 @@ "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC LIMIT ?": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, "select count(\"id\") as \"count\" from \"_emdash_comments\" where \"collection\" = ? and \"content_id\" = ? and \"status\" = ?": 1, "select count(*) as \"count\" from \"_emdash_collections\"": 1, "SELECT COUNT(*) as count FROM \"_emdash_migrations\"": 1, @@ -196,7 +196,7 @@ "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC LIMIT ?": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, "select count(\"id\") as \"count\" from \"_emdash_comments\" where \"collection\" = ? and \"content_id\" = ? and \"status\" = ?": 1 }, "GET /rss.xml (cold)": { diff --git a/scripts/query-counts.queries.sqlite.json b/scripts/query-counts.queries.sqlite.json index ce1f0152ae..875a298a10 100644 --- a/scripts/query-counts.queries.sqlite.json +++ b/scripts/query-counts.queries.sqlite.json @@ -78,7 +78,7 @@ "select * from \"_emdash_menu_items\" where \"menu_id\" = ? order by \"sort_order\" asc": 1, "select * from \"_emdash_menus\" where \"name\" = ? order by \"locale\" asc": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1 + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1 }, "GET /pages/about (warm)": { "select \"a\".\"id\" as \"a_id\", \"a\".\"name\" as \"a_name\", \"a\".\"label\" as \"a_label\", \"a\".\"description\" as \"a_description\", \"w\".\"id\" as \"w_id\", \"w\".\"type\" as \"w_type\", \"w\".\"title\" as \"w_title\", \"w\".\"content\" as \"w_content\", \"w\".\"menu_name\" as \"w_menu_name\", \"w\".\"component_id\" as \"w_component_id\", \"w\".\"component_props\" as \"w_component_props\", \"w\".\"area_id\" as \"w_area_id\", \"w\".\"sort_order\" as \"w_sort_order\", \"w\".\"created_at\" as \"w_created_at\" from \"_emdash_widget_areas\" as \"a\" left join \"_emdash_widgets\" as \"w\" on \"w\".\"area_id\" = \"a\".\"id\" where \"a\".\"name\" = ? order by \"w\".\"sort_order\" asc": 1, @@ -86,7 +86,7 @@ "select * from \"_emdash_menu_items\" where \"menu_id\" = ? order by \"sort_order\" asc": 1, "select * from \"_emdash_menus\" where \"name\" = ? order by \"locale\" asc": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1 + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1 }, "GET /posts (cold)": { "select \"a\".\"id\" as \"a_id\", \"a\".\"name\" as \"a_name\", \"a\".\"label\" as \"a_label\", \"a\".\"description\" as \"a_description\", \"w\".\"id\" as \"w_id\", \"w\".\"type\" as \"w_type\", \"w\".\"title\" as \"w_title\", \"w\".\"content\" as \"w_content\", \"w\".\"menu_name\" as \"w_menu_name\", \"w\".\"component_id\" as \"w_component_id\", \"w\".\"component_props\" as \"w_component_props\", \"w\".\"area_id\" as \"w_area_id\", \"w\".\"sort_order\" as \"w_sort_order\", \"w\".\"created_at\" as \"w_created_at\" from \"_emdash_widget_areas\" as \"a\" left join \"_emdash_widgets\" as \"w\" on \"w\".\"area_id\" = \"a\".\"id\" where \"a\".\"name\" = ? order by \"w\".\"sort_order\" asc": 1, @@ -117,7 +117,7 @@ "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC LIMIT ?": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, "select count(\"id\") as \"count\" from \"_emdash_comments\" where \"collection\" = ? and \"content_id\" = ? and \"status\" = ?": 1 }, "GET /posts/building-for-the-long-term (warm)": { @@ -133,7 +133,7 @@ "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_pages\".id AND t.locale = \"ec_pages\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_pages\".id AND b.locale = \"ec_pages\".locale) AS \"_emdash_bylines\" FROM \"ec_pages\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"created_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC": 1, "SELECT *, (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"ec_posts\".id AND t.locale = \"ec_posts\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"ec_posts\".id AND b.locale = \"ec_posts\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" WHERE deleted_at IS NULL AND (\"status\" = 'published' OR (\"status\" = 'scheduled' AND \"scheduled_at\" <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))) ORDER BY \"published_at\" DESC, \"id\" DESC LIMIT ?": 1, - "SELECT c.*, \"s\".\"seo_title\" AS \"_emdash_seo_title\", \"s\".\"seo_description\" AS \"_emdash_seo_description\", \"s\".\"seo_image\" AS \"_emdash_seo_image\", \"s\".\"seo_canonical\" AS \"_emdash_seo_canonical\", \"s\".\"seo_no_index\" AS \"_emdash_seo_no_index\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c LEFT JOIN \"_emdash_seo\" AS s ON s.collection = ? AND s.content_id = c.id WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, + "SELECT c.*, (SELECT json_object('seo_title', s.seo_title, 'seo_description', s.seo_description, 'seo_image', s.seo_image, 'seo_canonical', s.seo_canonical, 'seo_no_index', s.seo_no_index) FROM \"_emdash_seo\" AS s WHERE s.collection = ? AND s.content_id = \"c\".id LIMIT 1) AS \"_emdash_seo\", (SELECT json_group_array(json_object('id', t.id, 'name', t.name, 'slug', t.slug, 'label', t.label, 'parent_id', t.parent_id, 'locale', t.locale, 'translation_group', t.translation_group)) FROM \"content_taxonomies\" AS ct JOIN \"taxonomies\" AS t ON t.translation_group = ct.taxonomy_id WHERE ct.collection = ? AND ct.entry_id = \"c\".id AND t.locale = \"c\".locale) AS \"_emdash_terms\", (SELECT json_group_array(json_object('roleLabel', cb.role_label, 'sortOrder', cb.sort_order, 'byline', json_object('id', b.id, 'slug', b.slug, 'displayName', b.display_name, 'bio', b.bio, 'avatarMediaId', b.avatar_media_id, 'avatarStorageKey', m.storage_key, 'avatarAlt', m.alt, 'avatarBlurhash', m.blurhash, 'avatarDominantColor', m.dominant_color, 'websiteUrl', b.website_url, 'userId', b.user_id, 'isGuest', b.is_guest, 'createdAt', b.created_at, 'updatedAt', b.updated_at, 'locale', b.locale, 'translationGroup', b.translation_group))) FROM \"_emdash_content_bylines\" AS cb JOIN \"_emdash_bylines\" AS b ON b.translation_group = cb.byline_id LEFT JOIN \"media\" AS m ON m.id = b.avatar_media_id WHERE cb.collection_slug = ? AND cb.content_id = \"c\".id AND b.locale = \"c\".locale) AS \"_emdash_bylines\" FROM \"ec_posts\" AS c WHERE c.deleted_at IS NULL AND (c.slug = ? OR c.id = ?) LIMIT 1": 1, "select count(\"id\") as \"count\" from \"_emdash_comments\" where \"collection\" = ? and \"content_id\" = ? and \"status\" = ?": 1 }, "GET /rss.xml (cold)": { From 0b99ad15b59ddb14f3cd10b064df9fdd8a3d41df Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 01:18:32 +0100 Subject: [PATCH 06/29] fix(media): skip path-like local usage refs --- packages/core/src/media/usage/extractor.ts | 31 ++++++++++++++----- .../tests/unit/media/usage-extractor.test.ts | 10 ++++-- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/packages/core/src/media/usage/extractor.ts b/packages/core/src/media/usage/extractor.ts index 7273cf1be1..f2cd33a73c 100644 --- a/packages/core/src/media/usage/extractor.ts +++ b/packages/core/src/media/usage/extractor.ts @@ -108,7 +108,8 @@ function extractPortableTextOccurrences( for (const [blockIndex, block] of value.entries()) { if (!isRecord(block) || block._type !== "image" || !isRecord(block.asset)) continue; - const ref = readPortableTextAssetRef(block.asset); + const provider = normalizeProvider(block.asset.provider); + const ref = readPortableTextAssetRef(block.asset, provider); if (!ref) continue; addRefOccurrence(occurrences, seen, { @@ -117,7 +118,7 @@ function extractPortableTextOccurrences( referenceType: "portable_text_image", ref: buildMediaRef({ id: ref.id, - provider: readString(block.asset.provider) ?? "local", + provider, mimeType: normalizeMimeValue(block.asset.mimeType), fallbackKind: "image", }), @@ -182,18 +183,19 @@ function addRefOccurrence( function readMediaRef(value: unknown, fallbackKind: MediaKind | null): MediaRef | null { if (typeof value === "string") { - const id = normalizeStableId(value); + const id = normalizeLocalMediaId(value); return id ? buildMediaRef({ id, provider: "local", mimeType: null, fallbackKind }) : null; } if (!isRecord(value)) return null; - const id = normalizeStableId(value.id); + const provider = normalizeProvider(value.provider); + const id = provider === "local" ? normalizeLocalMediaId(value.id) : normalizeStableId(value.id); if (!id) return null; return buildMediaRef({ id, - provider: readString(value.provider) ?? "local", + provider, mimeType: normalizeMimeValue(value.mimeType), fallbackKind, }); @@ -205,7 +207,7 @@ function buildMediaRef(input: { mimeType: string | null; fallbackKind: MediaKind | null; }): MediaRef | null { - const provider = input.provider.trim() || "local"; + const provider = normalizeProvider(input.provider); if (provider === "external") return null; return { @@ -219,16 +221,29 @@ function buildMediaRef(input: { function readPortableTextAssetRef( asset: Record, + provider: string, ): { key: "_ref" | "id"; id: string } | null { - const ref = normalizeStableId(asset._ref); + const normalizeId = provider === "local" ? normalizeLocalMediaId : normalizeStableId; + const ref = normalizeId(asset._ref); if (ref) return { key: "_ref", id: ref }; - const id = normalizeStableId(asset.id); + const id = normalizeId(asset.id); if (id) return { key: "id", id }; return null; } +function normalizeProvider(value: unknown): string { + const provider = readString(value)?.trim(); + return provider || "local"; +} + +function normalizeLocalMediaId(value: unknown): string | null { + const id = normalizeStableId(value); + if (!id) return null; + return id.includes("/") ? null : id; +} + function normalizeStableId(value: unknown): string | null { if (typeof value !== "string") return null; const trimmed = value.trim(); diff --git a/packages/core/tests/unit/media/usage-extractor.test.ts b/packages/core/tests/unit/media/usage-extractor.test.ts index 265546788e..e6b5720ee6 100644 --- a/packages/core/tests/unit/media/usage-extractor.test.ts +++ b/packages/core/tests/unit/media/usage-extractor.test.ts @@ -62,6 +62,9 @@ describe("extractMediaUsageOccurrences", () => { field("hero", "image"), field("attachment", "file"), field("external", "image"), + field("protocolRelative", "image"), + field("rootRelative", "image"), + field("relativePath", "image"), field("internal", "image"), field("blank", "image"), ], @@ -69,6 +72,9 @@ describe("extractMediaUsageOccurrences", () => { hero: "media-hero", attachment: "media-file", external: "https://example.com/photo.jpg", + protocolRelative: "//cdn.example.com/photo.jpg", + rootRelative: "/images/photo.jpg", + relativePath: "images/photo.jpg", internal: "/_emdash/api/media/file/uploads/photo.jpg", blank: " ", }, @@ -105,7 +111,7 @@ describe("extractMediaUsageOccurrences", () => { fields: [field("hero", "image"), field("video", "file")], data: { hero: { - id: "cf-image-1", + id: "folder/cf-image-1", provider: "cloudflare-images", mimeType: "image/png", }, @@ -125,7 +131,7 @@ describe("extractMediaUsageOccurrences", () => { referenceType: "image_field", mediaId: null, provider: "cloudflare-images", - providerAssetId: "cf-image-1", + providerAssetId: "folder/cf-image-1", mediaKind: "image", mimeType: "image/png", }, From f477b815cf568415e8c0cd5d37203eb4040644a8 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 14:50:53 +0100 Subject: [PATCH 07/29] feat(media): add usage index status metadata --- .../047_media_usage_index_status.ts | 142 +++++++ .../core/src/database/migrations/runner.ts | 2 + .../src/database/repositories/media-usage.ts | 25 +- packages/core/src/database/types.ts | 22 ++ .../media-usage-status-migration.test.ts | 371 ++++++++++++++++++ .../integration/database/migrations.test.ts | 2 + 6 files changed, 562 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/database/migrations/047_media_usage_index_status.ts create mode 100644 packages/core/tests/integration/database/media-usage-status-migration.test.ts diff --git a/packages/core/src/database/migrations/047_media_usage_index_status.ts b/packages/core/src/database/migrations/047_media_usage_index_status.ts new file mode 100644 index 0000000000..3a1272e696 --- /dev/null +++ b/packages/core/src/database/migrations/047_media_usage_index_status.ts @@ -0,0 +1,142 @@ +import type { Kysely } from "kysely"; + +import { columnExists, currentTimestamp } from "../dialect-helpers.js"; + +const SOURCE_TABLE = "_emdash_media_usage_sources"; +const DUPLICATE_COLUMN_RE = /(?:duplicate column|column .* already exists|already exists.*column)/i; + +/** + * Media usage index metadata. + * + * DDL-only by design: adds freshness/completeness metadata and scope status + * tracking for later reference-index reconciliation. No content scans, no + * runtime imports, and no backfill. + */ +export async function up(db: Kysely): Promise { + await addSourceMetadataColumns(db); + + await db.schema + .createTable("_emdash_media_usage_index_status") + .ifNotExists() + .addColumn("adapter_id", "text", (c) => c.notNull()) + .addColumn("scope_type", "text", (c) => c.notNull()) + .addColumn("scope_key", "text", (c) => c.notNull()) + .addColumn("status", "text", (c) => c.notNull()) + .addColumn("schema_version", "integer", (c) => c.notNull().defaultTo(1)) + .addColumn("started_at", "text") + .addColumn("completed_at", "text") + .addColumn("cursor", "text") + .addColumn("indexed_source_count", "integer", (c) => c.notNull().defaultTo(0)) + .addColumn("failed_source_count", "integer", (c) => c.notNull().defaultTo(0)) + .addColumn("last_error_code", "text") + .addColumn("updated_at", "text", (c) => c.notNull().defaultTo(currentTimestamp(db))) + .addPrimaryKeyConstraint("_emdash_media_usage_index_status_pk", [ + "adapter_id", + "scope_type", + "scope_key", + ]) + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_sources_completeness") + .ifNotExists() + .on("_emdash_media_usage_sources") + .columns(["source_type", "collection_slug", "source_completeness"]) + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_sources_fingerprint") + .ifNotExists() + .on("_emdash_media_usage_sources") + .column("source_fingerprint") + .execute(); + + await db.schema + .createIndex("idx__emdash_media_usage_index_status_status") + .ifNotExists() + .on("_emdash_media_usage_index_status") + .columns(["adapter_id", "status"]) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex("idx__emdash_media_usage_index_status_status").ifExists().execute(); + await db.schema.dropIndex("idx__emdash_media_usage_sources_fingerprint").ifExists().execute(); + await db.schema.dropIndex("idx__emdash_media_usage_sources_completeness").ifExists().execute(); + await db.schema.dropTable("_emdash_media_usage_index_status").ifExists().execute(); + + await dropSourceMetadataColumns(db); +} + +async function addSourceMetadataColumns(db: Kysely): Promise { + await addColumnIfMissing(db, "source_updated_at", () => + db.schema.alterTable(SOURCE_TABLE).addColumn("source_updated_at", "text").execute(), + ); + await addColumnIfMissing(db, "source_version", () => + db.schema.alterTable(SOURCE_TABLE).addColumn("source_version", "integer").execute(), + ); + await addColumnIfMissing(db, "source_fingerprint", () => + db.schema.alterTable(SOURCE_TABLE).addColumn("source_fingerprint", "text").execute(), + ); + await addColumnIfMissing(db, "source_completeness", () => + db.schema + .alterTable(SOURCE_TABLE) + .addColumn("source_completeness", "text", (c) => c.notNull().defaultTo("unknown")) + .execute(), + ); + await addColumnIfMissing(db, "last_attempted_at", () => + db.schema.alterTable(SOURCE_TABLE).addColumn("last_attempted_at", "text").execute(), + ); + await addColumnIfMissing(db, "last_error_code", () => + db.schema.alterTable(SOURCE_TABLE).addColumn("last_error_code", "text").execute(), + ); +} + +async function addColumnIfMissing( + db: Kysely, + columnName: string, + addColumn: () => Promise, +): Promise { + if (await columnExists(db, SOURCE_TABLE, columnName)) return; + + try { + await addColumn(); + } catch (error) { + if (DUPLICATE_COLUMN_RE.test(deepErrorMessage(error))) { + if (await columnExists(db, SOURCE_TABLE, columnName)) return; + } + throw error; + } +} + +function deepErrorMessage(error: unknown): string { + if (error instanceof Error) { + const own = error.message ?? ""; + if (error.cause) { + const causeMessage = deepErrorMessage(error.cause); + return own ? `${own}: ${causeMessage}` : causeMessage; + } + return own; + } + if (typeof error === "string") return error; + try { + return JSON.stringify(error); + } catch { + return String(error); + } +} + +async function dropSourceMetadataColumns(db: Kysely): Promise { + for (const columnName of [ + "last_error_code", + "last_attempted_at", + "source_completeness", + "source_fingerprint", + "source_version", + "source_updated_at", + ] as const) { + if (await columnExists(db, SOURCE_TABLE, columnName)) { + await db.schema.alterTable(SOURCE_TABLE).dropColumn(columnName).execute(); + } + } +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 0c651816b0..73577cfa13 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -48,6 +48,7 @@ import * as m043 from "./043_content_references.js"; import * as m044 from "./044_comment_reactions.js"; import * as m045 from "./045_taxonomy_parent_group.js"; import * as m046 from "./046_media_usage_index.js"; +import * as m047 from "./047_media_usage_index_status.js"; const MIGRATIONS: Readonly> = Object.freeze({ "001_initial": m001, @@ -95,6 +96,7 @@ const MIGRATIONS: Readonly> = Object.freeze({ "044_comment_reactions": m044, "045_taxonomy_parent_group": m045, "046_media_usage_index": m046, + "047_media_usage_index_status": m047, }); /** Total number of registered migrations. Exported for use in tests. */ diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 5b7ea18ad0..d8743f6df4 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -4,7 +4,7 @@ import { ulid } from "ulidx"; import type { MediaKind, MediaUsageReferenceType } from "../../media/usage/types.js"; import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js"; import { withTransaction } from "../transaction.js"; -import type { Database, MediaUsageSourceTable, MediaUsageTable } from "../types.js"; +import type { Database, MediaUsageTable } from "../types.js"; type DatabaseExecutor = Kysely | Transaction; @@ -64,6 +64,27 @@ export interface MediaUsageSource { updatedAt: string; } +interface MediaUsageSourceRow { + source_key: string; + source_type: string; + collection_slug: string | null; + content_id: string | null; + source_variant: string; + locale: string | null; + translation_group: string | null; + content_slug: string | null; + content_title: string | null; + content_status: string | null; + content_scheduled_at: string | null; + content_deleted_at: string | null; + revision_id: string | null; + current_generation: string; + schema_version: number; + indexed_at: string; + created_at: string; + updated_at: string; +} + export interface MediaUsageOccurrence { id: string; sourceKey: string; @@ -362,7 +383,7 @@ const currentUsageSelect = [ "u.created_at as occurrence_created_at", ] as const; -function rowToSource(row: Selectable): MediaUsageSource { +function rowToSource(row: MediaUsageSourceRow): MediaUsageSource { return { sourceKey: row.source_key, sourceType: row.source_type, diff --git a/packages/core/src/database/types.ts b/packages/core/src/database/types.ts index f8278fb34b..9b5eeac3f4 100644 --- a/packages/core/src/database/types.ts +++ b/packages/core/src/database/types.ts @@ -76,6 +76,12 @@ export interface MediaUsageSourceTable { revision_id: string | null; current_generation: string; schema_version: Generated; + source_updated_at: Generated; + source_version: Generated; + source_fingerprint: Generated; + source_completeness: Generated; + last_attempted_at: Generated; + last_error_code: Generated; indexed_at: Generated; created_at: Generated; updated_at: Generated; @@ -97,6 +103,21 @@ export interface MediaUsageTable { created_at: Generated; } +export interface MediaUsageIndexStatusTable { + adapter_id: string; + scope_type: string; + scope_key: string; + status: string; + schema_version: Generated; + started_at: Generated; + completed_at: Generated; + cursor: Generated; + indexed_source_count: Generated; + failed_source_count: Generated; + last_error_code: Generated; + updated_at: Generated; +} + export interface UserTable { id: string; email: string; @@ -453,6 +474,7 @@ export interface Database { media: MediaTable; _emdash_media_usage_sources: MediaUsageSourceTable; _emdash_media_usage: MediaUsageTable; + _emdash_media_usage_index_status: MediaUsageIndexStatusTable; users: UserTable; credentials: CredentialTable; auth_tokens: AuthTokenTable; diff --git a/packages/core/tests/integration/database/media-usage-status-migration.test.ts b/packages/core/tests/integration/database/media-usage-status-migration.test.ts new file mode 100644 index 0000000000..0f626c93b8 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-status-migration.test.ts @@ -0,0 +1,371 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import type { Database } from "../../../src/database/types.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +const EXPECTED_SOURCE_COLUMNS = [ + "source_updated_at", + "source_version", + "source_fingerprint", + "source_completeness", + "last_attempted_at", + "last_error_code", +] as const; + +const EXPECTED_STATUS_COLUMNS = [ + "adapter_id", + "scope_type", + "scope_key", + "status", + "schema_version", + "started_at", + "completed_at", + "cursor", + "indexed_source_count", + "failed_source_count", + "last_error_code", + "updated_at", +] as const; + +const EXPECTED_INDEXES = [ + "idx__emdash_media_usage_sources_completeness", + "idx__emdash_media_usage_sources_fingerprint", + "idx__emdash_media_usage_index_status_status", +] as const; + +describeEachDialect("media usage index status migration", (dialect) => { + let ctx: DialectTestContext; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("is registered and creates metadata columns plus status table", async () => { + const migrations = await ctx.db.selectFrom("_emdash_migrations").select("name").execute(); + expect(migrations.map((row) => row.name)).toContain("047_media_usage_index_status"); + + const sourceColumns = await listColumnNames(ctx, "_emdash_media_usage_sources"); + for (const columnName of EXPECTED_SOURCE_COLUMNS) { + expect(sourceColumns.has(columnName), `missing source column ${columnName}`).toBe(true); + } + + const statusColumns = await listColumnNames(ctx, "_emdash_media_usage_index_status"); + for (const columnName of EXPECTED_STATUS_COLUMNS) { + expect(statusColumns.has(columnName), `missing status column ${columnName}`).toBe(true); + } + }); + + it("applies defaults for source completeness and status counters", async () => { + await ctx.db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: "content:posts:entry1:live", + source_type: "content", + collection_slug: "posts", + content_id: "entry1", + source_variant: "live", + locale: "en", + translation_group: "tg1", + content_slug: "hello-world", + content_title: "Hello World", + content_status: "published", + content_scheduled_at: null, + content_deleted_at: null, + revision_id: "rev1", + current_generation: "gen1", + }) + .execute(); + + const source = await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select([ + "source_completeness", + "source_updated_at", + "source_version", + "source_fingerprint", + "last_attempted_at", + "last_error_code", + ]) + .where("source_key", "=", "content:posts:entry1:live") + .executeTakeFirstOrThrow(); + + expect(source).toEqual({ + source_completeness: "unknown", + source_updated_at: null, + source_version: null, + source_fingerprint: null, + last_attempted_at: null, + last_error_code: null, + }); + + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "posts", + status: "never", + }) + .execute(); + + const status = await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select([ + "schema_version", + "indexed_source_count", + "failed_source_count", + "started_at", + "completed_at", + "cursor", + "last_error_code", + "updated_at", + ]) + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", "posts") + .executeTakeFirstOrThrow(); + + expect(status).toEqual({ + schema_version: 1, + indexed_source_count: 0, + failed_source_count: 0, + started_at: null, + completed_at: null, + cursor: null, + last_error_code: null, + updated_at: expect.any(String), + }); + }); + + it("rejects duplicate status rows for the same adapter scope", async () => { + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "posts", + status: "running", + }) + .execute(); + + await expect( + ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "posts", + status: "complete", + }) + .execute(), + ).rejects.toThrow(); + }); + + it("adds source completeness default for existing PR 1 source rows", async () => { + const migration = + await import("../../../src/database/migrations/047_media_usage_index_status.js"); + const sourceKey = "content:posts:pre047:live"; + + await migration.down(ctx.db); + + await ctx.db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: sourceKey, + source_type: "content", + collection_slug: "posts", + content_id: "pre047", + source_variant: "live", + locale: "en", + translation_group: "tg-pre047", + content_slug: "pre047", + content_title: "Pre 047", + content_status: "published", + content_scheduled_at: null, + content_deleted_at: null, + revision_id: "rev-pre047", + current_generation: "gen-pre047", + }) + .execute(); + + await migration.up(ctx.db); + + const source = await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select([ + "source_completeness", + "source_updated_at", + "source_version", + "source_fingerprint", + "last_attempted_at", + "last_error_code", + ]) + .where("source_key", "=", sourceKey) + .executeTakeFirstOrThrow(); + + expect(source).toEqual({ + source_completeness: "unknown", + source_updated_at: null, + source_version: null, + source_fingerprint: null, + last_attempted_at: null, + last_error_code: null, + }); + }); + + it("creates expected indexes", async () => { + const indexNames = await listIndexNames(ctx); + + for (const indexName of EXPECTED_INDEXES) { + expect(indexNames.has(indexName), `missing index ${indexName}`).toBe(true); + } + }); + + it("up() can run again after registered migrations complete", async () => { + const migration = + await import("../../../src/database/migrations/047_media_usage_index_status.js"); + const sourceKey = "content:posts:entry-preserve:live"; + const sourceMetadata = { + source_completeness: "complete", + source_updated_at: "2026-01-01T00:00:00.000Z", + source_version: 7, + source_fingerprint: "fingerprint-preserve", + last_attempted_at: "2026-01-01T00:00:01.000Z", + last_error_code: "PREVIOUS_ERROR", + }; + + await ctx.db + .insertInto("_emdash_media_usage_sources") + .values({ + source_key: sourceKey, + source_type: "content", + collection_slug: "posts", + content_id: "entry-preserve", + source_variant: "live", + locale: "en", + translation_group: "tg-preserve", + content_slug: "preserve", + content_title: "Preserve", + content_status: "published", + content_scheduled_at: null, + content_deleted_at: null, + revision_id: "rev-preserve", + current_generation: "gen-preserve", + ...sourceMetadata, + }) + .execute(); + + const statusMetadata = { + schema_version: 3, + started_at: "2026-01-02T00:00:00.000Z", + completed_at: "2026-01-02T00:00:10.000Z", + cursor: "cursor-preserve", + indexed_source_count: 12, + failed_source_count: 2, + last_error_code: "STATUS_ERROR", + updated_at: "2026-01-02T00:00:11.000Z", + }; + + await ctx.db + .insertInto("_emdash_media_usage_index_status") + .values({ + adapter_id: "content-media", + scope_type: "collection", + scope_key: "posts", + status: "partial", + ...statusMetadata, + }) + .execute(); + + await migration.up(ctx.db); + + const statusColumns = await listColumnNames(ctx, "_emdash_media_usage_index_status"); + expect(statusColumns.has("adapter_id")).toBe(true); + + const source = await ctx.db + .selectFrom("_emdash_media_usage_sources") + .select([ + "source_completeness", + "source_updated_at", + "source_version", + "source_fingerprint", + "last_attempted_at", + "last_error_code", + ]) + .where("source_key", "=", sourceKey) + .executeTakeFirstOrThrow(); + expect(source).toEqual(sourceMetadata); + + const status = await ctx.db + .selectFrom("_emdash_media_usage_index_status") + .select([ + "status", + "schema_version", + "started_at", + "completed_at", + "cursor", + "indexed_source_count", + "failed_source_count", + "last_error_code", + "updated_at", + ]) + .where("adapter_id", "=", "content-media") + .where("scope_type", "=", "collection") + .where("scope_key", "=", "posts") + .executeTakeFirstOrThrow(); + expect(status).toEqual({ status: "partial", ...statusMetadata }); + }); + + it("down() drops metadata and up() recreates it", async () => { + const migration = + await import("../../../src/database/migrations/047_media_usage_index_status.js"); + + await migration.down(ctx.db); + + await expect( + sql`SELECT 1 FROM _emdash_media_usage_index_status`.execute(ctx.db), + ).rejects.toThrow(); + const sourceColumnsAfterDown = await listColumnNames(ctx, "_emdash_media_usage_sources"); + expect(sourceColumnsAfterDown.has("source_completeness")).toBe(false); + + await migration.up(ctx.db); + + const statusColumns = await listColumnNames(ctx, "_emdash_media_usage_index_status"); + expect(statusColumns.has("adapter_id")).toBe(true); + const sourceColumnsAfterUp = await listColumnNames(ctx, "_emdash_media_usage_sources"); + expect(sourceColumnsAfterUp.has("source_completeness")).toBe(true); + }); +}); + +async function listColumnNames( + ctx: DialectTestContext, + tableName: keyof Database, +): Promise> { + const tables = await ctx.db.introspection.getTables(); + const table = tables.find((candidate) => candidate.name === tableName); + return new Set(table?.columns.map((column) => column.name) ?? []); +} + +async function listIndexNames(ctx: DialectTestContext): Promise> { + if (ctx.dialect === "sqlite") { + const result = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'index' + `.execute(ctx.db); + return new Set(result.rows.map((row) => row.name)); + } + + const result = await sql<{ name: string }>` + SELECT indexname AS name FROM pg_indexes WHERE schemaname = current_schema() + `.execute(ctx.db); + return new Set(result.rows.map((row) => row.name)); +} diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index 469ba12441..04d3a27a61 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -50,6 +50,7 @@ describe("Database Migrations (Integration)", () => { "_emdash_byline_field_group_values", "_emdash_media_usage_sources", "_emdash_media_usage", + "_emdash_media_usage_index_status", ]; for (const table of tables) { @@ -131,6 +132,7 @@ describe("Database Migrations (Integration)", () => { "044_comment_reactions", "045_taxonomy_parent_group", "046_media_usage_index", + "047_media_usage_index_status", ]; await db.deleteFrom("_emdash_migrations").where("name", "in", trailing).execute(); From 6b6c0114b6f96c2543e09763dd0f8ea1401729b4 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Wed, 1 Jul 2026 15:42:51 +0100 Subject: [PATCH 08/29] feat: enhance media usage index with new tracking and management features --- .changeset/media-usage-index-hardening.md | 5 + .../src/database/repositories/media-usage.ts | 532 ++++++++++++++++-- .../database/media-usage-repository.test.ts | 465 ++++++++++++++- 3 files changed, 961 insertions(+), 41 deletions(-) create mode 100644 .changeset/media-usage-index-hardening.md diff --git a/.changeset/media-usage-index-hardening.md b/.changeset/media-usage-index-hardening.md new file mode 100644 index 0000000000..dac00411e5 --- /dev/null +++ b/.changeset/media-usage-index-hardening.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Adds internal media usage index hardening for future reference tracking. diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index d8743f6df4..6b2faf6923 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -1,14 +1,20 @@ -import type { Kysely, Selectable, Transaction } from "kysely"; +import { sql, type Kysely, type Selectable, type Transaction, type Updateable } from "kysely"; import { ulid } from "ulidx"; import type { MediaKind, MediaUsageReferenceType } from "../../media/usage/types.js"; import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js"; import { withTransaction } from "../transaction.js"; -import type { Database, MediaUsageTable } from "../types.js"; +import type { + Database, + MediaUsageIndexStatusTable, + MediaUsageSourceTable, + MediaUsageTable, +} from "../types.js"; +import { decodeCursor, encodeCursor, type FindManyResult } from "./types.js"; type DatabaseExecutor = Kysely | Transaction; -const OCCURRENCE_BIND_COLUMNS = 12; +const OCCURRENCE_BIND_COLUMNS = 13; const OCCURRENCE_INSERT_BATCH_SIZE = Math.max( 1, Math.floor(SQL_BATCH_SIZE / OCCURRENCE_BIND_COLUMNS), @@ -29,6 +35,12 @@ export interface MediaUsageSourceInput { contentDeletedAt?: string | null; revisionId?: string | null; schemaVersion?: number; + sourceUpdatedAt?: string | null; + sourceVersion?: number | null; + sourceFingerprint?: string | null; + sourceCompleteness?: MediaUsageSourceCompleteness; + lastAttemptedAt?: string | null; + lastErrorCode?: string | null; } export interface MediaUsageOccurrenceInput { @@ -59,11 +71,67 @@ export interface MediaUsageSource { revisionId: string | null; currentGeneration: string; schemaVersion: number; + sourceUpdatedAt: string | null; + sourceVersion: number | null; + sourceFingerprint: string | null; + sourceCompleteness: string; + lastAttemptedAt: string | null; + lastErrorCode: string | null; indexedAt: string; createdAt: string; updatedAt: string; } +export type MediaUsageSourceCompleteness = + | "unknown" + | "complete" + | "partial" + | "failed" + | "unsupported"; + +export type MediaUsageIndexStatusValue = + | "never" + | "running" + | "complete" + | "partial" + | "failed" + | "stale"; + +export interface MediaUsageIndexStatusIdentity { + adapterId: string; + scopeType: string; + scopeKey: string; +} + +export interface MediaUsageIndexStatusInput extends MediaUsageIndexStatusIdentity { + status: MediaUsageIndexStatusValue; + schemaVersion?: number; + startedAt?: string | null; + completedAt?: string | null; + cursor?: string | null; + indexedSourceCount?: number; + failedSourceCount?: number; + lastErrorCode?: string | null; + updatedAt?: string; +} + +export interface MediaUsageIndexStatus extends MediaUsageIndexStatusIdentity { + status: string; + schemaVersion: number; + startedAt: string | null; + completedAt: string | null; + cursor: string | null; + indexedSourceCount: number; + failedSourceCount: number; + lastErrorCode: string | null; + updatedAt: string; +} + +export interface FindMediaUsageOptions { + limit?: number; + cursor?: string; +} + interface MediaUsageSourceRow { source_key: string; source_type: string; @@ -80,6 +148,12 @@ interface MediaUsageSourceRow { revision_id: string | null; current_generation: string; schema_version: number; + source_updated_at: string | null; + source_version: number | null; + source_fingerprint: string | null; + source_completeness: string; + last_attempted_at: string | null; + last_error_code: string | null; indexed_at: string; created_at: string; updated_at: string; @@ -122,9 +196,15 @@ interface JoinedUsageRow { revision_id: string | null; current_generation: string; schema_version: number; + source_updated_at: string | null; + source_version: number | null; + source_fingerprint: string | null; + source_completeness: string; + last_attempted_at: string | null; + last_error_code: string | null; indexed_at: string; source_created_at: string; - source_updated_at: string; + source_row_updated_at: string; occurrence_id: string; generation: string; field_slug: string; @@ -151,14 +231,8 @@ export class MediaUsageRepository { const now = new Date().toISOString(); await withTransaction(this.db, async (trx) => { - await this.insertOccurrences(trx, source.sourceKey, generation, occurrences); + await this.insertOccurrences(trx, source.sourceKey, generation, occurrences, now); await this.upsertSource(trx, source, generation, now); - - try { - await this.deleteStaleGenerations(trx, source.sourceKey, generation); - } catch (error) { - console.error("[media-usage] failed to delete stale generations:", error); - } }); const replaced = await this.findSource(source.sourceKey); @@ -178,6 +252,76 @@ export class MediaUsageRepository { return row ? rowToSource(row) : null; } + async markSourceAttempted(source: MediaUsageSourceInput): Promise { + const now = new Date().toISOString(); + const attemptedAt = source.lastAttemptedAt ?? now; + const row = { + source_key: source.sourceKey, + source_type: source.sourceType, + collection_slug: source.collectionSlug ?? null, + content_id: source.contentId ?? null, + source_variant: source.sourceVariant, + locale: source.locale ?? null, + translation_group: source.translationGroup ?? null, + content_slug: source.contentSlug ?? null, + content_title: source.contentTitle ?? null, + content_status: source.contentStatus ?? null, + content_scheduled_at: source.contentScheduledAt ?? null, + content_deleted_at: source.contentDeletedAt ?? null, + revision_id: source.revisionId ?? null, + current_generation: ulid(), + schema_version: source.schemaVersion ?? 1, + source_updated_at: source.sourceUpdatedAt ?? null, + source_version: source.sourceVersion ?? null, + source_fingerprint: source.sourceFingerprint ?? null, + source_completeness: + source.sourceCompleteness ?? (source.lastErrorCode ? "failed" : "unknown"), + last_attempted_at: attemptedAt, + last_error_code: source.lastErrorCode ?? null, + indexed_at: now, + updated_at: now, + }; + const updates: Updateable = { + source_type: row.source_type, + source_variant: row.source_variant, + source_completeness: row.source_completeness, + last_attempted_at: row.last_attempted_at, + last_error_code: row.last_error_code, + updated_at: row.updated_at, + }; + + if (source.collectionSlug !== undefined) updates.collection_slug = row.collection_slug; + if (source.contentId !== undefined) updates.content_id = row.content_id; + if (source.locale !== undefined) updates.locale = row.locale; + if (source.translationGroup !== undefined) updates.translation_group = row.translation_group; + if (source.contentSlug !== undefined) updates.content_slug = row.content_slug; + if (source.contentTitle !== undefined) updates.content_title = row.content_title; + if (source.contentStatus !== undefined) updates.content_status = row.content_status; + if (source.contentScheduledAt !== undefined) { + updates.content_scheduled_at = row.content_scheduled_at; + } + if (source.contentDeletedAt !== undefined) updates.content_deleted_at = row.content_deleted_at; + if (source.revisionId !== undefined) updates.revision_id = row.revision_id; + if (source.schemaVersion !== undefined) updates.schema_version = row.schema_version; + if (source.sourceUpdatedAt !== undefined) updates.source_updated_at = row.source_updated_at; + if (source.sourceVersion !== undefined) updates.source_version = row.source_version; + if (source.sourceFingerprint !== undefined) { + updates.source_fingerprint = row.source_fingerprint; + } + + await this.db + .insertInto("_emdash_media_usage_sources") + .values(row) + .onConflict((oc) => oc.column("source_key").doUpdateSet(updates)) + .execute(); + + const attempted = await this.findSource(source.sourceKey); + if (!attempted) { + throw new Error(`Media usage source ${source.sourceKey} was not persisted`); + } + return attempted; + } + async findCurrentUsageByMediaId(mediaId: string): Promise { const rows = await this.db .selectFrom("_emdash_media_usage_sources as s") @@ -218,15 +362,31 @@ export class MediaUsageRepository { return rows.map(rowToUsageRecord); } + async findCurrentUsagePageByMediaId( + mediaId: string, + options: FindMediaUsageOptions = {}, + ): Promise> { + return this.findCurrentUsagePage((query) => query.where("u.media_id", "=", mediaId), options); + } + + async findCurrentUsagePageByProviderAsset( + provider: string, + providerAssetId: string, + options: FindMediaUsageOptions = {}, + ): Promise> { + return this.findCurrentUsagePage( + (query) => + query.where("u.provider", "=", provider).where("u.provider_asset_id", "=", providerAssetId), + options, + ); + } + async deleteSource(sourceKey: string): Promise { - return withTransaction(this.db, async (trx) => { - await trx.deleteFrom("_emdash_media_usage").where("source_key", "=", sourceKey).execute(); - const result = await trx - .deleteFrom("_emdash_media_usage_sources") - .where("source_key", "=", sourceKey) - .executeTakeFirst(); - return Number(result.numDeletedRows ?? 0); - }); + return this.deleteSources([sourceKey]); + } + + async deleteSources(sourceKeys: readonly string[]): Promise { + return this.deleteSourceKeys(sourceKeys, "source-first"); } async deleteContentSources(collectionSlug: string, contentId: string): Promise { @@ -238,20 +398,277 @@ export class MediaUsageRepository { .where("content_id", "=", contentId) .execute(); const sourceKeys = sourceRows.map((row) => row.source_key); - if (sourceKeys.length === 0) return 0; + return this.deleteSourceKeys(sourceKeys, "usage-first"); + } + + async deleteCollectionSources(collectionSlug: string): Promise { + let deleted = 0; + while (true) { + const sourceRows = await this.db + .selectFrom("_emdash_media_usage_sources") + .select("source_key") + .where("source_type", "=", "content") + .where("collection_slug", "=", collectionSlug) + .orderBy("source_key", "asc") + .limit(SQL_BATCH_SIZE) + .execute(); + if (sourceRows.length === 0) break; + + deleted += await this.deleteSourceKeys( + sourceRows.map((row) => row.source_key), + "usage-first", + ); + } + return deleted; + } + + async deleteOrphanOccurrencesOlderThan(cutoff: string, limit: number): Promise { + const batchLimit = Math.floor(limit); + if (batchLimit <= 0) return 0; + + const rows = await this.db + .selectFrom("_emdash_media_usage as u") + .leftJoin("_emdash_media_usage_sources as s", (join) => + join.onRef("s.source_key", "=", "u.source_key"), + ) + .select("u.id") + .where("s.source_key", "is", null) + .where("u.created_at", "<", cutoff) + .orderBy("u.created_at", "asc") + .orderBy("u.id", "asc") + .limit(batchLimit) + .execute(); + + let deleted = 0; + for (const idBatch of chunks( + rows.map((row) => row.id), + SQL_BATCH_SIZE, + )) { + const result = await this.db + .deleteFrom("_emdash_media_usage") + .where("id", "in", idBatch) + .where("created_at", "<", cutoff) + .where( + sql`NOT EXISTS (SELECT 1 FROM _emdash_media_usage_sources s WHERE s.source_key = _emdash_media_usage.source_key)`, + ) + .executeTakeFirst(); + deleted += Number(result.numDeletedRows ?? 0); + } + return deleted; + } + + async deleteStaleGenerationsOlderThan(cutoff: string, limit: number): Promise { + const batchLimit = Math.floor(limit); + if (batchLimit <= 0) return 0; + + const rows = await this.db + .selectFrom("_emdash_media_usage as u") + .innerJoin("_emdash_media_usage_sources as s", (join) => + join.onRef("s.source_key", "=", "u.source_key"), + ) + .select("u.id") + .where("u.created_at", "<", cutoff) + .whereRef("u.generation", "!=", "s.current_generation") + .whereRef("u.created_at", "<", "s.indexed_at") + .orderBy("u.created_at", "asc") + .orderBy("u.id", "asc") + .limit(batchLimit) + .execute(); + + const ids = rows.map((row) => row.id); + if (ids.length === 0) return 0; + + let deleted = 0; + for (const idBatch of chunks(ids, SQL_BATCH_SIZE)) { + const result = await this.db + .deleteFrom("_emdash_media_usage") + .where("id", "in", idBatch) + .where("created_at", "<", cutoff) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_sources as s") + .select("s.source_key") + .whereRef("s.source_key", "=", "_emdash_media_usage.source_key") + .whereRef("s.current_generation", "!=", "_emdash_media_usage.generation") + .whereRef("_emdash_media_usage.created_at", "<", "s.indexed_at"), + ), + ) + .executeTakeFirst(); + deleted += Number(result.numDeletedRows ?? 0); + } + return deleted; + } + + async deleteAbandonedGenerationsOlderThan(cutoff: string, limit: number): Promise { + const batchLimit = Math.floor(limit); + if (batchLimit <= 0) return 0; + + const rows = await this.db + .selectFrom("_emdash_media_usage as u") + .innerJoin("_emdash_media_usage_sources as s", (join) => + join.onRef("s.source_key", "=", "u.source_key"), + ) + .select("u.id") + .where("u.created_at", "<", cutoff) + .whereRef("u.generation", "!=", "s.current_generation") + .whereRef("u.created_at", ">=", "s.indexed_at") + .orderBy("u.created_at", "asc") + .orderBy("u.id", "asc") + .limit(batchLimit) + .execute(); + + let deleted = 0; + for (const idBatch of chunks( + rows.map((row) => row.id), + SQL_BATCH_SIZE, + )) { + const result = await this.db + .deleteFrom("_emdash_media_usage") + .where("id", "in", idBatch) + .where("created_at", "<", cutoff) + .where((eb) => + eb.exists( + eb + .selectFrom("_emdash_media_usage_sources as s") + .select("s.source_key") + .whereRef("s.source_key", "=", "_emdash_media_usage.source_key") + .whereRef("s.current_generation", "!=", "_emdash_media_usage.generation") + .whereRef("_emdash_media_usage.created_at", ">=", "s.indexed_at"), + ), + ) + .executeTakeFirst(); + deleted += Number(result.numDeletedRows ?? 0); + } + return deleted; + } + + async upsertIndexStatus(input: MediaUsageIndexStatusInput): Promise { + const now = input.updatedAt ?? new Date().toISOString(); + const row = { + adapter_id: input.adapterId, + scope_type: input.scopeType, + scope_key: input.scopeKey, + status: input.status, + schema_version: input.schemaVersion ?? 1, + started_at: input.startedAt ?? null, + completed_at: input.completedAt ?? null, + cursor: input.cursor ?? null, + indexed_source_count: input.indexedSourceCount ?? 0, + failed_source_count: input.failedSourceCount ?? 0, + last_error_code: input.lastErrorCode ?? null, + updated_at: now, + }; + + await this.db + .insertInto("_emdash_media_usage_index_status") + .values(row) + .onConflict((oc) => + oc.columns(["adapter_id", "scope_type", "scope_key"]).doUpdateSet({ + status: row.status, + schema_version: row.schema_version, + started_at: row.started_at, + completed_at: row.completed_at, + cursor: row.cursor, + indexed_source_count: row.indexed_source_count, + failed_source_count: row.failed_source_count, + last_error_code: row.last_error_code, + updated_at: row.updated_at, + }), + ) + .execute(); + + const status = await this.findIndexStatus(input); + if (!status) { + throw new Error( + `Media usage index status ${input.adapterId}:${input.scopeType}:${input.scopeKey} was not persisted`, + ); + } + return status; + } + + async findIndexStatus( + identity: MediaUsageIndexStatusIdentity, + ): Promise { + const row = await this.db + .selectFrom("_emdash_media_usage_index_status") + .selectAll() + .where("adapter_id", "=", identity.adapterId) + .where("scope_type", "=", identity.scopeType) + .where("scope_key", "=", identity.scopeKey) + .executeTakeFirst(); + + return row ? rowToIndexStatus(row) : null; + } + + private async findCurrentUsagePage( + applyFilter: ( + query: ReturnType, + ) => ReturnType, + options: FindMediaUsageOptions, + ): Promise> { + const limit = Math.min(Math.max(1, options.limit ?? 50), 100); + let query = applyFilter(this.currentUsageBaseQuery()) + .orderBy("u.id", "asc") + .limit(limit + 1); + + if (options.cursor) { + const { id } = decodeCursor(options.cursor); + query = query.where("u.id", ">", id); + } + + const rows = await query.execute(); + const items = rows.slice(0, limit).map(rowToUsageRecord); + const result: FindManyResult = { items }; + + if (rows.length > limit && items.length > 0) { + const last = items.at(-1)!; + result.nextCursor = encodeCursor(last.occurrence.id, last.occurrence.id); + } + + return result; + } + + private currentUsageBaseQuery() { + return this.db + .selectFrom("_emdash_media_usage_sources as s") + .innerJoin("_emdash_media_usage as u", (join) => + join + .onRef("u.source_key", "=", "s.source_key") + .onRef("u.generation", "=", "s.current_generation"), + ) + .select(currentUsageSelect); + } + + private async deleteSourceKeys( + sourceKeys: readonly string[], + order: "source-first" | "usage-first", + ): Promise { + const uniqueSourceKeys = [...new Set(sourceKeys)]; + if (uniqueSourceKeys.length === 0) return 0; return withTransaction(this.db, async (trx) => { let deleted = 0; - for (const sourceKeyBatch of chunks(sourceKeys, SQL_BATCH_SIZE)) { - await trx - .deleteFrom("_emdash_media_usage") - .where("source_key", "in", sourceKeyBatch) - .execute(); + for (const sourceKeyBatch of chunks(uniqueSourceKeys, SQL_BATCH_SIZE)) { + if (order === "usage-first") { + await trx + .deleteFrom("_emdash_media_usage") + .where("source_key", "in", sourceKeyBatch) + .execute(); + } + const result = await trx .deleteFrom("_emdash_media_usage_sources") .where("source_key", "in", sourceKeyBatch) .executeTakeFirst(); deleted += Number(result.numDeletedRows ?? 0); + + if (order === "source-first") { + await trx + .deleteFrom("_emdash_media_usage") + .where("source_key", "in", sourceKeyBatch) + .execute(); + } } return deleted; }); @@ -262,6 +679,7 @@ export class MediaUsageRepository { sourceKey: string, generation: string, occurrences: readonly MediaUsageOccurrenceInput[], + now: string, ): Promise { if (occurrences.length === 0) return; @@ -278,6 +696,7 @@ export class MediaUsageRepository { provider_asset_id: occurrence.providerAssetId, media_kind: occurrence.mediaKind ?? null, mime_type: occurrence.mimeType ?? null, + created_at: now, })); for (const rowBatch of chunks(rows, OCCURRENCE_INSERT_BATCH_SIZE)) { @@ -307,6 +726,12 @@ export class MediaUsageRepository { revision_id: source.revisionId ?? null, current_generation: generation, schema_version: source.schemaVersion ?? 1, + source_updated_at: source.sourceUpdatedAt ?? null, + source_version: source.sourceVersion ?? null, + source_fingerprint: source.sourceFingerprint ?? null, + source_completeness: source.sourceCompleteness ?? "complete", + last_attempted_at: source.lastAttemptedAt ?? now, + last_error_code: null, indexed_at: now, updated_at: now, }; @@ -330,24 +755,18 @@ export class MediaUsageRepository { revision_id: row.revision_id, current_generation: row.current_generation, schema_version: row.schema_version, + source_updated_at: row.source_updated_at, + source_version: row.source_version, + source_fingerprint: row.source_fingerprint, + source_completeness: row.source_completeness, + last_attempted_at: row.last_attempted_at, + last_error_code: row.last_error_code, indexed_at: row.indexed_at, updated_at: row.updated_at, }), ) .execute(); } - - private async deleteStaleGenerations( - db: DatabaseExecutor, - sourceKey: string, - currentGeneration: string, - ): Promise { - await db - .deleteFrom("_emdash_media_usage") - .where("source_key", "=", sourceKey) - .where("generation", "!=", currentGeneration) - .execute(); - } } const currentUsageSelect = [ @@ -366,9 +785,15 @@ const currentUsageSelect = [ "s.revision_id as revision_id", "s.current_generation as current_generation", "s.schema_version as schema_version", + "s.source_updated_at as source_updated_at", + "s.source_version as source_version", + "s.source_fingerprint as source_fingerprint", + "s.source_completeness as source_completeness", + "s.last_attempted_at as last_attempted_at", + "s.last_error_code as last_error_code", "s.indexed_at as indexed_at", "s.created_at as source_created_at", - "s.updated_at as source_updated_at", + "s.updated_at as source_row_updated_at", "u.id as occurrence_id", "u.generation as generation", "u.field_slug as field_slug", @@ -400,6 +825,12 @@ function rowToSource(row: MediaUsageSourceRow): MediaUsageSource { revisionId: row.revision_id, currentGeneration: row.current_generation, schemaVersion: Number(row.schema_version), + sourceUpdatedAt: row.source_updated_at, + sourceVersion: row.source_version === null ? null : Number(row.source_version), + sourceFingerprint: row.source_fingerprint, + sourceCompleteness: row.source_completeness, + lastAttemptedAt: row.last_attempted_at, + lastErrorCode: row.last_error_code, indexedAt: row.indexed_at, createdAt: row.created_at, updatedAt: row.updated_at, @@ -442,9 +873,15 @@ function rowToUsageRecord(row: JoinedUsageRow): MediaUsageRecord { revision_id: row.revision_id, current_generation: row.current_generation, schema_version: row.schema_version, + source_updated_at: row.source_updated_at, + source_version: row.source_version, + source_fingerprint: row.source_fingerprint, + source_completeness: row.source_completeness, + last_attempted_at: row.last_attempted_at, + last_error_code: row.last_error_code, indexed_at: row.indexed_at, created_at: row.source_created_at, - updated_at: row.source_updated_at, + updated_at: row.source_row_updated_at, }), occurrence: rowToOccurrence({ id: row.occurrence_id, @@ -463,3 +900,20 @@ function rowToUsageRecord(row: JoinedUsageRow): MediaUsageRecord { }), }; } + +function rowToIndexStatus(row: Selectable): MediaUsageIndexStatus { + return { + adapterId: row.adapter_id, + scopeType: row.scope_type, + scopeKey: row.scope_key, + status: row.status, + schemaVersion: Number(row.schema_version), + startedAt: row.started_at, + completedAt: row.completed_at, + cursor: row.cursor, + indexedSourceCount: Number(row.indexed_source_count), + failedSourceCount: Number(row.failed_source_count), + lastErrorCode: row.last_error_code, + updatedAt: row.updated_at, + }; +} diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index b7a791991a..750924033b 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -61,7 +61,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { ]); }); - it("flips generations and removes stale occurrence rows", async () => { + it("flips generations without removing stale occurrence rows", async () => { const first = await repo.replaceSource(contentSource("entry1", "live"), [ occurrence("hero", "media-old"), ]); @@ -79,7 +79,282 @@ describeEachDialect("MediaUsageRepository", (dialect) => { .where("source_key", "=", "content:posts:entry1:live") .execute(); - expect(rows).toEqual([{ generation: second.currentGeneration, media_id: "media-new" }]); + expect(rows).toHaveLength(2); + expect(rows).toContainEqual({ generation: first.currentGeneration, media_id: "media-old" }); + expect(rows).toContainEqual({ generation: second.currentGeneration, media_id: "media-new" }); + }); + + it("writes ISO occurrence timestamps for safe cleanup cutoffs", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + + const row = await ctx.db + .selectFrom("_emdash_media_usage") + .select("created_at") + .where("source_key", "=", "content:posts:entry1:live") + .executeTakeFirstOrThrow(); + + expect(row.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/); + }); + + it("deletes stale generations by age and limit without deleting current usage", async () => { + const first = await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-old-1"), + occurrence("body", "media-old-2"), + ]); + const second = await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-current"), + ]); + + await ctx.db + .updateTable("_emdash_media_usage") + .set({ created_at: "2026-01-01T00:00:00.000Z" }) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + + expect(await repo.deleteStaleGenerationsOlderThan("2026-01-02T00:00:00.000Z", 1)).toBe(1); + expect(await repo.findCurrentUsageByMediaId("media-current")).toHaveLength(1); + + let rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select(["generation", "media_id"]) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + + expect(rows).toHaveLength(2); + expect(rows).toContainEqual({ + generation: second.currentGeneration, + media_id: "media-current", + }); + expect(rows.filter((row) => row.generation === first.currentGeneration)).toHaveLength(1); + + expect(await repo.deleteStaleGenerationsOlderThan("2026-01-02T00:00:00.000Z", 10)).toBe(1); + + rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select(["generation", "media_id"]) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + + expect(rows).toEqual([{ generation: second.currentGeneration, media_id: "media-current" }]); + }); + + it("does not delete in-flight generations that are newer than or equal to the published source", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-current"), + ]); + + await ctx.db + .updateTable("_emdash_media_usage_sources") + .set({ indexed_at: "2026-01-01T00:00:00.000Z" }) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + await ctx.db + .updateTable("_emdash_media_usage") + .set({ created_at: "2026-01-01T00:00:00.000Z" }) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + await ctx.db + .insertInto("_emdash_media_usage") + .values({ + id: "pending-occurrence", + source_key: "content:posts:entry1:live", + generation: "pending-generation", + field_slug: "hero", + field_path: "pendingHero", + occurrence_index: 0, + reference_type: "image_field", + media_id: "media-pending", + provider: "local", + provider_asset_id: "media-pending", + media_kind: "image", + mime_type: null, + created_at: "2026-01-01T00:00:01.000Z", + }) + .execute(); + await ctx.db + .insertInto("_emdash_media_usage") + .values({ + id: "pending-same-ms-occurrence", + source_key: "content:posts:entry1:live", + generation: "pending-same-ms-generation", + field_slug: "body", + field_path: "pendingSameMs", + occurrence_index: 0, + reference_type: "image_field", + media_id: "media-pending-same-ms", + provider: "local", + provider_asset_id: "media-pending-same-ms", + media_kind: "image", + mime_type: null, + created_at: "2026-01-01T00:00:00.000Z", + }) + .execute(); + + expect(await repo.deleteStaleGenerationsOlderThan("2026-01-02T00:00:00.000Z", 10)).toBe(0); + + const rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select(["generation", "media_id"]) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + + expect(rows).toContainEqual({ generation: "pending-generation", media_id: "media-pending" }); + expect(rows).toContainEqual({ + generation: "pending-same-ms-generation", + media_id: "media-pending-same-ms", + }); + expect(await repo.findCurrentUsageByMediaId("media-current")).toHaveLength(1); + }); + + it("deletes abandoned generations from failed partial replacements by age", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-current"), + ]); + + await ctx.db + .updateTable("_emdash_media_usage_sources") + .set({ indexed_at: "2026-01-01T00:00:00.000Z" }) + .where("source_key", "=", "content:posts:entry1:live") + .execute(); + await ctx.db + .insertInto("_emdash_media_usage") + .values({ + id: "abandoned-occurrence", + source_key: "content:posts:entry1:live", + generation: "abandoned-generation", + field_slug: "hero", + field_path: "abandonedHero", + occurrence_index: 0, + reference_type: "image_field", + media_id: "media-abandoned", + provider: "local", + provider_asset_id: "media-abandoned", + media_kind: "image", + mime_type: null, + created_at: "2026-01-01T00:00:01.000Z", + }) + .execute(); + + expect(await repo.deleteStaleGenerationsOlderThan("2026-01-02T00:00:00.000Z", 10)).toBe(0); + expect(await repo.deleteAbandonedGenerationsOlderThan("2026-01-02T00:00:00.000Z", 10)).toBe(1); + expect(await repo.findCurrentUsageByMediaId("media-current")).toHaveLength(1); + + const abandoned = await ctx.db + .selectFrom("_emdash_media_usage") + .select("id") + .where("id", "=", "abandoned-occurrence") + .execute(); + expect(abandoned).toEqual([]); + }); + + it("persists source freshness metadata and clears previous source errors", async () => { + await repo.markSourceAttempted( + contentSource("entry1", "live", { + sourceCompleteness: "failed", + lastAttemptedAt: "2026-01-01T00:00:00.000Z", + lastErrorCode: "EXTRACT_FAILED", + }), + ); + + const source = await repo.replaceSource( + contentSource("entry1", "live", { + sourceUpdatedAt: "2026-01-01T00:00:01.000Z", + sourceVersion: 7, + sourceFingerprint: "fingerprint-entry1-live", + sourceCompleteness: "complete", + lastAttemptedAt: "2026-01-01T00:00:02.000Z", + }), + [occurrence("hero", "media-hero")], + ); + + expect(source).toEqual( + expect.objectContaining({ + sourceUpdatedAt: "2026-01-01T00:00:01.000Z", + sourceVersion: 7, + sourceFingerprint: "fingerprint-entry1-live", + sourceCompleteness: "complete", + lastAttemptedAt: "2026-01-01T00:00:02.000Z", + lastErrorCode: null, + }), + ); + }); + + it("marks failed source attempts without replacing current usage", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + + const failed = await repo.markSourceAttempted( + contentSource("entry1", "live", { + sourceCompleteness: "failed", + lastAttemptedAt: "2026-01-01T00:00:00.000Z", + lastErrorCode: "LOAD_FAILED", + }), + ); + + expect(failed).toEqual( + expect.objectContaining({ + sourceCompleteness: "failed", + lastAttemptedAt: "2026-01-01T00:00:00.000Z", + lastErrorCode: "LOAD_FAILED", + }), + ); + expect(await repo.findCurrentUsageByMediaId("media-live")).toHaveLength(1); + + const neverIndexed = await repo.markSourceAttempted( + contentSource("entry-missing", "draft", { + lastAttemptedAt: "2026-01-01T00:00:01.000Z", + lastErrorCode: "MISSING_TABLE", + }), + ); + + expect(neverIndexed).toEqual( + expect.objectContaining({ + sourceKey: "content:posts:entry-missing:draft", + sourceCompleteness: "failed", + lastAttemptedAt: "2026-01-01T00:00:01.000Z", + lastErrorCode: "MISSING_TABLE", + }), + ); + + const rows = await ctx.db + .selectFrom("_emdash_media_usage") + .select("id") + .where("source_key", "=", "content:posts:entry-missing:draft") + .execute(); + expect(rows).toEqual([]); + }); + + it("preserves existing source metadata when marking a minimal failed attempt", async () => { + await repo.replaceSource( + contentSource("entry1", "live", { + contentTitle: "Existing title", + sourceUpdatedAt: "2026-01-01T00:00:00.000Z", + sourceVersion: 3, + sourceFingerprint: "fingerprint-existing", + }), + [occurrence("hero", "media-live")], + ); + + const failed = await repo.markSourceAttempted({ + sourceKey: "content:posts:entry1:live", + sourceType: "content", + sourceVariant: "live", + lastAttemptedAt: "2026-01-01T00:00:01.000Z", + lastErrorCode: "LOAD_FAILED", + }); + + expect(failed).toEqual( + expect.objectContaining({ + collectionSlug: "posts", + contentId: "entry1", + contentTitle: "Existing title", + sourceUpdatedAt: "2026-01-01T00:00:00.000Z", + sourceVersion: 3, + sourceFingerprint: "fingerprint-existing", + sourceCompleteness: "failed", + lastErrorCode: "LOAD_FAILED", + }), + ); + expect(await repo.findCurrentUsageByMediaId("media-live")).toHaveLength(1); }); it("supports empty replacement while preserving the source row", async () => { @@ -120,6 +395,84 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(await repo.findCurrentUsageByMediaId("media-page")).toHaveLength(1); }); + it("deletes content sources by collection", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry2", "draft"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry1", "live", { collectionSlug: "pages" }), [ + occurrence("hero", "media-page"), + ]); + + expect(await repo.deleteCollectionSources("posts")).toBe(2); + expect(await repo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-draft")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-page")).toHaveLength(1); + }); + + it("deletes specific source keys in D1-safe batches", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "draft"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry2", "live"), [occurrence("hero", "media-other")]); + + expect( + await repo.deleteSources([ + "content:posts:entry1:live", + "content:posts:entry1:draft", + "content:posts:entry1:live", + ]), + ).toBe(2); + expect(await repo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-draft")).toEqual([]); + expect(await repo.findCurrentUsageByMediaId("media-other")).toHaveLength(1); + }); + + it("deletes orphan occurrence rows by age in bounded batches", async () => { + await ctx.db + .insertInto("_emdash_media_usage") + .values([ + { + id: "orphan-1", + source_key: "missing-source-1", + generation: "generation-1", + field_slug: "hero", + field_path: "hero", + occurrence_index: 0, + reference_type: "image_field", + media_id: "media-orphan-1", + provider: "local", + provider_asset_id: "media-orphan-1", + media_kind: "image", + mime_type: null, + created_at: "2026-01-01T00:00:00.000Z", + }, + { + id: "orphan-newer", + source_key: "missing-source-2", + generation: "generation-1", + field_slug: "hero", + field_path: "hero", + occurrence_index: 0, + reference_type: "image_field", + media_id: "media-orphan-newer", + provider: "local", + provider_asset_id: "media-orphan-newer", + media_kind: "image", + mime_type: null, + created_at: "2026-01-02T00:00:00.000Z", + }, + ]) + .execute(); + + expect(await repo.deleteOrphanOccurrencesOlderThan("2026-01-01T12:00:00.000Z", 1)).toBe(1); + expect(await repo.deleteOrphanOccurrencesOlderThan("2026-01-01T12:00:00.000Z", 10)).toBe(0); + + const remaining = await ctx.db + .selectFrom("_emdash_media_usage") + .select("id") + .where("source_key", "=", "missing-source-2") + .execute(); + expect(remaining).toEqual([{ id: "orphan-newer" }]); + }); + it("finds current usage by provider asset", async () => { await repo.replaceSource(contentSource("entry1", "live"), [ occurrence("video", "mux-video-1", { @@ -161,6 +514,114 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(usage.map((row) => row.source.sourceVariant)).toEqual(["draft", "live"]); }); + it("paginates current media usage by occurrence id", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("hero", "media-shared"), + occurrence("body", "media-shared"), + ]); + await repo.replaceSource(contentSource("entry2", "live"), [occurrence("hero", "media-shared")]); + + const page1 = await repo.findCurrentUsagePageByMediaId("media-shared", { limit: 2 }); + expect(page1.items).toHaveLength(2); + expect(page1.nextCursor).toEqual(expect.any(String)); + + const page2 = await repo.findCurrentUsagePageByMediaId("media-shared", { + limit: 2, + cursor: page1.nextCursor, + }); + expect(page2.items).toHaveLength(1); + expect(page2.nextCursor).toBeUndefined(); + + const occurrenceIds = [...page1.items, ...page2.items].map((record) => record.occurrence.id); + expect(occurrenceIds).toEqual(occurrenceIds.toSorted()); + }); + + it("paginates current provider-asset usage by occurrence id", async () => { + await repo.replaceSource(contentSource("entry1", "live"), [ + occurrence("video", "mux-video-1", { + provider: "mux", + providerAssetId: "mux-video-1", + mediaId: null, + }), + occurrence("video2", "mux-video-1", { + provider: "mux", + providerAssetId: "mux-video-1", + mediaId: null, + }), + ]); + + const page1 = await repo.findCurrentUsagePageByProviderAsset("mux", "mux-video-1", { + limit: 1, + }); + const page2 = await repo.findCurrentUsagePageByProviderAsset("mux", "mux-video-1", { + limit: 1, + cursor: page1.nextCursor, + }); + + expect(page1.items).toHaveLength(1); + expect(page2.items).toHaveLength(1); + expect(page2.items[0]!.occurrence.id > page1.items[0]!.occurrence.id).toBe(true); + }); + + it("upserts and reads index status rows", async () => { + const running = await repo.upsertIndexStatus({ + adapterId: "content-media", + scopeType: "collection", + scopeKey: "posts", + status: "running", + startedAt: "2026-01-01T00:00:00.000Z", + cursor: "cursor-1", + indexedSourceCount: 2, + failedSourceCount: 1, + lastErrorCode: "LOAD_FAILED", + updatedAt: "2026-01-01T00:00:01.000Z", + }); + + expect(running).toEqual({ + adapterId: "content-media", + scopeType: "collection", + scopeKey: "posts", + status: "running", + schemaVersion: 1, + startedAt: "2026-01-01T00:00:00.000Z", + completedAt: null, + cursor: "cursor-1", + indexedSourceCount: 2, + failedSourceCount: 1, + lastErrorCode: "LOAD_FAILED", + updatedAt: "2026-01-01T00:00:01.000Z", + }); + + const complete = await repo.upsertIndexStatus({ + adapterId: "content-media", + scopeType: "collection", + scopeKey: "posts", + status: "complete", + startedAt: "2026-01-01T00:00:00.000Z", + completedAt: "2026-01-01T00:00:02.000Z", + indexedSourceCount: 3, + updatedAt: "2026-01-01T00:00:02.000Z", + }); + + expect(complete).toEqual( + expect.objectContaining({ + status: "complete", + completedAt: "2026-01-01T00:00:02.000Z", + cursor: null, + indexedSourceCount: 3, + failedSourceCount: 0, + lastErrorCode: null, + }), + ); + expect( + await repo.findIndexStatus({ + adapterId: "content-media", + scopeType: "collection", + scopeKey: "posts", + }), + ).toEqual(complete); + }); + it("replaces more occurrences than one D1 insert batch", async () => { const occurrences = Array.from({ length: SQL_BATCH_SIZE + 7 }, (_, index) => occurrence(`gallery-${index}`, `media-${index}`, { From 0066d192ef4e08274d9771e8d1236fa45fc15bd4 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 14:06:55 +0100 Subject: [PATCH 09/29] feat(media): Rename Repository Fixtures To Storage Variants --- .../database/media-usage-migration.test.ts | 4 +- .../database/media-usage-repository.test.ts | 137 +++++++++--------- .../media-usage-status-migration.test.ts | 14 +- 3 files changed, 79 insertions(+), 76 deletions(-) diff --git a/packages/core/tests/integration/database/media-usage-migration.test.ts b/packages/core/tests/integration/database/media-usage-migration.test.ts index ce9797ffb2..03f9200024 100644 --- a/packages/core/tests/integration/database/media-usage-migration.test.ts +++ b/packages/core/tests/integration/database/media-usage-migration.test.ts @@ -43,7 +43,7 @@ describeEachDialect("media usage index migration", (dialect) => { }); it("accepts a content usage source and one occurrence", async () => { - const sourceKey = "content:posts:entry1:live"; + const sourceKey = "content:posts:entry1:columns"; const generation = "gen1"; await ctx.db @@ -53,7 +53,7 @@ describeEachDialect("media usage index migration", (dialect) => { source_type: "content", collection_slug: "posts", content_id: "entry1", - source_variant: "live", + source_variant: "columns", content_slug: "hello-world", content_title: "Hello World", locale: "en", diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index 750924033b..260a536a30 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -23,7 +23,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("replaces a source with a current generation of occurrences", async () => { - const source = await repo.replaceSource(contentSource("entry1", "live"), [ + const source = await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-hero", { mimeType: "image/jpeg", mediaKind: "image" }), occurrence("attachment", "media-file", { referenceType: "file_field", @@ -33,14 +33,14 @@ describeEachDialect("MediaUsageRepository", (dialect) => { ]); expect(source.currentGeneration).toEqual(expect.any(String)); - expect(source.sourceKey).toBe("content:posts:entry1:live"); - expect(source.sourceVariant).toBe("live"); + expect(source.sourceKey).toBe("content:posts:entry1:columns"); + expect(source.sourceVariant).toBe("columns"); const usage = await repo.findCurrentUsageByMediaId("media-hero"); expect(usage).toEqual([ { source: expect.objectContaining({ - sourceKey: "content:posts:entry1:live", + sourceKey: "content:posts:entry1:columns", collectionSlug: "posts", contentId: "entry1", contentSlug: "hello-world", @@ -62,10 +62,10 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("flips generations without removing stale occurrence rows", async () => { - const first = await repo.replaceSource(contentSource("entry1", "live"), [ + const first = await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-old"), ]); - const second = await repo.replaceSource(contentSource("entry1", "live"), [ + const second = await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-new"), ]); @@ -76,7 +76,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { const rows = await ctx.db .selectFrom("_emdash_media_usage") .select(["generation", "media_id"]) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); expect(rows).toHaveLength(2); @@ -85,30 +85,30 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("writes ISO occurrence timestamps for safe cleanup cutoffs", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); const row = await ctx.db .selectFrom("_emdash_media_usage") .select("created_at") - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .executeTakeFirstOrThrow(); expect(row.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T.*Z$/); }); it("deletes stale generations by age and limit without deleting current usage", async () => { - const first = await repo.replaceSource(contentSource("entry1", "live"), [ + const first = await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-old-1"), occurrence("body", "media-old-2"), ]); - const second = await repo.replaceSource(contentSource("entry1", "live"), [ + const second = await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-current"), ]); await ctx.db .updateTable("_emdash_media_usage") .set({ created_at: "2026-01-01T00:00:00.000Z" }) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); expect(await repo.deleteStaleGenerationsOlderThan("2026-01-02T00:00:00.000Z", 1)).toBe(1); @@ -117,7 +117,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { let rows = await ctx.db .selectFrom("_emdash_media_usage") .select(["generation", "media_id"]) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); expect(rows).toHaveLength(2); @@ -132,32 +132,32 @@ describeEachDialect("MediaUsageRepository", (dialect) => { rows = await ctx.db .selectFrom("_emdash_media_usage") .select(["generation", "media_id"]) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); expect(rows).toEqual([{ generation: second.currentGeneration, media_id: "media-current" }]); }); it("does not delete in-flight generations that are newer than or equal to the published source", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [ + await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-current"), ]); await ctx.db .updateTable("_emdash_media_usage_sources") .set({ indexed_at: "2026-01-01T00:00:00.000Z" }) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); await ctx.db .updateTable("_emdash_media_usage") .set({ created_at: "2026-01-01T00:00:00.000Z" }) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); await ctx.db .insertInto("_emdash_media_usage") .values({ id: "pending-occurrence", - source_key: "content:posts:entry1:live", + source_key: "content:posts:entry1:columns", generation: "pending-generation", field_slug: "hero", field_path: "pendingHero", @@ -175,7 +175,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { .insertInto("_emdash_media_usage") .values({ id: "pending-same-ms-occurrence", - source_key: "content:posts:entry1:live", + source_key: "content:posts:entry1:columns", generation: "pending-same-ms-generation", field_slug: "body", field_path: "pendingSameMs", @@ -195,7 +195,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { const rows = await ctx.db .selectFrom("_emdash_media_usage") .select(["generation", "media_id"]) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); expect(rows).toContainEqual({ generation: "pending-generation", media_id: "media-pending" }); @@ -207,20 +207,20 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes abandoned generations from failed partial replacements by age", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [ + await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-current"), ]); await ctx.db .updateTable("_emdash_media_usage_sources") .set({ indexed_at: "2026-01-01T00:00:00.000Z" }) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .execute(); await ctx.db .insertInto("_emdash_media_usage") .values({ id: "abandoned-occurrence", - source_key: "content:posts:entry1:live", + source_key: "content:posts:entry1:columns", generation: "abandoned-generation", field_slug: "hero", field_path: "abandonedHero", @@ -249,7 +249,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { it("persists source freshness metadata and clears previous source errors", async () => { await repo.markSourceAttempted( - contentSource("entry1", "live", { + contentSource("entry1", "columns", { sourceCompleteness: "failed", lastAttemptedAt: "2026-01-01T00:00:00.000Z", lastErrorCode: "EXTRACT_FAILED", @@ -257,7 +257,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { ); const source = await repo.replaceSource( - contentSource("entry1", "live", { + contentSource("entry1", "columns", { sourceUpdatedAt: "2026-01-01T00:00:01.000Z", sourceVersion: 7, sourceFingerprint: "fingerprint-entry1-live", @@ -280,10 +280,10 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("marks failed source attempts without replacing current usage", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); const failed = await repo.markSourceAttempted( - contentSource("entry1", "live", { + contentSource("entry1", "columns", { sourceCompleteness: "failed", lastAttemptedAt: "2026-01-01T00:00:00.000Z", lastErrorCode: "LOAD_FAILED", @@ -300,7 +300,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(await repo.findCurrentUsageByMediaId("media-live")).toHaveLength(1); const neverIndexed = await repo.markSourceAttempted( - contentSource("entry-missing", "draft", { + contentSource("entry-missing", "draft_overlay", { lastAttemptedAt: "2026-01-01T00:00:01.000Z", lastErrorCode: "MISSING_TABLE", }), @@ -308,7 +308,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(neverIndexed).toEqual( expect.objectContaining({ - sourceKey: "content:posts:entry-missing:draft", + sourceKey: "content:posts:entry-missing:draft_overlay", sourceCompleteness: "failed", lastAttemptedAt: "2026-01-01T00:00:01.000Z", lastErrorCode: "MISSING_TABLE", @@ -318,14 +318,14 @@ describeEachDialect("MediaUsageRepository", (dialect) => { const rows = await ctx.db .selectFrom("_emdash_media_usage") .select("id") - .where("source_key", "=", "content:posts:entry-missing:draft") + .where("source_key", "=", "content:posts:entry-missing:draft_overlay") .execute(); expect(rows).toEqual([]); }); it("preserves existing source metadata when marking a minimal failed attempt", async () => { await repo.replaceSource( - contentSource("entry1", "live", { + contentSource("entry1", "columns", { contentTitle: "Existing title", sourceUpdatedAt: "2026-01-01T00:00:00.000Z", sourceVersion: 3, @@ -335,9 +335,9 @@ describeEachDialect("MediaUsageRepository", (dialect) => { ); const failed = await repo.markSourceAttempted({ - sourceKey: "content:posts:entry1:live", + sourceKey: "content:posts:entry1:columns", sourceType: "content", - sourceVariant: "live", + sourceVariant: "columns", lastAttemptedAt: "2026-01-01T00:00:01.000Z", lastErrorCode: "LOAD_FAILED", }); @@ -358,33 +358,33 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("supports empty replacement while preserving the source row", async () => { - const first = await repo.replaceSource(contentSource("entry1", "live"), [ + const first = await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-old"), ]); - const second = await repo.replaceSource(contentSource("entry1", "live"), []); + const second = await repo.replaceSource(contentSource("entry1", "columns"), []); expect(second.currentGeneration).not.toBe(first.currentGeneration); - expect(await repo.findSource("content:posts:entry1:live")).toEqual( + expect(await repo.findSource("content:posts:entry1:columns")).toEqual( expect.objectContaining({ currentGeneration: second.currentGeneration }), ); expect(await repo.findCurrentUsageByMediaId("media-old")).toEqual([]); }); it("deletes a single source and its occurrences", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry1", "draft"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [occurrence("hero", "media-draft")]); - expect(await repo.deleteSource("content:posts:entry1:live")).toBe(1); - expect(await repo.findSource("content:posts:entry1:live")).toBeNull(); + expect(await repo.deleteSource("content:posts:entry1:columns")).toBe(1); + expect(await repo.findSource("content:posts:entry1:columns")).toBeNull(); expect(await repo.findCurrentUsageByMediaId("media-live")).toEqual([]); expect(await repo.findCurrentUsageByMediaId("media-draft")).toHaveLength(1); }); it("deletes all content sources for one collection and content id", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry1", "draft"), [occurrence("hero", "media-draft")]); - await repo.replaceSource(contentSource("entry2", "live"), [occurrence("hero", "media-other")]); - await repo.replaceSource(contentSource("entry1", "live", { collectionSlug: "pages" }), [ + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry2", "columns"), [occurrence("hero", "media-other")]); + await repo.replaceSource(contentSource("entry1", "columns", { collectionSlug: "pages" }), [ occurrence("hero", "media-page"), ]); @@ -396,9 +396,9 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes content sources by collection", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry2", "draft"), [occurrence("hero", "media-draft")]); - await repo.replaceSource(contentSource("entry1", "live", { collectionSlug: "pages" }), [ + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry2", "draft_overlay"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry1", "columns", { collectionSlug: "pages" }), [ occurrence("hero", "media-page"), ]); @@ -409,15 +409,15 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes specific source keys in D1-safe batches", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry1", "draft"), [occurrence("hero", "media-draft")]); - await repo.replaceSource(contentSource("entry2", "live"), [occurrence("hero", "media-other")]); + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry2", "columns"), [occurrence("hero", "media-other")]); expect( await repo.deleteSources([ - "content:posts:entry1:live", - "content:posts:entry1:draft", - "content:posts:entry1:live", + "content:posts:entry1:columns", + "content:posts:entry1:draft_overlay", + "content:posts:entry1:columns", ]), ).toBe(2); expect(await repo.findCurrentUsageByMediaId("media-live")).toEqual([]); @@ -474,7 +474,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("finds current usage by provider asset", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [ + await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("video", "mux-video-1", { referenceType: "file_field", provider: "mux", @@ -487,7 +487,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(await repo.findCurrentUsageByProviderAsset("mux", "mux-video-1")).toEqual([ { - source: expect.objectContaining({ sourceKey: "content:posts:entry1:live" }), + source: expect.objectContaining({ sourceKey: "content:posts:entry1:columns" }), occurrence: expect.objectContaining({ mediaId: null, provider: "mux", @@ -499,27 +499,30 @@ describeEachDialect("MediaUsageRepository", (dialect) => { ]); }); - it("keeps live and draft source keys separate for the same content", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [occurrence("hero", "media-shared")]); - await repo.replaceSource(contentSource("entry1", "draft"), [ + it("keeps columns and draft overlay source keys separate for the same content", async () => { + await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-shared")]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ occurrence("draftHero", "media-shared", { fieldPath: "draftHero" }), ]); const usage = await repo.findCurrentUsageByMediaId("media-shared"); expect(usage.map((row) => row.source.sourceKey)).toEqual([ - "content:posts:entry1:draft", - "content:posts:entry1:live", + "content:posts:entry1:columns", + "content:posts:entry1:draft_overlay", + ]); + expect(usage.map((row) => row.source.sourceVariant)).toEqual([ + "columns", + "draft_overlay", ]); - expect(usage.map((row) => row.source.sourceVariant)).toEqual(["draft", "live"]); }); it("paginates current media usage by occurrence id", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [ + await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-shared"), occurrence("body", "media-shared"), ]); - await repo.replaceSource(contentSource("entry2", "live"), [occurrence("hero", "media-shared")]); + await repo.replaceSource(contentSource("entry2", "columns"), [occurrence("hero", "media-shared")]); const page1 = await repo.findCurrentUsagePageByMediaId("media-shared", { limit: 2 }); expect(page1.items).toHaveLength(2); @@ -537,7 +540,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("paginates current provider-asset usage by occurrence id", async () => { - await repo.replaceSource(contentSource("entry1", "live"), [ + await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("video", "mux-video-1", { provider: "mux", providerAssetId: "mux-video-1", @@ -629,7 +632,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }), ); - const source = await repo.replaceSource(contentSource("entry1", "draft"), occurrences); + const source = await repo.replaceSource(contentSource("entry1", "draft_overlay"), occurrences); const rows = await ctx.db .selectFrom("_emdash_media_usage") .select(["generation", "media_id"]) @@ -644,7 +647,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { function contentSource( contentId: string, - variant: "live" | "draft", + variant: "columns" | "draft_overlay", overrides: Partial[0]> = {}, ): Parameters[0] { const collectionSlug = overrides.collectionSlug ?? "posts"; @@ -658,7 +661,7 @@ function contentSource( translationGroup: `tg-${contentId}`, contentSlug: "hello-world", contentTitle: "Hello World", - contentStatus: variant === "live" ? "published" : "draft", + contentStatus: variant === "columns" ? "published" : "draft", contentScheduledAt: null, contentDeletedAt: null, revisionId: `rev-${contentId}-${variant}`, diff --git a/packages/core/tests/integration/database/media-usage-status-migration.test.ts b/packages/core/tests/integration/database/media-usage-status-migration.test.ts index 0f626c93b8..a37bcb01f7 100644 --- a/packages/core/tests/integration/database/media-usage-status-migration.test.ts +++ b/packages/core/tests/integration/database/media-usage-status-migration.test.ts @@ -69,11 +69,11 @@ describeEachDialect("media usage index status migration", (dialect) => { await ctx.db .insertInto("_emdash_media_usage_sources") .values({ - source_key: "content:posts:entry1:live", + source_key: "content:posts:entry1:columns", source_type: "content", collection_slug: "posts", content_id: "entry1", - source_variant: "live", + source_variant: "columns", locale: "en", translation_group: "tg1", content_slug: "hello-world", @@ -96,7 +96,7 @@ describeEachDialect("media usage index status migration", (dialect) => { "last_attempted_at", "last_error_code", ]) - .where("source_key", "=", "content:posts:entry1:live") + .where("source_key", "=", "content:posts:entry1:columns") .executeTakeFirstOrThrow(); expect(source).toEqual({ @@ -174,7 +174,7 @@ describeEachDialect("media usage index status migration", (dialect) => { it("adds source completeness default for existing PR 1 source rows", async () => { const migration = await import("../../../src/database/migrations/047_media_usage_index_status.js"); - const sourceKey = "content:posts:pre047:live"; + const sourceKey = "content:posts:pre047:columns"; await migration.down(ctx.db); @@ -185,7 +185,7 @@ describeEachDialect("media usage index status migration", (dialect) => { source_type: "content", collection_slug: "posts", content_id: "pre047", - source_variant: "live", + source_variant: "columns", locale: "en", translation_group: "tg-pre047", content_slug: "pre047", @@ -234,7 +234,7 @@ describeEachDialect("media usage index status migration", (dialect) => { it("up() can run again after registered migrations complete", async () => { const migration = await import("../../../src/database/migrations/047_media_usage_index_status.js"); - const sourceKey = "content:posts:entry-preserve:live"; + const sourceKey = "content:posts:entry-preserve:columns"; const sourceMetadata = { source_completeness: "complete", source_updated_at: "2026-01-01T00:00:00.000Z", @@ -251,7 +251,7 @@ describeEachDialect("media usage index status migration", (dialect) => { source_type: "content", collection_slug: "posts", content_id: "entry-preserve", - source_variant: "live", + source_variant: "columns", locale: "en", translation_group: "tg-preserve", content_slug: "preserve", From 81158b2046766a9b13baa0321da51d5e63de7e29 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 14:11:59 +0100 Subject: [PATCH 10/29] feat(media): Add Source-Key Contract Tests --- packages/core/src/media/usage/source-key.ts | 28 +++++++++++++++ .../database/media-usage-repository.test.ts | 12 +++++-- .../tests/unit/media/usage-source-key.test.ts | 35 +++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/media/usage/source-key.ts create mode 100644 packages/core/tests/unit/media/usage-source-key.test.ts diff --git a/packages/core/src/media/usage/source-key.ts b/packages/core/src/media/usage/source-key.ts new file mode 100644 index 0000000000..7ae749d11f --- /dev/null +++ b/packages/core/src/media/usage/source-key.ts @@ -0,0 +1,28 @@ +export const MEDIA_USAGE_CONTENT_SOURCE_VARIANTS = ["columns", "draft_overlay"] as const; + +export type MediaUsageContentSourceVariant = + (typeof MEDIA_USAGE_CONTENT_SOURCE_VARIANTS)[number]; + +export interface ContentMediaUsageSourceKeyInput { + collectionSlug: string; + contentId: string; + sourceVariant: MediaUsageContentSourceVariant; +} + +export function isMediaUsageContentSourceVariant( + value: unknown, +): value is MediaUsageContentSourceVariant { + return ( + typeof value === "string" && + (MEDIA_USAGE_CONTENT_SOURCE_VARIANTS as readonly string[]).includes(value) + ); +} + +export function buildContentMediaUsageSourceKey( + input: ContentMediaUsageSourceKeyInput, +): string { + if (!isMediaUsageContentSourceVariant(input.sourceVariant)) { + throw new Error(`Invalid media usage content source variant: ${input.sourceVariant}`); + } + return `content:${input.collectionSlug}:${input.contentId}:${input.sourceVariant}`; +} diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index 260a536a30..b2fc6ab2da 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { + buildContentMediaUsageSourceKey, + type MediaUsageContentSourceVariant, +} from "../../../src/media/usage/source-key.js"; import { SQL_BATCH_SIZE } from "../../../src/utils/chunks.js"; import { describeEachDialect, @@ -647,12 +651,16 @@ describeEachDialect("MediaUsageRepository", (dialect) => { function contentSource( contentId: string, - variant: "columns" | "draft_overlay", + variant: MediaUsageContentSourceVariant, overrides: Partial[0]> = {}, ): Parameters[0] { const collectionSlug = overrides.collectionSlug ?? "posts"; return { - sourceKey: `content:${collectionSlug}:${contentId}:${variant}`, + sourceKey: buildContentMediaUsageSourceKey({ + collectionSlug, + contentId, + sourceVariant: variant, + }), sourceType: "content", collectionSlug, contentId, diff --git a/packages/core/tests/unit/media/usage-source-key.test.ts b/packages/core/tests/unit/media/usage-source-key.test.ts new file mode 100644 index 0000000000..87d0002a12 --- /dev/null +++ b/packages/core/tests/unit/media/usage-source-key.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; + +import { + buildContentMediaUsageSourceKey, + isMediaUsageContentSourceVariant, + MEDIA_USAGE_CONTENT_SOURCE_VARIANTS, +} from "../../../src/media/usage/source-key.js"; + +describe("media usage content source keys", () => { + it("uses the content namespace and storage source variant", () => { + expect( + buildContentMediaUsageSourceKey({ + collectionSlug: "posts", + contentId: "entry1", + sourceVariant: "columns", + }), + ).toBe("content:posts:entry1:columns"); + + expect( + buildContentMediaUsageSourceKey({ + collectionSlug: "posts", + contentId: "entry1", + sourceVariant: "draft_overlay", + }), + ).toBe("content:posts:entry1:draft_overlay"); + }); + + it("keeps publish states out of source variant identity", () => { + expect(MEDIA_USAGE_CONTENT_SOURCE_VARIANTS).toEqual(["columns", "draft_overlay"]); + expect(isMediaUsageContentSourceVariant("columns")).toBe(true); + expect(isMediaUsageContentSourceVariant("draft_overlay")).toBe(true); + expect(isMediaUsageContentSourceVariant("live")).toBe(false); + expect(isMediaUsageContentSourceVariant("draft")).toBe(false); + }); +}); From c66ca0f2bb977f2aafd4182ed8731cb24403edf1 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 14:16:10 +0100 Subject: [PATCH 11/29] feat(media): refine extractor to focus on image subfields and update tests --- packages/core/src/media/usage/extractor.ts | 6 +- .../tests/unit/media/usage-extractor.test.ts | 63 ++++++++----------- 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/packages/core/src/media/usage/extractor.ts b/packages/core/src/media/usage/extractor.ts index f2cd33a73c..7b9c487b5b 100644 --- a/packages/core/src/media/usage/extractor.ts +++ b/packages/core/src/media/usage/extractor.ts @@ -84,14 +84,14 @@ function extractRepeaterOccurrences( if (!isRecord(item)) continue; for (const subField of subFields) { - if (subField.type !== "image" && subField.type !== "file") continue; + if (subField.type !== "image") continue; addOccurrence(occurrences, seen, { fieldSlug, fieldPath: `${fieldSlug}[${itemIndex}].${subField.slug}`, - referenceType: subField.type === "image" ? "image_field" : "file_field", + referenceType: "image_field", value: item[subField.slug], - fallbackKind: subField.type === "image" ? "image" : null, + fallbackKind: "image", }); } } diff --git a/packages/core/tests/unit/media/usage-extractor.test.ts b/packages/core/tests/unit/media/usage-extractor.test.ts index e6b5720ee6..384c77b251 100644 --- a/packages/core/tests/unit/media/usage-extractor.test.ts +++ b/packages/core/tests/unit/media/usage-extractor.test.ts @@ -149,29 +149,18 @@ describe("extractMediaUsageOccurrences", () => { ]); }); - it("extracts repeater image and defensive file subfields with stable paths", () => { + it("extracts repeater image subfields with stable paths", () => { const occurrences = extractMediaUsageOccurrences({ fields: [ field("sections", "repeater", { - subFields: [ - { slug: "image", type: "image", label: "Image" }, - { slug: "download", type: "file", label: "Download" }, - ], + subFields: [{ slug: "image", type: "image", label: "Image" }], }), ], data: { sections: [ - { - image: { id: "image-1", mimeType: "image/webp" }, - download: { id: "file-1", mimeType: "application/zip" }, - }, + { image: { id: "image-1", mimeType: "image/webp" } }, { image: "image-2", - download: { - id: "video-1", - provider: "mux", - mimeType: "video/mp4", - }, }, ], }, @@ -189,17 +178,6 @@ describe("extractMediaUsageOccurrences", () => { mediaKind: "image", mimeType: "image/webp", }, - { - fieldSlug: "sections", - fieldPath: "sections[0].download", - occurrenceIndex: 0, - referenceType: "file_field", - mediaId: "file-1", - provider: "local", - providerAssetId: "file-1", - mediaKind: "archive", - mimeType: "application/zip", - }, { fieldSlug: "sections", fieldPath: "sections[1].image", @@ -211,20 +189,33 @@ describe("extractMediaUsageOccurrences", () => { mediaKind: "image", mimeType: null, }, - { - fieldSlug: "sections", - fieldPath: "sections[1].download", - occurrenceIndex: 0, - referenceType: "file_field", - mediaId: null, - provider: "mux", - providerAssetId: "video-1", - mediaKind: "video", - mimeType: "video/mp4", - }, ]); }); + it("ignores unsupported repeater file subfields", () => { + const occurrences = extractMediaUsageOccurrences({ + fields: [ + field("sections", "repeater", { + subFields: [{ slug: "download", type: "file", label: "Download" }], + }), + ], + data: { + sections: [ + { download: { id: "file-1", mimeType: "application/zip" } }, + { + download: { + id: "video-1", + provider: "mux", + mimeType: "video/mp4", + }, + }, + ], + }, + }); + + expect(occurrences).toEqual([]); + }); + it("extracts Portable Text image block asset refs", () => { const occurrences = extractMediaUsageOccurrences({ fields: [field("body", "portableText")], From f7590e148fecf8ad0927191566f5203de17598a0 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 14:36:25 +0100 Subject: [PATCH 12/29] feat(media): tightened the source-variant write/input type seam. --- packages/core/src/database/repositories/media-usage.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 6b2faf6923..1849aed6b3 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -1,6 +1,7 @@ import { sql, type Kysely, type Selectable, type Transaction, type Updateable } from "kysely"; import { ulid } from "ulidx"; +import type { MediaUsageContentSourceVariant } from "../../media/usage/source-key.js"; import type { MediaKind, MediaUsageReferenceType } from "../../media/usage/types.js"; import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js"; import { withTransaction } from "../transaction.js"; @@ -25,7 +26,7 @@ export interface MediaUsageSourceInput { sourceType: string; collectionSlug?: string | null; contentId?: string | null; - sourceVariant: string; + sourceVariant: MediaUsageContentSourceVariant; locale?: string | null; translationGroup?: string | null; contentSlug?: string | null; From 918ac41c4f6bf192eeee6c0561ca0e9ce3f375cb Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 15:14:28 +0100 Subject: [PATCH 13/29] feat(media): implement content media usage field discovery and associated tests --- .../core/src/media/usage/content-fields.ts | 119 ++++++++++++++ .../media-usage-content-fields.test.ts | 155 ++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 packages/core/src/media/usage/content-fields.ts create mode 100644 packages/core/tests/integration/database/media-usage-content-fields.test.ts diff --git a/packages/core/src/media/usage/content-fields.ts b/packages/core/src/media/usage/content-fields.ts new file mode 100644 index 0000000000..e139918423 --- /dev/null +++ b/packages/core/src/media/usage/content-fields.ts @@ -0,0 +1,119 @@ +import type { Kysely } from "kysely"; + +import type { Database } from "../../database/types.js"; +import { validateIdentifier } from "../../database/validate.js"; +import type { MediaUsageExtractionField, MediaUsageExtractionSubField } from "./types.js"; + +export type ContentMediaUsageField = MediaUsageExtractionField; + +export interface ContentMediaUsageFieldDiscovery { + extractionFields: ContentMediaUsageField[]; + displayFieldSlugs: string[]; +} + +export class MediaUsageFieldDiscoveryError extends Error { + constructor( + message: string, + public code: "INVALID_REPEATER_VALIDATION", + ) { + super(message); + this.name = "MediaUsageFieldDiscoveryError"; + } +} + +interface FieldDiscoveryRow { + slug: string; + type: string; + validation: string | null; +} + +const DISPLAY_FIELD_SLUGS = ["title", "name"] as const; +const SUPPORTED_TOP_LEVEL_TYPES = ["file", "image", "portableText"] as const; + +type SupportedTopLevelType = (typeof SUPPORTED_TOP_LEVEL_TYPES)[number]; + +export async function loadContentMediaUsageFields( + db: Kysely, + collectionSlug: string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + + const rows = await db + .selectFrom("_emdash_fields") + .innerJoin("_emdash_collections", "_emdash_collections.id", "_emdash_fields.collection_id") + .select(["_emdash_fields.slug", "_emdash_fields.type", "_emdash_fields.validation"]) + .where("_emdash_collections.slug", "=", collectionSlug) + .execute(); + + const extractionFields: ContentMediaUsageField[] = []; + const rowBySlug = new Map(); + + for (const row of rows) { + rowBySlug.set(row.slug, row); + if (isSupportedTopLevelType(row.type)) { + validateIdentifier(row.slug, "media usage field slug"); + extractionFields.push({ slug: row.slug, type: row.type }); + continue; + } + + if (row.type === "repeater") { + validateIdentifier(row.slug, "media usage field slug"); + const subFields = normalizeRepeaterImageSubFields(row.validation); + if (subFields.length > 0) { + extractionFields.push({ + slug: row.slug, + type: "repeater", + validation: { subFields }, + }); + } + } + } + + extractionFields.sort((a, b) => a.slug.localeCompare(b.slug)); + + return { + extractionFields, + displayFieldSlugs: DISPLAY_FIELD_SLUGS.filter((slug) => { + if (!rowBySlug.has(slug)) return false; + validateIdentifier(slug, "media usage display field slug"); + return true; + }), + }; +} + +function normalizeRepeaterImageSubFields( + rawValidation: string | null, +): MediaUsageExtractionSubField[] { + const validation = parseValidation(rawValidation); + if (!isRecord(validation) || !Array.isArray(validation.subFields)) return []; + + const subFields: MediaUsageExtractionSubField[] = []; + for (const subField of validation.subFields) { + if (!isRecord(subField) || subField.type !== "image") continue; + if (typeof subField.slug !== "string") continue; + validateIdentifier(subField.slug, "media usage repeater sub-field slug"); + subFields.push({ slug: subField.slug, type: "image" }); + } + + return subFields.toSorted((a, b) => a.slug.localeCompare(b.slug)); +} + +function parseValidation(rawValidation: string | null): unknown { + if (!rawValidation) return null; + try { + return JSON.parse(rawValidation); + } catch { + throw new MediaUsageFieldDiscoveryError( + "Repeater field validation must be valid JSON before media usage can be discovered", + "INVALID_REPEATER_VALIDATION", + ); + } +} + +function isSupportedTopLevelType(value: string): value is SupportedTopLevelType { + return (SUPPORTED_TOP_LEVEL_TYPES as readonly string[]).includes(value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/core/tests/integration/database/media-usage-content-fields.test.ts b/packages/core/tests/integration/database/media-usage-content-fields.test.ts new file mode 100644 index 0000000000..eee9d89044 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-content-fields.test.ts @@ -0,0 +1,155 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { IdentifierError } from "../../../src/database/validate.js"; +import { + loadContentMediaUsageFields, + MediaUsageFieldDiscoveryError, +} from "../../../src/media/usage/content-fields.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("content media usage field discovery", (dialect) => { + let ctx: DialectTestContext; + let registry: SchemaRegistry; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "posts", label: "Posts" }); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("discovers V1 extraction fields and display fields separately", async () => { + await registry.createField("posts", { slug: "name", label: "Name", type: "string" }); + await registry.createField("posts", { slug: "body", label: "Body", type: "portableText" }); + await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); + await registry.createField("posts", { slug: "raw_data", label: "Raw Data", type: "json" }); + await registry.createField("posts", { slug: "attachment", label: "Attachment", type: "file" }); + await registry.createField("posts", { slug: "hero", label: "Hero", type: "image" }); + await registry.createField("posts", { + slug: "sections", + label: "Sections", + type: "repeater", + validation: { + subFields: [ + { slug: "caption", type: "text", label: "Caption" }, + { slug: "image", type: "image", label: "Image" }, + ], + }, + }); + + const discovery = await loadContentMediaUsageFields(ctx.db, "posts"); + + expect(discovery.displayFieldSlugs).toEqual(["title", "name"]); + expect(discovery.extractionFields).toEqual([ + { slug: "attachment", type: "file" }, + { slug: "body", type: "portableText" }, + { slug: "hero", type: "image" }, + { + slug: "sections", + type: "repeater", + validation: { subFields: [{ slug: "image", type: "image" }] }, + }, + ]); + }); + + it("filters unsupported repeater subfields and excludes repeaters without images", async () => { + await registry.createField("posts", { + slug: "sections", + label: "Sections", + type: "repeater", + validation: { subFields: [{ slug: "image", type: "image", label: "Image" }] }, + }); + await registry.createField("posts", { + slug: "downloads", + label: "Downloads", + type: "repeater", + validation: { subFields: [{ slug: "placeholder", type: "image", label: "Placeholder" }] }, + }); + + await ctx.db + .updateTable("_emdash_fields") + .set({ + validation: JSON.stringify({ + subFields: [ + { slug: "download", type: "file", label: "Download" }, + { slug: "image", type: "image", label: "Image" }, + { slug: "caption", type: "text", label: "Caption" }, + ], + }), + }) + .where("slug", "=", "sections") + .execute(); + await ctx.db + .updateTable("_emdash_fields") + .set({ + validation: JSON.stringify({ + subFields: [{ slug: "download", type: "file", label: "Download" }], + }), + }) + .where("slug", "=", "downloads") + .execute(); + + const discovery = await loadContentMediaUsageFields(ctx.db, "posts"); + + expect(discovery.extractionFields).toEqual([ + { + slug: "sections", + type: "repeater", + validation: { subFields: [{ slug: "image", type: "image" }] }, + }, + ]); + }); + + it("fails closed on malformed repeater validation", async () => { + await registry.createField("posts", { + slug: "sections", + label: "Sections", + type: "repeater", + validation: { subFields: [{ slug: "image", type: "image", label: "Image" }] }, + }); + await ctx.db + .updateTable("_emdash_fields") + .set({ validation: "{" }) + .where("slug", "=", "sections") + .execute(); + + await expect(loadContentMediaUsageFields(ctx.db, "posts")).rejects.toThrow( + MediaUsageFieldDiscoveryError, + ); + }); + + it("rejects supported fields with invalid slugs before they can become column refs", async () => { + const collection = await registry.getCollection("posts"); + expect(collection).not.toBeNull(); + + await ctx.db + .insertInto("_emdash_fields") + .values({ + id: "invalid-media-field", + collection_id: collection!.id, + slug: "bad-slug", + label: "Bad Slug", + type: "image", + column_type: "TEXT", + required: 0, + unique: 0, + default_value: null, + validation: null, + widget: null, + options: null, + sort_order: 0, + }) + .execute(); + + await expect(loadContentMediaUsageFields(ctx.db, "posts")).rejects.toThrow(IdentifierError); + }); +}); From 5a662f509b94b2ac2d6dea99befdccce17fe4e50 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 15:24:22 +0100 Subject: [PATCH 14/29] feat(media): add content media usage snapshots loader and associated tests --- .../core/src/media/usage/content-snapshots.ts | 186 ++++++++++++++++++ .../media-usage-content-snapshots.test.ts | 114 +++++++++++ 2 files changed, 300 insertions(+) create mode 100644 packages/core/src/media/usage/content-snapshots.ts create mode 100644 packages/core/tests/integration/database/media-usage-content-snapshots.test.ts diff --git a/packages/core/src/media/usage/content-snapshots.ts b/packages/core/src/media/usage/content-snapshots.ts new file mode 100644 index 0000000000..31a43abe1a --- /dev/null +++ b/packages/core/src/media/usage/content-snapshots.ts @@ -0,0 +1,186 @@ +import { sql, type Kysely } from "kysely"; + +import type { MediaUsageOccurrenceInput, MediaUsageSourceInput } from "../../database/repositories/media-usage.js"; +import type { Database } from "../../database/types.js"; +import { validateIdentifier } from "../../database/validate.js"; +import { extractMediaUsageOccurrences } from "./extractor.js"; +import { + loadContentMediaUsageFields, + type ContentMediaUsageField, +} from "./content-fields.js"; +import { buildContentMediaUsageSourceKey } from "./source-key.js"; + +const CONTENT_SOURCE_SCHEMA_VERSION = 1; + +const CONTENT_SYSTEM_COLUMNS = [ + "id", + "slug", + "status", + "created_at", + "updated_at", + "published_at", + "scheduled_at", + "deleted_at", + "version", + "live_revision_id", + "draft_revision_id", + "locale", + "translation_group", +] as const; + +export type LoadContentMediaUsageSnapshotsResult = + | { success: true; snapshots: ContentMediaUsageSnapshot[] } + | { + success: false; + error: "CONTENT_NOT_FOUND" | "DRAFT_REVISION_NOT_FOUND" | "DRAFT_REVISION_MISMATCH"; + source?: MediaUsageSourceInput; + }; + +export interface ContentMediaUsageSnapshot { + source: MediaUsageSourceInput; + occurrences: MediaUsageOccurrenceInput[]; + fields: readonly ContentMediaUsageField[]; +} + +export async function loadContentMediaUsageSnapshots( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + const discovery = await loadContentMediaUsageFields(db, collectionSlug); + const row = await loadContentRow(db, collectionSlug, contentId, [ + ...discovery.extractionFields.map((field) => field.slug), + ...discovery.displayFieldSlugs, + ]); + + if (!row) return { success: false, error: "CONTENT_NOT_FOUND" }; + + const columnsData = projectData(row, discovery.extractionFields.map((field) => field.slug)); + const displayData = projectData(row, discovery.displayFieldSlugs); + const occurrences = extractMediaUsageOccurrences({ + fields: discovery.extractionFields, + data: columnsData, + }); + + return { + success: true, + snapshots: [ + { + source: buildColumnsSource(collectionSlug, row, displayData), + occurrences, + fields: discovery.extractionFields, + }, + ], + }; +} + +async function loadContentRow( + db: Kysely, + collectionSlug: string, + contentId: string, + fieldSlugs: readonly string[], +): Promise | null> { + const tableName = getContentTableName(collectionSlug); + const columns = uniqueColumns([...CONTENT_SYSTEM_COLUMNS, ...fieldSlugs]); + const columnRefs = columns.map((column) => sql.ref(column)); + const result = await sql>` + SELECT ${sql.join(columnRefs, sql`, `)} + FROM ${sql.ref(tableName)} + WHERE id = ${contentId} + LIMIT 1 + `.execute(db); + + return result.rows[0] ?? null; +} + +function buildColumnsSource( + collectionSlug: string, + row: Record, + displayData: Record, +): MediaUsageSourceInput { + const contentId = readString(row.id) ?? ""; + const contentSlug = readNullableString(row.slug); + return { + sourceKey: buildContentMediaUsageSourceKey({ + collectionSlug, + contentId, + sourceVariant: "columns", + }), + sourceType: "content", + collectionSlug, + contentId, + sourceVariant: "columns", + locale: readNullableString(row.locale), + translationGroup: readNullableString(row.translation_group), + contentSlug, + contentTitle: deriveContentTitle(displayData, contentSlug, contentId), + contentStatus: readNullableString(row.status), + contentScheduledAt: readNullableString(row.scheduled_at), + contentDeletedAt: readNullableString(row.deleted_at), + revisionId: readNullableString(row.live_revision_id), + schemaVersion: CONTENT_SOURCE_SCHEMA_VERSION, + sourceUpdatedAt: readNullableString(row.updated_at), + sourceVersion: readNumber(row.version), + }; +} + +function projectData(row: Record, fieldSlugs: readonly string[]): Record { + const data: Record = {}; + for (const fieldSlug of fieldSlugs) { + data[fieldSlug] = deserializeValue(row[fieldSlug] ?? null); + } + return data; +} + +function uniqueColumns(columns: readonly string[]): string[] { + const unique = [...new Set(columns)]; + for (const column of unique) validateIdentifier(column, "content media usage column"); + return unique; +} + +function getContentTableName(collectionSlug: string): string { + validateIdentifier(collectionSlug, "collection slug"); + return `ec_${collectionSlug}`; +} + +function deserializeValue(value: unknown): unknown { + if (typeof value === "string" && (value.startsWith("{") || value.startsWith("["))) { + try { + return JSON.parse(value); + } catch { + return value; + } + } + return value; +} + +function deriveContentTitle( + displayData: Record, + contentSlug: string | null, + contentId: string, +): string | null { + for (const fieldSlug of ["title", "name"] as const) { + const value = displayData[fieldSlug]; + if (typeof value === "string" && value.trim()) return value; + } + return contentSlug ?? contentId; +} + +function readString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function readNullableString(value: unknown): string | null { + return value === null || value === undefined ? null : readString(value); +} + +function readNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "bigint") return Number(value); + if (typeof value === "string" && value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; +} diff --git a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts new file mode 100644 index 0000000000..abe0fe8c82 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts @@ -0,0 +1,114 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { ContentRepository } from "../../../src/database/repositories/content.js"; +import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; +import { loadContentMediaUsageSnapshots } from "../../../src/media/usage/content-snapshots.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("content media usage snapshots", (dialect) => { + let ctx: DialectTestContext; + let registry: SchemaRegistry; + let contentRepo: ContentRepository; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + registry = new SchemaRegistry(ctx.db); + contentRepo = new ContentRepository(ctx.db); + + await registry.createCollection({ slug: "posts", label: "Posts" }); + await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); + await registry.createField("posts", { slug: "hero", label: "Hero", type: "image" }); + await registry.createField("posts", { slug: "attachment", label: "Attachment", type: "file" }); + await registry.createField("posts", { + slug: "sections", + label: "Sections", + type: "repeater", + validation: { subFields: [{ slug: "image", type: "image", label: "Image" }] }, + }); + await registry.createField("posts", { slug: "body", label: "Body", type: "portableText" }); + await registry.createField("posts", { slug: "raw_data", label: "Raw Data", type: "json" }); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("builds a columns snapshot from stored content fields", async () => { + const item = await contentRepo.create({ + type: "posts", + slug: "hello-world", + status: "published", + locale: "en", + data: { + title: "Hello World", + hero: { id: "media-hero", provider: "local", mimeType: "image/webp" }, + attachment: { id: "media-file", provider: "local", mimeType: "application/pdf" }, + sections: [{ image: { id: "media-section", provider: "local" } }], + body: [{ _type: "image", asset: { _ref: "media-body" } }], + raw_data: { id: "media-ignored" }, + }, + }); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + expect(result.snapshots).toHaveLength(1); + const snapshot = result.snapshots[0]!; + + expect(snapshot.source).toEqual( + expect.objectContaining({ + sourceKey: buildContentMediaUsageSourceKey({ + collectionSlug: "posts", + contentId: item.id, + sourceVariant: "columns", + }), + sourceType: "content", + collectionSlug: "posts", + contentId: item.id, + sourceVariant: "columns", + locale: "en", + translationGroup: item.translationGroup, + contentSlug: "hello-world", + contentTitle: "Hello World", + contentStatus: "published", + contentScheduledAt: null, + contentDeletedAt: null, + revisionId: null, + sourceUpdatedAt: item.updatedAt, + sourceVersion: item.version, + }), + ); + expect(snapshot.fields).toEqual([ + { slug: "attachment", type: "file" }, + { slug: "body", type: "portableText" }, + { slug: "hero", type: "image" }, + { + slug: "sections", + type: "repeater", + validation: { subFields: [{ slug: "image", type: "image" }] }, + }, + ]); + expect(snapshot.occurrences).toEqual([ + expect.objectContaining({ fieldPath: "attachment", mediaId: "media-file" }), + expect.objectContaining({ fieldPath: "body[0].asset._ref", mediaId: "media-body" }), + expect.objectContaining({ fieldPath: "hero", mediaId: "media-hero" }), + expect.objectContaining({ fieldPath: "sections[0].image", mediaId: "media-section" }), + ]); + expect(snapshot.occurrences).not.toEqual( + expect.arrayContaining([expect.objectContaining({ mediaId: "media-ignored" })]), + ); + }); + + it("returns a typed not-found result for missing content", async () => { + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", "missing-content"); + + expect(result).toEqual({ success: false, error: "CONTENT_NOT_FOUND" }); + }); +}); From 35c90cf122e2f38df12d9149e31fcf1449253edf Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 16:07:09 +0100 Subject: [PATCH 15/29] feat(media): enhance content media usage snapshots with draft revision handling and fingerprinting --- .../core/src/media/usage/content-snapshots.ts | 284 ++++++++++- .../media-usage-content-snapshots.test.ts | 465 +++++++++++++++++- 2 files changed, 717 insertions(+), 32 deletions(-) diff --git a/packages/core/src/media/usage/content-snapshots.ts b/packages/core/src/media/usage/content-snapshots.ts index 31a43abe1a..cc1fc87772 100644 --- a/packages/core/src/media/usage/content-snapshots.ts +++ b/packages/core/src/media/usage/content-snapshots.ts @@ -1,16 +1,20 @@ import { sql, type Kysely } from "kysely"; -import type { MediaUsageOccurrenceInput, MediaUsageSourceInput } from "../../database/repositories/media-usage.js"; +import type { + MediaUsageOccurrenceInput, + MediaUsageSourceInput, +} from "../../database/repositories/media-usage.js"; import type { Database } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; +import { hashString } from "../../utils/hash.js"; +import { loadContentMediaUsageFields, type ContentMediaUsageField } from "./content-fields.js"; import { extractMediaUsageOccurrences } from "./extractor.js"; import { - loadContentMediaUsageFields, - type ContentMediaUsageField, -} from "./content-fields.js"; -import { buildContentMediaUsageSourceKey } from "./source-key.js"; + buildContentMediaUsageSourceKey, + type MediaUsageContentSourceVariant, +} from "./source-key.js"; -const CONTENT_SOURCE_SCHEMA_VERSION = 1; +export const CONTENT_SOURCE_SCHEMA_VERSION = 1; const CONTENT_SYSTEM_COLUMNS = [ "id", @@ -32,9 +36,13 @@ export type LoadContentMediaUsageSnapshotsResult = | { success: true; snapshots: ContentMediaUsageSnapshot[] } | { success: false; - error: "CONTENT_NOT_FOUND" | "DRAFT_REVISION_NOT_FOUND" | "DRAFT_REVISION_MISMATCH"; + error: + | "CONTENT_NOT_FOUND" + | "DRAFT_REVISION_NOT_FOUND" + | "DRAFT_REVISION_MISMATCH" + | "DRAFT_REVISION_INVALID"; source?: MediaUsageSourceInput; - }; + }; export interface ContentMediaUsageSnapshot { source: MediaUsageSourceInput; @@ -56,25 +64,117 @@ export async function loadContentMediaUsageSnapshots( if (!row) return { success: false, error: "CONTENT_NOT_FOUND" }; - const columnsData = projectData(row, discovery.extractionFields.map((field) => field.slug)); + const columnsData = projectData( + row, + discovery.extractionFields.map((field) => field.slug), + ); const displayData = projectData(row, discovery.displayFieldSlugs); const occurrences = extractMediaUsageOccurrences({ fields: discovery.extractionFields, data: columnsData, }); + const columnsRevisionId = readNullableString(row.live_revision_id); + const columnsFingerprint = await buildSourceFingerprint({ + collectionSlug, + sourceVariant: "columns", + revisionId: columnsRevisionId, + fields: discovery.extractionFields, + data: columnsData, + }); + const snapshots: ContentMediaUsageSnapshot[] = [ + { + source: buildContentSource({ + collectionSlug, + row, + displayData, + sourceVariant: "columns", + revisionId: columnsRevisionId, + sourceFingerprint: columnsFingerprint, + }), + occurrences, + fields: discovery.extractionFields, + }, + ]; + + const draftRevisionId = readNullableString(row.draft_revision_id); + if (draftRevisionId) { + const attemptedDraftSource = buildContentSource({ + collectionSlug, + row, + displayData, + sourceVariant: "draft_overlay", + revisionId: draftRevisionId, + }); + const revisionResult = await loadRevisionRow(db, draftRevisionId); + if (!revisionResult) { + return { + success: false, + error: "DRAFT_REVISION_NOT_FOUND", + source: attemptedDraftSource, + }; + } + if (!revisionResult.success) { + return { + success: false, + error: "DRAFT_REVISION_INVALID", + source: attemptedDraftSource, + }; + } + const revision = revisionResult.revision; + if (revision.collection !== collectionSlug || revision.entryId !== row.id) { + return { + success: false, + error: "DRAFT_REVISION_MISMATCH", + source: attemptedDraftSource, + }; + } + + const revisionData = stripRevisionMetadata(revision.data); + const draftOverlayData = { ...columnsData, ...revisionData }; + const draftDisplayData = { + ...displayData, + ...projectPresentData(revisionData, discovery.displayFieldSlugs), + }; + const draftContentSlug = + readNullableString(revision.data._slug) ?? readNullableString(row.slug); + const draftFingerprint = await buildSourceFingerprint({ + collectionSlug, + sourceVariant: "draft_overlay", + revisionId: draftRevisionId, + fields: discovery.extractionFields, + data: draftOverlayData, + }); + snapshots.push({ + source: buildContentSource({ + collectionSlug, + row, + displayData: draftDisplayData, + sourceVariant: "draft_overlay", + revisionId: draftRevisionId, + contentSlug: draftContentSlug, + sourceFingerprint: draftFingerprint, + }), + occurrences: extractMediaUsageOccurrences({ + fields: discovery.extractionFields, + data: draftOverlayData, + }), + fields: discovery.extractionFields, + }); + } return { success: true, - snapshots: [ - { - source: buildColumnsSource(collectionSlug, row, displayData), - occurrences, - fields: discovery.extractionFields, - }, - ], + snapshots, }; } +interface RevisionSnapshotRow { + id: string; + collection: string; + entryId: string; + data: Record; +} + async function loadContentRow( db: Kysely, collectionSlug: string, @@ -94,23 +194,51 @@ async function loadContentRow( return result.rows[0] ?? null; } -function buildColumnsSource( - collectionSlug: string, - row: Record, - displayData: Record, -): MediaUsageSourceInput { +async function loadRevisionRow( + db: Kysely, + revisionId: string, +): Promise<{ success: true; revision: RevisionSnapshotRow } | { success: false } | null> { + const row = await db + .selectFrom("revisions") + .select(["id", "collection", "entry_id", "data"]) + .where("id", "=", revisionId) + .executeTakeFirst(); + if (!row) return null; + const data = parseRevisionData(row.data); + if (!data) return { success: false }; + return { + success: true, + revision: { + id: row.id, + collection: row.collection, + entryId: row.entry_id, + data, + }, + }; +} + +function buildContentSource(input: { + collectionSlug: string; + row: Record; + displayData: Record; + sourceVariant: MediaUsageContentSourceVariant; + revisionId: string | null; + contentSlug?: string | null; + sourceFingerprint?: string | null; +}): MediaUsageSourceInput { + const { collectionSlug, row, displayData, sourceVariant, revisionId } = input; const contentId = readString(row.id) ?? ""; - const contentSlug = readNullableString(row.slug); + const contentSlug = input.contentSlug ?? readNullableString(row.slug); return { sourceKey: buildContentMediaUsageSourceKey({ collectionSlug, contentId, - sourceVariant: "columns", + sourceVariant, }), sourceType: "content", collectionSlug, contentId, - sourceVariant: "columns", + sourceVariant, locale: readNullableString(row.locale), translationGroup: readNullableString(row.translation_group), contentSlug, @@ -118,14 +246,83 @@ function buildColumnsSource( contentStatus: readNullableString(row.status), contentScheduledAt: readNullableString(row.scheduled_at), contentDeletedAt: readNullableString(row.deleted_at), - revisionId: readNullableString(row.live_revision_id), + revisionId, schemaVersion: CONTENT_SOURCE_SCHEMA_VERSION, sourceUpdatedAt: readNullableString(row.updated_at), sourceVersion: readNumber(row.version), + sourceFingerprint: input.sourceFingerprint ?? null, }; } -function projectData(row: Record, fieldSlugs: readonly string[]): Record { +async function buildSourceFingerprint(input: { + collectionSlug: string; + sourceVariant: MediaUsageContentSourceVariant; + revisionId: string | null; + fields: readonly ContentMediaUsageField[]; + data: Record; +}): Promise { + return hashString( + canonicalJson({ + schemaVersion: CONTENT_SOURCE_SCHEMA_VERSION, + collectionSlug: input.collectionSlug, + sourceVariant: input.sourceVariant, + fields: normalizeFingerprintFields(input.fields), + values: projectFingerprintData(input.data, input.fields), + revisionId: input.sourceVariant === "draft_overlay" ? input.revisionId : null, + }), + ); +} + +function normalizeFingerprintFields( + fields: readonly ContentMediaUsageField[], +): Record[] { + return fields + .map((field) => { + if (field.type !== "repeater") return { slug: field.slug, type: field.type }; + return { + slug: field.slug, + type: field.type, + subFields: (field.validation?.subFields ?? []) + .map((subField) => ({ slug: subField.slug, type: subField.type })) + .toSorted((a, b) => a.slug.localeCompare(b.slug)), + }; + }) + .toSorted((a, b) => String(a.slug).localeCompare(String(b.slug))); +} + +function projectFingerprintData( + data: Record, + fields: readonly ContentMediaUsageField[], +): Record { + const projected: Record = {}; + for (const field of fields) { + projected[field.slug] = Object.hasOwn(data, field.slug) ? data[field.slug] : null; + } + return projected; +} + +function canonicalJson(value: unknown): string { + return JSON.stringify(canonicalize(value)); +} + +function canonicalize(value: unknown): unknown { + if (value === undefined) return null; + if (typeof value === "bigint") return value.toString(); + if (typeof value === "number") return Number.isFinite(value) ? value : null; + if (Array.isArray(value)) return value.map((item) => canonicalize(item)); + if (!isRecord(value)) return value; + + const canonical: Record = {}; + for (const key of Object.keys(value).toSorted()) { + canonical[key] = canonicalize(value[key]); + } + return canonical; +} + +function projectData( + row: Record, + fieldSlugs: readonly string[], +): Record { const data: Record = {}; for (const fieldSlug of fieldSlugs) { data[fieldSlug] = deserializeValue(row[fieldSlug] ?? null); @@ -133,6 +330,17 @@ function projectData(row: Record, fieldSlugs: readonly string[] return data; } +function projectPresentData( + row: Record, + fieldSlugs: readonly string[], +): Record { + const data: Record = {}; + for (const fieldSlug of fieldSlugs) { + if (Object.hasOwn(row, fieldSlug)) data[fieldSlug] = row[fieldSlug]; + } + return data; +} + function uniqueColumns(columns: readonly string[]): string[] { const unique = [...new Set(columns)]; for (const column of unique) validateIdentifier(column, "content media usage column"); @@ -155,6 +363,26 @@ function deserializeValue(value: unknown): unknown { return value; } +function parseRevisionData(value: unknown): Record | null { + if (typeof value === "string") { + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) ? parsed : null; + } catch { + return null; + } + } + return isRecord(value) ? value : null; +} + +function stripRevisionMetadata(data: Record): Record { + const stripped: Record = {}; + for (const [key, value] of Object.entries(data)) { + if (!key.startsWith("_")) stripped[key] = value; + } + return stripped; +} + function deriveContentTitle( displayData: Record, contentSlug: string | null, @@ -184,3 +412,7 @@ function readNumber(value: unknown): number | null { } return null; } + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts index abe0fe8c82..a41a9f6ecc 100644 --- a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts @@ -1,8 +1,10 @@ +import { sql } from "kysely"; +import { ulid } from "ulidx"; import { afterEach, beforeEach, expect, it } from "vitest"; -import { ContentRepository } from "../../../src/database/repositories/content.js"; -import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; +import { RevisionRepository } from "../../../src/database/repositories/revision.js"; import { loadContentMediaUsageSnapshots } from "../../../src/media/usage/content-snapshots.js"; +import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { describeEachDialect, @@ -14,12 +16,12 @@ import { describeEachDialect("content media usage snapshots", (dialect) => { let ctx: DialectTestContext; let registry: SchemaRegistry; - let contentRepo: ContentRepository; + let revisionRepo: RevisionRepository; beforeEach(async () => { ctx = await setupForDialect(dialect); registry = new SchemaRegistry(ctx.db); - contentRepo = new ContentRepository(ctx.db); + revisionRepo = new RevisionRepository(ctx.db); await registry.createCollection({ slug: "posts", label: "Posts" }); await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); @@ -40,8 +42,7 @@ describeEachDialect("content media usage snapshots", (dialect) => { }); it("builds a columns snapshot from stored content fields", async () => { - const item = await contentRepo.create({ - type: "posts", + const item = await insertPost(ctx, { slug: "hello-world", status: "published", locale: "en", @@ -111,4 +112,456 @@ describeEachDialect("content media usage snapshots", (dialect) => { expect(result).toEqual({ success: false, error: "CONTENT_NOT_FOUND" }); }); + + it("builds columns and draft overlay snapshots for a pending draft revision", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { + _slug: "draft-post", + title: "Draft Title", + hero: { id: "media-draft", provider: "local", mimeType: "image/webp" }, + }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + expect(result.snapshots.map((snapshot) => snapshot.source.sourceVariant)).toEqual([ + "columns", + "draft_overlay", + ]); + + const columns = result.snapshots.find( + (snapshot) => snapshot.source.sourceVariant === "columns", + )!; + const overlay = result.snapshots.find( + (snapshot) => snapshot.source.sourceVariant === "draft_overlay", + )!; + + expect(columns.source).toEqual( + expect.objectContaining({ + contentSlug: "live-post", + contentTitle: "Live Title", + revisionId: null, + sourceKey: buildContentMediaUsageSourceKey({ + collectionSlug: "posts", + contentId: item.id, + sourceVariant: "columns", + }), + }), + ); + expect(columns.occurrences).toEqual([ + expect.objectContaining({ fieldPath: "hero", mediaId: "media-live" }), + ]); + + expect(overlay.source).toEqual( + expect.objectContaining({ + contentSlug: "draft-post", + contentTitle: "Draft Title", + revisionId: draft.id, + sourceKey: buildContentMediaUsageSourceKey({ + collectionSlug: "posts", + contentId: item.id, + sourceVariant: "draft_overlay", + }), + }), + ); + expect(overlay.occurrences).toEqual([ + expect.objectContaining({ fieldPath: "hero", mediaId: "media-draft" }), + ]); + }); + + it("merges draft overlays over columns when the draft changes unrelated fields", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { title: "Draft Title" }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + const overlay = result.snapshots.find( + (snapshot) => snapshot.source.sourceVariant === "draft_overlay", + )!; + expect(overlay.source.contentTitle).toBe("Draft Title"); + expect(overlay.occurrences).toEqual([ + expect.objectContaining({ fieldPath: "hero", mediaId: "media-live" }), + ]); + }); + + it("preserves column display fields when a draft changes only media", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { hero: { id: "media-draft", provider: "local", mimeType: "image/webp" } }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + const overlay = result.snapshots.find( + (snapshot) => snapshot.source.sourceVariant === "draft_overlay", + )!; + expect(overlay.source.contentTitle).toBe("Live Title"); + expect(overlay.occurrences).toEqual([ + expect.objectContaining({ fieldPath: "hero", mediaId: "media-draft" }), + ]); + }); + + it("keeps JSON-looking revision display strings as strings", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { title: "Live Title" }, + }); + const draftTitle = '{"headline":"Draft"}'; + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { title: draftTitle }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + const overlay = result.snapshots.find( + (snapshot) => snapshot.source.sourceVariant === "draft_overlay", + )!; + expect(overlay.source.contentTitle).toBe(draftTitle); + }); + + it("honors draft nulls that clear media fields", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { hero: null }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + const overlay = result.snapshots.find( + (snapshot) => snapshot.source.sourceVariant === "draft_overlay", + )!; + expect(overlay.occurrences).toEqual([]); + }); + + it("fails when draft_revision_id belongs to another content row", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { title: "Live Title" }, + }); + const other = await insertPost(ctx, { + slug: "other-post", + status: "published", + data: { title: "Other Title" }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: other.id, + data: { title: "Other Draft" }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result).toEqual( + expect.objectContaining({ + success: false, + error: "DRAFT_REVISION_MISMATCH", + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ); + }); + + it("fails when draft revision data is invalid JSON", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { title: "Live Title" }, + }); + const revisionId = ulid(); + await sql` + INSERT INTO revisions (id, collection, entry_id, data, author_id) + VALUES (${revisionId}, ${"posts"}, ${item.id}, ${"{"}, ${null}) + `.execute(ctx.db); + await setDraftRevision(ctx, item.id, revisionId); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result).toEqual( + expect.objectContaining({ + success: false, + error: "DRAFT_REVISION_INVALID", + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ); + }); + + it("adds stable source schema versions and fingerprints to snapshots", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { + title: "Draft Title", + hero: { id: "media-draft", provider: "local", mimeType: "image/webp" }, + }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const firstResult = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + const secondResult = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(firstResult.success).toBe(true); + expect(secondResult.success).toBe(true); + if (!firstResult.success) throw new Error(firstResult.error); + if (!secondResult.success) throw new Error(secondResult.error); + const firstColumns = getSnapshot(firstResult, "columns"); + const firstOverlay = getSnapshot(firstResult, "draft_overlay"); + const secondColumns = getSnapshot(secondResult, "columns"); + const secondOverlay = getSnapshot(secondResult, "draft_overlay"); + + expect(firstColumns.source.schemaVersion).toBe(1); + expect(firstOverlay.source.schemaVersion).toBe(1); + expect(firstColumns.source.sourceFingerprint).toEqual(expect.stringMatching(/^[a-f0-9]{16}$/)); + expect(firstOverlay.source.sourceFingerprint).toEqual(expect.stringMatching(/^[a-f0-9]{16}$/)); + expect(firstOverlay.source.sourceFingerprint).not.toBe(firstColumns.source.sourceFingerprint); + expect(secondColumns.source.sourceFingerprint).toBe(firstColumns.source.sourceFingerprint); + expect(secondOverlay.source.sourceFingerprint).toBe(firstOverlay.source.sourceFingerprint); + }); + + it("changes fingerprints when extraction-relevant values or fields change", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const initial = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + expect(initial.success).toBe(true); + if (!initial.success) throw new Error(initial.error); + const initialFingerprint = getSnapshot(initial, "columns").source.sourceFingerprint; + + await updatePostHero(ctx, item.id, { + id: "media-updated", + provider: "local", + mimeType: "image/webp", + }); + const valueChanged = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + expect(valueChanged.success).toBe(true); + if (!valueChanged.success) throw new Error(valueChanged.error); + const valueChangedFingerprint = getSnapshot(valueChanged, "columns").source.sourceFingerprint; + + await registry.createField("posts", { + slug: "thumbnail", + label: "Thumbnail", + type: "image", + }); + const fieldChanged = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + expect(fieldChanged.success).toBe(true); + if (!fieldChanged.success) throw new Error(fieldChanged.error); + const fieldChangedFingerprint = getSnapshot(fieldChanged, "columns").source.sourceFingerprint; + + expect(valueChangedFingerprint).not.toBe(initialFingerprint); + expect(fieldChangedFingerprint).not.toBe(valueChangedFingerprint); + }); + + it("keeps fingerprints stable for non-extraction schema metadata changes", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + attachment: { id: "file-live", provider: "local", mimeType: "application/pdf" }, + }, + }); + const before = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + expect(before.success).toBe(true); + if (!before.success) throw new Error(before.error); + const beforeFingerprint = getSnapshot(before, "columns").source.sourceFingerprint; + + await ctx.db + .updateTable("_emdash_fields") + .set({ + label: "Hero Image", + required: 1, + validation: JSON.stringify({ allowedMimeTypes: ["image/png"] }), + sort_order: 999, + }) + .where("slug", "=", "hero") + .execute(); + await ctx.db + .updateTable("_emdash_fields") + .set({ sort_order: -999 }) + .where("slug", "=", "attachment") + .execute(); + + const after = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + expect(after.success).toBe(true); + if (!after.success) throw new Error(after.error); + + expect(getSnapshot(after, "columns").source.sourceFingerprint).toBe(beforeFingerprint); + }); }); + +interface TestPostInput { + slug: string; + status: string; + locale?: string; + data: Record; +} + +interface TestPost { + id: string; + slug: string; + status: string; + updatedAt: string; + version: number; + translationGroup: string; +} + +async function insertPost(ctx: DialectTestContext, input: TestPostInput): Promise { + const id = ulid(); + const now = new Date().toISOString(); + await sql` + INSERT INTO ${sql.ref("ec_posts")} ( + id, + slug, + status, + created_at, + updated_at, + version, + locale, + translation_group, + title, + hero, + attachment, + sections, + body, + raw_data + ) VALUES ( + ${id}, + ${input.slug}, + ${input.status}, + ${now}, + ${now}, + ${1}, + ${input.locale ?? "en"}, + ${id}, + ${serializeFieldValue(input.data.title)}, + ${serializeFieldValue(input.data.hero)}, + ${serializeFieldValue(input.data.attachment)}, + ${serializeFieldValue(input.data.sections)}, + ${serializeFieldValue(input.data.body)}, + ${serializeFieldValue(input.data.raw_data)} + ) + `.execute(ctx.db); + + return { + id, + slug: input.slug, + status: input.status, + updatedAt: now, + version: 1, + translationGroup: id, + }; +} + +async function setDraftRevision( + ctx: DialectTestContext, + contentId: string, + revisionId: string, +): Promise { + await sql` + UPDATE ${sql.ref("ec_posts")} + SET draft_revision_id = ${revisionId}, + updated_at = ${new Date().toISOString()} + WHERE id = ${contentId} + `.execute(ctx.db); +} + +async function updatePostHero( + ctx: DialectTestContext, + contentId: string, + hero: Record, +): Promise { + await sql` + UPDATE ${sql.ref("ec_posts")} + SET hero = ${serializeFieldValue(hero)}, + updated_at = ${new Date().toISOString()} + WHERE id = ${contentId} + `.execute(ctx.db); +} + +function getSnapshot( + result: Extract>, { success: true }>, + sourceVariant: "columns" | "draft_overlay", +) { + const snapshot = result.snapshots.find( + (candidate) => candidate.source.sourceVariant === sourceVariant, + ); + if (!snapshot) throw new Error(`Missing ${sourceVariant} snapshot`); + return snapshot; +} + +function serializeFieldValue(value: unknown): unknown { + if (value === null || value === undefined) return null; + if (typeof value === "object") return JSON.stringify(value); + return value; +} From 647361d93defa6c4f5e5503d62b9ff32cf8c8f08 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 16:23:21 +0100 Subject: [PATCH 16/29] feat(media): update buildContentSource to conditionally include sourceFingerprint --- packages/core/src/media/usage/content-snapshots.ts | 5 +++-- .../database/media-usage-content-snapshots.test.ts | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/core/src/media/usage/content-snapshots.ts b/packages/core/src/media/usage/content-snapshots.ts index cc1fc87772..7d05623ad6 100644 --- a/packages/core/src/media/usage/content-snapshots.ts +++ b/packages/core/src/media/usage/content-snapshots.ts @@ -229,7 +229,7 @@ function buildContentSource(input: { const { collectionSlug, row, displayData, sourceVariant, revisionId } = input; const contentId = readString(row.id) ?? ""; const contentSlug = input.contentSlug ?? readNullableString(row.slug); - return { + const source: MediaUsageSourceInput = { sourceKey: buildContentMediaUsageSourceKey({ collectionSlug, contentId, @@ -250,8 +250,9 @@ function buildContentSource(input: { schemaVersion: CONTENT_SOURCE_SCHEMA_VERSION, sourceUpdatedAt: readNullableString(row.updated_at), sourceVersion: readNumber(row.version), - sourceFingerprint: input.sourceFingerprint ?? null, }; + if (input.sourceFingerprint !== undefined) source.sourceFingerprint = input.sourceFingerprint; + return source; } async function buildSourceFingerprint(input: { diff --git a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts index a41a9f6ecc..b1ce39e424 100644 --- a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts @@ -341,6 +341,7 @@ describeEachDialect("content media usage snapshots", (dialect) => { source: expect.objectContaining({ sourceVariant: "draft_overlay" }), }), ); + expect(result.source).not.toHaveProperty("sourceFingerprint"); }); it("adds stable source schema versions and fingerprints to snapshots", async () => { From 7466f64a3642f081e99c300d87b97817e4799a13 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 16:58:14 +0100 Subject: [PATCH 17/29] feat(media): Add Content Usage Refresh Service --- .../core/src/media/usage/content-refresh.ts | 249 +++++++++++ .../media-usage-content-refresh.test.ts | 410 ++++++++++++++++++ 2 files changed, 659 insertions(+) create mode 100644 packages/core/src/media/usage/content-refresh.ts create mode 100644 packages/core/tests/integration/database/media-usage-content-refresh.test.ts diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts new file mode 100644 index 0000000000..ad60926154 --- /dev/null +++ b/packages/core/src/media/usage/content-refresh.ts @@ -0,0 +1,249 @@ +import type { Kysely } from "kysely"; + +import { MediaUsageRepository } from "../../database/repositories/media-usage.js"; +import type { Database } from "../../database/types.js"; +import { validateIdentifier } from "../../database/validate.js"; +import { + CONTENT_SOURCE_SCHEMA_VERSION, + loadContentMediaUsageSnapshots, +} from "./content-snapshots.js"; +import { + buildContentMediaUsageSourceKey, + MEDIA_USAGE_CONTENT_SOURCE_VARIANTS, +} from "./source-key.js"; + +export const CONTENT_MEDIA_USAGE_ADAPTER_ID = "content-media"; +export const CONTENT_MEDIA_USAGE_COLLECTION_SCOPE = "collection"; + +const CONTENT_USAGE_LOCKS_KEY = Symbol.for("emdash.mediaUsage.contentLocks"); + +export type ContentMediaUsageRefreshErrorCode = + | "CONTENT_NOT_FOUND" + | "DRAFT_REVISION_NOT_FOUND" + | "DRAFT_REVISION_MISMATCH" + | "DRAFT_REVISION_INVALID" + | "CONTENT_USAGE_REFRESH_ERROR" + | "CONTENT_USAGE_DELETE_ERROR" + | "CONTENT_USAGE_STALE"; + +export interface ContentMediaUsageRefreshResult { + success: boolean; + refreshedSourceCount: number; + deletedSourceCount: number; + failedSourceCount: number; + errorCode?: ContentMediaUsageRefreshErrorCode; +} + +const ZERO_RESULT: ContentMediaUsageRefreshResult = { + success: true, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: 0, +}; + +export async function refreshContentMediaUsage( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + return withContentUsageLock(collectionSlug, contentId, () => + refreshContentMediaUsageUnlocked(db, collectionSlug, contentId), + ); +} + +async function refreshContentMediaUsageUnlocked( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + try { + const snapshotsResult = await loadContentMediaUsageSnapshots(db, collectionSlug, contentId); + if (!snapshotsResult.success) { + return markSnapshotFailure(db, collectionSlug, snapshotsResult); + } + + const repo = new MediaUsageRepository(db); + for (const snapshot of snapshotsResult.snapshots) { + await repo.replaceSource(snapshot.source, snapshot.occurrences); + } + + const expectedSourceKeys = new Set( + snapshotsResult.snapshots.map((snapshot) => snapshot.source.sourceKey), + ); + const absentSourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) => + buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }), + ).filter((sourceKey) => !expectedSourceKeys.has(sourceKey)); + const deletedSourceCount = await repo.deleteSources(absentSourceKeys); + + return { + success: true, + refreshedSourceCount: snapshotsResult.snapshots.length, + deletedSourceCount, + failedSourceCount: 0, + }; + } catch (error) { + console.error(`[media-usage] Failed to refresh ${collectionSlug}/${contentId}:`, error); + await markContentMediaUsageCollectionStaleSafely( + db, + collectionSlug, + "CONTENT_USAGE_REFRESH_ERROR", + ); + return { + success: false, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_REFRESH_ERROR", + }; + } +} + +export async function deleteContentMediaUsage( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + return withContentUsageLock(collectionSlug, contentId, () => + deleteContentMediaUsageUnlocked(db, collectionSlug, contentId), + ); +} + +async function deleteContentMediaUsageUnlocked( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + try { + const deletedSourceCount = await new MediaUsageRepository(db).deleteContentSources( + collectionSlug, + contentId, + ); + return { ...ZERO_RESULT, deletedSourceCount }; + } catch (error) { + console.error( + `[media-usage] Failed to delete usage for ${collectionSlug}/${contentId}:`, + error, + ); + await markContentMediaUsageCollectionStaleSafely( + db, + collectionSlug, + "CONTENT_USAGE_DELETE_ERROR", + ); + return { + success: false, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_DELETE_ERROR", + }; + } +} + +export async function refreshContentMediaUsageAfterWrite( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + const result = await refreshContentMediaUsage(db, collectionSlug, contentId); + if (!result.success) { + console.error( + `[media-usage] Usage refresh for ${collectionSlug}/${contentId} finished with ${result.errorCode}`, + ); + } +} + +export async function markContentMediaUsageCollectionStale( + db: Kysely, + collectionSlug: string, + lastErrorCode: ContentMediaUsageRefreshErrorCode | string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + const repo = new MediaUsageRepository(db); + const identity = { + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: collectionSlug, + }; + const existing = await repo.findIndexStatus(identity); + await repo.upsertIndexStatus({ + ...identity, + status: "stale", + schemaVersion: existing?.schemaVersion ?? CONTENT_SOURCE_SCHEMA_VERSION, + startedAt: existing?.startedAt ?? null, + completedAt: existing?.completedAt ?? null, + cursor: existing?.cursor ?? null, + indexedSourceCount: existing?.indexedSourceCount ?? 0, + failedSourceCount: existing?.failedSourceCount ?? 0, + lastErrorCode, + }); +} + +async function markSnapshotFailure( + db: Kysely, + collectionSlug: string, + result: Exclude>, { success: true }>, +): Promise { + const repo = new MediaUsageRepository(db); + if (result.source) { + await repo.markSourceAttempted({ + ...result.source, + sourceCompleteness: "failed", + lastErrorCode: result.error, + }); + } + await markContentMediaUsageCollectionStale(db, collectionSlug, result.error); + return { + success: false, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: result.source ? 1 : 0, + errorCode: result.error, + }; +} + +async function markContentMediaUsageCollectionStaleSafely( + db: Kysely, + collectionSlug: string, + lastErrorCode: ContentMediaUsageRefreshErrorCode, +): Promise { + try { + await markContentMediaUsageCollectionStale(db, collectionSlug, lastErrorCode); + } catch (error) { + console.error(`[media-usage] Failed to mark ${collectionSlug} stale:`, error); + } +} + +async function withContentUsageLock( + collectionSlug: string, + contentId: string, + fn: () => Promise, +): Promise { + const locks = getContentUsageLocks(); + const lockKey = `${collectionSlug}\0${contentId}`; + const previous = locks.get(lockKey) ?? Promise.resolve(); + let releaseCurrent!: () => void; + const current = new Promise((resolve) => { + releaseCurrent = resolve; + }); + const next = previous.catch(() => {}).then(() => current); + locks.set(lockKey, next); + + try { + await previous.catch(() => {}); + return await fn(); + } finally { + releaseCurrent(); + if (locks.get(lockKey) === next) locks.delete(lockKey); + } +} + +function getContentUsageLocks(): Map> { + const global = globalThis as typeof globalThis & Record; + const existing = global[CONTENT_USAGE_LOCKS_KEY]; + if (existing instanceof Map) return existing as Map>; + const locks = new Map>(); + global[CONTENT_USAGE_LOCKS_KEY] = locks; + return locks; +} diff --git a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts new file mode 100644 index 0000000000..4cf86af5db --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts @@ -0,0 +1,410 @@ +import { sql } from "kysely"; +import { ulid } from "ulidx"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { RevisionRepository } from "../../../src/database/repositories/revision.js"; +import { + CONTENT_MEDIA_USAGE_ADAPTER_ID, + CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + deleteContentMediaUsage, + markContentMediaUsageCollectionStale, + refreshContentMediaUsage, +} from "../../../src/media/usage/content-refresh.js"; +import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("content media usage refresh", (dialect) => { + let ctx: DialectTestContext; + let registry: SchemaRegistry; + let usageRepo: MediaUsageRepository; + let revisionRepo: RevisionRepository; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + registry = new SchemaRegistry(ctx.db); + usageRepo = new MediaUsageRepository(ctx.db); + revisionRepo = new RevisionRepository(ctx.db); + + await registry.createCollection({ slug: "posts", label: "Posts" }); + await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); + await registry.createField("posts", { slug: "hero", label: "Hero", type: "image" }); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("refreshes a columns source and replaces its current usage", async () => { + const item = await insertPost(ctx, { + slug: "hello-world", + status: "published", + data: { + title: "Hello World", + hero: { id: "media-old", provider: "local", mimeType: "image/webp" }, + }, + }); + const columnsKey = sourceKey(item.id, "columns"); + + const first = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(first).toEqual({ + success: true, + refreshedSourceCount: 1, + deletedSourceCount: 0, + failedSourceCount: 0, + }); + const firstSource = await usageRepo.findSource(columnsKey); + expect(firstSource).toEqual( + expect.objectContaining({ + sourceKey: columnsKey, + sourceCompleteness: "complete", + contentTitle: "Hello World", + sourceFingerprint: expect.stringMatching(/^[a-f0-9]{16}$/), + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-old")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceKey: columnsKey }), + occurrence: expect.objectContaining({ fieldPath: "hero", mediaId: "media-old" }), + }), + ]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toBeNull(); + + await updatePostHero(ctx, item.id, { + id: "media-new", + provider: "local", + mimeType: "image/webp", + }); + const second = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(second).toEqual({ + success: true, + refreshedSourceCount: 1, + deletedSourceCount: 0, + failedSourceCount: 0, + }); + expect((await usageRepo.findSource(columnsKey))?.currentGeneration).not.toBe( + firstSource?.currentGeneration, + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-old")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-new")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceKey: columnsKey }), + occurrence: expect.objectContaining({ fieldPath: "hero", mediaId: "media-new" }), + }), + ]); + }); + + it("refreshes columns and draft overlay sources when a draft exists", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { + title: "Draft Title", + hero: { id: "media-draft", provider: "local", mimeType: "image/webp" }, + }, + }); + await setDraftRevision(ctx, item.id, draft.id); + + const result = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: true, + refreshedSourceCount: 2, + deletedSourceCount: 0, + failedSourceCount: 0, + }); + expect(await usageRepo.findSource(sourceKey(item.id, "columns"))).toEqual( + expect.objectContaining({ sourceVariant: "columns", contentTitle: "Live Title" }), + ); + expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).toEqual( + expect.objectContaining({ + sourceVariant: "draft_overlay", + contentTitle: "Draft Title", + revisionId: draft.id, + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]); + }); + + it("deletes a stale draft overlay source after a successful columns-only refresh", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { hero: { id: "media-draft", provider: "local", mimeType: "image/webp" } }, + }); + await setDraftRevision(ctx, item.id, draft.id); + await refreshContentMediaUsage(ctx.db, "posts", item.id); + expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).not.toBeNull(); + + await clearDraftRevision(ctx, item.id); + const result = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: true, + refreshedSourceCount: 1, + deletedSourceCount: 1, + failedSourceCount: 0, + }); + expect(await usageRepo.findSource(sourceKey(item.id, "columns"))).not.toBeNull(); + expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([]); + }); + + it("deletes every source for a content item", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { hero: { id: "media-draft", provider: "local", mimeType: "image/webp" } }, + }); + await setDraftRevision(ctx, item.id, draft.id); + await refreshContentMediaUsage(ctx.db, "posts", item.id); + + const result = await deleteContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: true, + refreshedSourceCount: 0, + deletedSourceCount: 2, + failedSourceCount: 0, + }); + expect(await usageRepo.findSource(sourceKey(item.id, "columns"))).toBeNull(); + expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([]); + }); + + it("marks draft snapshot failures without replacing current usage", async () => { + const item = await insertPost(ctx, { + slug: "live-post", + status: "published", + data: { + title: "Live Title", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + await refreshContentMediaUsage(ctx.db, "posts", item.id); + const revisionId = ulid(); + await sql` + INSERT INTO revisions (id, collection, entry_id, data, author_id) + VALUES (${revisionId}, ${"posts"}, ${item.id}, ${"{"}, ${null}) + `.execute(ctx.db); + await setDraftRevision(ctx, item.id, revisionId); + + const result = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: false, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: 1, + errorCode: "DRAFT_REVISION_INVALID", + }); + expect(await usageRepo.findSource(sourceKey(item.id, "columns"))).toEqual( + expect.objectContaining({ sourceCompleteness: "complete" }), + ); + expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).toEqual( + expect.objectContaining({ + sourceCompleteness: "failed", + lastErrorCode: "DRAFT_REVISION_INVALID", + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toEqual( + expect.objectContaining({ + status: "stale", + lastErrorCode: "DRAFT_REVISION_INVALID", + }), + ); + }); + + it("marks collection coverage stale while preserving existing status metadata", async () => { + await usageRepo.upsertIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + status: "partial", + schemaVersion: 7, + startedAt: "2026-01-01T00:00:00.000Z", + completedAt: "2026-01-01T00:00:02.000Z", + cursor: "cursor-1", + indexedSourceCount: 12, + failedSourceCount: 3, + lastErrorCode: "OLD_ERROR", + updatedAt: "2026-01-01T00:00:03.000Z", + }); + + await markContentMediaUsageCollectionStale(ctx.db, "posts", "SCHEMA_FIELD_CHANGED"); + + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toEqual( + expect.objectContaining({ + status: "stale", + schemaVersion: 7, + startedAt: "2026-01-01T00:00:00.000Z", + completedAt: "2026-01-01T00:00:02.000Z", + cursor: "cursor-1", + indexedSourceCount: 12, + failedSourceCount: 3, + lastErrorCode: "SCHEMA_FIELD_CHANGED", + }), + ); + }); +}); + +interface TestPostInput { + slug: string; + status: string; + locale?: string; + data: Record; +} + +interface TestPost { + id: string; + slug: string; + status: string; + translationGroup: string; +} + +async function insertPost(ctx: DialectTestContext, input: TestPostInput): Promise { + const id = ulid(); + const now = new Date().toISOString(); + await sql` + INSERT INTO ${sql.ref("ec_posts")} ( + id, + slug, + status, + created_at, + updated_at, + version, + locale, + translation_group, + title, + hero + ) VALUES ( + ${id}, + ${input.slug}, + ${input.status}, + ${now}, + ${now}, + ${1}, + ${input.locale ?? "en"}, + ${id}, + ${serializeFieldValue(input.data.title)}, + ${serializeFieldValue(input.data.hero)} + ) + `.execute(ctx.db); + + return { + id, + slug: input.slug, + status: input.status, + translationGroup: id, + }; +} + +async function updatePostHero( + ctx: DialectTestContext, + contentId: string, + hero: Record, +): Promise { + await sql` + UPDATE ${sql.ref("ec_posts")} + SET hero = ${serializeFieldValue(hero)}, + updated_at = ${new Date().toISOString()} + WHERE id = ${contentId} + `.execute(ctx.db); +} + +async function setDraftRevision( + ctx: DialectTestContext, + contentId: string, + revisionId: string, +): Promise { + await sql` + UPDATE ${sql.ref("ec_posts")} + SET draft_revision_id = ${revisionId}, + updated_at = ${new Date().toISOString()} + WHERE id = ${contentId} + `.execute(ctx.db); +} + +async function clearDraftRevision(ctx: DialectTestContext, contentId: string): Promise { + await sql` + UPDATE ${sql.ref("ec_posts")} + SET draft_revision_id = ${null}, + updated_at = ${new Date().toISOString()} + WHERE id = ${contentId} + `.execute(ctx.db); +} + +function sourceKey(contentId: string, sourceVariant: "columns" | "draft_overlay"): string { + return buildContentMediaUsageSourceKey({ + collectionSlug: "posts", + contentId, + sourceVariant, + }); +} + +function serializeFieldValue(value: unknown): unknown { + if (value === null || value === undefined) return null; + if (typeof value === "object") return JSON.stringify(value); + return value; +} From 07058607f3af648451e3461a182303a4a6771b35 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 17:10:03 +0100 Subject: [PATCH 18/29] feat(media): Wire Runtime Create, Update, Duplicate, Revision Restore --- packages/core/src/emdash-runtime.ts | 38 +++- .../media-usage-runtime-refresh.test.ts | 210 ++++++++++++++++++ 2 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index fde5c3bb19..47d1ef661a 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -31,6 +31,7 @@ import type { import { validateIdentifier } from "./database/validate.js"; import { normalizeMediaValue } from "./media/normalize.js"; import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js"; +import { refreshContentMediaUsageAfterWrite } from "./media/usage/content-refresh.js"; import type { SandboxedPluginInstance, SandboxRunner } from "./plugins/sandbox/types.js"; import type { ResolvedPlugin, @@ -2570,6 +2571,9 @@ export class EmDashRuntime { authorId: body.authorId, bylines: body.bylines, }); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } // Run afterSave hooks (fire-and-forget) if (result.success && result.data) { @@ -2738,6 +2742,9 @@ export class EmDashRuntime { // supporting collections, that's the just-saved draft, not the live // columns. const hydrated = await this.hydrateDraftData(result); + if (hydrated.success && hydrated.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [resolvedId]); + } // Run afterSave hooks (fire-and-forget) if (hydrated.success && hydrated.data) { @@ -2823,7 +2830,11 @@ export class EmDashRuntime { } async handleContentDuplicate(collection: string, id: string, authorId?: string) { - return handleContentDuplicate(this.db, collection, id, authorId); + const result = await handleContentDuplicate(this.db, collection, id, authorId); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } + return result; } // ========================================================================= @@ -3011,6 +3022,9 @@ export class EmDashRuntime { // behavior for collections that opt out of the draft model. if (!usesDraftRevisions) { const result = await handleRevisionRestore(this.db, revisionId, callerUserId); + if (result.success) { + await this.refreshContentUsageAfterSuccessfulWrite(revision.collection, [revision.entryId]); + } return this.hydrateDraftData(result); } @@ -3046,7 +3060,11 @@ export class EmDashRuntime { // columns and the next `content_get` would surface different // values (the bug that motivated this rewrite). const refetched = await handleContentGet(this.db, revision.collection, revision.entryId); - return this.hydrateDraftData(refetched); + const hydrated = await this.hydrateDraftData(refetched); + if (hydrated.success) { + await this.refreshContentUsageAfterSuccessfulWrite(revision.collection, [revision.entryId]); + } + return hydrated; } catch (error) { console.error("[emdash] revision restore failed:", error); return { @@ -3059,6 +3077,22 @@ export class EmDashRuntime { } } + private async refreshContentUsageAfterSuccessfulWrite( + collection: string, + contentIds: readonly string[], + ): Promise { + for (const contentId of new Set(contentIds)) { + try { + await refreshContentMediaUsageAfterWrite(this.db, collection, contentId); + } catch (error) { + console.error( + `[media-usage] Failed after content write ${collection}/${contentId}:`, + error, + ); + } + } + } + // ========================================================================= // Plugin Routes // ========================================================================= diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts new file mode 100644 index 0000000000..fd72647ad3 --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -0,0 +1,210 @@ +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { RevisionRepository } from "../../../src/database/repositories/revision.js"; +import type { EmDashRuntime } from "../../../src/emdash-runtime.js"; +import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { createTestRuntime } from "../../utils/mcp-runtime.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("runtime content media usage refresh", (dialect) => { + let ctx: DialectTestContext; + let runtime: EmDashRuntime; + let usageRepo: MediaUsageRepository; + let revisionRepo: RevisionRepository; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + const registry = new SchemaRegistry(ctx.db); + await registry.createCollection({ slug: "posts", label: "Posts" }); + await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); + await registry.createField("posts", { slug: "hero", label: "Hero", type: "image" }); + await registry.createCollection({ slug: "plain_posts", label: "Plain Posts", supports: [] }); + await registry.createField("plain_posts", { + slug: "title", + label: "Title", + type: "string", + }); + await registry.createField("plain_posts", { slug: "hero", label: "Hero", type: "image" }); + + runtime = createTestRuntime(ctx.db); + usageRepo = new MediaUsageRepository(ctx.db); + revisionRepo = new RevisionRepository(ctx.db); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("refreshes columns usage after runtime content create", async () => { + const created = await runtime.handleContentCreate("plain_posts", { + slug: "created-post", + data: { + title: "Created Post", + hero: mediaRef("media-created"), + }, + }); + + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + const contentId = created.data.item.id; + expect(await usageRepo.findSource(sourceKey("plain_posts", contentId, "columns"))).toEqual( + expect.objectContaining({ + contentTitle: "Created Post", + sourceCompleteness: "complete", + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-created")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ contentId, sourceVariant: "columns" }), + occurrence: expect.objectContaining({ fieldPath: "hero", mediaId: "media-created" }), + }), + ]); + }); + + it("refreshes columns usage after runtime non-revision content update", async () => { + const created = await runtime.handleContentCreate("plain_posts", { + slug: "updated-post", + data: { + title: "Updated Post", + hero: mediaRef("media-old"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + + const updated = await runtime.handleContentUpdate("plain_posts", created.data.item.id, { + data: { hero: mediaRef("media-new") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-old")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-new")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ + contentId: created.data.item.id, + sourceVariant: "columns", + }), + }), + ]); + }); + + it("refreshes draft overlay usage after runtime revision-enabled content update", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "drafted-post", + data: { + title: "Drafted Post", + hero: mediaRef("media-live"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + + const updated = await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-draft") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findSource(sourceKey("posts", created.data.item.id, "columns"))).toEqual( + expect.objectContaining({ sourceVariant: "columns", contentTitle: "Drafted Post" }), + ); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toEqual(expect.objectContaining({ sourceVariant: "draft_overlay" })); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]); + }); + + it("refreshes columns usage for duplicated content", async () => { + const created = await runtime.handleContentCreate("plain_posts", { + slug: "original-post", + data: { + title: "Original Post", + hero: mediaRef("media-copy"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + + const duplicated = await runtime.handleContentDuplicate("plain_posts", created.data.item.id); + + expect(duplicated.success).toBe(true); + if (!duplicated.success) throw new Error(duplicated.error.message); + expect(await usageRepo.findCurrentUsageByMediaId("media-copy")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ + contentId: duplicated.data.item.id, + sourceVariant: "columns", + }), + }), + ]), + ); + }); + + it("refreshes draft overlay usage after runtime revision restore", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "restored-post", + data: { + title: "Restored Post", + hero: mediaRef("media-live"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + + const firstDraft = await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-restored") }, + }); + expect(firstDraft.success).toBe(true); + const revisionToRestore = ( + await revisionRepo.findByEntry("posts", created.data.item.id, { limit: 1 }) + )[0]; + expect(revisionToRestore).toBeDefined(); + const secondDraft = await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-current-draft") }, + }); + expect(secondDraft.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-current-draft")).toHaveLength(1); + + const restored = await runtime.handleRevisionRestore(revisionToRestore!.id, "user-1"); + + expect(restored.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-current-draft")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-restored")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]); + }); +}); + +function mediaRef(id: string): Record { + return { + id, + provider: "local", + mimeType: "image/webp", + width: 100, + height: 100, + }; +} + +function sourceKey( + collectionSlug: string, + contentId: string, + sourceVariant: "columns" | "draft_overlay", +): string { + return buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }); +} From cf64e8a7a07fd29964cf953a4dbc105b60858128 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 17:18:49 +0100 Subject: [PATCH 19/29] Wire Publish, Unpublish, Schedule, Unschedule, Discard Draft --- packages/core/src/emdash-runtime.ts | 24 ++- .../media-usage-runtime-refresh.test.ts | 167 ++++++++++++++++++ 2 files changed, 188 insertions(+), 3 deletions(-) diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index 47d1ef661a..a85bad254f 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -2847,6 +2847,9 @@ export class EmDashRuntime { options: { publishedAt?: string; requireScheduledDue?: boolean } = {}, ) { const result = await handleContentPublish(this.db, collection, id, options); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } // Run afterPublish hooks (fire-and-forget) if (result.success && result.data) { @@ -2858,6 +2861,9 @@ export class EmDashRuntime { async handleContentUnpublish(collection: string, id: string) { const result = await handleContentUnpublish(this.db, collection, id); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } // Run afterUnpublish hooks (fire-and-forget) if (result.success && result.data) { @@ -2868,11 +2874,19 @@ export class EmDashRuntime { } async handleContentSchedule(collection: string, id: string, scheduledAt: string) { - return handleContentSchedule(this.db, collection, id, scheduledAt); + const result = await handleContentSchedule(this.db, collection, id, scheduledAt); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } + return result; } async handleContentUnschedule(collection: string, id: string) { - return handleContentUnschedule(this.db, collection, id); + const result = await handleContentUnschedule(this.db, collection, id); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } + return result; } async handleContentCountScheduled(collection: string) { @@ -2880,7 +2894,11 @@ export class EmDashRuntime { } async handleContentDiscardDraft(collection: string, id: string) { - return handleContentDiscardDraft(this.db, collection, id); + const result = await handleContentDiscardDraft(this.db, collection, id); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } + return result; } async handleContentCompare(collection: string, id: string) { diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts index fd72647ad3..77b5f31f0d 100644 --- a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -1,3 +1,4 @@ +import { sql } from "kysely"; import { afterEach, beforeEach, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; @@ -189,6 +190,172 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { }), ]); }); + + it("refreshes usage when publishing a draft overlay", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "publish-post", + data: { + title: "Publish Post", + hero: mediaRef("media-live"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-draft") }, + }); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]); + + const published = await runtime.handleContentPublish("posts", created.data.item.id); + + expect(published.success).toBe(true); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + }); + + it("refreshes usage when unpublishing creates a draft overlay", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "unpublish-post", + data: { + title: "Unpublish Post", + hero: mediaRef("media-unpublish"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + const published = await runtime.handleContentPublish("posts", created.data.item.id); + expect(published.success).toBe(true); + + const unpublished = await runtime.handleContentUnpublish("posts", created.data.item.id); + + expect(unpublished.success).toBe(true); + expect(await usageRepo.findSource(sourceKey("posts", created.data.item.id, "columns"))).toEqual( + expect.objectContaining({ contentStatus: "draft", sourceVariant: "columns" }), + ); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toEqual(expect.objectContaining({ contentStatus: "draft", sourceVariant: "draft_overlay" })); + expect(await usageRepo.findCurrentUsageByMediaId("media-unpublish")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]), + ); + }); + + it("refreshes schedule metadata on schedule and unschedule", async () => { + const created = await runtime.handleContentCreate("plain_posts", { + slug: "scheduled-post", + data: { + title: "Scheduled Post", + hero: mediaRef("media-schedule"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + const scheduledAt = new Date(Date.now() + 86_400_000).toISOString(); + + const scheduled = await runtime.handleContentSchedule( + "plain_posts", + created.data.item.id, + scheduledAt, + ); + + expect(scheduled.success).toBe(true); + expect( + await usageRepo.findSource(sourceKey("plain_posts", created.data.item.id, "columns")), + ).toEqual( + expect.objectContaining({ + contentStatus: "scheduled", + contentScheduledAt: scheduledAt, + }), + ); + + const unscheduled = await runtime.handleContentUnschedule("plain_posts", created.data.item.id); + + expect(unscheduled.success).toBe(true); + expect( + await usageRepo.findSource(sourceKey("plain_posts", created.data.item.id, "columns")), + ).toEqual( + expect.objectContaining({ + contentStatus: "draft", + contentScheduledAt: null, + }), + ); + }); + + it("refreshes usage when discarding a draft overlay", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "discard-post", + data: { + title: "Discard Post", + hero: mediaRef("media-live"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-discard") }, + }); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).not.toBeNull(); + + const discarded = await runtime.handleContentDiscardDraft("posts", created.data.item.id); + + expect(discarded.success).toBe(true); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-discard")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + }); + + it("refreshes usage when scheduled publish runs through the runtime", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "scheduled-publish-post", + data: { + title: "Scheduled Publish Post", + hero: mediaRef("media-live"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-scheduled-draft") }, + }); + const dueAt = new Date(Date.now() - 60_000).toISOString(); + await sql` + UPDATE ${sql.ref("ec_posts")} + SET status = ${"scheduled"}, + scheduled_at = ${dueAt} + WHERE id = ${created.data.item.id} + `.execute(ctx.db); + + const result = await runtime.publishScheduled(); + + expect(result).toEqual([{ collection: "posts", id: created.data.item.id }]); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-live")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-scheduled-draft")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + }); }); function mediaRef(id: string): Record { From 1ecf9f5d5efc1448922cfec5851687aa4d9a32cb Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 17:35:29 +0100 Subject: [PATCH 20/29] Wire Trash And Permanent Delete --- packages/core/src/api/handlers/content.ts | 17 ++- packages/core/src/emdash-runtime.ts | 33 ++++- .../media-usage-runtime-refresh.test.ts | 123 ++++++++++++++++++ 3 files changed, 165 insertions(+), 8 deletions(-) diff --git a/packages/core/src/api/handlers/content.ts b/packages/core/src/api/handlers/content.ts index e2863c73e4..80424a67b2 100644 --- a/packages/core/src/api/handlers/content.ts +++ b/packages/core/src/api/handlers/content.ts @@ -1046,15 +1046,18 @@ export async function handleContentDelete( db: Kysely, collection: string, id: string, -): Promise> { +): Promise> { try { - const deleted = await withTransaction(db, async (trx) => { + const result = await withTransaction(db, async (trx) => { const repo = new ContentRepository(trx); const resolvedId = (await resolveId(repo, collection, id)) ?? id; - return repo.delete(collection, resolvedId); + return { + id: resolvedId, + deleted: await repo.delete(collection, resolvedId), + }; }); - if (!deleted) { + if (!result.deleted) { return { success: false, error: { @@ -1066,7 +1069,7 @@ export async function handleContentDelete( return { success: true, - data: { deleted: true }, + data: { deleted: true, id: result.id }, }; } catch (error) { console.error("Content delete error:", error); @@ -1129,7 +1132,7 @@ export async function handleContentPermanentDelete( db: Kysely, collection: string, id: string, -): Promise> { +): Promise> { try { const repo = new ContentRepository(db); const resolvedId = (await resolveIdIncludingTrashed(repo, collection, id)) ?? id; @@ -1166,7 +1169,7 @@ export async function handleContentPermanentDelete( return { success: true, - data: { deleted: true }, + data: { deleted: true, id: resolvedId }, }; } catch (error) { console.error("Content permanent delete error:", error); diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index a85bad254f..ee54872384 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -31,7 +31,10 @@ import type { import { validateIdentifier } from "./database/validate.js"; import { normalizeMediaValue } from "./media/normalize.js"; import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js"; -import { refreshContentMediaUsageAfterWrite } from "./media/usage/content-refresh.js"; +import { + deleteContentMediaUsage, + refreshContentMediaUsageAfterWrite, +} from "./media/usage/content-refresh.js"; import type { SandboxedPluginInstance, SandboxRunner } from "./plugins/sandbox/types.js"; import type { ResolvedPlugin, @@ -2783,6 +2786,9 @@ export class EmDashRuntime { // Delete the content const result = await handleContentDelete(this.db, collection, id); + if (result.success) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.id]); + } // Run afterDelete hooks (fire-and-forget) if (result.success) { @@ -2805,6 +2811,9 @@ export class EmDashRuntime { async handleContentRestore(collection: string, id: string) { const result = await handleContentRestore(this.db, collection, id); + if (result.success && result.data) { + await this.refreshContentUsageAfterSuccessfulWrite(collection, [result.data.item.id]); + } // Run afterRestore hooks (fire-and-forget) if (result.success) { @@ -2816,6 +2825,9 @@ export class EmDashRuntime { async handleContentPermanentDelete(collection: string, id: string) { const result = await handleContentPermanentDelete(this.db, collection, id); + if (result.success) { + await this.deleteContentUsageAfterSuccessfulPermanentDelete(collection, result.data.id); + } // Run afterDelete hooks so plugins (e.g. AI Search) can clean up if (result.success) { @@ -3111,6 +3123,25 @@ export class EmDashRuntime { } } + private async deleteContentUsageAfterSuccessfulPermanentDelete( + collection: string, + contentId: string, + ): Promise { + try { + const result = await deleteContentMediaUsage(this.db, collection, contentId); + if (!result.success) { + console.error( + `[media-usage] Usage delete for ${collection}/${contentId} finished with ${result.errorCode}`, + ); + } + } catch (error) { + console.error( + `[media-usage] Failed after permanent content delete ${collection}/${contentId}:`, + error, + ); + } + } + // ========================================================================= // Plugin Routes // ========================================================================= diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts index 77b5f31f0d..b3e87d2de0 100644 --- a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -356,6 +356,129 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), ]); }); + + it("refreshes trash metadata while preserving usage on soft delete", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "trashed-post", + data: { + title: "Trashed Post", + hero: mediaRef("media-live-trash"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-draft-trash") }, + }); + + const deleted = await runtime.handleContentDelete("posts", "trashed-post"); + + expect(deleted.success).toBe(true); + if (!deleted.success) throw new Error(deleted.error.message); + expect(deleted.data.id).toBe(created.data.item.id); + expect(await usageRepo.findSource(sourceKey("posts", created.data.item.id, "columns"))).toEqual( + expect.objectContaining({ + contentDeletedAt: expect.any(String), + sourceVariant: "columns", + }), + ); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toEqual( + expect.objectContaining({ + contentDeletedAt: expect.any(String), + sourceVariant: "draft_overlay", + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-live-trash")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-trash")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]); + }); + + it("refreshes trash metadata when restoring content", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "restore-trash-post", + data: { + title: "Restore Trash Post", + hero: mediaRef("media-live-restore"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-draft-restore") }, + }); + const deleted = await runtime.handleContentDelete("posts", created.data.item.id); + expect(deleted.success).toBe(true); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toEqual(expect.objectContaining({ contentDeletedAt: expect.any(String) })); + + const restored = await runtime.handleContentRestore("posts", "restore-trash-post"); + + expect(restored.success).toBe(true); + expect(await usageRepo.findSource(sourceKey("posts", created.data.item.id, "columns"))).toEqual( + expect.objectContaining({ + contentDeletedAt: null, + sourceVariant: "columns", + }), + ); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toEqual( + expect.objectContaining({ + contentDeletedAt: null, + sourceVariant: "draft_overlay", + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-live-restore")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ sourceVariant: "columns" }) }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-restore")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ sourceVariant: "draft_overlay" }), + }), + ]); + }); + + it("deletes usage sources and current occurrences on permanent delete", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "permanent-delete-post", + data: { + title: "Permanent Delete Post", + hero: mediaRef("media-live-permanent"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + await runtime.handleContentUpdate("posts", created.data.item.id, { + data: { hero: mediaRef("media-draft-permanent") }, + }); + const deleted = await runtime.handleContentDelete("posts", created.data.item.id); + expect(deleted.success).toBe(true); + + const permanentlyDeleted = await runtime.handleContentPermanentDelete( + "posts", + "permanent-delete-post", + ); + + expect(permanentlyDeleted.success).toBe(true); + if (!permanentlyDeleted.success) throw new Error(permanentlyDeleted.error.message); + expect(permanentlyDeleted.data.id).toBe(created.data.item.id); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "columns")), + ).toBeNull(); + expect( + await usageRepo.findSource(sourceKey("posts", created.data.item.id, "draft_overlay")), + ).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-live-permanent")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-permanent")).toEqual([]); + }); }); function mediaRef(id: string): Record { From 24e5333414ec3038567dc7345914a9b977c08f0e Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Thu, 2 Jul 2026 17:59:22 +0100 Subject: [PATCH 21/29] Refresh I18n Non-Translatable Siblings --- packages/core/src/emdash-runtime.ts | 32 ++- .../core/src/media/usage/content-refresh.ts | 52 +++- .../media-usage-runtime-refresh.test.ts | 267 ++++++++++++++++++ 3 files changed, 349 insertions(+), 2 deletions(-) diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index ee54872384..f49bd84930 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -33,6 +33,8 @@ import { normalizeMediaValue } from "./media/normalize.js"; import type { MediaProvider, MediaProviderCapabilities } from "./media/types.js"; import { deleteContentMediaUsage, + findNonTranslatableSiblingContentIds, + markContentMediaUsageCollectionStale, refreshContentMediaUsageAfterWrite, } from "./media/usage/content-refresh.js"; import type { SandboxedPluginInstance, SandboxRunner } from "./plugins/sandbox/types.js"; @@ -2746,7 +2748,35 @@ export class EmDashRuntime { // columns. const hydrated = await this.hydrateDraftData(result); if (hydrated.success && hydrated.data) { - await this.refreshContentUsageAfterSuccessfulWrite(collection, [resolvedId]); + const contentIdsToRefresh = [resolvedId]; + if (!usesDraftRevisions && processedData) { + try { + contentIdsToRefresh.push( + ...(await findNonTranslatableSiblingContentIds( + this.db, + collection, + resolvedId, + hydrated.data.item.translationGroup, + processedData, + )), + ); + } catch (error) { + console.error( + `[media-usage] Failed to discover synced i18n siblings for ${collection}/${resolvedId}:`, + error, + ); + try { + await markContentMediaUsageCollectionStale( + this.db, + collection, + "CONTENT_USAGE_REFRESH_ERROR", + ); + } catch (staleError) { + console.error(`[media-usage] Failed to mark ${collection} stale:`, staleError); + } + } + } + await this.refreshContentUsageAfterSuccessfulWrite(collection, contentIdsToRefresh); } // Run afterSave hooks (fire-and-forget) diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts index ad60926154..0dee3c844a 100644 --- a/packages/core/src/media/usage/content-refresh.ts +++ b/packages/core/src/media/usage/content-refresh.ts @@ -1,8 +1,10 @@ -import type { Kysely } from "kysely"; +import { sql, type Kysely } from "kysely"; import { MediaUsageRepository } from "../../database/repositories/media-usage.js"; import type { Database } from "../../database/types.js"; import { validateIdentifier } from "../../database/validate.js"; +import { isI18nEnabled } from "../../i18n/config.js"; +import { loadContentMediaUsageFields } from "./content-fields.js"; import { CONTENT_SOURCE_SCHEMA_VERSION, loadContentMediaUsageSnapshots, @@ -180,6 +182,54 @@ export async function markContentMediaUsageCollectionStale( }); } +export async function findNonTranslatableSiblingContentIds( + db: Kysely, + collectionSlug: string, + updatedContentId: string, + translationGroup: string | null | undefined, + updatedData: Record | undefined, +): Promise { + if (!isI18nEnabled() || !updatedData || !translationGroup) return []; + + validateIdentifier(collectionSlug, "collection slug"); + const collection = await db + .selectFrom("_emdash_collections") + .select("id") + .where("slug", "=", collectionSlug) + .executeTakeFirst(); + if (!collection) return []; + + const fields = await db + .selectFrom("_emdash_fields") + .select("slug") + .where("collection_id", "=", collection.id) + .where("translatable", "=", 0) + .execute(); + + const touchedNonTranslatableSlugs = fields + .filter((field) => field.slug in updatedData) + .map((field) => field.slug); + if (touchedNonTranslatableSlugs.length === 0) return []; + + const usageFields = await loadContentMediaUsageFields(db, collectionSlug); + const usageRelevantSlugs = new Set([ + ...usageFields.extractionFields.map((field) => field.slug), + ...usageFields.displayFieldSlugs, + ]); + if (!touchedNonTranslatableSlugs.some((slug) => usageRelevantSlugs.has(slug))) return []; + + const tableName = `ec_${collectionSlug}`; + const rows = await sql<{ id: string }>` + SELECT id + FROM ${sql.ref(tableName)} + WHERE translation_group = ${translationGroup} + AND id != ${updatedContentId} + ORDER BY id ASC + `.execute(db); + + return rows.rows.map((row) => row.id); +} + async function markSnapshotFailure( db: Kysely, collectionSlug: string, diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts index b3e87d2de0..d0a715465e 100644 --- a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, expect, it } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; import { RevisionRepository } from "../../../src/database/repositories/revision.js"; import type { EmDashRuntime } from "../../../src/emdash-runtime.js"; +import { setI18nConfig } from "../../../src/i18n/config.js"; import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { createTestRuntime } from "../../utils/mcp-runtime.js"; @@ -21,11 +22,18 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { let revisionRepo: RevisionRepository; beforeEach(async () => { + setI18nConfig(null); ctx = await setupForDialect(dialect); const registry = new SchemaRegistry(ctx.db); await registry.createCollection({ slug: "posts", label: "Posts" }); await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); await registry.createField("posts", { slug: "hero", label: "Hero", type: "image" }); + await registry.createField("posts", { + slug: "shared_hero", + label: "Shared Hero", + type: "image", + translatable: false, + }); await registry.createCollection({ slug: "plain_posts", label: "Plain Posts", supports: [] }); await registry.createField("plain_posts", { slug: "title", @@ -33,6 +41,34 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { type: "string", }); await registry.createField("plain_posts", { slug: "hero", label: "Hero", type: "image" }); + await registry.createCollection({ + slug: "localized_posts", + label: "Localized Posts", + supports: [], + }); + await registry.createField("localized_posts", { + slug: "title", + label: "Title", + type: "string", + translatable: false, + }); + await registry.createField("localized_posts", { + slug: "hero", + label: "Hero", + type: "image", + }); + await registry.createField("localized_posts", { + slug: "shared_hero", + label: "Shared Hero", + type: "image", + translatable: false, + }); + await registry.createField("localized_posts", { + slug: "summary", + label: "Summary", + type: "string", + translatable: false, + }); runtime = createTestRuntime(ctx.db); usageRepo = new MediaUsageRepository(ctx.db); @@ -40,6 +76,7 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { }); afterEach(async () => { + setI18nConfig(null); await teardownForDialect(ctx); }); @@ -479,8 +516,238 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { expect(await usageRepo.findCurrentUsageByMediaId("media-live-permanent")).toEqual([]); expect(await usageRepo.findCurrentUsageByMediaId("media-draft-permanent")).toEqual([]); }); + + it("refreshes i18n siblings when a non-translatable image syncs across locales", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const { enId, frId } = await createLocalizedPostsPair(runtime, "shared-image", { + enSharedHero: "media-shared-old-en", + frSharedHero: "media-shared-old-fr", + }); + + const updated = await runtime.handleContentUpdate("localized_posts", enId, { + data: { shared_hero: mediaRef("media-shared-new") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-shared-old-en")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-shared-old-fr")).toEqual([]); + expect(await usageRepo.findCurrentUsageByMediaId("media-shared-new")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ contentId: enId, sourceVariant: "columns" }), + occurrence: expect.objectContaining({ fieldPath: "shared_hero" }), + }), + expect.objectContaining({ + source: expect.objectContaining({ contentId: frId, sourceVariant: "columns" }), + occurrence: expect.objectContaining({ fieldPath: "shared_hero" }), + }), + ]), + ); + }); + + it("does not refresh i18n siblings for translatable image updates", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const { enId, frId } = await createLocalizedPostsPair(runtime, "translatable-image", { + enHero: "media-hero-old-en", + frHero: "media-hero-old-fr", + }); + const frSourceBefore = await usageRepo.findSource( + sourceKey("localized_posts", frId, "columns"), + ); + expect(frSourceBefore).not.toBeNull(); + + const updated = await runtime.handleContentUpdate("localized_posts", enId, { + data: { hero: mediaRef("media-hero-new-en") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-hero-new-en")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ contentId: enId, sourceVariant: "columns" }), + }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-hero-old-fr")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ contentId: frId, sourceVariant: "columns" }), + }), + ]); + expect( + (await usageRepo.findSource(sourceKey("localized_posts", frId, "columns"))) + ?.currentGeneration, + ).toBe(frSourceBefore?.currentGeneration); + }); + + it("does not refresh i18n siblings for non-usage non-translatable updates", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const { enId, frId } = await createLocalizedPostsPair(runtime, "non-usage-sync"); + const frSourceBefore = await usageRepo.findSource( + sourceKey("localized_posts", frId, "columns"), + ); + expect(frSourceBefore).not.toBeNull(); + + const updated = await runtime.handleContentUpdate("localized_posts", enId, { + data: { summary: "Shared summary" }, + }); + + expect(updated.success).toBe(true); + expect( + (await usageRepo.findSource(sourceKey("localized_posts", frId, "columns"))) + ?.currentGeneration, + ).toBe(frSourceBefore?.currentGeneration); + }); + + it("does not refresh i18n siblings for revision-enabled draft saves", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const en = await runtime.handleContentCreate("posts", { + slug: "draft-sibling-en", + locale: "en", + data: { + title: "English Draft Sibling", + shared_hero: mediaRef("media-draft-sibling-old-en"), + }, + }); + expect(en.success).toBe(true); + if (!en.success) throw new Error(en.error.message); + const fr = await runtime.handleContentCreate("posts", { + slug: "draft-sibling-fr", + locale: "fr", + translationOf: en.data.item.id, + data: { + title: "French Draft Sibling", + shared_hero: mediaRef("media-draft-sibling-old-fr"), + }, + }); + expect(fr.success).toBe(true); + if (!fr.success) throw new Error(fr.error.message); + const frSourceBefore = await usageRepo.findSource( + sourceKey("posts", fr.data.item.id, "columns"), + ); + expect(frSourceBefore).not.toBeNull(); + + const updated = await runtime.handleContentUpdate("posts", en.data.item.id, { + data: { shared_hero: mediaRef("media-draft-sibling-new") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-sibling-new")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ + contentId: en.data.item.id, + sourceVariant: "draft_overlay", + }), + }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-sibling-old-fr")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ + contentId: fr.data.item.id, + sourceVariant: "columns", + }), + }), + ]); + expect( + (await usageRepo.findSource(sourceKey("posts", fr.data.item.id, "columns"))) + ?.currentGeneration, + ).toBe(frSourceBefore?.currentGeneration); + }); + + it("refreshes trashed i18n siblings when non-translatable fields sync to them", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const { enId, frId } = await createLocalizedPostsPair(runtime, "trashed-sibling", { + enSharedHero: "media-trash-old-en", + frSharedHero: "media-trash-old-fr", + }); + const deleted = await runtime.handleContentDelete("localized_posts", frId); + expect(deleted.success).toBe(true); + + const updated = await runtime.handleContentUpdate("localized_posts", enId, { + data: { shared_hero: mediaRef("media-trash-new") }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findCurrentUsageByMediaId("media-trash-old-fr")).toEqual([]); + expect(await usageRepo.findSource(sourceKey("localized_posts", frId, "columns"))).toEqual( + expect.objectContaining({ + contentDeletedAt: expect.any(String), + contentId: frId, + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-trash-new")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + source: expect.objectContaining({ contentId: enId, contentDeletedAt: null }), + }), + expect.objectContaining({ + source: expect.objectContaining({ + contentId: frId, + contentDeletedAt: expect.any(String), + }), + }), + ]), + ); + }); + + it("refreshes i18n sibling source metadata for non-translatable display fields", async () => { + setI18nConfig({ defaultLocale: "en", locales: ["en", "fr"] }); + const { enId, frId } = await createLocalizedPostsPair(runtime, "shared-title", { + enTitle: "English Title", + frTitle: "French Title", + }); + + const updated = await runtime.handleContentUpdate("localized_posts", enId, { + data: { title: "Shared Title" }, + }); + + expect(updated.success).toBe(true); + expect(await usageRepo.findSource(sourceKey("localized_posts", enId, "columns"))).toEqual( + expect.objectContaining({ contentTitle: "Shared Title" }), + ); + expect(await usageRepo.findSource(sourceKey("localized_posts", frId, "columns"))).toEqual( + expect.objectContaining({ contentTitle: "Shared Title" }), + ); + }); }); +async function createLocalizedPostsPair( + runtime: EmDashRuntime, + slugPrefix: string, + input: { + enTitle?: string; + frTitle?: string; + enHero?: string; + frHero?: string; + enSharedHero?: string; + frSharedHero?: string; + } = {}, +): Promise<{ enId: string; frId: string }> { + const en = await runtime.handleContentCreate("localized_posts", { + slug: `${slugPrefix}-en`, + locale: "en", + data: { + title: input.enTitle ?? "English Post", + hero: mediaRef(input.enHero ?? `${slugPrefix}-hero-en`), + shared_hero: mediaRef(input.enSharedHero ?? `${slugPrefix}-shared-en`), + }, + }); + expect(en.success).toBe(true); + if (!en.success) throw new Error(en.error.message); + + const fr = await runtime.handleContentCreate("localized_posts", { + slug: `${slugPrefix}-fr`, + locale: "fr", + translationOf: en.data.item.id, + data: { + title: input.frTitle ?? "French Post", + hero: mediaRef(input.frHero ?? `${slugPrefix}-hero-fr`), + shared_hero: mediaRef(input.frSharedHero ?? `${slugPrefix}-shared-fr`), + }, + }); + expect(fr.success).toBe(true); + if (!fr.success) throw new Error(fr.error.message); + + return { enId: en.data.item.id, frId: fr.data.item.id }; +} + function mediaRef(id: string): Record { return { id, From 7b6dbb91eb4d7f79e919fec0d4c484a36d7f4229 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 14:35:04 +0100 Subject: [PATCH 22/29] Mark Bypass And Schema Paths Stale --- .../api/import/wordpress/rewrite-urls.ts | 220 +++++----- .../src/database/repositories/media-usage.ts | 10 + .../core/src/media/usage/content-refresh.ts | 113 ++++- packages/core/src/plugins/context.ts | 171 ++++---- packages/core/src/schema/registry.ts | 394 +++++++++++------- packages/core/src/seed/apply.ts | 255 +++++++----- .../media-usage-content-refresh.test.ts | 7 +- .../database/media-usage-stale-bypass.test.ts | 355 ++++++++++++++++ 8 files changed, 1098 insertions(+), 427 deletions(-) create mode 100644 packages/core/tests/integration/database/media-usage-stale-bypass.test.ts diff --git a/packages/core/src/astro/routes/api/import/wordpress/rewrite-urls.ts b/packages/core/src/astro/routes/api/import/wordpress/rewrite-urls.ts index 182fd91c5c..87ae6da3d4 100644 --- a/packages/core/src/astro/routes/api/import/wordpress/rewrite-urls.ts +++ b/packages/core/src/astro/routes/api/import/wordpress/rewrite-urls.ts @@ -20,6 +20,7 @@ import { wpRewriteUrlsBody } from "#api/schemas.js"; import { validateIdentifier } from "#db/validate.js"; import { normalizeMediaValue } from "#media/normalize.js"; import type { MediaProvider } from "#media/types.js"; +import { markContentMediaUsageCollectionStaleSafely } from "#media/usage/content-refresh.js"; import type { EmDashHandlers } from "#types"; import { @@ -77,11 +78,12 @@ export const POST: APIRoute = async ({ request, locals }) => { } }; -async function rewriteUrls( +export async function rewriteUrls( db: NonNullable, urlMap: Record, getProvider: (id: string) => MediaProvider | undefined, collections?: string[], + markUsageCollectionStale: typeof markContentMediaUsageCollectionStaleSafely = markContentMediaUsageCollectionStaleSafely, ): Promise { const { SchemaRegistry } = await import("#schema/registry.js"); const registry = new SchemaRegistry(db); @@ -92,6 +94,18 @@ async function rewriteUrls( urlsRewritten: 0, errors: [], }; + const staleMarkedCollections = new Set(); + const staleMarkFailedCollections = new Set(); + const markCollectionStale = async (collectionSlug: string): Promise => { + if (staleMarkedCollections.has(collectionSlug)) return; + const marked = await markUsageCollectionStale(db, collectionSlug, "CONTENT_USAGE_STALE"); + if (marked) { + staleMarkedCollections.add(collectionSlug); + staleMarkFailedCollections.delete(collectionSlug); + } else { + staleMarkFailedCollections.add(collectionSlug); + } + }; // Build base URL map for flexible matching const baseMap = buildBaseUrlMap(urlMap); @@ -102,52 +116,67 @@ async function rewriteUrls( ? allCollections.filter((c) => collections.includes(c.slug)) : allCollections; - for (const collection of targetCollections) { - // Get fields that might contain URLs - const fields = await registry.listFields(collection.id); - const portableTextFields = fields.filter((f) => f.type === "portableText"); - const stringFields = fields.filter((f) => ["text", "string"].includes(f.type)); - // Image and file fields store URLs directly as TEXT - const mediaFields = fields.filter((f) => ["image", "file"].includes(f.type)); - - if (portableTextFields.length === 0 && stringFields.length === 0 && mediaFields.length === 0) - continue; - - // Get table name - validateIdentifier(collection.slug, "collection slug"); - const tableName = `ec_${collection.slug}`; - - try { - // Query all rows - const rows = await sql<{ id: string; [key: string]: unknown }>` + try { + for (const collection of targetCollections) { + // Get fields that might contain URLs + const fields = await registry.listFields(collection.id); + const portableTextFields = fields.filter((f) => f.type === "portableText"); + const stringFields = fields.filter((f) => ["text", "string"].includes(f.type)); + // Image and file fields store URLs directly as TEXT + const mediaFields = fields.filter((f) => ["image", "file"].includes(f.type)); + + if (portableTextFields.length === 0 && stringFields.length === 0 && mediaFields.length === 0) + continue; + + // Get table name + validateIdentifier(collection.slug, "collection slug"); + const tableName = `ec_${collection.slug}`; + + try { + // Query all rows + const rows = await sql<{ id: string; [key: string]: unknown }>` SELECT * FROM ${sql.ref(tableName)} WHERE deleted_at IS NULL `.execute(db); - for (const row of rows.rows) { - let rowUpdated = false; - const updates: Record = {}; - let rowUrlsRewritten = 0; + for (const row of rows.rows) { + let rowUpdated = false; + const updates: Record = {}; + let rowUrlsRewritten = 0; - // Handle Portable Text fields - parse JSON and rewrite URLs in blocks - for (const field of portableTextFields) { - const value = row[field.slug]; - if (!value || typeof value !== "string") continue; + // Handle Portable Text fields - parse JSON and rewrite URLs in blocks + for (const field of portableTextFields) { + const value = row[field.slug]; + if (!value || typeof value !== "string") continue; - try { - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- JSON.parse returns unknown; validated by Array.isArray below - const blocks = JSON.parse(value) as PortableTextBlock[]; - if (!Array.isArray(blocks)) continue; + try { + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- JSON.parse returns unknown; validated by Array.isArray below + const blocks = JSON.parse(value) as PortableTextBlock[]; + if (!Array.isArray(blocks)) continue; - const rewriteResult = rewritePortableTextUrls(blocks, urlMap, baseMap); + const rewriteResult = rewritePortableTextUrls(blocks, urlMap, baseMap); - if (rewriteResult.changed) { - updates[field.slug] = JSON.stringify(blocks); - rowUpdated = true; - rowUrlsRewritten += rewriteResult.urlsRewritten; + if (rewriteResult.changed) { + updates[field.slug] = JSON.stringify(blocks); + rowUpdated = true; + rowUrlsRewritten += rewriteResult.urlsRewritten; + } + } catch { + // Not valid JSON, try string replacement as fallback + const stringResult = rewriteStringUrls(value, urlMap, baseMap); + if (stringResult.changed) { + updates[field.slug] = stringResult.newValue; + rowUpdated = true; + rowUrlsRewritten += stringResult.urlsRewritten; + } } - } catch { - // Not valid JSON, try string replacement as fallback + } + + // Handle string/text fields - simple string replacement + for (const field of stringFields) { + const value = row[field.slug]; + if (!value || typeof value !== "string") continue; + const stringResult = rewriteStringUrls(value, urlMap, baseMap); if (stringResult.changed) { updates[field.slug] = stringResult.newValue; @@ -155,74 +184,69 @@ async function rewriteUrls( rowUrlsRewritten += stringResult.urlsRewritten; } } - } - - // Handle string/text fields - simple string replacement - for (const field of stringFields) { - const value = row[field.slug]; - if (!value || typeof value !== "string") continue; - - const stringResult = rewriteStringUrls(value, urlMap, baseMap); - if (stringResult.changed) { - updates[field.slug] = stringResult.newValue; - rowUpdated = true; - rowUrlsRewritten += stringResult.urlsRewritten; - } - } - // Handle image/file fields - normalize to MediaValue objects - for (const field of mediaFields) { - const value = row[field.slug]; - if (!value || typeof value !== "string") continue; - - // Values are stored as JSON MediaValue objects (e.g. featured_image from - // import normalizes to {"provider":"external","src":""}). Match on the - // inner `src`, falling back to the raw value for legacy bare-URL rows. - const newUrl = findMatchingUrl(extractMediaUrl(value), urlMap, baseMap); - if (newUrl) { - // Normalize into a proper MediaValue instead of storing a bare URL - try { - const normalized = await normalizeMediaValue(newUrl, getProvider); - updates[field.slug] = normalized ? JSON.stringify(normalized) : newUrl; - } catch { - updates[field.slug] = newUrl; + // Handle image/file fields - normalize to MediaValue objects + for (const field of mediaFields) { + const value = row[field.slug]; + if (!value || typeof value !== "string") continue; + + // Values are stored as JSON MediaValue objects (e.g. featured_image from + // import normalizes to {"provider":"external","src":""}). Match on the + // inner `src`, falling back to the raw value for legacy bare-URL rows. + const newUrl = findMatchingUrl(extractMediaUrl(value), urlMap, baseMap); + if (newUrl) { + // Normalize into a proper MediaValue instead of storing a bare URL + try { + const normalized = await normalizeMediaValue(newUrl, getProvider); + updates[field.slug] = normalized ? JSON.stringify(normalized) : newUrl; + } catch { + updates[field.slug] = newUrl; + } + rowUpdated = true; + rowUrlsRewritten++; } - rowUpdated = true; - rowUrlsRewritten++; } - } - - if (rowUpdated) { - try { - // Build update query dynamically - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- Kysely dynamic table requires type assertion - let query = db.updateTable(tableName as any).where("id", "=", row.id); - for (const [key, value] of Object.entries(updates)) { - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- Kysely dynamic column update requires type assertion - query = query.set({ [key]: value } as any); + if (rowUpdated) { + try { + // Build update query dynamically + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- Kysely dynamic table requires type assertion + let query = db.updateTable(tableName as any).where("id", "=", row.id); + + for (const [key, value] of Object.entries(updates)) { + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- Kysely dynamic column update requires type assertion + query = query.set({ [key]: value } as any); + } + + await query.execute(); + + result.updated++; + result.urlsRewritten += rowUrlsRewritten; + result.byCollection[collection.slug] = + (result.byCollection[collection.slug] || 0) + 1; + await markCollectionStale(collection.slug); + } catch (updateError) { + result.errors.push({ + collection: collection.slug, + id: row.id, + error: updateError instanceof Error ? updateError.message : "Update failed", + }); } - - await query.execute(); - - result.updated++; - result.urlsRewritten += rowUrlsRewritten; - result.byCollection[collection.slug] = (result.byCollection[collection.slug] || 0) + 1; - } catch (updateError) { - result.errors.push({ - collection: collection.slug, - id: row.id, - error: updateError instanceof Error ? updateError.message : "Update failed", - }); } } + } catch (queryError) { + result.errors.push({ + collection: collection.slug, + id: "*", + error: queryError instanceof Error ? queryError.message : "Query failed for collection", + }); } - } catch (queryError) { - result.errors.push({ - collection: collection.slug, - id: "*", - error: queryError instanceof Error ? queryError.message : "Query failed for collection", - }); + } + } finally { + for (const collectionSlug of staleMarkFailedCollections) { + if (staleMarkedCollections.has(collectionSlug)) continue; + const marked = await markUsageCollectionStale(db, collectionSlug, "CONTENT_USAGE_STALE"); + if (marked) staleMarkedCollections.add(collectionSlug); } } diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 1849aed6b3..e4cd06d49c 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -602,6 +602,16 @@ export class MediaUsageRepository { return row ? rowToIndexStatus(row) : null; } + async deleteIndexStatus(identity: MediaUsageIndexStatusIdentity): Promise { + const result = await this.db + .deleteFrom("_emdash_media_usage_index_status") + .where("adapter_id", "=", identity.adapterId) + .where("scope_type", "=", identity.scopeType) + .where("scope_key", "=", identity.scopeKey) + .executeTakeFirst(); + return Number(result.numDeletedRows ?? 0); + } + private async findCurrentUsagePage( applyFilter: ( query: ReturnType, diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts index 0dee3c844a..96a2f627b6 100644 --- a/packages/core/src/media/usage/content-refresh.ts +++ b/packages/core/src/media/usage/content-refresh.ts @@ -18,6 +18,7 @@ export const CONTENT_MEDIA_USAGE_ADAPTER_ID = "content-media"; export const CONTENT_MEDIA_USAGE_COLLECTION_SCOPE = "collection"; const CONTENT_USAGE_LOCKS_KEY = Symbol.for("emdash.mediaUsage.contentLocks"); +const CONTENT_USAGE_COLLECTION_LOCKS_KEY = Symbol.for("emdash.mediaUsage.collectionLocks"); export type ContentMediaUsageRefreshErrorCode = | "CONTENT_NOT_FOUND" @@ -49,8 +50,10 @@ export async function refreshContentMediaUsage( contentId: string, ): Promise { validateIdentifier(collectionSlug, "collection slug"); - return withContentUsageLock(collectionSlug, contentId, () => - refreshContentMediaUsageUnlocked(db, collectionSlug, contentId), + return withContentUsageCollectionLock(collectionSlug, () => + withContentUsageLock(collectionSlug, contentId, () => + refreshContentMediaUsageUnlocked(db, collectionSlug, contentId), + ), ); } @@ -66,9 +69,18 @@ async function refreshContentMediaUsageUnlocked( } const repo = new MediaUsageRepository(db); + if (!(await contentCollectionExists(db, collectionSlug))) { + const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); + return { ...ZERO_RESULT, deletedSourceCount }; + } + for (const snapshot of snapshotsResult.snapshots) { await repo.replaceSource(snapshot.source, snapshot.occurrences); } + if (!(await contentCollectionExists(db, collectionSlug))) { + const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); + return { ...ZERO_RESULT, deletedSourceCount }; + } const expectedSourceKeys = new Set( snapshotsResult.snapshots.map((snapshot) => snapshot.source.sourceKey), @@ -101,6 +113,18 @@ async function refreshContentMediaUsageUnlocked( } } +async function contentCollectionExists( + db: Kysely, + collectionSlug: string, +): Promise { + const row = await db + .selectFrom("_emdash_collections") + .select("id") + .where("slug", "=", collectionSlug) + .executeTakeFirst(); + return row !== undefined; +} + export async function deleteContentMediaUsage( db: Kysely, collectionSlug: string, @@ -143,6 +167,53 @@ async function deleteContentMediaUsageUnlocked( } } +export async function deleteContentMediaUsageCollection( + db: Kysely, + collectionSlug: string, +): Promise { + validateIdentifier(collectionSlug, "collection slug"); + return withContentUsageCollectionLock(collectionSlug, () => + deleteContentMediaUsageCollectionUnlocked(db, collectionSlug), + ); +} + +async function deleteContentMediaUsageCollectionUnlocked( + db: Kysely, + collectionSlug: string, +): Promise { + try { + const repo = new MediaUsageRepository(db); + const deletedSourceCount = await repo.deleteCollectionSources(collectionSlug); + await repo.deleteIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: collectionSlug, + }); + return { ...ZERO_RESULT, deletedSourceCount }; + } catch (error) { + console.error(`[media-usage] Failed to delete usage for collection ${collectionSlug}:`, error); + try { + await new MediaUsageRepository(db).deleteIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: collectionSlug, + }); + } catch (statusError) { + console.error( + `[media-usage] Failed to clear usage status for deleted collection ${collectionSlug}:`, + statusError, + ); + } + return { + success: false, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_DELETE_ERROR", + }; + } +} + export async function refreshContentMediaUsageAfterWrite( db: Kysely, collectionSlug: string, @@ -253,15 +324,17 @@ async function markSnapshotFailure( }; } -async function markContentMediaUsageCollectionStaleSafely( +export async function markContentMediaUsageCollectionStaleSafely( db: Kysely, collectionSlug: string, lastErrorCode: ContentMediaUsageRefreshErrorCode, -): Promise { +): Promise { try { await markContentMediaUsageCollectionStale(db, collectionSlug, lastErrorCode); + return true; } catch (error) { console.error(`[media-usage] Failed to mark ${collectionSlug} stale:`, error); + return false; } } @@ -289,6 +362,29 @@ async function withContentUsageLock( } } +async function withContentUsageCollectionLock( + collectionSlug: string, + fn: () => Promise, +): Promise { + // Coarse by design: row refreshes and collection source deletes must not interleave. + const locks = getContentUsageCollectionLocks(); + const previous = locks.get(collectionSlug) ?? Promise.resolve(); + let releaseCurrent!: () => void; + const current = new Promise((resolve) => { + releaseCurrent = resolve; + }); + const next = previous.catch(() => {}).then(() => current); + locks.set(collectionSlug, next); + + try { + await previous.catch(() => {}); + return await fn(); + } finally { + releaseCurrent(); + if (locks.get(collectionSlug) === next) locks.delete(collectionSlug); + } +} + function getContentUsageLocks(): Map> { const global = globalThis as typeof globalThis & Record; const existing = global[CONTENT_USAGE_LOCKS_KEY]; @@ -297,3 +393,12 @@ function getContentUsageLocks(): Map> { global[CONTENT_USAGE_LOCKS_KEY] = locks; return locks; } + +function getContentUsageCollectionLocks(): Map> { + const global = globalThis as typeof globalThis & Record; + const existing = global[CONTENT_USAGE_COLLECTION_LOCKS_KEY]; + if (existing instanceof Map) return existing as Map>; + const locks = new Map>(); + global[CONTENT_USAGE_COLLECTION_LOCKS_KEY] = locks; + return locks; +} diff --git a/packages/core/src/plugins/context.ts b/packages/core/src/plugins/context.ts index c7a9b53396..123a362b7f 100644 --- a/packages/core/src/plugins/context.ts +++ b/packages/core/src/plugins/context.ts @@ -22,6 +22,7 @@ import { stripCredentialHeaders, } from "../import/ssrf.js"; import { enrichImageMetadata } from "../media/enrich.js"; +import { markContentMediaUsageCollectionStaleSafely } from "../media/usage/content-refresh.js"; import { invalidateSiteSettingsCache } from "../settings/index.js"; import type { Storage } from "../storage/types.js"; import { CronAccessImpl } from "./cron.js"; @@ -294,90 +295,118 @@ export function createContentAccessWithWrite(db: Kysely): ContentAcces async create(collection: string, data: ContentWriteInput): Promise { const { fields, seo } = splitSeoFromInput(data); + let contentMutated = false; - return withTransaction(db, async (trx) => { - const trxContentRepo = new ContentRepository(trx); - const trxSeoRepo = new SeoRepository(trx); - - const hasSeo = await assertSeoEnabled(trxSeoRepo, collection, seo); - - const item = await trxContentRepo.create({ - type: collection, - data: fields, + try { + const created = await withTransaction(db, async (trx) => { + const trxContentRepo = new ContentRepository(trx); + const trxSeoRepo = new SeoRepository(trx); + + const hasSeo = await assertSeoEnabled(trxSeoRepo, collection, seo); + + const item = await trxContentRepo.create({ + type: collection, + data: fields, + }); + contentMutated = true; + + const result: ContentItem = { + id: item.id, + type: item.type, + slug: item.slug, + status: item.status, + data: item.data, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + locale: item.locale, + publishedAt: item.publishedAt, + }; + + if (hasSeo) { + result.seo = + seo !== undefined + ? await trxSeoRepo.upsert(collection, item.id, seo) + : await trxSeoRepo.get(collection, item.id); + } + + return result; }); - - const result: ContentItem = { - id: item.id, - type: item.type, - slug: item.slug, - status: item.status, - data: item.data, - createdAt: item.createdAt, - updatedAt: item.updatedAt, - locale: item.locale, - publishedAt: item.publishedAt, - }; - - if (hasSeo) { - result.seo = - seo !== undefined - ? await trxSeoRepo.upsert(collection, item.id, seo) - : await trxSeoRepo.get(collection, item.id); + await markContentMediaUsageCollectionStaleSafely(db, collection, "CONTENT_USAGE_STALE"); + return created; + } catch (error) { + if (contentMutated) { + await markContentMediaUsageCollectionStaleSafely(db, collection, "CONTENT_USAGE_STALE"); } - - return result; - }); + throw error; + } }, async update(collection: string, id: string, data: ContentWriteInput): Promise { const { fields, seo } = splitSeoFromInput(data); + const hasFieldUpdates = Object.keys(fields).length > 0; + let contentMutated = false; - return withTransaction(db, async (trx) => { - const trxContentRepo = new ContentRepository(trx); - const trxSeoRepo = new SeoRepository(trx); - - const hasSeo = await assertSeoEnabled(trxSeoRepo, collection, seo); - - // Pass the `data` payload to ContentRepository.update only when - // there are field updates — passing an empty object would still - // bump updated_at/version, but we want a seo-only call to touch - // only the SEO table. ContentRepository.update handles the no-op - // path by returning the current row. - const hasFieldUpdates = Object.keys(fields).length > 0; - const item = hasFieldUpdates - ? await trxContentRepo.update(collection, id, { data: fields }) - : await (async () => { - const existing = await trxContentRepo.findById(collection, id); - if (!existing) throw new Error("Content not found"); - return existing; - })(); - - const result: ContentItem = { - id: item.id, - type: item.type, - slug: item.slug, - status: item.status, - data: item.data, - createdAt: item.createdAt, - updatedAt: item.updatedAt, - locale: item.locale, - publishedAt: item.publishedAt, - }; - - if (hasSeo) { - result.seo = - seo !== undefined - ? await trxSeoRepo.upsert(collection, item.id, seo) - : await trxSeoRepo.get(collection, item.id); + try { + const updated = await withTransaction(db, async (trx) => { + const trxContentRepo = new ContentRepository(trx); + const trxSeoRepo = new SeoRepository(trx); + + const hasSeo = await assertSeoEnabled(trxSeoRepo, collection, seo); + + // Pass the `data` payload to ContentRepository.update only when + // there are field updates — passing an empty object would still + // bump updated_at/version, but we want a seo-only call to touch + // only the SEO table. ContentRepository.update handles the no-op + // path by returning the current row. + const item = hasFieldUpdates + ? await trxContentRepo.update(collection, id, { data: fields }) + : await (async () => { + const existing = await trxContentRepo.findById(collection, id); + if (!existing) throw new Error("Content not found"); + return existing; + })(); + if (hasFieldUpdates) contentMutated = true; + + const result: ContentItem = { + id: item.id, + type: item.type, + slug: item.slug, + status: item.status, + data: item.data, + createdAt: item.createdAt, + updatedAt: item.updatedAt, + locale: item.locale, + publishedAt: item.publishedAt, + }; + + if (hasSeo) { + result.seo = + seo !== undefined + ? await trxSeoRepo.upsert(collection, item.id, seo) + : await trxSeoRepo.get(collection, item.id); + } + + return result; + }); + if (hasFieldUpdates) { + await markContentMediaUsageCollectionStaleSafely(db, collection, "CONTENT_USAGE_STALE"); } - - return result; - }); + return updated; + } catch (error) { + if (contentMutated) { + await markContentMediaUsageCollectionStaleSafely(db, collection, "CONTENT_USAGE_STALE"); + } + throw error; + } }, async delete(collection: string, id: string): Promise { const contentRepo = new ContentRepository(db); - return contentRepo.delete(collection, id); + const deleted = await contentRepo.delete(collection, id); + if (deleted) { + await markContentMediaUsageCollectionStaleSafely(db, collection, "CONTENT_USAGE_STALE"); + } + return deleted; }, }; } diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index 390624086f..a98672eefc 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -7,6 +7,10 @@ import { currentTimestamp, listTablesLike, tableExists } from "../database/diale import { withTransaction } from "../database/transaction.js"; import type { CollectionTable, Database, FieldTable } from "../database/types.js"; import { validateIdentifier } from "../database/validate.js"; +import { + deleteContentMediaUsageCollection, + markContentMediaUsageCollectionStaleSafely, +} from "../media/usage/content-refresh.js"; import { FTSManager } from "../search/fts-manager.js"; import { chunks, SQL_BATCH_SIZE } from "../utils/chunks.js"; import { @@ -365,18 +369,28 @@ export class SchemaRegistry { } } - await withTransaction(this.db, async (trx) => { - // Drop FTS table and triggers before dropping the content table - const ftsManager = new FTSManager(trx); - await ftsManager.dropFtsTable(slug); - - // Drop the content table - const tableName = this.getTableName(slug); - await sql`DROP TABLE IF EXISTS ${sql.ref(tableName)}`.execute(trx); - - // Delete the collection record (fields will cascade) - await trx.deleteFrom("_emdash_collections").where("id", "=", existing.id).execute(); - }); + let contentTableDropped = false; + try { + await withTransaction(this.db, async (trx) => { + // Drop FTS table and triggers before dropping the content table + const ftsManager = new FTSManager(trx); + await ftsManager.dropFtsTable(slug); + + // Drop the content table + const tableName = this.getTableName(slug); + await sql`DROP TABLE IF EXISTS ${sql.ref(tableName)}`.execute(trx); + contentTableDropped = true; + + // Delete the collection record (fields will cascade) + await trx.deleteFrom("_emdash_collections").where("id", "=", existing.id).execute(); + }); + await deleteContentMediaUsageCollection(this.db, slug); + } catch (error) { + if (contentTableDropped && !(await tableExists(this.db, this.getTableName(slug)))) { + await deleteContentMediaUsageCollection(this.db, slug); + } + throw error; + } } // ============================================ @@ -451,63 +465,82 @@ export class SchemaRegistry { const sortOrder = input.sortOrder ?? (maxSort?.max ?? -1) + 1; - return withTransaction(this.db, async (trx) => { - // Insert field record - await trx - .insertInto("_emdash_fields") - .values({ - id, - collection_id: collection.id, - slug: input.slug, - label: input.label, - type: input.type, - column_type: columnType, - required: input.required ? 1 : 0, - unique: input.unique ? 1 : 0, - default_value: - input.defaultValue !== undefined ? JSON.stringify(input.defaultValue) : null, - validation: input.validation ? JSON.stringify(input.validation) : null, - widget: input.widget ?? null, - options: input.options ? JSON.stringify(input.options) : null, - sort_order: sortOrder, - searchable: input.searchable ? 1 : 0, - translatable: input.translatable === false ? 0 : 1, - }) - .execute(); + let schemaMutated = false; + try { + const created = await withTransaction(this.db, async (trx) => { + // Insert field record + await trx + .insertInto("_emdash_fields") + .values({ + id, + collection_id: collection.id, + slug: input.slug, + label: input.label, + type: input.type, + column_type: columnType, + required: input.required ? 1 : 0, + unique: input.unique ? 1 : 0, + default_value: + input.defaultValue !== undefined ? JSON.stringify(input.defaultValue) : null, + validation: input.validation ? JSON.stringify(input.validation) : null, + widget: input.widget ?? null, + options: input.options ? JSON.stringify(input.options) : null, + sort_order: sortOrder, + searchable: input.searchable ? 1 : 0, + translatable: input.translatable === false ? 0 : 1, + }) + .execute(); + schemaMutated = true; + + // Add column to content table — pass trx to stay on the same connection + await this.addColumn( + collectionSlug, + input.slug, + input.type, + { + required: input.required, + defaultValue: input.defaultValue, + }, + trx, + ); - // Add column to content table — pass trx to stay on the same connection - await this.addColumn( - collectionSlug, - input.slug, - input.type, - { - required: input.required, - defaultValue: input.defaultValue, - }, - trx, - ); + // Read the created field via trx (not this.db) to avoid connection mutex deadlock + const fieldRow = await trx + .selectFrom("_emdash_fields") + .where("collection_id", "=", collection.id) + .where("slug", "=", input.slug) + .selectAll() + .executeTakeFirst(); - // Read the created field via trx (not this.db) to avoid connection mutex deadlock - const fieldRow = await trx - .selectFrom("_emdash_fields") - .where("collection_id", "=", collection.id) - .where("slug", "=", input.slug) - .selectAll() - .executeTakeFirst(); + if (!fieldRow) { + throw new SchemaError("Failed to create field", "CREATE_FAILED"); + } - if (!fieldRow) { - throw new SchemaError("Failed to create field", "CREATE_FAILED"); - } + const field = this.mapFieldRow(fieldRow); - const field = this.mapFieldRow(fieldRow); + // Sync search state if this field is searchable; support checks are handled by syncSearchState() + if (input.searchable) { + await this.syncSearchState(collectionSlug, trx); + } - // Sync search state if this field is searchable; support checks are handled by syncSearchState() - if (input.searchable) { - await this.syncSearchState(collectionSlug, trx); + return field; + }); + await markContentMediaUsageCollectionStaleSafely( + this.db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); + return created; + } catch (error) { + if (schemaMutated) { + await markContentMediaUsageCollectionStaleSafely( + this.db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); } - - return field; - }); + throw error; + } } /** @@ -553,67 +586,92 @@ export class SchemaRegistry { nextColumnType = newColumnType; } - return withTransaction(this.db, async (trx) => { - await trx - .updateTable("_emdash_fields") - .set({ - type: nextType, - column_type: nextColumnType, - label: input.label ?? field.label, - required: - input.required !== undefined ? (input.required ? 1 : 0) : field.required ? 1 : 0, - unique: input.unique !== undefined ? (input.unique ? 1 : 0) : field.unique ? 1 : 0, - searchable: - input.searchable !== undefined ? (input.searchable ? 1 : 0) : field.searchable ? 1 : 0, - translatable: - input.translatable !== undefined - ? input.translatable - ? 1 - : 0 - : field.translatable - ? 1 - : 0, - default_value: - input.defaultValue !== undefined - ? JSON.stringify(input.defaultValue) - : field.defaultValue !== undefined - ? JSON.stringify(field.defaultValue) + let schemaMutated = false; + try { + const updatedField = await withTransaction(this.db, async (trx) => { + await trx + .updateTable("_emdash_fields") + .set({ + type: nextType, + column_type: nextColumnType, + label: input.label ?? field.label, + required: + input.required !== undefined ? (input.required ? 1 : 0) : field.required ? 1 : 0, + unique: input.unique !== undefined ? (input.unique ? 1 : 0) : field.unique ? 1 : 0, + searchable: + input.searchable !== undefined + ? input.searchable + ? 1 + : 0 + : field.searchable + ? 1 + : 0, + translatable: + input.translatable !== undefined + ? input.translatable + ? 1 + : 0 + : field.translatable + ? 1 + : 0, + default_value: + input.defaultValue !== undefined + ? JSON.stringify(input.defaultValue) + : field.defaultValue !== undefined + ? JSON.stringify(field.defaultValue) + : null, + validation: nextValidation ? JSON.stringify(nextValidation) : null, + widget: input.widget ?? field.widget ?? null, + options: input.options + ? JSON.stringify(input.options) + : field.options + ? JSON.stringify(field.options) : null, - validation: nextValidation ? JSON.stringify(nextValidation) : null, - widget: input.widget ?? field.widget ?? null, - options: input.options - ? JSON.stringify(input.options) - : field.options - ? JSON.stringify(field.options) - : null, - sort_order: input.sortOrder ?? field.sortOrder, - }) - .where("id", "=", field.id) - .execute(); - - // Read the updated field via trx (not this.db) to avoid connection mutex deadlock - const updatedRow = await trx - .selectFrom("_emdash_fields") - .where("collection_id", "=", field.collectionId) - .where("slug", "=", fieldSlug) - .selectAll() - .executeTakeFirst(); + sort_order: input.sortOrder ?? field.sortOrder, + }) + .where("id", "=", field.id) + .execute(); + schemaMutated = true; + + // Read the updated field via trx (not this.db) to avoid connection mutex deadlock + const updatedRow = await trx + .selectFrom("_emdash_fields") + .where("collection_id", "=", field.collectionId) + .where("slug", "=", fieldSlug) + .selectAll() + .executeTakeFirst(); + + if (!updatedRow) { + throw new SchemaError("Failed to update field", "UPDATE_FAILED"); + } - if (!updatedRow) { - throw new SchemaError("Failed to update field", "UPDATE_FAILED"); - } + const updated = this.mapFieldRow(updatedRow); - const updated = this.mapFieldRow(updatedRow); + // If searchable changed, sync FTS state for this collection + const searchableChanged = + input.searchable !== undefined && input.searchable !== field.searchable; + if (searchableChanged) { + await this.syncSearchState(collectionSlug, trx); + } - // If searchable changed, sync FTS state for this collection - const searchableChanged = - input.searchable !== undefined && input.searchable !== field.searchable; - if (searchableChanged) { - await this.syncSearchState(collectionSlug, trx); + return updated; + }); + await markContentMediaUsageCollectionStaleSafely( + this.db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); + return updatedField; + } catch (error) { + if (schemaMutated) { + await markContentMediaUsageCollectionStaleSafely( + this.db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); } - - return updated; - }); + throw error; + } } /** @@ -666,22 +724,40 @@ export class SchemaRegistry { ); } - await withTransaction(this.db, async (trx) => { - // Delete the field record first so syncSearchState sees the updated field list. - // This ordering matters for searchable fields: SQLite prevents dropping a column - // that is still referenced by a trigger. syncSearchState drops and recreates the - // FTS triggers based on the remaining searchable fields, clearing the dependency - // before we attempt the ALTER TABLE DROP COLUMN below. - await trx.deleteFrom("_emdash_fields").where("id", "=", field.id).execute(); - - // If the deleted field was searchable, sync FTS state (removes old triggers) - if (field.searchable) { - await this.syncSearchState(collectionSlug, trx); - } + let schemaMutated = false; + try { + await withTransaction(this.db, async (trx) => { + // Delete the field record first so syncSearchState sees the updated field list. + // This ordering matters for searchable fields: SQLite prevents dropping a column + // that is still referenced by a trigger. syncSearchState drops and recreates the + // FTS triggers based on the remaining searchable fields, clearing the dependency + // before we attempt the ALTER TABLE DROP COLUMN below. + await trx.deleteFrom("_emdash_fields").where("id", "=", field.id).execute(); + schemaMutated = true; + + // If the deleted field was searchable, sync FTS state (removes old triggers) + if (field.searchable) { + await this.syncSearchState(collectionSlug, trx); + } - // Drop column from content table — safe now because FTS triggers are gone - await this.dropColumn(collectionSlug, fieldSlug, trx); - }); + // Drop column from content table — safe now because FTS triggers are gone + await this.dropColumn(collectionSlug, fieldSlug, trx); + }); + await markContentMediaUsageCollectionStaleSafely( + this.db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); + } catch (error) { + if (schemaMutated) { + await markContentMediaUsageCollectionStaleSafely( + this.db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); + } + throw error; + } } /** @@ -1142,28 +1218,38 @@ export class SchemaRegistry { const id = ulid(); const label = options?.label || this.slugToLabel(slug); - await this.db - .insertInto("_emdash_collections") - .values({ - id, - slug, - label, - label_singular: options?.labelSingular ?? null, - description: options?.description ?? null, - icon: null, - supports: JSON.stringify([]), - source: "discovered", - has_seo: 0, - url_pattern: null, - }) - .execute(); + let collectionRegistered = false; + try { + await this.db + .insertInto("_emdash_collections") + .values({ + id, + slug, + label, + label_singular: options?.labelSingular ?? null, + description: options?.description ?? null, + icon: null, + supports: JSON.stringify([]), + source: "discovered", + has_seo: 0, + url_pattern: null, + }) + .execute(); + collectionRegistered = true; - const collection = await this.getCollection(slug); - if (!collection) { - throw new SchemaError("Failed to register orphaned table", "REGISTER_FAILED"); - } + const collection = await this.getCollection(slug); + if (!collection) { + throw new SchemaError("Failed to register orphaned table", "REGISTER_FAILED"); + } + await markContentMediaUsageCollectionStaleSafely(this.db, slug, "CONTENT_USAGE_STALE"); - return collection; + return collection; + } catch (error) { + if (collectionRegistered) { + await markContentMediaUsageCollectionStaleSafely(this.db, slug, "CONTENT_USAGE_STALE"); + } + throw error; + } } /** diff --git a/packages/core/src/seed/apply.ts b/packages/core/src/seed/apply.ts index f060a93db2..3ab8774e8a 100644 --- a/packages/core/src/seed/apply.ts +++ b/packages/core/src/seed/apply.ts @@ -21,6 +21,7 @@ import type { Database } from "../database/types.js"; import type { MediaValue } from "../fields/types.js"; import { getI18nConfig } from "../i18n/config.js"; import { ssrfSafeFetch, validateExternalUrl } from "../import/ssrf.js"; +import { markContentMediaUsageCollectionStaleSafely } from "../media/usage/content-refresh.js"; import { SchemaRegistry } from "../schema/registry.js"; import { FTSManager } from "../search/fts-manager.js"; import { setSiteSettings } from "../settings/index.js"; @@ -114,6 +115,8 @@ export async function applySeed( // Track seed content IDs for reference resolution (shared across content and menus) const seedIdMap = new Map(); // seed id -> real entry id const seedBylineIdMap = new Map(); // seed byline id -> real byline id + const staleMarkedContentCollections = new Set(); + const failedStaleContentCollections = new Set(); // Fallback locale for rows that omit an explicit `locale`. Prefer the runtime // config (runtime-driven seeds), then the seed's self-described `defaultLocale` @@ -121,6 +124,33 @@ export async function applySeed( // seed-carried default, a non-`en` single-locale project would be rewritten to // `en` on apply (#1421). const defaultLocale = getI18nConfig()?.defaultLocale ?? seed.defaultLocale ?? "en"; + const markSeedContentCollectionStale = async (collectionSlug: string): Promise => { + if (staleMarkedContentCollections.has(collectionSlug)) return; + const marked = await markContentMediaUsageCollectionStaleSafely( + db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); + if (marked) { + staleMarkedContentCollections.add(collectionSlug); + failedStaleContentCollections.delete(collectionSlug); + } else { + failedStaleContentCollections.add(collectionSlug); + } + }; + const retryFailedSeedContentStaleMarks = async (): Promise => { + for (const collectionSlug of failedStaleContentCollections) { + const marked = await markContentMediaUsageCollectionStaleSafely( + db, + collectionSlug, + "CONTENT_USAGE_STALE", + ); + if (marked) { + staleMarkedContentCollections.add(collectionSlug); + failedStaleContentCollections.delete(collectionSlug); + } + } + }; // 1. Site settings if (seed.settings) { @@ -422,132 +452,159 @@ export async function applySeed( if (includeContent && seed.content) { const contentRepo = new ContentRepository(db); - // Create content entries - for (const [collectionSlug, entries] of Object.entries(seed.content)) { - for (const entry of entries) { - // Resolve the entry's locale up front so a non-`en` single-locale - // export (which omits `locale`) is filed under the project default - // rather than `en` (#1421). - const entryLocale = entry.locale ?? defaultLocale; - - // Check if entry exists (by slug + locale for locale-aware lookup) - const existing = await contentRepo.findBySlug(collectionSlug, entry.slug, entryLocale); - - if (existing) { - if (onConflict === "error") { - throw new Error( - `Conflict: content "${entry.slug}" in "${collectionSlug}" already exists`, - ); + try { + // Create content entries + for (const [collectionSlug, entries] of Object.entries(seed.content)) { + for (const entry of entries) { + // Resolve the entry's locale up front so a non-`en` single-locale + // export (which omits `locale`) is filed under the project default + // rather than `en` (#1421). + const entryLocale = entry.locale ?? defaultLocale; + + // Check if entry exists (by slug + locale for locale-aware lookup) + const existing = await contentRepo.findBySlug(collectionSlug, entry.slug, entryLocale); + + if (existing) { + if (onConflict === "error") { + throw new Error( + `Conflict: content "${entry.slug}" in "${collectionSlug}" already exists`, + ); + } + + if (onConflict === "update") { + // Resolve $ref and $media in data + const resolvedData = await resolveReferences( + entry.data, + seedIdMap, + mediaContext, + result, + ); + + // Update content + bylines + taxonomies atomically + const status = entry.status || "published"; + let contentMutated = false; + try { + await withTransaction(db, async (trx) => { + const trxContentRepo = new ContentRepository(trx); + const trxBylineRepo = new BylineRepository(trx); + const trxRevisionRepo = new RevisionRepository(trx); + + await trxContentRepo.update(collectionSlug, existing.id, { + status, + data: resolvedData, + }); + contentMutated = true; + + await applyContentBylines( + trxBylineRepo, + collectionSlug, + existing.id, + entry, + seedBylineIdMap, + true, + ); + await applyContentTaxonomies(trx, collectionSlug, existing.id, entry, true); + + // Seed is declarative — when status is "published", promote to a live + // revision so the admin UI shows "Unpublish" instead of "Save & Publish" + // and `live_revision_id` is populated for downstream queries. + // + // Create a fresh revision from the updated data and stage it as the + // draft so `publish()` picks it up instead of re-syncing stale data + // from an existing live revision. + if (status === "published") { + const draft = await trxRevisionRepo.create({ + collection: collectionSlug, + entryId: existing.id, + data: resolvedData, + }); + await trxContentRepo.setDraftRevision(collectionSlug, existing.id, draft.id); + await trxContentRepo.publish(collectionSlug, existing.id); + } + }); + } catch (error) { + if (contentMutated) await markSeedContentCollectionStale(collectionSlug); + throw error; + } + + seedIdMap.set(entry.id, existing.id); + result.content.updated++; + await markSeedContentCollectionStale(collectionSlug); + continue; + } + + // skip + result.content.skipped++; + seedIdMap.set(entry.id, existing.id); + continue; } - if (onConflict === "update") { - // Resolve $ref and $media in data - const resolvedData = await resolveReferences( - entry.data, - seedIdMap, - mediaContext, - result, - ); + // Resolve $ref and $media in data + const resolvedData = await resolveReferences(entry.data, seedIdMap, mediaContext, result); - // Update content + bylines + taxonomies atomically - const status = entry.status || "published"; - await withTransaction(db, async (trx) => { + // Resolve translationOf: map from seed-local ID to real EmDash ID + let translationOf: string | undefined; + if (entry.translationOf) { + const sourceId = seedIdMap.get(entry.translationOf); + if (!sourceId) { + console.warn( + `content.${collectionSlug}: translationOf "${entry.translationOf}" not found (not yet created or missing). Skipping translation link.`, + ); + } else { + translationOf = sourceId; + } + } + + // Create entry + bylines + taxonomies atomically + const status = entry.status || "published"; + let contentMutated = false; + let created: Awaited>; + try { + created = await withTransaction(db, async (trx) => { const trxContentRepo = new ContentRepository(trx); const trxBylineRepo = new BylineRepository(trx); - const trxRevisionRepo = new RevisionRepository(trx); - await trxContentRepo.update(collectionSlug, existing.id, { + const item = await trxContentRepo.create({ + type: collectionSlug, + slug: entry.slug, status, data: resolvedData, + locale: entryLocale, + translationOf, + publishedAt: status === "published" ? new Date().toISOString() : null, }); + contentMutated = true; await applyContentBylines( trxBylineRepo, collectionSlug, - existing.id, + item.id, entry, seedBylineIdMap, - true, ); - await applyContentTaxonomies(trx, collectionSlug, existing.id, entry, true); + await applyContentTaxonomies(trx, collectionSlug, item.id, entry, false); // Seed is declarative — when status is "published", promote to a live // revision so the admin UI shows "Unpublish" instead of "Save & Publish" // and `live_revision_id` is populated for downstream queries. - // - // Create a fresh revision from the updated data and stage it as the - // draft so `publish()` picks it up instead of re-syncing stale data - // from an existing live revision. if (status === "published") { - const draft = await trxRevisionRepo.create({ - collection: collectionSlug, - entryId: existing.id, - data: resolvedData, - }); - await trxContentRepo.setDraftRevision(collectionSlug, existing.id, draft.id); - await trxContentRepo.publish(collectionSlug, existing.id); + await trxContentRepo.publish(collectionSlug, item.id); } - }); - seedIdMap.set(entry.id, existing.id); - result.content.updated++; - continue; + return item; + }); + } catch (error) { + if (contentMutated) await markSeedContentCollectionStale(collectionSlug); + throw error; } - // skip - result.content.skipped++; - seedIdMap.set(entry.id, existing.id); - continue; + seedIdMap.set(entry.id, created.id); + result.content.created++; + await markSeedContentCollectionStale(collectionSlug); } - - // Resolve $ref and $media in data - const resolvedData = await resolveReferences(entry.data, seedIdMap, mediaContext, result); - - // Resolve translationOf: map from seed-local ID to real EmDash ID - let translationOf: string | undefined; - if (entry.translationOf) { - const sourceId = seedIdMap.get(entry.translationOf); - if (!sourceId) { - console.warn( - `content.${collectionSlug}: translationOf "${entry.translationOf}" not found (not yet created or missing). Skipping translation link.`, - ); - } else { - translationOf = sourceId; - } - } - - // Create entry + bylines + taxonomies atomically - const status = entry.status || "published"; - const created = await withTransaction(db, async (trx) => { - const trxContentRepo = new ContentRepository(trx); - const trxBylineRepo = new BylineRepository(trx); - - const item = await trxContentRepo.create({ - type: collectionSlug, - slug: entry.slug, - status, - data: resolvedData, - locale: entryLocale, - translationOf, - publishedAt: status === "published" ? new Date().toISOString() : null, - }); - - await applyContentBylines(trxBylineRepo, collectionSlug, item.id, entry, seedBylineIdMap); - await applyContentTaxonomies(trx, collectionSlug, item.id, entry, false); - - // Seed is declarative — when status is "published", promote to a live - // revision so the admin UI shows "Unpublish" instead of "Save & Publish" - // and `live_revision_id` is populated for downstream queries. - if (status === "published") { - await trxContentRepo.publish(collectionSlug, item.id); - } - - return item; - }); - - seedIdMap.set(entry.id, created.id); - result.content.created++; } + } finally { + await retryFailedSeedContentStaleMarks(); } } diff --git a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts index 4cf86af5db..6c0b3b252e 100644 --- a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts @@ -81,7 +81,12 @@ describeEachDialect("content media usage refresh", (dialect) => { scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, scopeKey: "posts", }), - ).toBeNull(); + ).toEqual( + expect.objectContaining({ + status: "stale", + lastErrorCode: "CONTENT_USAGE_STALE", + }), + ); await updatePostHero(ctx, item.id, { id: "media-new", diff --git a/packages/core/tests/integration/database/media-usage-stale-bypass.test.ts b/packages/core/tests/integration/database/media-usage-stale-bypass.test.ts new file mode 100644 index 0000000000..039050b8bf --- /dev/null +++ b/packages/core/tests/integration/database/media-usage-stale-bypass.test.ts @@ -0,0 +1,355 @@ +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it } from "vitest"; + +import { rewriteUrls } from "../../../src/astro/routes/api/import/wordpress/rewrite-urls.js"; +import { ContentRepository } from "../../../src/database/repositories/content.js"; +import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { + CONTENT_MEDIA_USAGE_ADAPTER_ID, + CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + markContentMediaUsageCollectionStaleSafely, +} from "../../../src/media/usage/content-refresh.js"; +import { + buildContentMediaUsageSourceKey, + type MediaUsageContentSourceVariant, +} from "../../../src/media/usage/source-key.js"; +import { createContentAccessWithWrite } from "../../../src/plugins/context.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { applySeed } from "../../../src/seed/apply.js"; +import type { SeedFile } from "../../../src/seed/types.js"; +import { + describeEachDialect, + setupForDialect, + teardownForDialect, + type DialectTestContext, +} from "../../utils/test-db.js"; + +describeEachDialect("media usage stale marking for bypass writes", (dialect) => { + let ctx: DialectTestContext; + let registry: SchemaRegistry; + let usageRepo: MediaUsageRepository; + + beforeEach(async () => { + ctx = await setupForDialect(dialect); + registry = new SchemaRegistry(ctx.db); + usageRepo = new MediaUsageRepository(ctx.db); + await createCollectionWithFields("posts"); + await createCollectionWithFields("pages"); + }); + + afterEach(async () => { + await teardownForDialect(ctx); + }); + + it("marks touched collections stale after seed content writes", async () => { + await markComplete("posts"); + const seed: SeedFile = { + version: "1", + content: { + posts: [ + { + id: "seed-post", + slug: "seed-post", + data: { + title: "Seed Post", + hero: mediaRef("media-seed"), + }, + }, + ], + }, + }; + + await applySeed(ctx.db, seed, { includeContent: true }); + + await expectCollectionStatus("posts", "stale"); + }); + + it("marks seed-touched collections stale even when a later content entry fails", async () => { + await markComplete("posts"); + const seed: SeedFile = { + version: "1", + content: { + posts: [ + { + id: "seed-created-before-failure", + slug: "duplicate-seed-slug", + data: { title: "Created Before Failure" }, + }, + { + id: "seed-conflict", + slug: "duplicate-seed-slug", + data: { title: "Conflict" }, + }, + ], + }, + }; + + await expect( + applySeed(ctx.db, seed, { includeContent: true, onConflict: "error" }), + ).rejects.toThrow(/Conflict: content/); + + await expectCollectionStatus("posts", "stale"); + }); + + it("marks collections stale after plugin content direct writes", async () => { + await markComplete("posts"); + const content = createContentAccessWithWrite(ctx.db); + + await content.create("posts", { + title: "Plugin Post", + hero: mediaRef("media-plugin-create"), + }); + + await expectCollectionStatus("posts", "stale"); + await markComplete("posts"); + + const repo = new ContentRepository(ctx.db); + const item = await repo.create({ + type: "posts", + slug: "plugin-update", + data: { title: "Plugin Update", hero: mediaRef("media-plugin-old") }, + }); + + await content.update("posts", item.id, { hero: mediaRef("media-plugin-new") }); + + await expectCollectionStatus("posts", "stale"); + await markComplete("posts"); + + expect(await content.delete("posts", item.id)).toBe(true); + + await expectCollectionStatus("posts", "stale"); + }); + + it("marks collections stale after schema field mutations", async () => { + await markComplete("posts"); + + await registry.createField("posts", { slug: "deck", label: "Deck", type: "string" }); + + await expectCollectionStatus("posts", "stale"); + await markComplete("posts"); + + await registry.updateField("posts", "hero", { label: "Hero Image" }); + + await expectCollectionStatus("posts", "stale"); + await markComplete("posts"); + + await registry.deleteField("posts", "deck"); + + await expectCollectionStatus("posts", "stale"); + }); + + it("marks registered orphaned tables stale", async () => { + await sql`CREATE TABLE ec_orphan_posts (id text primary key)`.execute(ctx.db); + + await registry.registerOrphanedTable("orphan_posts"); + + await expectCollectionStatus("orphan_posts", "stale"); + }); + + it("deletes collection usage sources after collection deletion", async () => { + await markComplete("posts"); + await usageRepo.replaceSource(contentSource("posts", "entry-1", "columns"), [ + occurrence("hero", "media-collection-delete"), + ]); + expect(await usageRepo.findSource(sourceKey("posts", "entry-1", "columns"))).not.toBeNull(); + + await registry.deleteCollection("posts", { force: true }); + + expect(await usageRepo.findSource(sourceKey("posts", "entry-1", "columns"))).toBeNull(); + expect(await usageRepo.findCurrentUsageByMediaId("media-collection-delete")).toEqual([]); + expect(await findCollectionStatus("posts")).toBeNull(); + }); + + it("retries failed WordPress rewrite stale marks once after the rewrite pass", async () => { + const repo = new ContentRepository(ctx.db); + const oldUrl = "https://example.com/wp-content/uploads/2026/01/hero.jpg"; + await repo.create({ + type: "posts", + slug: "rewrite-retry-post", + data: { title: "Rewrite Retry Post", body: `` }, + }); + await markComplete("posts"); + let attempts = 0; + + const result = await rewriteUrls( + ctx.db, + { [oldUrl]: "/_emdash/media/file/imported/hero.jpg" }, + () => undefined, + ["posts"], + async (db, collectionSlug, lastErrorCode) => { + attempts++; + if (attempts === 1) return false; + return markContentMediaUsageCollectionStaleSafely(db, collectionSlug, lastErrorCode); + }, + ); + + expect(result.byCollection).toEqual({ posts: 1 }); + expect(attempts).toBe(2); + await expectCollectionStatus("posts", "stale"); + }); + + it("marks only rewritten WordPress URL collections stale", async () => { + const repo = new ContentRepository(ctx.db); + const oldUrl = "https://example.com/wp-content/uploads/2026/01/hero.jpg"; + await repo.create({ + type: "posts", + slug: "rewrite-post", + data: { title: "Rewrite Post", body: `` }, + }); + await repo.create({ + type: "pages", + slug: "clean-page", + data: { title: "Clean Page", body: "No matching media URL" }, + }); + await markComplete("posts"); + await markComplete("pages"); + + const result = await rewriteUrls( + ctx.db, + { [oldUrl]: "/_emdash/media/file/imported/hero.jpg" }, + () => undefined, + ); + + expect(result.byCollection).toEqual({ posts: 1 }); + await expectCollectionStatus("posts", "stale"); + await expectCollectionStatus("pages", "complete"); + }); + + it("marks earlier WordPress rewrite collections stale when a later collection fails", async () => { + const repo = new ContentRepository(ctx.db); + const oldUrl = "https://example.com/wp-content/uploads/2026/01/hero.jpg"; + await repo.create({ + type: "posts", + slug: "rewrite-before-error", + data: { title: "Rewrite Before Error", body: `` }, + }); + await registry.createCollection({ slug: "zz_broken", label: "Broken" }); + const broken = await registry.getCollection("zz_broken"); + expect(broken).not.toBeNull(); + await ctx.db + .insertInto("_emdash_fields") + .values({ + id: "broken_field", + collection_id: broken!.id, + slug: "bad_repeater", + label: "Bad Repeater", + type: "repeater", + column_type: "JSON", + required: 0, + unique: 0, + default_value: null, + validation: "{", + widget: null, + options: null, + sort_order: 0, + searchable: 0, + translatable: 1, + }) + .execute(); + await markComplete("posts"); + let staleMarkAttempts = 0; + + await expect( + rewriteUrls( + ctx.db, + { [oldUrl]: "/_emdash/media/file/imported/hero.jpg" }, + () => undefined, + ["posts", "zz_broken"], + async (db, collectionSlug, lastErrorCode) => { + if (collectionSlug !== "posts") { + return markContentMediaUsageCollectionStaleSafely(db, collectionSlug, lastErrorCode); + } + staleMarkAttempts++; + if (staleMarkAttempts === 1) return false; + return markContentMediaUsageCollectionStaleSafely(db, collectionSlug, lastErrorCode); + }, + ), + ).rejects.toThrow(); + + expect(staleMarkAttempts).toBe(2); + await expectCollectionStatus("posts", "stale"); + }); + + async function createCollectionWithFields(slug: string) { + await registry.createCollection({ slug, label: slug }); + await registry.createField(slug, { slug: "title", label: "Title", type: "string" }); + await registry.createField(slug, { slug: "body", label: "Body", type: "text" }); + await registry.createField(slug, { slug: "hero", label: "Hero", type: "image" }); + } + + async function markComplete(collectionSlug: string) { + await usageRepo.upsertIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: collectionSlug, + status: "complete", + schemaVersion: 1, + indexedSourceCount: 1, + failedSourceCount: 0, + }); + } + + async function expectCollectionStatus(collectionSlug: string, status: string) { + await expect(findCollectionStatus(collectionSlug)).resolves.toEqual( + expect.objectContaining({ status }), + ); + } + + async function findCollectionStatus(collectionSlug: string) { + return usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: collectionSlug, + }); + } +}); + +function mediaRef(id: string): Record { + return { + id, + provider: "local", + mimeType: "image/webp", + width: 100, + height: 100, + }; +} + +function sourceKey( + collectionSlug: string, + contentId: string, + sourceVariant: MediaUsageContentSourceVariant, +): string { + return buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }); +} + +function contentSource( + collectionSlug: string, + contentId: string, + sourceVariant: MediaUsageContentSourceVariant, +) { + return { + sourceKey: sourceKey(collectionSlug, contentId, sourceVariant), + sourceType: "content", + collectionSlug, + contentId, + sourceVariant, + contentSlug: "hello-world", + contentTitle: "Hello World", + contentStatus: "published", + schemaVersion: 1, + sourceCompleteness: "complete" as const, + }; +} + +function occurrence(fieldSlug: string, mediaId: string) { + return { + fieldSlug, + fieldPath: fieldSlug, + referenceType: "image_field" as const, + mediaId, + provider: "local", + providerAssetId: mediaId, + mediaKind: "image" as const, + mimeType: "image/webp", + }; +} From 18cdf127b4adc46fd5250d406ae4777954e268df Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 14:45:43 +0100 Subject: [PATCH 23/29] Add Refresh Failure Path Coverage --- .../media-usage-runtime-refresh.test.ts | 116 +++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts index d0a715465e..26bd85b7e0 100644 --- a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -1,10 +1,14 @@ import { sql } from "kysely"; -import { afterEach, beforeEach, expect, it } from "vitest"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; import { RevisionRepository } from "../../../src/database/repositories/revision.js"; import type { EmDashRuntime } from "../../../src/emdash-runtime.js"; import { setI18nConfig } from "../../../src/i18n/config.js"; +import { + CONTENT_MEDIA_USAGE_ADAPTER_ID, + CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, +} from "../../../src/media/usage/content-refresh.js"; import { buildContentMediaUsageSourceKey } from "../../../src/media/usage/source-key.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { createTestRuntime } from "../../utils/mcp-runtime.js"; @@ -165,6 +169,82 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { ]); }); + it("keeps runtime updates successful when draft overlay usage refresh fails", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "failed-refresh-post", + data: { + title: "Failed Refresh Post", + hero: mediaRef("media-live-before-failure"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + const contentId = created.data.item.id; + + const firstDraft = await runtime.handleContentUpdate("posts", contentId, { + data: { hero: mediaRef("media-draft-before-failure") }, + }); + expect(firstDraft.success).toBe(true); + const draftSourceBefore = await usageRepo.findSource( + sourceKey("posts", contentId, "draft_overlay"), + ); + expect(draftSourceBefore).toEqual( + expect.objectContaining({ + sourceCompleteness: "complete", + lastErrorCode: null, + }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-before-failure")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ contentId, sourceVariant: "draft_overlay" }), + }), + ]); + await corruptFuturePostDraftRevisionSnapshots(ctx); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + + const updated = await runtime + .handleContentUpdate("posts", contentId, { + data: { hero: mediaRef("media-draft-after-failure") }, + }) + .finally(() => { + consoleError.mockRestore(); + }); + + expect(updated.success).toBe(true); + const draftSourceAfter = await usageRepo.findSource( + sourceKey("posts", contentId, "draft_overlay"), + ); + expect(draftSourceAfter).toEqual( + expect.objectContaining({ + sourceCompleteness: "failed", + lastErrorCode: "DRAFT_REVISION_INVALID", + }), + ); + expect(draftSourceAfter?.currentGeneration).toBe(draftSourceBefore?.currentGeneration); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-before-failure")).toEqual([ + expect.objectContaining({ + source: expect.objectContaining({ + contentId, + sourceVariant: "draft_overlay", + lastErrorCode: "DRAFT_REVISION_INVALID", + }), + }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-draft-after-failure")).toEqual([]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toEqual( + expect.objectContaining({ + status: "stale", + lastErrorCode: "DRAFT_REVISION_INVALID", + }), + ); + }); + it("refreshes columns usage for duplicated content", async () => { const created = await runtime.handleContentCreate("plain_posts", { slug: "original-post", @@ -765,3 +845,37 @@ function sourceKey( ): string { return buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }); } + +async function corruptFuturePostDraftRevisionSnapshots(ctx: DialectTestContext): Promise { + if (ctx.dialect === "postgres") { + await sql` + CREATE FUNCTION corrupt_posts_draft_revision() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.draft_revision_id IS NOT NULL THEN + UPDATE revisions SET data = '{' WHERE id = NEW.draft_revision_id; + END IF; + RETURN NEW; + END; + $$ + `.execute(ctx.db); + await sql` + CREATE TRIGGER corrupt_posts_draft_revision + AFTER UPDATE OF draft_revision_id ON ec_posts + FOR EACH ROW + EXECUTE FUNCTION corrupt_posts_draft_revision() + `.execute(ctx.db); + return; + } + + await sql` + CREATE TRIGGER corrupt_posts_draft_revision + AFTER UPDATE OF draft_revision_id ON ec_posts + WHEN NEW.draft_revision_id IS NOT NULL + BEGIN + UPDATE revisions SET data = '{' WHERE id = NEW.draft_revision_id; + END + `.execute(ctx.db); +} From 770f8a2a3180f20a51c2297fe0e4cce57a6db6f3 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 15:45:10 +0100 Subject: [PATCH 24/29] Implement guarded replace and delete methods in MediaUsageRepository with generation checks --- .../src/database/repositories/media-usage.ts | 154 +++++++-- .../core/src/media/usage/content-refresh.ts | 93 +++++- .../media-usage-content-refresh.test.ts | 297 ++++++++++++++++++ .../database/media-usage-repository.test.ts | 128 ++++++-- 4 files changed, 619 insertions(+), 53 deletions(-) diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index e4cd06d49c..fea702700a 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -83,6 +83,16 @@ export interface MediaUsageSource { updatedAt: string; } +export interface MediaUsageGuardedReplaceResult { + replaced: boolean; + source: MediaUsageSource | null; +} + +export interface MediaUsageGuardedDeleteResult { + deleted: boolean; + source: MediaUsageSource | null; +} + export type MediaUsageSourceCompleteness = | "unknown" | "complete" @@ -243,6 +253,31 @@ export class MediaUsageRepository { return replaced; } + async replaceSourceIfCurrent( + source: MediaUsageSourceInput, + occurrences: readonly MediaUsageOccurrenceInput[], + expectedCurrentGeneration: string | null, + ): Promise { + const generation = ulid(); + const now = new Date().toISOString(); + const row = this.buildSourceRow(source, generation, now); + + await withTransaction(this.db, async (trx) => { + await this.insertOccurrences(trx, source.sourceKey, generation, occurrences, now); + if (expectedCurrentGeneration === null) { + await this.insertSourceIfAbsent(trx, row); + return; + } + await this.updateSourceIfGeneration(trx, row, expectedCurrentGeneration); + }); + + const current = await this.findSource(source.sourceKey); + return { + replaced: current?.currentGeneration === generation, + source: current, + }; + } + async findSource(sourceKey: string): Promise { const row = await this.db .selectFrom("_emdash_media_usage_sources") @@ -386,6 +421,28 @@ export class MediaUsageRepository { return this.deleteSources([sourceKey]); } + async deleteSourceIfCurrent( + sourceKey: string, + expectedCurrentGeneration: string, + ): Promise { + let deleted = false; + await withTransaction(this.db, async (trx) => { + const result = await trx + .deleteFrom("_emdash_media_usage_sources") + .where("source_key", "=", sourceKey) + .where("current_generation", "=", expectedCurrentGeneration) + .executeTakeFirst(); + deleted = Number(result.numDeletedRows ?? 0) > 0; + if (!deleted) return; + await trx.deleteFrom("_emdash_media_usage").where("source_key", "=", sourceKey).execute(); + }); + + return { + deleted, + source: await this.findSource(sourceKey), + }; + } + async deleteSources(sourceKeys: readonly string[]): Promise { return this.deleteSourceKeys(sourceKeys, "source-first"); } @@ -721,7 +778,42 @@ export class MediaUsageRepository { generation: string, now: string, ): Promise { - const row = { + const row = this.buildSourceRow(source, generation, now); + + await db + .insertInto("_emdash_media_usage_sources") + .values(row) + .onConflict((oc) => oc.column("source_key").doUpdateSet(this.sourceUpdateSet(row))) + .execute(); + } + + private async insertSourceIfAbsent( + db: DatabaseExecutor, + row: ReturnType, + ): Promise { + await db + .insertInto("_emdash_media_usage_sources") + .values(row) + .onConflict((oc) => oc.column("source_key").doNothing()) + .execute(); + } + + private async updateSourceIfGeneration( + db: DatabaseExecutor, + row: ReturnType, + expectedCurrentGeneration: string, + ): Promise { + const result = await db + .updateTable("_emdash_media_usage_sources") + .set(this.sourceUpdateSet(row)) + .where("source_key", "=", row.source_key) + .where("current_generation", "=", expectedCurrentGeneration) + .executeTakeFirst(); + return Number(result.numUpdatedRows ?? 0) > 0; + } + + private buildSourceRow(source: MediaUsageSourceInput, generation: string, now: string) { + return { source_key: source.sourceKey, source_type: source.sourceType, collection_slug: source.collectionSlug ?? null, @@ -740,43 +832,43 @@ export class MediaUsageRepository { source_updated_at: source.sourceUpdatedAt ?? null, source_version: source.sourceVersion ?? null, source_fingerprint: source.sourceFingerprint ?? null, + // Complete means this source was fully refreshed for the extractor's current + // schema/version coverage, not that every possible reference shape is known. source_completeness: source.sourceCompleteness ?? "complete", last_attempted_at: source.lastAttemptedAt ?? now, last_error_code: null, indexed_at: now, updated_at: now, }; + } - await db - .insertInto("_emdash_media_usage_sources") - .values(row) - .onConflict((oc) => - oc.column("source_key").doUpdateSet({ - source_type: row.source_type, - collection_slug: row.collection_slug, - content_id: row.content_id, - source_variant: row.source_variant, - locale: row.locale, - translation_group: row.translation_group, - content_slug: row.content_slug, - content_title: row.content_title, - content_status: row.content_status, - content_scheduled_at: row.content_scheduled_at, - content_deleted_at: row.content_deleted_at, - revision_id: row.revision_id, - current_generation: row.current_generation, - schema_version: row.schema_version, - source_updated_at: row.source_updated_at, - source_version: row.source_version, - source_fingerprint: row.source_fingerprint, - source_completeness: row.source_completeness, - last_attempted_at: row.last_attempted_at, - last_error_code: row.last_error_code, - indexed_at: row.indexed_at, - updated_at: row.updated_at, - }), - ) - .execute(); + private sourceUpdateSet( + row: ReturnType, + ): Updateable { + return { + source_type: row.source_type, + collection_slug: row.collection_slug, + content_id: row.content_id, + source_variant: row.source_variant, + locale: row.locale, + translation_group: row.translation_group, + content_slug: row.content_slug, + content_title: row.content_title, + content_status: row.content_status, + content_scheduled_at: row.content_scheduled_at, + content_deleted_at: row.content_deleted_at, + revision_id: row.revision_id, + current_generation: row.current_generation, + schema_version: row.schema_version, + source_updated_at: row.source_updated_at, + source_version: row.source_version, + source_fingerprint: row.source_fingerprint, + source_completeness: row.source_completeness, + last_attempted_at: row.last_attempted_at, + last_error_code: row.last_error_code, + indexed_at: row.indexed_at, + updated_at: row.updated_at, + }; } } diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts index 96a2f627b6..87779ba306 100644 --- a/packages/core/src/media/usage/content-refresh.ts +++ b/packages/core/src/media/usage/content-refresh.ts @@ -20,6 +20,9 @@ export const CONTENT_MEDIA_USAGE_COLLECTION_SCOPE = "collection"; const CONTENT_USAGE_LOCKS_KEY = Symbol.for("emdash.mediaUsage.contentLocks"); const CONTENT_USAGE_COLLECTION_LOCKS_KEY = Symbol.for("emdash.mediaUsage.collectionLocks"); +// These maps only de-dupe usage work inside the current isolate/process. Cross-worker +// correctness comes from expected-generation guards on repository writes. + export type ContentMediaUsageRefreshErrorCode = | "CONTENT_NOT_FOUND" | "DRAFT_REVISION_NOT_FOUND" @@ -27,6 +30,7 @@ export type ContentMediaUsageRefreshErrorCode = | "DRAFT_REVISION_INVALID" | "CONTENT_USAGE_REFRESH_ERROR" | "CONTENT_USAGE_DELETE_ERROR" + | "CONTENT_USAGE_GENERATION_CONFLICT" | "CONTENT_USAGE_STALE"; export interface ContentMediaUsageRefreshResult { @@ -63,19 +67,36 @@ async function refreshContentMediaUsageUnlocked( contentId: string, ): Promise { try { + const repo = new MediaUsageRepository(db); + const observedGenerations = await loadObservedContentSourceGenerations( + db, + collectionSlug, + contentId, + ); const snapshotsResult = await loadContentMediaUsageSnapshots(db, collectionSlug, contentId); if (!snapshotsResult.success) { return markSnapshotFailure(db, collectionSlug, snapshotsResult); } - const repo = new MediaUsageRepository(db); if (!(await contentCollectionExists(db, collectionSlug))) { const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); return { ...ZERO_RESULT, deletedSourceCount }; } + let refreshedSourceCount = 0; for (const snapshot of snapshotsResult.snapshots) { - await repo.replaceSource(snapshot.source, snapshot.occurrences); + const result = await repo.replaceSourceIfCurrent( + snapshot.source, + snapshot.occurrences, + observedGenerations.get(snapshot.source.sourceKey) ?? null, + ); + if (!result.replaced) { + return markGenerationConflict(db, collectionSlug, { + refreshedSourceCount, + deletedSourceCount: 0, + }); + } + refreshedSourceCount++; } if (!(await contentCollectionExists(db, collectionSlug))) { const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); @@ -88,11 +109,27 @@ async function refreshContentMediaUsageUnlocked( const absentSourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) => buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }), ).filter((sourceKey) => !expectedSourceKeys.has(sourceKey)); - const deletedSourceCount = await repo.deleteSources(absentSourceKeys); + let deletedSourceCount = 0; + for (const sourceKey of absentSourceKeys) { + const expectedGeneration = observedGenerations.get(sourceKey) ?? null; + if (expectedGeneration === null) continue; + + const result = await repo.deleteSourceIfCurrent(sourceKey, expectedGeneration); + if (result.deleted) { + deletedSourceCount++; + continue; + } + if (result.source) { + return markGenerationConflict(db, collectionSlug, { + refreshedSourceCount, + deletedSourceCount, + }); + } + } return { success: true, - refreshedSourceCount: snapshotsResult.snapshots.length, + refreshedSourceCount, deletedSourceCount, failedSourceCount: 0, }; @@ -113,6 +150,54 @@ async function refreshContentMediaUsageUnlocked( } } +async function loadObservedContentSourceGenerations( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise> { + const generations = new Map(); + const sourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) => + buildContentMediaUsageSourceKey({ + collectionSlug, + contentId, + sourceVariant, + }), + ); + for (const sourceKey of sourceKeys) { + generations.set(sourceKey, null); + } + + const rows = await db + .selectFrom("_emdash_media_usage_sources") + .select(["source_key", "current_generation"]) + .where("source_key", "in", sourceKeys) + .execute(); + for (const row of rows) { + generations.set(row.source_key, row.current_generation); + } + + return generations; +} + +async function markGenerationConflict( + db: Kysely, + collectionSlug: string, + counts: Pick, +): Promise { + await markContentMediaUsageCollectionStaleSafely( + db, + collectionSlug, + "CONTENT_USAGE_GENERATION_CONFLICT", + ); + return { + success: false, + refreshedSourceCount: counts.refreshedSourceCount, + deletedSourceCount: counts.deletedSourceCount, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_GENERATION_CONFLICT", + }; +} + async function contentCollectionExists( db: Kysely, collectionSlug: string, diff --git a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts index 6c0b3b252e..5833df0e8a 100644 --- a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts @@ -192,6 +192,98 @@ describeEachDialect("content media usage refresh", (dialect) => { expect(await usageRepo.findCurrentUsageByMediaId("media-draft")).toEqual([]); }); + it("marks coverage stale instead of clobbering a newer source generation", async () => { + const item = await insertPost(ctx, { + slug: "guarded-replace-post", + status: "published", + data: { + title: "Guarded Replace Post", + hero: { id: "media-old", provider: "local", mimeType: "image/webp" }, + }, + }); + await refreshContentMediaUsage(ctx.db, "posts", item.id); + await installSourceReplacementConflictTrigger(ctx); + await updatePostHero(ctx, item.id, { + id: "media-stale-refresh", + provider: "local", + mimeType: "image/webp", + }); + + const result = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: false, + refreshedSourceCount: 0, + deletedSourceCount: 0, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_GENERATION_CONFLICT", + }); + expect(await usageRepo.findCurrentUsageByMediaId("media-concurrent-generation")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ contentId: item.id }) }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-stale-refresh")).toEqual([]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toEqual( + expect.objectContaining({ + status: "stale", + lastErrorCode: "CONTENT_USAGE_GENERATION_CONFLICT", + }), + ); + }); + + it("does not delete a draft overlay source that changed after observation", async () => { + const item = await insertPost(ctx, { + slug: "guarded-delete-post", + status: "published", + data: { + title: "Guarded Delete Post", + hero: { id: "media-live", provider: "local", mimeType: "image/webp" }, + }, + }); + const draft = await revisionRepo.create({ + collection: "posts", + entryId: item.id, + data: { hero: { id: "media-draft", provider: "local", mimeType: "image/webp" } }, + }); + await setDraftRevision(ctx, item.id, draft.id); + await refreshContentMediaUsage(ctx.db, "posts", item.id); + await clearDraftRevision(ctx, item.id); + await installDraftOverlayDeletionConflictTrigger(ctx); + + const result = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: false, + refreshedSourceCount: 1, + deletedSourceCount: 0, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_GENERATION_CONFLICT", + }); + expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).toEqual( + expect.objectContaining({ currentGeneration: "concurrent-draft-generation" }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-concurrent-draft-generation")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ contentId: item.id }) }), + ]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toEqual( + expect.objectContaining({ + status: "stale", + lastErrorCode: "CONTENT_USAGE_GENERATION_CONFLICT", + }), + ); + }); + it("deletes every source for a content item", async () => { const item = await insertPost(ctx, { slug: "live-post", @@ -400,6 +492,211 @@ async function clearDraftRevision(ctx: DialectTestContext, contentId: string): P `.execute(ctx.db); } +async function installSourceReplacementConflictTrigger(ctx: DialectTestContext): Promise { + if (ctx.dialect === "postgres") { + await sql` + CREATE FUNCTION media_usage_replace_conflict() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.generation <> 'concurrent-generation' + AND NEW.source_key LIKE 'content:posts:%:columns' THEN + INSERT INTO _emdash_media_usage ( + id, + source_key, + generation, + field_slug, + field_path, + occurrence_index, + reference_type, + media_id, + provider, + provider_asset_id, + media_kind, + mime_type, + created_at + ) VALUES ( + 'concurrent-generation-occurrence', + NEW.source_key, + 'concurrent-generation', + 'hero', + 'hero', + 0, + 'image_field', + 'media-concurrent-generation', + 'local', + 'media-concurrent-generation', + 'image', + 'image/webp', + '2026-01-01T00:00:00.000Z' + ) + ON CONFLICT (id) DO NOTHING; + + UPDATE _emdash_media_usage_sources + SET current_generation = 'concurrent-generation' + WHERE source_key = NEW.source_key; + END IF; + RETURN NEW; + END; + $$ + `.execute(ctx.db); + await sql` + CREATE TRIGGER media_usage_replace_conflict + AFTER INSERT ON _emdash_media_usage + FOR EACH ROW + EXECUTE FUNCTION media_usage_replace_conflict() + `.execute(ctx.db); + return; + } + + await sql` + CREATE TRIGGER media_usage_replace_conflict + AFTER INSERT ON _emdash_media_usage + WHEN NEW.generation != 'concurrent-generation' + AND NEW.source_key LIKE 'content:posts:%:columns' + BEGIN + INSERT OR IGNORE INTO _emdash_media_usage ( + id, + source_key, + generation, + field_slug, + field_path, + occurrence_index, + reference_type, + media_id, + provider, + provider_asset_id, + media_kind, + mime_type, + created_at + ) VALUES ( + 'concurrent-generation-occurrence', + NEW.source_key, + 'concurrent-generation', + 'hero', + 'hero', + 0, + 'image_field', + 'media-concurrent-generation', + 'local', + 'media-concurrent-generation', + 'image', + 'image/webp', + '2026-01-01T00:00:00.000Z' + ); + + UPDATE _emdash_media_usage_sources + SET current_generation = 'concurrent-generation' + WHERE source_key = NEW.source_key; + END + `.execute(ctx.db); +} + +async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContext): Promise { + if (ctx.dialect === "postgres") { + await sql` + CREATE FUNCTION media_usage_draft_delete_conflict() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + DECLARE + draft_source_key text; + BEGIN + IF NEW.generation <> 'concurrent-draft-generation' + AND NEW.source_key LIKE 'content:posts:%:columns' THEN + draft_source_key := replace(NEW.source_key, ':columns', ':draft_overlay'); + INSERT INTO _emdash_media_usage ( + id, + source_key, + generation, + field_slug, + field_path, + occurrence_index, + reference_type, + media_id, + provider, + provider_asset_id, + media_kind, + mime_type, + created_at + ) VALUES ( + 'concurrent-draft-generation-occurrence', + draft_source_key, + 'concurrent-draft-generation', + 'hero', + 'hero', + 0, + 'image_field', + 'media-concurrent-draft-generation', + 'local', + 'media-concurrent-draft-generation', + 'image', + 'image/webp', + '2026-01-01T00:00:00.000Z' + ) + ON CONFLICT (id) DO NOTHING; + + UPDATE _emdash_media_usage_sources + SET current_generation = 'concurrent-draft-generation' + WHERE source_key = draft_source_key; + END IF; + RETURN NEW; + END; + $$ + `.execute(ctx.db); + await sql` + CREATE TRIGGER media_usage_draft_delete_conflict + AFTER INSERT ON _emdash_media_usage + FOR EACH ROW + EXECUTE FUNCTION media_usage_draft_delete_conflict() + `.execute(ctx.db); + return; + } + + await sql` + CREATE TRIGGER media_usage_draft_delete_conflict + AFTER INSERT ON _emdash_media_usage + WHEN NEW.generation != 'concurrent-draft-generation' + AND NEW.source_key LIKE 'content:posts:%:columns' + BEGIN + INSERT OR IGNORE INTO _emdash_media_usage ( + id, + source_key, + generation, + field_slug, + field_path, + occurrence_index, + reference_type, + media_id, + provider, + provider_asset_id, + media_kind, + mime_type, + created_at + ) VALUES ( + 'concurrent-draft-generation-occurrence', + replace(NEW.source_key, ':columns', ':draft_overlay'), + 'concurrent-draft-generation', + 'hero', + 'hero', + 0, + 'image_field', + 'media-concurrent-draft-generation', + 'local', + 'media-concurrent-draft-generation', + 'image', + 'image/webp', + '2026-01-01T00:00:00.000Z' + ); + + UPDATE _emdash_media_usage_sources + SET current_generation = 'concurrent-draft-generation' + WHERE source_key = replace(NEW.source_key, ':columns', ':draft_overlay'); + END + `.execute(ctx.db); +} + function sourceKey(contentId: string, sourceVariant: "columns" | "draft_overlay"): string { return buildContentMediaUsageSourceKey({ collectionSlug: "posts", diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index b2fc6ab2da..2a21ac4579 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -88,8 +88,77 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(rows).toContainEqual({ generation: second.currentGeneration, media_id: "media-new" }); }); + it("does not replace a source when the expected generation is stale", async () => { + const first = await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-old"), + ]); + const second = await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-concurrent"), + ]); + + const stale = await repo.replaceSourceIfCurrent( + contentSource("entry1", "columns"), + [occurrence("hero", "media-stale")], + first.currentGeneration, + ); + + expect(stale.replaced).toBe(false); + expect(stale.source).toEqual( + expect.objectContaining({ currentGeneration: second.currentGeneration }), + ); + expect((await repo.findSource("content:posts:entry1:columns"))?.currentGeneration).toBe( + second.currentGeneration, + ); + expect(await repo.findCurrentUsageByMediaId("media-concurrent")).toHaveLength(1); + expect(await repo.findCurrentUsageByMediaId("media-stale")).toEqual([]); + }); + + it("does not create a source observed absent when another writer created it first", async () => { + const concurrent = await repo.replaceSource(contentSource("entry-new", "columns"), [ + occurrence("hero", "media-concurrent"), + ]); + + const stale = await repo.replaceSourceIfCurrent( + contentSource("entry-new", "columns"), + [occurrence("hero", "media-stale")], + null, + ); + + expect(stale.replaced).toBe(false); + expect(stale.source).toEqual( + expect.objectContaining({ currentGeneration: concurrent.currentGeneration }), + ); + expect(await repo.findCurrentUsageByMediaId("media-concurrent")).toHaveLength(1); + expect(await repo.findCurrentUsageByMediaId("media-stale")).toEqual([]); + }); + + it("does not delete a source when the expected generation is stale", async () => { + const first = await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ + occurrence("hero", "media-old-draft"), + ]); + const second = await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ + occurrence("hero", "media-concurrent-draft"), + ]); + + const stale = await repo.deleteSourceIfCurrent( + "content:posts:entry1:draft_overlay", + first.currentGeneration, + ); + + expect(stale.deleted).toBe(false); + expect(stale.source).toEqual( + expect.objectContaining({ currentGeneration: second.currentGeneration }), + ); + expect(await repo.findSource("content:posts:entry1:draft_overlay")).toEqual( + expect.objectContaining({ currentGeneration: second.currentGeneration }), + ); + expect(await repo.findCurrentUsageByMediaId("media-concurrent-draft")).toHaveLength(1); + }); + it("writes ISO occurrence timestamps for safe cleanup cutoffs", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); const row = await ctx.db .selectFrom("_emdash_media_usage") @@ -284,7 +353,9 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("marks failed source attempts without replacing current usage", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); const failed = await repo.markSourceAttempted( contentSource("entry1", "columns", { @@ -375,8 +446,12 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes a single source and its occurrences", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry1", "draft_overlay"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ + occurrence("hero", "media-draft"), + ]); expect(await repo.deleteSource("content:posts:entry1:columns")).toBe(1); expect(await repo.findSource("content:posts:entry1:columns")).toBeNull(); @@ -385,9 +460,15 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes all content sources for one collection and content id", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry1", "draft_overlay"), [occurrence("hero", "media-draft")]); - await repo.replaceSource(contentSource("entry2", "columns"), [occurrence("hero", "media-other")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ + occurrence("hero", "media-draft"), + ]); + await repo.replaceSource(contentSource("entry2", "columns"), [ + occurrence("hero", "media-other"), + ]); await repo.replaceSource(contentSource("entry1", "columns", { collectionSlug: "pages" }), [ occurrence("hero", "media-page"), ]); @@ -400,8 +481,12 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes content sources by collection", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry2", "draft_overlay"), [occurrence("hero", "media-draft")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); + await repo.replaceSource(contentSource("entry2", "draft_overlay"), [ + occurrence("hero", "media-draft"), + ]); await repo.replaceSource(contentSource("entry1", "columns", { collectionSlug: "pages" }), [ occurrence("hero", "media-page"), ]); @@ -413,9 +498,15 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("deletes specific source keys in D1-safe batches", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-live")]); - await repo.replaceSource(contentSource("entry1", "draft_overlay"), [occurrence("hero", "media-draft")]); - await repo.replaceSource(contentSource("entry2", "columns"), [occurrence("hero", "media-other")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); + await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ + occurrence("hero", "media-draft"), + ]); + await repo.replaceSource(contentSource("entry2", "columns"), [ + occurrence("hero", "media-other"), + ]); expect( await repo.deleteSources([ @@ -504,7 +595,9 @@ describeEachDialect("MediaUsageRepository", (dialect) => { }); it("keeps columns and draft overlay source keys separate for the same content", async () => { - await repo.replaceSource(contentSource("entry1", "columns"), [occurrence("hero", "media-shared")]); + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-shared"), + ]); await repo.replaceSource(contentSource("entry1", "draft_overlay"), [ occurrence("draftHero", "media-shared", { fieldPath: "draftHero" }), ]); @@ -515,10 +608,7 @@ describeEachDialect("MediaUsageRepository", (dialect) => { "content:posts:entry1:columns", "content:posts:entry1:draft_overlay", ]); - expect(usage.map((row) => row.source.sourceVariant)).toEqual([ - "columns", - "draft_overlay", - ]); + expect(usage.map((row) => row.source.sourceVariant)).toEqual(["columns", "draft_overlay"]); }); it("paginates current media usage by occurrence id", async () => { @@ -526,7 +616,9 @@ describeEachDialect("MediaUsageRepository", (dialect) => { occurrence("hero", "media-shared"), occurrence("body", "media-shared"), ]); - await repo.replaceSource(contentSource("entry2", "columns"), [occurrence("hero", "media-shared")]); + await repo.replaceSource(contentSource("entry2", "columns"), [ + occurrence("hero", "media-shared"), + ]); const page1 = await repo.findCurrentUsagePageByMediaId("media-shared", { limit: 2 }); expect(page1.items).toHaveLength(2); From f30f1dec603829a256cdc8319b66bdf22955c34f Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 15:52:05 +0100 Subject: [PATCH 25/29] Fix media usage lint issues --- packages/core/src/media/usage/content-refresh.ts | 4 +++- packages/core/src/media/usage/source-key.ts | 10 ++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts index 87779ba306..1261ec368e 100644 --- a/packages/core/src/media/usage/content-refresh.ts +++ b/packages/core/src/media/usage/content-refresh.ts @@ -315,7 +315,7 @@ export async function refreshContentMediaUsageAfterWrite( export async function markContentMediaUsageCollectionStale( db: Kysely, collectionSlug: string, - lastErrorCode: ContentMediaUsageRefreshErrorCode | string, + lastErrorCode: string, ): Promise { validateIdentifier(collectionSlug, "collection slug"); const repo = new MediaUsageRepository(db); @@ -473,6 +473,7 @@ async function withContentUsageCollectionLock( function getContentUsageLocks(): Map> { const global = globalThis as typeof globalThis & Record; const existing = global[CONTENT_USAGE_LOCKS_KEY]; + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- globalThis symbol slot stores only this map if (existing instanceof Map) return existing as Map>; const locks = new Map>(); global[CONTENT_USAGE_LOCKS_KEY] = locks; @@ -482,6 +483,7 @@ function getContentUsageLocks(): Map> { function getContentUsageCollectionLocks(): Map> { const global = globalThis as typeof globalThis & Record; const existing = global[CONTENT_USAGE_COLLECTION_LOCKS_KEY]; + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- globalThis symbol slot stores only this map if (existing instanceof Map) return existing as Map>; const locks = new Map>(); global[CONTENT_USAGE_COLLECTION_LOCKS_KEY] = locks; diff --git a/packages/core/src/media/usage/source-key.ts b/packages/core/src/media/usage/source-key.ts index 7ae749d11f..e685476c5a 100644 --- a/packages/core/src/media/usage/source-key.ts +++ b/packages/core/src/media/usage/source-key.ts @@ -1,7 +1,6 @@ export const MEDIA_USAGE_CONTENT_SOURCE_VARIANTS = ["columns", "draft_overlay"] as const; -export type MediaUsageContentSourceVariant = - (typeof MEDIA_USAGE_CONTENT_SOURCE_VARIANTS)[number]; +export type MediaUsageContentSourceVariant = (typeof MEDIA_USAGE_CONTENT_SOURCE_VARIANTS)[number]; export interface ContentMediaUsageSourceKeyInput { collectionSlug: string; @@ -18,11 +17,6 @@ export function isMediaUsageContentSourceVariant( ); } -export function buildContentMediaUsageSourceKey( - input: ContentMediaUsageSourceKeyInput, -): string { - if (!isMediaUsageContentSourceVariant(input.sourceVariant)) { - throw new Error(`Invalid media usage content source variant: ${input.sourceVariant}`); - } +export function buildContentMediaUsageSourceKey(input: ContentMediaUsageSourceKeyInput): string { return `content:${input.collectionSlug}:${input.contentId}:${input.sourceVariant}`; } From 1446a658208b81192cd2d7c9285efc2d5b9ccab2 Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 16:16:53 +0100 Subject: [PATCH 26/29] Fix media usage display title extraction --- .../core/src/media/usage/content-snapshots.ts | 13 ++++++++++++- packages/core/src/media/usage/extractor.ts | 2 +- .../media-usage-content-snapshots.test.ts | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/core/src/media/usage/content-snapshots.ts b/packages/core/src/media/usage/content-snapshots.ts index 7d05623ad6..acf6349e48 100644 --- a/packages/core/src/media/usage/content-snapshots.ts +++ b/packages/core/src/media/usage/content-snapshots.ts @@ -68,7 +68,7 @@ export async function loadContentMediaUsageSnapshots( row, discovery.extractionFields.map((field) => field.slug), ); - const displayData = projectData(row, discovery.displayFieldSlugs); + const displayData = projectRawData(row, discovery.displayFieldSlugs); const occurrences = extractMediaUsageOccurrences({ fields: discovery.extractionFields, data: columnsData, @@ -331,6 +331,17 @@ function projectData( return data; } +function projectRawData( + row: Record, + fieldSlugs: readonly string[], +): Record { + const data: Record = {}; + for (const fieldSlug of fieldSlugs) { + data[fieldSlug] = row[fieldSlug] ?? null; + } + return data; +} + function projectPresentData( row: Record, fieldSlugs: readonly string[], diff --git a/packages/core/src/media/usage/extractor.ts b/packages/core/src/media/usage/extractor.ts index 7b9c487b5b..fc6747cc63 100644 --- a/packages/core/src/media/usage/extractor.ts +++ b/packages/core/src/media/usage/extractor.ts @@ -1,4 +1,5 @@ import { normalizeMime } from "../mime.js"; +import { INTERNAL_MEDIA_PREFIX } from "../normalize.js"; import type { ExtractedMediaUsageOccurrence, ExtractMediaUsageOccurrencesInput, @@ -7,7 +8,6 @@ import type { MediaUsageReferenceType, } from "./types.js"; -const INTERNAL_MEDIA_PREFIX = "/_emdash/api/media/file/"; const URL_LIKE_RE = /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i; interface MediaRef { diff --git a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts index b1ce39e424..d237431da9 100644 --- a/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-snapshots.test.ts @@ -113,6 +113,21 @@ describeEachDialect("content media usage snapshots", (dialect) => { expect(result).toEqual({ success: false, error: "CONTENT_NOT_FOUND" }); }); + it("keeps JSON-looking stored display strings as strings", async () => { + const title = '{"headline":"Columns"}'; + const item = await insertPost(ctx, { + slug: "json-title", + status: "published", + data: { title }, + }); + + const result = await loadContentMediaUsageSnapshots(ctx.db, "posts", item.id); + + expect(result.success).toBe(true); + if (!result.success) throw new Error(result.error); + expect(getSnapshot(result, "columns").source.contentTitle).toBe(title); + }); + it("builds columns and draft overlay snapshots for a pending draft revision", async () => { const item = await insertPost(ctx, { slug: "live-post", From fc980c383cc3e559485ac02d08e40ce25152c20e Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 16:43:03 +0100 Subject: [PATCH 27/29] update guarded replace result handling and add test for replacement success signal --- .../src/database/repositories/media-usage.ts | 18 +++++----- .../database/media-usage-repository.test.ts | 33 ++++++++++++++++++- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index e20a1d7bf7..4837811b0f 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -85,6 +85,7 @@ export interface MediaUsageSource { export interface MediaUsageGuardedReplaceResult { replaced: boolean; + /** Populated only when a guarded replacement did not win the current source row. */ source: MediaUsageSource | null; } @@ -260,20 +261,20 @@ export class MediaUsageRepository { const generation = ulid(); const now = new Date().toISOString(); const row = this.buildSourceRow(source, generation, now); + let replaced = false; await withTransaction(this.db, async (trx) => { await this.insertOccurrences(trx, source.sourceKey, generation, occurrences, now); if (expectedCurrentGeneration === null) { - await this.insertSourceIfAbsent(trx, row); + replaced = await this.insertSourceIfAbsent(trx, row); return; } - await this.updateSourceIfGeneration(trx, row, expectedCurrentGeneration); + replaced = await this.updateSourceIfGeneration(trx, row, expectedCurrentGeneration); }); - const current = await this.findSource(source.sourceKey); return { - replaced: current?.currentGeneration === generation, - source: current, + replaced, + source: replaced ? null : await this.findSource(source.sourceKey), }; } @@ -789,12 +790,13 @@ export class MediaUsageRepository { private async insertSourceIfAbsent( db: DatabaseExecutor, row: ReturnType, - ): Promise { - await db + ): Promise { + const result = await db .insertInto("_emdash_media_usage_sources") .values(row) .onConflict((oc) => oc.column("source_key").doNothing()) - .execute(); + .executeTakeFirst(); + return (result.numInsertedOrUpdatedRows ?? 0n) > 0n; } private async updateSourceIfGeneration( diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index d0d624b0cb..9143aa2e6d 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, expect, it } from "vitest"; -import { MediaUsageRepository } from "../../../src/database/repositories/media-usage.js"; +import { + MediaUsageRepository, + type MediaUsageSource, +} from "../../../src/database/repositories/media-usage.js"; import { buildContentMediaUsageSourceKey, type MediaUsageContentSourceVariant, @@ -113,6 +116,34 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(await repo.findCurrentUsageByMediaId("media-stale")).toEqual([]); }); + it("uses the guarded write result as the replacement success signal", async () => { + const first = await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-old"), + ]); + + class StaleReadRepository extends MediaUsageRepository { + override async findSource(sourceKey: string): Promise { + const source = await super.findSource(sourceKey); + if (sourceKey !== first.sourceKey || !source) return source; + return { ...source, currentGeneration: first.currentGeneration }; + } + } + + const staleReadRepo = new StaleReadRepository(ctx.db); + const result = await staleReadRepo.replaceSourceIfCurrent( + contentSource("entry1", "columns"), + [occurrence("hero", "media-new")], + first.currentGeneration, + ); + + expect(result.replaced).toBe(true); + expect(result.source).toBeNull(); + expect((await repo.findSource(first.sourceKey))?.currentGeneration).not.toBe( + first.currentGeneration, + ); + expect(await repo.findCurrentUsageByMediaId("media-new")).toHaveLength(1); + }); + it("does not create a source observed absent when another writer created it first", async () => { const concurrent = await repo.replaceSource(contentSource("entry-new", "columns"), [ occurrence("hero", "media-concurrent"), From 6f496fbfef8432636f5816b65070a4fa55dd37bc Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 17:53:11 +0100 Subject: [PATCH 28/29] Implement retry logic for content media usage refresh to handle generation conflicts --- .../core/src/media/usage/content-refresh.ts | 159 +++++++++------ .../media-usage-content-refresh.test.ts | 183 ++++++++++++++++-- 2 files changed, 263 insertions(+), 79 deletions(-) diff --git a/packages/core/src/media/usage/content-refresh.ts b/packages/core/src/media/usage/content-refresh.ts index 1261ec368e..8082a2f2e8 100644 --- a/packages/core/src/media/usage/content-refresh.ts +++ b/packages/core/src/media/usage/content-refresh.ts @@ -19,6 +19,7 @@ export const CONTENT_MEDIA_USAGE_COLLECTION_SCOPE = "collection"; const CONTENT_USAGE_LOCKS_KEY = Symbol.for("emdash.mediaUsage.contentLocks"); const CONTENT_USAGE_COLLECTION_LOCKS_KEY = Symbol.for("emdash.mediaUsage.collectionLocks"); +const CONTENT_USAGE_REFRESH_MAX_ATTEMPTS = 2; // These maps only de-dupe usage work inside the current isolate/process. Cross-worker // correctness comes from expected-generation guards on repository writes. @@ -67,72 +68,17 @@ async function refreshContentMediaUsageUnlocked( contentId: string, ): Promise { try { - const repo = new MediaUsageRepository(db); - const observedGenerations = await loadObservedContentSourceGenerations( - db, - collectionSlug, - contentId, - ); - const snapshotsResult = await loadContentMediaUsageSnapshots(db, collectionSlug, contentId); - if (!snapshotsResult.success) { - return markSnapshotFailure(db, collectionSlug, snapshotsResult); - } - - if (!(await contentCollectionExists(db, collectionSlug))) { - const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); - return { ...ZERO_RESULT, deletedSourceCount }; + let conflictResult: ContentMediaUsageRefreshResult | null = null; + for (let attempt = 0; attempt < CONTENT_USAGE_REFRESH_MAX_ATTEMPTS; attempt++) { + const result = await refreshContentMediaUsageAttempt(db, collectionSlug, contentId); + if (result.errorCode !== "CONTENT_USAGE_GENERATION_CONFLICT") return result; + conflictResult = result; } - let refreshedSourceCount = 0; - for (const snapshot of snapshotsResult.snapshots) { - const result = await repo.replaceSourceIfCurrent( - snapshot.source, - snapshot.occurrences, - observedGenerations.get(snapshot.source.sourceKey) ?? null, - ); - if (!result.replaced) { - return markGenerationConflict(db, collectionSlug, { - refreshedSourceCount, - deletedSourceCount: 0, - }); - } - refreshedSourceCount++; - } - if (!(await contentCollectionExists(db, collectionSlug))) { - const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); - return { ...ZERO_RESULT, deletedSourceCount }; - } - - const expectedSourceKeys = new Set( - snapshotsResult.snapshots.map((snapshot) => snapshot.source.sourceKey), - ); - const absentSourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) => - buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }), - ).filter((sourceKey) => !expectedSourceKeys.has(sourceKey)); - let deletedSourceCount = 0; - for (const sourceKey of absentSourceKeys) { - const expectedGeneration = observedGenerations.get(sourceKey) ?? null; - if (expectedGeneration === null) continue; - - const result = await repo.deleteSourceIfCurrent(sourceKey, expectedGeneration); - if (result.deleted) { - deletedSourceCount++; - continue; - } - if (result.source) { - return markGenerationConflict(db, collectionSlug, { - refreshedSourceCount, - deletedSourceCount, - }); - } - } - - return { - success: true, - refreshedSourceCount, - deletedSourceCount, - failedSourceCount: 0, - }; + return markGenerationConflict(db, collectionSlug, { + refreshedSourceCount: conflictResult?.refreshedSourceCount ?? 0, + deletedSourceCount: conflictResult?.deletedSourceCount ?? 0, + }); } catch (error) { console.error(`[media-usage] Failed to refresh ${collectionSlug}/${contentId}:`, error); await markContentMediaUsageCollectionStaleSafely( @@ -150,6 +96,79 @@ async function refreshContentMediaUsageUnlocked( } } +async function refreshContentMediaUsageAttempt( + db: Kysely, + collectionSlug: string, + contentId: string, +): Promise { + const repo = new MediaUsageRepository(db); + const observedGenerations = await loadObservedContentSourceGenerations( + db, + collectionSlug, + contentId, + ); + const snapshotsResult = await loadContentMediaUsageSnapshots(db, collectionSlug, contentId); + if (!snapshotsResult.success) { + return markSnapshotFailure(db, collectionSlug, snapshotsResult); + } + + if (!(await contentCollectionExists(db, collectionSlug))) { + const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); + return { ...ZERO_RESULT, deletedSourceCount }; + } + + let refreshedSourceCount = 0; + for (const snapshot of snapshotsResult.snapshots) { + const result = await repo.replaceSourceIfCurrent( + snapshot.source, + snapshot.occurrences, + observedGenerations.get(snapshot.source.sourceKey) ?? null, + ); + if (!result.replaced) { + return generationConflictResult({ + refreshedSourceCount, + deletedSourceCount: 0, + }); + } + refreshedSourceCount++; + } + if (!(await contentCollectionExists(db, collectionSlug))) { + const deletedSourceCount = await repo.deleteContentSources(collectionSlug, contentId); + return { ...ZERO_RESULT, deletedSourceCount }; + } + + const expectedSourceKeys = new Set( + snapshotsResult.snapshots.map((snapshot) => snapshot.source.sourceKey), + ); + const absentSourceKeys = MEDIA_USAGE_CONTENT_SOURCE_VARIANTS.map((sourceVariant) => + buildContentMediaUsageSourceKey({ collectionSlug, contentId, sourceVariant }), + ).filter((sourceKey) => !expectedSourceKeys.has(sourceKey)); + let deletedSourceCount = 0; + for (const sourceKey of absentSourceKeys) { + const expectedGeneration = observedGenerations.get(sourceKey) ?? null; + if (expectedGeneration === null) continue; + + const result = await repo.deleteSourceIfCurrent(sourceKey, expectedGeneration); + if (result.deleted) { + deletedSourceCount++; + continue; + } + if (result.source) { + return generationConflictResult({ + refreshedSourceCount, + deletedSourceCount, + }); + } + } + + return { + success: true, + refreshedSourceCount, + deletedSourceCount, + failedSourceCount: 0, + }; +} + async function loadObservedContentSourceGenerations( db: Kysely, collectionSlug: string, @@ -198,6 +217,18 @@ async function markGenerationConflict( }; } +function generationConflictResult( + counts: Pick, +): ContentMediaUsageRefreshResult { + return { + success: false, + refreshedSourceCount: counts.refreshedSourceCount, + deletedSourceCount: counts.deletedSourceCount, + failedSourceCount: 0, + errorCode: "CONTENT_USAGE_GENERATION_CONFLICT", + }; +} + async function contentCollectionExists( db: Kysely, collectionSlug: string, diff --git a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts index 5833df0e8a..0c59dc58d2 100644 --- a/packages/core/tests/integration/database/media-usage-content-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-content-refresh.test.ts @@ -236,6 +236,44 @@ describeEachDialect("content media usage refresh", (dialect) => { ); }); + it("retries a replace generation conflict before marking coverage stale", async () => { + const item = await insertPost(ctx, { + slug: "retry-guarded-replace-post", + status: "published", + data: { + title: "Retry Guarded Replace Post", + hero: { id: "media-old", provider: "local", mimeType: "image/webp" }, + }, + }); + await refreshContentMediaUsage(ctx.db, "posts", item.id); + await installOneTimeSourceReplacementConflictTrigger(ctx); + await updatePostHero(ctx, item.id, { + id: "media-after-retry", + provider: "local", + mimeType: "image/webp", + }); + + const result = await refreshContentMediaUsage(ctx.db, "posts", item.id); + + expect(result).toEqual({ + success: true, + refreshedSourceCount: 1, + deletedSourceCount: 0, + failedSourceCount: 0, + }); + expect(await usageRepo.findCurrentUsageByMediaId("media-after-retry")).toEqual([ + expect.objectContaining({ source: expect.objectContaining({ contentId: item.id }) }), + ]); + expect(await usageRepo.findCurrentUsageByMediaId("media-concurrent-generation")).toEqual([]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).not.toEqual(expect.objectContaining({ lastErrorCode: "CONTENT_USAGE_GENERATION_CONFLICT" })); + }); + it("does not delete a draft overlay source that changed after observation", async () => { const item = await insertPost(ctx, { slug: "guarded-delete-post", @@ -265,7 +303,9 @@ describeEachDialect("content media usage refresh", (dialect) => { errorCode: "CONTENT_USAGE_GENERATION_CONFLICT", }); expect(await usageRepo.findSource(sourceKey(item.id, "draft_overlay"))).toEqual( - expect.objectContaining({ currentGeneration: "concurrent-draft-generation" }), + expect.objectContaining({ + currentGeneration: expect.stringMatching(/^concurrent-draft-generation-/), + }), ); expect(await usageRepo.findCurrentUsageByMediaId("media-concurrent-draft-generation")).toEqual([ expect.objectContaining({ source: expect.objectContaining({ contentId: item.id }) }), @@ -499,9 +539,12 @@ async function installSourceReplacementConflictTrigger(ctx: DialectTestContext): RETURNS trigger LANGUAGE plpgsql AS $$ + DECLARE + conflict_generation text; BEGIN - IF NEW.generation <> 'concurrent-generation' + IF NEW.generation NOT LIKE 'concurrent-generation-%' AND NEW.source_key LIKE 'content:posts:%:columns' THEN + conflict_generation := 'concurrent-generation-' || NEW.generation; INSERT INTO _emdash_media_usage ( id, source_key, @@ -517,9 +560,9 @@ async function installSourceReplacementConflictTrigger(ctx: DialectTestContext): mime_type, created_at ) VALUES ( - 'concurrent-generation-occurrence', + 'concurrent-generation-occurrence-' || NEW.generation, NEW.source_key, - 'concurrent-generation', + conflict_generation, 'hero', 'hero', 0, @@ -534,7 +577,7 @@ async function installSourceReplacementConflictTrigger(ctx: DialectTestContext): ON CONFLICT (id) DO NOTHING; UPDATE _emdash_media_usage_sources - SET current_generation = 'concurrent-generation' + SET current_generation = conflict_generation WHERE source_key = NEW.source_key; END IF; RETURN NEW; @@ -553,10 +596,118 @@ async function installSourceReplacementConflictTrigger(ctx: DialectTestContext): await sql` CREATE TRIGGER media_usage_replace_conflict AFTER INSERT ON _emdash_media_usage + WHEN NEW.generation NOT LIKE 'concurrent-generation-%' + AND NEW.source_key LIKE 'content:posts:%:columns' + BEGIN + INSERT INTO _emdash_media_usage ( + id, + source_key, + generation, + field_slug, + field_path, + occurrence_index, + reference_type, + media_id, + provider, + provider_asset_id, + media_kind, + mime_type, + created_at + ) VALUES ( + 'concurrent-generation-occurrence-' || NEW.generation, + NEW.source_key, + 'concurrent-generation-' || NEW.generation, + 'hero', + 'hero', + 0, + 'image_field', + 'media-concurrent-generation', + 'local', + 'media-concurrent-generation', + 'image', + 'image/webp', + '2026-01-01T00:00:00.000Z' + ); + + UPDATE _emdash_media_usage_sources + SET current_generation = 'concurrent-generation-' || NEW.generation + WHERE source_key = NEW.source_key; + END + `.execute(ctx.db); +} + +async function installOneTimeSourceReplacementConflictTrigger( + ctx: DialectTestContext, +): Promise { + if (ctx.dialect === "postgres") { + await sql` + CREATE FUNCTION media_usage_replace_conflict_once() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.generation <> 'concurrent-generation' + AND NEW.source_key LIKE 'content:posts:%:columns' + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage WHERE id = 'concurrent-generation-occurrence' + ) THEN + INSERT INTO _emdash_media_usage ( + id, + source_key, + generation, + field_slug, + field_path, + occurrence_index, + reference_type, + media_id, + provider, + provider_asset_id, + media_kind, + mime_type, + created_at + ) VALUES ( + 'concurrent-generation-occurrence', + NEW.source_key, + 'concurrent-generation', + 'hero', + 'hero', + 0, + 'image_field', + 'media-concurrent-generation', + 'local', + 'media-concurrent-generation', + 'image', + 'image/webp', + '2026-01-01T00:00:00.000Z' + ); + + UPDATE _emdash_media_usage_sources + SET current_generation = 'concurrent-generation' + WHERE source_key = NEW.source_key; + END IF; + RETURN NEW; + END; + $$ + `.execute(ctx.db); + await sql` + CREATE TRIGGER media_usage_replace_conflict_once + AFTER INSERT ON _emdash_media_usage + FOR EACH ROW + EXECUTE FUNCTION media_usage_replace_conflict_once() + `.execute(ctx.db); + return; + } + + await sql` + CREATE TRIGGER media_usage_replace_conflict_once + AFTER INSERT ON _emdash_media_usage WHEN NEW.generation != 'concurrent-generation' AND NEW.source_key LIKE 'content:posts:%:columns' + AND NOT EXISTS ( + SELECT 1 FROM _emdash_media_usage WHERE id = 'concurrent-generation-occurrence' + ) BEGIN - INSERT OR IGNORE INTO _emdash_media_usage ( + INSERT INTO _emdash_media_usage ( id, source_key, generation, @@ -602,10 +753,12 @@ async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContex AS $$ DECLARE draft_source_key text; + conflict_generation text; BEGIN - IF NEW.generation <> 'concurrent-draft-generation' + IF NEW.generation NOT LIKE 'concurrent-draft-generation-%' AND NEW.source_key LIKE 'content:posts:%:columns' THEN draft_source_key := replace(NEW.source_key, ':columns', ':draft_overlay'); + conflict_generation := 'concurrent-draft-generation-' || NEW.generation; INSERT INTO _emdash_media_usage ( id, source_key, @@ -621,9 +774,9 @@ async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContex mime_type, created_at ) VALUES ( - 'concurrent-draft-generation-occurrence', + 'concurrent-draft-generation-occurrence-' || NEW.generation, draft_source_key, - 'concurrent-draft-generation', + conflict_generation, 'hero', 'hero', 0, @@ -638,7 +791,7 @@ async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContex ON CONFLICT (id) DO NOTHING; UPDATE _emdash_media_usage_sources - SET current_generation = 'concurrent-draft-generation' + SET current_generation = conflict_generation WHERE source_key = draft_source_key; END IF; RETURN NEW; @@ -657,10 +810,10 @@ async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContex await sql` CREATE TRIGGER media_usage_draft_delete_conflict AFTER INSERT ON _emdash_media_usage - WHEN NEW.generation != 'concurrent-draft-generation' + WHEN NEW.generation NOT LIKE 'concurrent-draft-generation-%' AND NEW.source_key LIKE 'content:posts:%:columns' BEGIN - INSERT OR IGNORE INTO _emdash_media_usage ( + INSERT INTO _emdash_media_usage ( id, source_key, generation, @@ -675,9 +828,9 @@ async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContex mime_type, created_at ) VALUES ( - 'concurrent-draft-generation-occurrence', + 'concurrent-draft-generation-occurrence-' || NEW.generation, replace(NEW.source_key, ':columns', ':draft_overlay'), - 'concurrent-draft-generation', + 'concurrent-draft-generation-' || NEW.generation, 'hero', 'hero', 0, @@ -691,7 +844,7 @@ async function installDraftOverlayDeletionConflictTrigger(ctx: DialectTestContex ); UPDATE _emdash_media_usage_sources - SET current_generation = 'concurrent-draft-generation' + SET current_generation = 'concurrent-draft-generation-' || NEW.generation WHERE source_key = replace(NEW.source_key, ':columns', ':draft_overlay'); END `.execute(ctx.db); From 042c0e59f5a01ab1253428adedbcee55ff13c4de Mon Sep 17 00:00:00 2001 From: Noah Pham Date: Fri, 3 Jul 2026 18:33:41 +0100 Subject: [PATCH 29/29] Harden media usage failure handling --- .../src/database/repositories/media-usage.ts | 31 +++------ packages/core/src/emdash-runtime.ts | 9 +++ .../database/media-usage-repository.test.ts | 69 ++++++++++++++++++- .../media-usage-runtime-refresh.test.ts | 43 ++++++++++++ 4 files changed, 128 insertions(+), 24 deletions(-) diff --git a/packages/core/src/database/repositories/media-usage.ts b/packages/core/src/database/repositories/media-usage.ts index 4837811b0f..c35c8f7cb3 100644 --- a/packages/core/src/database/repositories/media-usage.ts +++ b/packages/core/src/database/repositories/media-usage.ts @@ -444,7 +444,7 @@ export class MediaUsageRepository { } async deleteSources(sourceKeys: readonly string[]): Promise { - return this.deleteSourceKeys(sourceKeys, "source-first"); + return this.deleteSourceKeys(sourceKeys); } async deleteContentSources(collectionSlug: string, contentId: string): Promise { @@ -456,7 +456,7 @@ export class MediaUsageRepository { .where("content_id", "=", contentId) .execute(); const sourceKeys = sourceRows.map((row) => row.source_key); - return this.deleteSourceKeys(sourceKeys, "usage-first"); + return this.deleteSourceKeys(sourceKeys); } async deleteCollectionSources(collectionSlug: string): Promise { @@ -472,10 +472,7 @@ export class MediaUsageRepository { .execute(); if (sourceRows.length === 0) break; - deleted += await this.deleteSourceKeys( - sourceRows.map((row) => row.source_key), - "usage-first", - ); + deleted += await this.deleteSourceKeys(sourceRows.map((row) => row.source_key)); } return deleted; } @@ -708,35 +705,23 @@ export class MediaUsageRepository { .select(currentUsageSelect); } - private async deleteSourceKeys( - sourceKeys: readonly string[], - order: "source-first" | "usage-first", - ): Promise { + private async deleteSourceKeys(sourceKeys: readonly string[]): Promise { const uniqueSourceKeys = [...new Set(sourceKeys)]; if (uniqueSourceKeys.length === 0) return 0; return withTransaction(this.db, async (trx) => { let deleted = 0; for (const sourceKeyBatch of chunks(uniqueSourceKeys, SQL_BATCH_SIZE)) { - if (order === "usage-first") { - await trx - .deleteFrom("_emdash_media_usage") - .where("source_key", "in", sourceKeyBatch) - .execute(); - } - const result = await trx .deleteFrom("_emdash_media_usage_sources") .where("source_key", "in", sourceKeyBatch) .executeTakeFirst(); deleted += Number(result.numDeletedRows ?? 0); - if (order === "source-first") { - await trx - .deleteFrom("_emdash_media_usage") - .where("source_key", "in", sourceKeyBatch) - .execute(); - } + await trx + .deleteFrom("_emdash_media_usage") + .where("source_key", "in", sourceKeyBatch) + .execute(); } return deleted; }); diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index e670543e18..757cf4e188 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -2673,6 +2673,7 @@ export class EmDashRuntime { // Content table columns = published data (never written by saves). // Draft data lives only in the revisions table. let usesDraftRevisions = false; + let draftStorageChanged = false; if (processedData) { try { const collectionInfo = await this.schemaRegistry.getCollectionWithFields(collection); @@ -2702,6 +2703,7 @@ export class EmDashRuntime { if (bodyWithoutRev.skipRevision && existing.draftRevisionId) { // Autosave: update existing draft revision in place await revisionRepo.updateData(existing.draftRevisionId, mergedData); + draftStorageChanged = true; } else { // Create new draft revision const revision = await revisionRepo.create({ @@ -2720,6 +2722,7 @@ export class EmDashRuntime { updated_at = ${new Date().toISOString()} WHERE id = ${resolvedId} `.execute(this.db); + draftStorageChanged = true; // Fire-and-forget: prune old revisions to prevent unbounded growth void revisionRepo.pruneOldRevisions(collection, resolvedId, 50).catch(() => {}); @@ -2777,6 +2780,12 @@ export class EmDashRuntime { } } await this.refreshContentUsageAfterSuccessfulWrite(collection, contentIdsToRefresh); + } else if (draftStorageChanged) { + try { + await markContentMediaUsageCollectionStale(this.db, collection, "CONTENT_USAGE_STALE"); + } catch (error) { + console.error(`[media-usage] Failed to mark ${collection} stale:`, error); + } } // Run afterSave hooks (fire-and-forget) diff --git a/packages/core/tests/integration/database/media-usage-repository.test.ts b/packages/core/tests/integration/database/media-usage-repository.test.ts index 9143aa2e6d..6ac8852ca2 100644 --- a/packages/core/tests/integration/database/media-usage-repository.test.ts +++ b/packages/core/tests/integration/database/media-usage-repository.test.ts @@ -1,4 +1,5 @@ -import { afterEach, beforeEach, expect, it } from "vitest"; +import { sql } from "kysely"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { MediaUsageRepository, @@ -510,6 +511,25 @@ describeEachDialect("MediaUsageRepository", (dialect) => { expect(await repo.findCurrentUsageByMediaId("media-other")).toHaveLength(1); expect(await repo.findCurrentUsageByMediaId("media-page")).toHaveLength(1); }); + + it("keeps current usage intact when source deletion fails without transactions", async () => { + await repo.replaceSource(contentSource("entry1", "columns"), [ + occurrence("hero", "media-live"), + ]); + await installSourceDeleteFailureTrigger(ctx); + vi.resetModules(); + const { MediaUsageRepository: D1LikeMediaUsageRepository } = + await import("../../../src/database/repositories/media-usage.js"); + const d1LikeRepo = new D1LikeMediaUsageRepository(withoutTransactions(ctx.db)); + + await expect(d1LikeRepo.deleteContentSources("posts", "entry1")).rejects.toThrow( + "source delete failed", + ); + + expect(await repo.findSource("content:posts:entry1:columns")).not.toBeNull(); + expect(await repo.findCurrentUsageByMediaId("media-live")).toHaveLength(1); + }); + it("deletes content sources by collection", async () => { await repo.replaceSource(contentSource("entry1", "columns"), [ occurrence("hero", "media-live"), @@ -815,3 +835,50 @@ function occurrence( ...overrides, }; } + +function withoutTransactions(db: DialectTestContext["db"]): DialectTestContext["db"] { + return new Proxy(db, { + get(target, property, receiver) { + if (property === "transaction") { + return () => ({ + execute: async () => { + throw new Error("transactions are not supported"); + }, + }); + } + + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as DialectTestContext["db"]; +} + +async function installSourceDeleteFailureTrigger(ctx: DialectTestContext): Promise { + if (ctx.dialect === "postgres") { + await sql` + CREATE FUNCTION media_usage_source_delete_failure() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + RAISE EXCEPTION 'source delete failed'; + END; + $$ + `.execute(ctx.db); + await sql` + CREATE TRIGGER media_usage_source_delete_failure + BEFORE DELETE ON _emdash_media_usage_sources + FOR EACH ROW + EXECUTE FUNCTION media_usage_source_delete_failure() + `.execute(ctx.db); + return; + } + + await sql` + CREATE TRIGGER media_usage_source_delete_failure + BEFORE DELETE ON _emdash_media_usage_sources + BEGIN + SELECT RAISE(ABORT, 'source delete failed'); + END + `.execute(ctx.db); +} diff --git a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts index 26bd85b7e0..28058491b9 100644 --- a/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts +++ b/packages/core/tests/integration/database/media-usage-runtime-refresh.test.ts @@ -169,6 +169,49 @@ describeEachDialect("runtime content media usage refresh", (dialect) => { ]); }); + it("marks coverage stale when a failed draft update has already advanced stored draft data", async () => { + const created = await runtime.handleContentCreate("posts", { + slug: "failed-metadata-draft-post", + data: { + title: "Failed Metadata Draft Post", + hero: mediaRef("media-live"), + }, + }); + expect(created.success).toBe(true); + if (!created.success) throw new Error(created.error.message); + const contentId = created.data.item.id; + await usageRepo.upsertIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + status: "complete", + lastErrorCode: null, + }); + + const updated = await runtime.handleContentUpdate("posts", contentId, { + data: { hero: mediaRef("media-unrefreshed-draft") }, + bylines: [{ bylineId: "missing-byline" }], + }); + + expect(updated.success).toBe(false); + expect((await revisionRepo.findByEntry("posts", contentId, { limit: 1 }))[0]?.data).toEqual( + expect.objectContaining({ hero: mediaRef("media-unrefreshed-draft") }), + ); + expect(await usageRepo.findCurrentUsageByMediaId("media-unrefreshed-draft")).toEqual([]); + expect( + await usageRepo.findIndexStatus({ + adapterId: CONTENT_MEDIA_USAGE_ADAPTER_ID, + scopeType: CONTENT_MEDIA_USAGE_COLLECTION_SCOPE, + scopeKey: "posts", + }), + ).toEqual( + expect.objectContaining({ + status: "stale", + lastErrorCode: "CONTENT_USAGE_STALE", + }), + ); + }); + it("keeps runtime updates successful when draft overlay usage refresh fails", async () => { const created = await runtime.handleContentCreate("posts", { slug: "failed-refresh-post",