diff --git a/.changeset/collection-list-columns.md b/.changeset/collection-list-columns.md new file mode 100644 index 0000000000..a9c058ef0e --- /dev/null +++ b/.changeset/collection-list-columns.md @@ -0,0 +1,6 @@ +--- +"emdash": minor +"@emdash-cms/admin": minor +--- + +Adds collection-configured custom field columns to admin content lists. Collection seeds and schema APIs can declare up to four supported fields through `admin.listColumns`; EmDash validates them when building the manifest and renders their stored values between the title and status columns. diff --git a/.changeset/indexed-custom-field-filtering.md b/.changeset/indexed-custom-field-filtering.md new file mode 100644 index 0000000000..b6f6700b91 --- /dev/null +++ b/.changeset/indexed-custom-field-filtering.md @@ -0,0 +1,5 @@ +--- +"emdash": minor +--- + +Adds server-side filtering for indexed custom fields through the REST client and plugin content list APIs. diff --git a/.changeset/indexed-custom-field-sorting.md b/.changeset/indexed-custom-field-sorting.md new file mode 100644 index 0000000000..9e2ea31d4a --- /dev/null +++ b/.changeset/indexed-custom-field-sorting.md @@ -0,0 +1,6 @@ +--- +"emdash": minor +"@emdash-cms/admin": minor +--- + +Adds opt-in database indexes for scalar custom fields and stable cursor pagination when ordering content lists by those fields. diff --git a/docs/src/content/docs/concepts/collections.mdx b/docs/src/content/docs/concepts/collections.mdx index 6e70f43d1a..5d6b86cab9 100644 --- a/docs/src/content/docs/concepts/collections.mdx +++ b/docs/src/content/docs/concepts/collections.mdx @@ -216,6 +216,7 @@ Every field supports these properties: | `type` | `FieldType` | One of the 16 field types | | `required` | `boolean` | Whether the field must have a value | | `unique` | `boolean` | Whether values must be unique across entries | +| `indexed` | `boolean` | Allow efficient sorting and filtering by this field | | `defaultValue` | `unknown` | Default value for new entries | | `validation` | `object` | Type-specific validation rules | | `widget` | `string` | Custom widget identifier | @@ -228,6 +229,16 @@ Every field supports these properties: `version`, `live_revision_id`, `draft_revision_id`, `terms`, `bylines`, `byline`. +Set `indexed: true` when a collection query needs to use a custom field in `orderBy` or a field +filter. Indexes are +supported for `string`, `url`, `number`, `integer`, `boolean`, `datetime`, `select`, `reference`, +and `slug` fields. EmDash rejects indexed JSON, rich content, and other non-scalar field types. + + + ## Validation rules The `validation` object varies by field type. Its full shape is: diff --git a/docs/src/content/docs/plugins/creating-plugins/api-routes.mdx b/docs/src/content/docs/plugins/creating-plugins/api-routes.mdx index 52b00de50f..b042aab2fe 100644 --- a/docs/src/content/docs/plugins/creating-plugins/api-routes.mdx +++ b/docs/src/content/docs/plugins/creating-plugins/api-routes.mdx @@ -53,6 +53,34 @@ export default { - `routeCtx` carries request-shaped data: `{ input, request, requestMeta }`. - `ctx` is the same `PluginContext` you get inside hooks — `ctx.storage`, `ctx.kv`, `ctx.content`, `ctx.http`, `ctx.log`, etc. +### Filtering indexed content fields + +Plugins with the `content:read` capability can filter custom fields that a collection marks as +`indexed`. Filters run in the database and combine with `AND` semantics: + +```typescript +const result = await ctx.content.list("items", { + where: { + fieldFilters: { + priority: { in: ["urgent", "high"] }, + score: { gte: 80 }, + resolved: false, + }, + }, +}); +``` + +Scalar values use exact matching. Use `null` for null matching, `{ in: [...] }` for a set of exact +values, or `gt`, `gte`, `lt`, and `lte` for range comparisons. EmDash rejects filters for fields +that are not indexed, values that do not match the field type, and more than 20 field filters per +query. An `in` filter accepts at most 50 values, and all exact values, range bounds, and `in` +members together have a 50-operand budget per query. Null matches do not consume that budget. + + + ## Route URLs Routes mount at `/_emdash/api/plugins//`. Route names can include slashes for nested paths. diff --git a/docs/src/content/docs/reference/field-types.mdx b/docs/src/content/docs/reference/field-types.mdx index 7ffad4e700..30396db2c5 100644 --- a/docs/src/content/docs/reference/field-types.mdx +++ b/docs/src/content/docs/reference/field-types.mdx @@ -386,12 +386,18 @@ All fields support these common properties: | `type` | `FieldType` | Field type (required) | | `required` | `boolean` | Require a value (default: false) | | `unique` | `boolean` | Enforce uniqueness (default: false) | +| `indexed` | `boolean` | Enable indexed field sorting/filtering | | `defaultValue` | `unknown` | Default value for new entries | | `validation` | `object` | Type-specific validation rules | | `widget` | `string` | Custom widget override | | `options` | `object` | Widget configuration | | `sortOrder` | `number` | Display order in admin | +`indexed` is available for scalar fields: `string`, `url`, `number`, `integer`, `boolean`, +`datetime`, `select`, `reference`, and `slug`. An indexed field can be passed as the `orderBy` +field or used in `fieldFilters` in content list queries. Avoid indexing fields that are not used +for sorting or filtering because every index adds storage and write overhead. + ## Reserved field slugs These slugs are reserved and cannot be used: diff --git a/docs/src/content/docs/reference/mcp-server.mdx b/docs/src/content/docs/reference/mcp-server.mdx index 68b8eb45c2..3b387b4345 100644 --- a/docs/src/content/docs/reference/mcp-server.mdx +++ b/docs/src/content/docs/reference/mcp-server.mdx @@ -351,6 +351,7 @@ Add a new field to a collection's schema. This adds a column to the database tab | `validation` | `object` | No | Constraints: `min`, `max`, `minLength`, `maxLength`, `pattern`, `options` | | `options` | `object` | No | Widget config: `collection` (for references), `rows` (for textarea) | | `searchable` | `boolean` | No | Include in full-text search index | +| `indexed` | `boolean` | No | Enable indexed sorting by this field | | `translatable` | `boolean` | No | Whether this field is translatable (default true) | Field types: `string`, `text`, `number`, `integer`, `boolean`, `datetime`, `select`, `multiSelect`, `portableText`, `image`, `file`, `reference`, `json`, `slug`. diff --git a/docs/src/content/docs/themes/seed-files.mdx b/docs/src/content/docs/themes/seed-files.mdx index 9ce91bf6b5..99072bf38b 100644 --- a/docs/src/content/docs/themes/seed-files.mdx +++ b/docs/src/content/docs/themes/seed-files.mdx @@ -132,6 +132,7 @@ Each collection definition creates a content type in the database: | `type` | `string` | Yes | Field type | | `required` | `boolean` | No | Validation: field must have a value | | `unique` | `boolean` | No | Validation: value must be unique | +| `indexed` | `boolean` | No | Enable indexed sorting by this field | | `defaultValue` | `any` | No | Default value for new entries | | `validation` | `object` | No | Additional validation rules | | `widget` | `string` | No | Admin UI widget override | diff --git a/packages/admin/src/components/ContentList.tsx b/packages/admin/src/components/ContentList.tsx index 67e87bc7e4..3d5bb4c86a 100644 --- a/packages/admin/src/components/ContentList.tsx +++ b/packages/admin/src/components/ContentList.tsx @@ -48,6 +48,13 @@ export interface ContentListSort { direction: "asc" | "desc"; } +export interface ContentListColumn { + slug: string; + label: string; + kind: string; + options?: Array<{ value: string; label: string }>; +} + /** Status filter values. `"all"` clears the status filter. */ export type ContentStatusFilter = "all" | "published" | "draft" | "scheduled" | "archived"; @@ -69,6 +76,8 @@ export interface ContentListProps { collection: string; collectionLabel: string; items: ContentItem[]; + /** Validated custom-field columns from the collection manifest. */ + listColumns?: ContentListColumn[]; trashedItems?: TrashedContentItem[]; isLoading?: boolean; isTrashedLoading?: boolean; @@ -163,6 +172,7 @@ export function ContentList({ collection, collectionLabel, items, + listColumns = [], trashedItems = [], isLoading, isTrashedLoading, @@ -321,7 +331,7 @@ export function ContentList({ } })(); }; - const colSpan = (i18n ? 5 : 4) + (bulkEnabled ? 1 : 0); + const colSpan = (i18n ? 5 : 4) + listColumns.length + (bulkEnabled ? 1 : 0); return (
@@ -511,6 +521,15 @@ export function ContentList({ onSortChange={onSortChange} label={t`Title`} /> + {listColumns.map((column) => ( + + {column.label} + + ))} void; showLocale?: boolean; urlPattern?: string; + listColumns: ContentListColumn[]; selectable?: boolean; selected?: boolean; onToggleSelect?: (id: string) => void; @@ -967,6 +988,7 @@ function ContentListItem({ onDuplicate, showLocale, urlPattern, + listColumns, selectable, selected, onToggleSelect, @@ -996,6 +1018,9 @@ function ContentListItem({ {title} + {listColumns.map((column) => ( + + ))} + + {text} + + + ); +} + +interface ListColumnFormatOptions { + emptyLabel: string; + falseLabel: string; + locale: string; + trueLabel: string; +} + +function formatListColumnValue( + column: ContentListColumn, + value: unknown, + { emptyLabel, falseLabel, locale, trueLabel }: ListColumnFormatOptions, +): string { + if (value === null || value === undefined || value === "") return emptyLabel; + + const optionLabel = (optionValue: unknown): string => { + const text = scalarListColumnValue(optionValue); + if (text === undefined) return emptyLabel; + return column.options?.find((option) => option.value === text)?.label ?? text; + }; + + switch (column.kind) { + case "select": + return optionLabel(value); + case "multiSelect": { + let values: unknown[]; + if (Array.isArray(value)) { + values = value; + } else if (typeof value === "string") { + try { + const parsed: unknown = JSON.parse(value); + values = Array.isArray(parsed) ? parsed : [value]; + } catch { + values = [value]; + } + } else { + values = [value]; + } + return values.length > 0 + ? new Intl.ListFormat(locale, { style: "short", type: "unit" }).format( + values.map(optionLabel), + ) + : emptyLabel; + } + case "boolean": + return value === true || value === 1 || value === "1" || value === "true" + ? trueLabel + : falseLabel; + case "datetime": { + const text = scalarListColumnValue(value); + if (text === undefined) return emptyLabel; + const date = new Date(text); + return Number.isNaN(date.getTime()) ? text : new Intl.DateTimeFormat(locale).format(date); + } + case "number": { + if (typeof value === "number" || typeof value === "bigint") { + return new Intl.NumberFormat(locale).format(value); + } + return scalarListColumnValue(value) ?? emptyLabel; + } + case "string": + default: + return scalarListColumnValue(value) ?? emptyLabel; + } +} + +function scalarListColumnValue(value: unknown): string | undefined { + if (typeof value === "string") return value; + if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") { + return String(value); + } + return undefined; +} + interface TrashedListItemProps { item: TrashedContentItem; onRestore?: (id: string) => void; diff --git a/packages/admin/src/components/FieldEditor.tsx b/packages/admin/src/components/FieldEditor.tsx index 435ea9dddb..63c295ebd9 100644 --- a/packages/admin/src/components/FieldEditor.tsx +++ b/packages/admin/src/components/FieldEditor.tsx @@ -32,6 +32,32 @@ import { AllowedTypesEditor } from "./AllowedTypesEditor"; const SLUG_INVALID_CHARS_REGEX = /[^a-z0-9]+/g; const SLUG_LEADING_TRAILING_REGEX = /^_|_$/g; +const SEARCHABLE_FIELD_TYPES = new Set([ + "string", + "text", + "portableText", + "slug", + "url", +]); +const INDEXABLE_FIELD_TYPES = new Set([ + "string", + "url", + "number", + "integer", + "boolean", + "datetime", + "select", + "reference", + "slug", +]); + +function isSearchableFieldType(type: FieldType | null): type is FieldType { + return type !== null && SEARCHABLE_FIELD_TYPES.has(type); +} + +function isIndexableFieldType(type: FieldType | null): type is FieldType { + return type !== null && INDEXABLE_FIELD_TYPES.has(type); +} // ============================================================================ // Types @@ -67,6 +93,7 @@ interface FieldFormState { required: boolean; unique: boolean; searchable: boolean; + indexed: boolean; minLength: string; maxLength: string; min: string; @@ -89,6 +116,7 @@ function getInitialFormState(field?: SchemaField): FieldFormState { required: field.required, unique: field.unique, searchable: field.searchable, + indexed: field.indexed ?? false, minLength: field.validation?.minLength?.toString() ?? "", maxLength: field.validation?.maxLength?.toString() ?? "", min: field.validation?.min?.toString() ?? "", @@ -111,6 +139,7 @@ function getInitialFormState(field?: SchemaField): FieldFormState { required: false, unique: false, searchable: false, + indexed: false, minLength: "", maxLength: "", min: "", @@ -138,7 +167,7 @@ export function FieldEditor({ open, onOpenChange, field, onSave, isSaving }: Fie } }, [open, field]); - const { step, selectedType, slug, label, required, unique, searchable } = formState; + const { step, selectedType, slug, label, required, unique, searchable, indexed } = formState; const { minLength, maxLength, min, max, pattern, options } = formState; const setField = (key: K, value: FieldFormState[K]) => setFormState((prev) => ({ ...prev, [key]: value })); @@ -312,12 +341,8 @@ export function FieldEditor({ open, onOpenChange, field, onSave, isSaving }: Fie } // Only include searchable for text-based fields - const isSearchableType = - selectedType === "string" || - selectedType === "text" || - selectedType === "portableText" || - selectedType === "slug" || - selectedType === "url"; + const isSearchableType = isSearchableFieldType(selectedType); + const isIndexableType = isIndexableFieldType(selectedType); const input: CreateFieldInput = { slug, @@ -326,6 +351,7 @@ export function FieldEditor({ open, onOpenChange, field, onSave, isSaving }: Fie required, unique, searchable: isSearchableType ? searchable : undefined, + indexed: isIndexableType ? indexed : false, validation: Object.keys(validation).length > 0 ? validation : null, }; @@ -441,17 +467,20 @@ export function FieldEditor({ open, onOpenChange, field, onSave, isSaving }: Fie onCheckedChange={(checked) => setField("unique", checked)} label={{t`Unique`}} /> - {(selectedType === "string" || - selectedType === "text" || - selectedType === "portableText" || - selectedType === "slug" || - selectedType === "url") && ( + {isSearchableFieldType(selectedType) && ( setField("searchable", checked)} label={{t`Searchable`}} /> )} + {isIndexableFieldType(selectedType) && ( + setField("indexed", checked)} + label={{t`Indexed`}} + /> + )}
{/* Type-specific validation */} diff --git a/packages/admin/src/components/index.ts b/packages/admin/src/components/index.ts index 46d6ac15c1..a98f5a2b78 100644 --- a/packages/admin/src/components/index.ts +++ b/packages/admin/src/components/index.ts @@ -5,7 +5,7 @@ export { Header } from "./Header"; // Page components export { Dashboard, type DashboardProps } from "./Dashboard"; -export { ContentList, type ContentListProps } from "./ContentList"; +export { ContentList, type ContentListColumn, type ContentListProps } from "./ContentList.js"; export { ContentEditor, type ContentEditorProps, type FieldDescriptor } from "./ContentEditor"; export { MediaLibrary, type MediaLibraryProps } from "./MediaLibrary"; export { MediaPickerModal, type MediaPickerModalProps } from "./MediaPickerModal"; diff --git a/packages/admin/src/lib/api/client.ts b/packages/admin/src/lib/api/client.ts index 9b0ff18ef2..e557267bcc 100644 --- a/packages/admin/src/lib/api/client.ts +++ b/packages/admin/src/lib/api/client.ts @@ -93,6 +93,7 @@ export interface AdminManifest { supports: string[]; hasSeo: boolean; urlPattern?: string; + listColumns?: string[]; fields: Record< string, { diff --git a/packages/admin/src/lib/api/schema.ts b/packages/admin/src/lib/api/schema.ts index 1b991befa4..f050d846ad 100644 --- a/packages/admin/src/lib/api/schema.ts +++ b/packages/admin/src/lib/api/schema.ts @@ -32,6 +32,7 @@ export interface SchemaCollection { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports: string[]; source?: string; urlPattern?: string; @@ -44,6 +45,10 @@ export interface SchemaCollection { updatedAt: string; } +export interface CollectionAdminConfig { + listColumns?: string[]; +} + export interface SchemaField { id: string; collectionId: string; @@ -54,6 +59,7 @@ export interface SchemaField { required: boolean; unique: boolean; searchable: boolean; + indexed: boolean; defaultValue?: unknown; validation?: { min?: number; @@ -80,6 +86,7 @@ export interface CreateCollectionInput { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports?: string[]; urlPattern?: string; hasSeo?: boolean; @@ -90,6 +97,7 @@ export interface UpdateCollectionInput { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports?: string[]; urlPattern?: string; hasSeo?: boolean; @@ -106,6 +114,7 @@ export interface CreateFieldInput { required?: boolean; unique?: boolean; searchable?: boolean; + indexed?: boolean; defaultValue?: unknown; validation?: { min?: number; @@ -125,6 +134,7 @@ export interface UpdateFieldInput { required?: boolean; unique?: boolean; searchable?: boolean; + indexed?: boolean; defaultValue?: unknown; validation?: { min?: number; diff --git a/packages/admin/src/router.tsx b/packages/admin/src/router.tsx index 18e7a82aba..1dabe40430 100644 --- a/packages/admin/src/router.tsx +++ b/packages/admin/src/router.tsx @@ -572,6 +572,19 @@ function ContentListPage() { return ; } + const listColumns = (collectionConfig.listColumns ?? []).flatMap((slug) => { + const field = collectionConfig.fields[slug]; + if (!field) return []; + return [ + { + slug, + label: field.label ?? slug, + kind: field.kind, + options: Array.isArray(field.options) ? field.options : undefined, + }, + ]; + }); + const handleLocaleChange = (locale: string) => { // Update URL search params without full navigation void navigate({ @@ -586,6 +599,7 @@ function ContentListPage() { collection={collection} collectionLabel={collectionConfig.label} items={items} + listColumns={listColumns} trashedItems={trashedData?.items || []} isLoading={isLoading || isFetchingNextPage} isTrashedLoading={isTrashedLoading} diff --git a/packages/admin/tests/components/ContentList.test.tsx b/packages/admin/tests/components/ContentList.test.tsx index 85133a941d..eae67d7c39 100644 --- a/packages/admin/tests/components/ContentList.test.tsx +++ b/packages/admin/tests/components/ContentList.test.tsx @@ -111,6 +111,98 @@ describe("ContentList", () => { await expect.element(screen.getByText("Second")).toBeInTheDocument(); await expect.element(screen.getByText("Third")).toBeInTheDocument(); }); + + it("renders configured custom fields using their field metadata", async () => { + const items = [ + makeItem({ + data: { + title: "Support request", + ticket_number: "SUP-1042", + priority: "urgent", + labels: ["bug", "unknown"], + vip: true, + }, + }), + ]; + const screen = await render( + , + ); + + await expect.element(screen.getByText("SUP-1042")).toBeInTheDocument(); + await expect.element(screen.getByText("Urgent")).toBeInTheDocument(); + await expect.element(screen.getByText("Bug, unknown")).toBeInTheDocument(); + await expect.element(screen.getByText("Yes")).toBeInTheDocument(); + }); + + it("formats numeric, datetime, boolean, and missing custom-field values", async () => { + const openedAt = "2025-01-02T00:00:00.000Z"; + const screen = await render( + , + ); + + await expect + .element(screen.getByText(new Intl.NumberFormat("en").format(12345.67))) + .toBeInTheDocument(); + await expect + .element(screen.getByText(new Intl.DateTimeFormat("en").format(new Date(openedAt)))) + .toBeInTheDocument(); + await expect.element(screen.getByText("No", { exact: true })).toBeInTheDocument(); + await expect.element(screen.getByText("Not set")).toBeInTheDocument(); + }); + + it("does not expose unconfigured custom-field data", async () => { + const screen = await render( + , + ); + + await expect.element(screen.getByText("urgent")).toBeInTheDocument(); + expect(screen.getByText("Hidden").query()).toBeNull(); + }); }); describe("empty states", () => { @@ -666,5 +758,22 @@ describe("ContentList", () => { // The header must not render as a button — it's just a label. expect(screen.getByRole("button", { name: "Title" }).query()).toBeNull(); }); + + it("keeps configured custom columns display-only", async () => { + const screen = await render( + , + ); + + await expect + .element(screen.getByRole("columnheader", { name: "Priority" })) + .toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Priority" }).query()).toBeNull(); + }); }); }); diff --git a/packages/admin/tests/components/ContentTypeEditor.test.tsx b/packages/admin/tests/components/ContentTypeEditor.test.tsx index 30420a1943..0129298d4b 100644 --- a/packages/admin/tests/components/ContentTypeEditor.test.tsx +++ b/packages/admin/tests/components/ContentTypeEditor.test.tsx @@ -42,6 +42,7 @@ function makeField(overrides: Partial = {}): SchemaField { required: false, unique: false, searchable: false, + indexed: false, sortOrder: 0, createdAt: "2025-01-01T00:00:00Z", ...overrides, diff --git a/packages/admin/tests/components/FieldEditor.test.tsx b/packages/admin/tests/components/FieldEditor.test.tsx index 007e2e1993..e383658b34 100644 --- a/packages/admin/tests/components/FieldEditor.test.tsx +++ b/packages/admin/tests/components/FieldEditor.test.tsx @@ -42,6 +42,7 @@ function makeField(overrides: Partial = {}): SchemaField { required: true, unique: false, searchable: true, + indexed: false, sortOrder: 0, createdAt: new Date().toISOString(), ...overrides, @@ -127,6 +128,7 @@ describe("FieldEditor", () => { it("shows searchable checkbox for string type", async () => { const screen = await render(); await expect.element(screen.getByText("Searchable")).toBeInTheDocument(); + await expect.element(screen.getByText("Indexed")).toBeInTheDocument(); }); it("shows min/max length validation for string type", async () => { @@ -162,6 +164,7 @@ describe("FieldEditor", () => { const screen = await render(); await expect.element(screen.getByLabelText("Min Value")).toBeInTheDocument(); await expect.element(screen.getByLabelText("Max Value")).toBeInTheDocument(); + await expect.element(screen.getByText("Indexed")).toBeInTheDocument(); }); it("does not show searchable for number type", async () => { @@ -201,6 +204,7 @@ describe("FieldEditor", () => { it("shows searchable checkbox for text type", async () => { const screen = await render(); await expect.element(screen.getByText("Searchable")).toBeInTheDocument(); + expect(screen.getByText("Indexed").query()).toBeNull(); }); }); @@ -338,6 +342,35 @@ describe("FieldEditor", () => { }); }); + describe("indexed flag", () => { + const save = async (screen: Awaited>) => { + const button = screen.getByRole("button", { name: "Update Field" }); + await expect.element(button).toBeEnabled(); + button.element().click(); + }; + + it("clears the flag when the field type cannot be indexed", async () => { + const onSave = vi.fn(); + const field = makeField({ slug: "body", label: "Body", type: "text", indexed: true }); + const screen = await render(); + + expect(screen.getByText("Indexed").query()).toBeNull(); + await save(screen); + + expect(onSave).toHaveBeenCalledWith(expect.objectContaining({ indexed: false })); + }); + + it("keeps the flag for a field type that can be indexed", async () => { + const onSave = vi.fn(); + const field = makeField({ slug: "priority", label: "Priority", indexed: true }); + const screen = await render(); + + await save(screen); + + expect(onSave).toHaveBeenCalledWith(expect.objectContaining({ indexed: true })); + }); + }); + describe("config step (file field)", () => { const fileField = makeField({ slug: "attachment", diff --git a/packages/core/src/api/handlers/content.ts b/packages/core/src/api/handlers/content.ts index 8a16b88fa4..3a7bad5128 100644 --- a/packages/core/src/api/handlers/content.ts +++ b/packages/core/src/api/handlers/content.ts @@ -5,6 +5,7 @@ import type { Kysely } from "kysely"; import { sql } from "kysely"; +import type { ContentFieldFilters } from "../../content-list-query.js"; import { isSqlite } from "../../database/dialect-helpers.js"; import { BylineRepository } from "../../database/repositories/byline.js"; import type { ContentBylineInput } from "../../database/repositories/byline.js"; @@ -430,6 +431,7 @@ export async function handleContentList( dateField?: ContentDateField; dateFrom?: string; dateTo?: string; + fieldFilters?: ContentFieldFilters; }, ): Promise> { try { @@ -438,6 +440,9 @@ export async function handleContentList( if (params.status) where.status = params.status; if (params.locale) where.locale = resolveConfiguredLocale(params.locale); if (params.authorId) where.authorId = params.authorId; + if (params.fieldFilters && Object.keys(params.fieldFilters).length > 0) { + where.fieldFilters = params.fieldFilters; + } // A date range requires a target column; ignore stray from/to without // a field so a half-specified filter doesn't silently drop all rows. diff --git a/packages/core/src/api/schemas/content.ts b/packages/core/src/api/schemas/content.ts index fafab339b2..2ae965cd42 100644 --- a/packages/core/src/api/schemas/content.ts +++ b/packages/core/src/api/schemas/content.ts @@ -1,5 +1,6 @@ import { z } from "zod"; +import { SQL_BATCH_SIZE } from "../../utils/chunks.js"; import { bylineSummarySchema, bylineCreditSchema, contentBylineInputSchema } from "./bylines.js"; import { cursorPaginationQuery, httpUrl, localeCode } from "./common.js"; @@ -26,6 +27,74 @@ const contentDateBound = z ]) .optional(); +const contentFieldComparable = z.union([z.string().max(2048), z.number().finite()]); +const contentFieldFilterScalar = z.union([contentFieldComparable, z.boolean(), z.null()]); +const contentFieldFilterValue = z.union([ + contentFieldFilterScalar, + z + .object({ + in: z + .array(z.union([contentFieldComparable, z.boolean()])) + .min(1) + .max(SQL_BATCH_SIZE), + }) + .strict(), + z + .object({ + gt: contentFieldComparable.optional(), + gte: contentFieldComparable.optional(), + lt: contentFieldComparable.optional(), + lte: contentFieldComparable.optional(), + }) + .strict() + .refine((value) => Object.values(value).some((bound) => bound !== undefined), { + message: "Range filter must include at least one bound", + }), +]); + +function contentFieldFilterOperandCount(value: unknown): number { + if (value === null) return 0; + if (!value || typeof value !== "object" || Array.isArray(value)) return 1; + if ("in" in value && Array.isArray(value.in)) return value.in.length; + return Object.values(value).filter((bound) => bound !== undefined).length; +} + +/** AND-combined filters over custom fields explicitly marked as indexed. */ +export const contentFieldFiltersSchema = z + .record( + z + .string() + .max(128) + .regex(/^[a-z][a-z0-9_]*$/, "must be a safe field identifier"), + contentFieldFilterValue, + ) + .refine((filters) => Object.keys(filters).length <= 20, { + message: "At most 20 indexed field filters are allowed", + }) + .refine( + (filters) => + Object.values(filters).reduce( + (total, value) => total + contentFieldFilterOperandCount(value), + 0, + ) <= SQL_BATCH_SIZE, + { + message: `Indexed field filters have a total operand budget of ${SQL_BATCH_SIZE}`, + }, + ); + +const contentFieldFiltersQuery = z + .string() + .max(8192) + .transform((value, ctx): unknown => { + try { + return JSON.parse(value); + } catch { + ctx.addIssue({ code: "custom", message: "must be valid JSON" }); + return z.NEVER; + } + }) + .pipe(contentFieldFiltersSchema); + export const contentListQuery = cursorPaginationQuery .extend({ status: z.string().optional(), @@ -42,6 +111,8 @@ export const contentListQuery = cursorPaginationQuery dateFrom: contentDateBound, /** Inclusive upper bound for the date range. Requires `dateField`. */ dateTo: contentDateBound, + /** JSON-encoded indexed custom-field filters, combined with AND semantics. */ + fieldFilters: contentFieldFiltersQuery.optional(), }) .meta({ id: "ContentListQuery" }); diff --git a/packages/core/src/api/schemas/schema.ts b/packages/core/src/api/schemas/schema.ts index f2922289e8..176228f71f 100644 --- a/packages/core/src/api/schemas/schema.ts +++ b/packages/core/src/api/schemas/schema.ts @@ -10,6 +10,12 @@ const collectionSupportValues = z.enum(["drafts", "revisions", "preview", "sched const collectionSourcePattern = /^(template:.+|import:.+|manual|discovered|seed)$/; +const collectionAdminConfig = z.object({ + listColumns: z + .array(z.string().min(1).max(63).regex(slugPattern, "Invalid field slug format")) + .optional(), +}); + const fieldTypeValues = z.enum([ "string", "text", @@ -82,6 +88,7 @@ export const createCollectionBody = z labelSingular: z.string().optional(), description: z.string().optional(), icon: z.string().optional(), + admin: collectionAdminConfig.optional(), supports: z.array(collectionSupportValues).optional(), source: z.string().regex(collectionSourcePattern).optional(), urlPattern: z.string().optional(), @@ -95,6 +102,7 @@ export const updateCollectionBody = z labelSingular: z.string().optional(), description: z.string().optional(), icon: z.string().optional(), + admin: collectionAdminConfig.optional(), supports: z.array(collectionSupportValues).optional(), urlPattern: z.string().nullish(), hasSeo: z.boolean().optional(), @@ -118,6 +126,7 @@ export const createFieldBody = z options: fieldWidgetOptions, sortOrder: z.number().int().min(0).optional(), searchable: z.boolean().optional(), + indexed: z.boolean().optional(), translatable: z.boolean().optional(), }) .meta({ id: "CreateFieldBody" }); @@ -134,6 +143,7 @@ export const updateFieldBody = z options: fieldWidgetOptions, sortOrder: z.number().int().min(0).optional(), searchable: z.boolean().optional(), + indexed: z.boolean().optional(), translatable: z.boolean().optional(), }) .meta({ id: "UpdateFieldBody" }); @@ -175,6 +185,7 @@ export const collectionSchema = z labelSingular: z.string().nullable(), description: z.string().nullable(), icon: z.string().nullable(), + admin: collectionAdminConfig.optional(), supports: z.array(z.string()), source: z.string().nullable(), urlPattern: z.string().nullable(), @@ -199,6 +210,7 @@ export const fieldSchema = z options: z.record(z.string(), z.unknown()).nullable(), sortOrder: z.number().int(), searchable: z.boolean(), + indexed: z.boolean(), translatable: z.boolean(), createdAt: z.string(), updatedAt: z.string(), diff --git a/packages/core/src/astro/types.ts b/packages/core/src/astro/types.ts index 3ed85d1e3c..857f04520f 100644 --- a/packages/core/src/astro/types.ts +++ b/packages/core/src/astro/types.ts @@ -8,6 +8,7 @@ import type { Element } from "@emdash-cms/blocks"; import type { Kysely } from "kysely"; +import type { ContentFieldFilters } from "../content-list-query.js"; import type { RouteMeta } from "../plugins/routes.js"; // Re-export core types @@ -31,6 +32,8 @@ export interface ManifestCollection { supports: string[]; hasSeo: boolean; urlPattern?: string; + /** Valid custom field slugs to render in the admin content list. */ + listColumns?: string[]; fields: Record< string, { @@ -241,6 +244,7 @@ export interface EmDashHandlers { dateField?: "createdAt" | "updatedAt" | "publishedAt"; dateFrom?: string; dateTo?: string; + fieldFilters?: ContentFieldFilters; }, ) => Promise; diff --git a/packages/core/src/cli/commands/export-seed.ts b/packages/core/src/cli/commands/export-seed.ts index 707a8a122c..d7ceecab74 100644 --- a/packages/core/src/cli/commands/export-seed.ts +++ b/packages/core/src/cli/commands/export-seed.ts @@ -314,6 +314,7 @@ async function exportCollections(db: Kysely): Promise 0 ? collection.supports : undefined, urlPattern: collection.urlPattern || undefined, fields: fields.map( @@ -324,6 +325,7 @@ async function exportCollections(db: Kysely): Promise> { const params = new URLSearchParams(); @@ -506,6 +509,9 @@ export class EmDashClient { if (options?.orderBy) params.set("orderBy", options.orderBy); if (options?.order) params.set("order", options.order); if (options?.locale) params.set("locale", options.locale); + if (options?.fieldFilters && Object.keys(options.fieldFilters).length > 0) { + params.set("fieldFilters", JSON.stringify(options.fieldFilters)); + } const qs = params.toString(); const path = `/content/${encodeURIComponent(collection)}${qs ? `?${qs}` : ""}`; @@ -521,6 +527,8 @@ export class EmDashClient { orderBy?: string; order?: "asc" | "desc"; locale?: string; + /** AND-combined filters over custom fields explicitly marked as indexed. */ + fieldFilters?: ContentFieldFilters; }, ): AsyncGenerator { let cursor: string | undefined; diff --git a/packages/core/src/content-list-query.ts b/packages/core/src/content-list-query.ts new file mode 100644 index 0000000000..075fbabc17 --- /dev/null +++ b/packages/core/src/content-list-query.ts @@ -0,0 +1,29 @@ +/** Scalar values accepted by indexed content-field filters. */ +export type ContentFieldFilterScalar = string | number | boolean | null; + +/** Inclusive or exclusive bounds for an indexed scalar field. */ +export interface ContentFieldRangeFilter { + gt?: string | number; + gte?: string | number; + lt?: string | number; + lte?: string | number; +} + +/** Match any one of the supplied values. */ +export interface ContentFieldInFilter { + in: Array; +} + +/** A single indexed content-field condition. */ +export type ContentFieldFilterValue = + | ContentFieldFilterScalar + | ContentFieldRangeFilter + | ContentFieldInFilter; + +/** + * Indexed custom-field filters, combined with AND semantics. + * + * A scalar performs an exact match, `null` matches missing values, `{ in: [...] }` + * performs membership matching, and range bounds can be combined. + */ +export type ContentFieldFilters = Record; diff --git a/packages/core/src/database/migrations/055_collection_admin_config.ts b/packages/core/src/database/migrations/055_collection_admin_config.ts new file mode 100644 index 0000000000..22d51b26b4 --- /dev/null +++ b/packages/core/src/database/migrations/055_collection_admin_config.ts @@ -0,0 +1,16 @@ +import type { Kysely } from "kysely"; + +import { columnExists } from "../dialect-helpers.js"; + +/** Persist collection-level admin presentation options. */ +export async function up(db: Kysely): Promise { + if (!(await columnExists(db, "_emdash_collections", "admin_config"))) { + await db.schema.alterTable("_emdash_collections").addColumn("admin_config", "text").execute(); + } +} + +export async function down(db: Kysely): Promise { + if (await columnExists(db, "_emdash_collections", "admin_config")) { + await db.schema.alterTable("_emdash_collections").dropColumn("admin_config").execute(); + } +} diff --git a/packages/core/src/database/migrations/056_indexed_content_fields.ts b/packages/core/src/database/migrations/056_indexed_content_fields.ts new file mode 100644 index 0000000000..998185c85d --- /dev/null +++ b/packages/core/src/database/migrations/056_indexed_content_fields.ts @@ -0,0 +1,36 @@ +import { sql, type Kysely } from "kysely"; + +import { columnExists } from "../dialect-helpers.js"; + +const FIELD_ID_PATTERN = /^[0-9A-Z]{26}$/; + +/** Mark custom fields whose structured list queries are backed by a physical index. */ +export async function up(db: Kysely): Promise { + if (!(await columnExists(db, "_emdash_fields", "indexed"))) { + await db.schema + .alterTable("_emdash_fields") + .addColumn("indexed", "integer", (column) => column.notNull().defaultTo(0)) + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (!(await columnExists(db, "_emdash_fields", "indexed"))) { + return; + } + + const indexedFields = await sql<{ id: string }>` + SELECT id FROM _emdash_fields WHERE indexed = 1 + `.execute(db); + + for (const field of indexedFields.rows) { + if (!FIELD_ID_PATTERN.test(field.id)) { + throw new Error(`Invalid indexed field id "${field.id}"`); + } + await sql` + DROP INDEX IF EXISTS ${sql.ref(`idx_cf_${field.id.toLowerCase()}`)} + `.execute(db); + } + + await db.schema.alterTable("_emdash_fields").dropColumn("indexed").execute(); +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 5c1c7764ff..a7f2cd443f 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -57,6 +57,8 @@ import * as m051 from "./051_content_taxonomies_denorm.js"; import * as m052 from "./052_media_usage_read_index.js"; import * as m053 from "./053_plugin_mcp_tools.js"; import * as m054 from "./054_media_upload_attempts.js"; +import * as m055 from "./055_collection_admin_config.js"; +import * as m056 from "./056_indexed_content_fields.js"; const MIGRATIONS: Readonly> = Object.freeze({ "001_initial": m001, @@ -112,6 +114,8 @@ const MIGRATIONS: Readonly> = Object.freeze({ "052_media_usage_read_index": m052, "053_plugin_mcp_tools": m053, "054_media_upload_attempts": m054, + "055_collection_admin_config": m055, + "056_indexed_content_fields": m056, }); /** Total number of registered migrations. Exported for use in tests. */ diff --git a/packages/core/src/database/repositories/content.ts b/packages/core/src/database/repositories/content.ts index 0ea9d8fab3..e979e3230e 100644 --- a/packages/core/src/database/repositories/content.ts +++ b/packages/core/src/database/repositories/content.ts @@ -1,7 +1,9 @@ import { sql, type Kysely } from "kysely"; import { ulid } from "ulidx"; +import type { ContentFieldFilterValue, ContentFieldFilters } from "../../content-list-query.js"; import { invalidateCollectionCache } from "../../object-cache/index.js"; +import { isIndexableFieldType, type FieldType } from "../../schema/types.js"; import { buildFtsPrefixMatch, buildSlugGlobPrefix } from "../../search/match.js"; import { chunks, SQL_BATCH_SIZE } from "../../utils/chunks.js"; import { isMissingTableError } from "../../utils/db-errors.js"; @@ -20,6 +22,7 @@ import type { import { ContentMutationConflictError, EmDashValidationError, + InvalidCursorError, ScheduledNotDueError, encodeCursor, decodeCursor, @@ -31,6 +34,21 @@ const SQLSTATE_PATTERN = /^[0-9A-Z]{5}$/; // LIKE wildcards that must be escaped so user search input is matched literally. const LIKE_WILDCARD_RE = /[\\%_]/g; +const MAX_INDEXED_FIELD_FILTERS = 20; +const MAX_IN_FILTER_VALUES = SQL_BATCH_SIZE; +const MAX_FILTER_STRING_LENGTH = 2048; + +type NormalizedFilterScalar = string | number; + +type ResolvedFieldFilter = + | { column: string; kind: "null" } + | { column: string; kind: "exact"; value: NormalizedFilterScalar } + | { column: string; kind: "in"; values: NormalizedFilterScalar[] } + | { + column: string; + kind: "range"; + bounds: Partial>; + }; function nullableColumnMatch(column: string, value: string | null): ReturnType { validateIdentifier(column, "content column"); @@ -85,6 +103,51 @@ function isConfirmedStatementFailure(error: unknown): boolean { return typeof code === "string" && (code.startsWith("SQLITE_") || SQLSTATE_PATTERN.test(code)); } +interface ResolvedOrderField { + column: string; + indexedCustomField: boolean; +} + +type IndexedOrderValue = string | number | null; + +interface IndexedFieldCursorPayload { + version: 1; + field: string; + value: IndexedOrderValue; +} + +function encodeIndexedFieldCursor(field: string, value: IndexedOrderValue, id: string): string { + const payload: IndexedFieldCursorPayload = { version: 1, field, value }; + return encodeCursor(JSON.stringify(payload), id); +} + +function decodeIndexedFieldCursor( + cursor: string, + field: string, +): { value: IndexedOrderValue; id: string } { + const { orderValue, id } = decodeCursor(cursor); + let payload: unknown; + try { + payload = JSON.parse(orderValue); + } catch { + throw new InvalidCursorError(cursor); + } + + if (payload === null || typeof payload !== "object") { + throw new InvalidCursorError(cursor); + } + const candidate = payload as Partial; + const validValue = + candidate.value === null || + typeof candidate.value === "string" || + typeof candidate.value === "number"; + if (candidate.version !== 1 || candidate.field !== field || !validValue) { + throw new InvalidCursorError(cursor); + } + + return { value: candidate.value as IndexedOrderValue, id }; +} + /** * Whitelist mapping a public date-filter field to its physical column. Keeping * this separate from `mapOrderField` makes the filterable set explicit and @@ -557,7 +620,9 @@ export class ContentRepository { // Determine ordering const orderField = options.orderBy?.field || "createdAt"; const orderDirection = options.orderBy?.direction || "desc"; - const dbField = this.mapOrderField(orderField); + const resolvedOrderField = await this.resolveOrderField(type, orderField); + const dbField = resolvedOrderField.column; + const resolvedFieldFilters = await this.resolveFieldFilters(type, options.where?.fieldFilters); // Validate order direction to prevent injection const safeOrderDirection = orderDirection.toLowerCase() === "asc" ? "ASC" : "DESC"; @@ -584,40 +649,88 @@ export class ContentRepository { query = this.applySearchFilter(query, options.where, type); query = this.applyDateFilter(query, options.where); + query = this.applyFieldFilters(query, resolvedFieldFilters); // Handle cursor pagination — decodeCursor throws InvalidCursorError // on malformed input; let it propagate so handlers surface a // structured INVALID_CURSOR rather than silently returning page 1. if (options.cursor) { - const { orderValue, id: cursorId } = decodeCursor(options.cursor); - - if (safeOrderDirection === "DESC") { - query = query.where((eb) => - eb.or([ - eb(dbField as any, "<", orderValue), - eb.and([eb(dbField as any, "=", orderValue), eb("id", "<", cursorId)]), - ]), - ); + if (resolvedOrderField.indexedCustomField) { + const { value, id: cursorId } = decodeIndexedFieldCursor(options.cursor, orderField); + const isPresent = sql`${sql.ref(dbField)} IS NOT NULL`; + const falseLiteral = sql`FALSE`; + const trueLiteral = sql`TRUE`; + if (safeOrderDirection === "ASC" && value === null) { + query = query.where(sql` + (${isPresent}) > ${falseLiteral} + OR ((${isPresent}) = ${falseLiteral} AND ${sql.ref("id")} > ${cursorId}) + `); + } else if (safeOrderDirection === "DESC" && value === null) { + query = query.where(sql` + (${isPresent}) = ${falseLiteral} AND ${sql.ref("id")} < ${cursorId} + `); + } else if (safeOrderDirection === "ASC") { + query = query.where(sql` + (${isPresent}, ${sql.ref(dbField)}, ${sql.ref("id")}) + > (${trueLiteral}, ${value}, ${cursorId}) + `); + } else { + query = query.where(sql` + (${isPresent}, ${sql.ref(dbField)}, ${sql.ref("id")}) + < (${trueLiteral}, ${value}, ${cursorId}) + `); + } } else { - query = query.where((eb) => - eb.or([ - eb(dbField as any, ">", orderValue), - eb.and([eb(dbField as any, "=", orderValue), eb("id", ">", cursorId)]), - ]), - ); + const { orderValue, id: cursorId } = decodeCursor(options.cursor); + + if (safeOrderDirection === "DESC") { + query = query.where((eb) => + eb.or([ + eb(dbField as any, "<", orderValue), + eb.and([eb(dbField as any, "=", orderValue), eb("id", "<", cursorId)]), + ]), + ); + } else { + query = query.where((eb) => + eb.or([ + eb(dbField as any, ">", orderValue), + eb.and([eb(dbField as any, "=", orderValue), eb("id", ">", cursorId)]), + ]), + ); + } } } // Apply ordering and limit - query = query - .orderBy(dbField as any, safeOrderDirection === "ASC" ? "asc" : "desc") - .orderBy("id", safeOrderDirection === "ASC" ? "asc" : "desc") - .limit(limit + 1); + const indexedOrderFilter = resolvedOrderField.indexedCustomField + ? resolvedFieldFilters.find((filter) => filter.column === dbField) + : undefined; + if (resolvedOrderField.indexedCustomField && !indexedOrderFilter) { + query = query.orderBy( + sql`${sql.ref(dbField)} IS NOT NULL`, + safeOrderDirection === "ASC" ? "asc" : "desc", + ); + } + if (indexedOrderFilter?.kind !== "null") { + query = query.orderBy(dbField as any, safeOrderDirection === "ASC" ? "asc" : "desc"); + } + query = query.orderBy("id", safeOrderDirection === "ASC" ? "asc" : "desc").limit(limit + 1); // Run the page fetch and the unbounded count together — the UI needs // both to render a stable denominator (kept on every page intentionally), // and issuing them in parallel on SQLite is essentially free. - const [rows, total] = await Promise.all([query.execute(), this.count(type, options.where)]); + // + // A collection whose table is missing rejects both. The query that loses + // the race stays in flight holding a pooled connection, and a Postgres + // pool destroyed in that window never finishes closing. + const [rowsResult, countResult] = await Promise.allSettled([ + query.execute(), + this.countWithResolvedFilters(type, options.where, resolvedFieldFilters), + ]); + if (rowsResult.status === "rejected") throw rowsResult.reason; + if (countResult.status === "rejected") throw countResult.reason; + const rows = rowsResult.value; + const total = countResult.value; const hasMore = rows.length > limit; const items = rows.slice(0, limit); @@ -629,11 +742,26 @@ export class ContentRepository { if (hasMore && items.length > 0) { const lastRow = items.at(-1) as Record; const lastOrderValue = lastRow[dbField]; - const orderStr = - typeof lastOrderValue === "string" || typeof lastOrderValue === "number" - ? String(lastOrderValue) - : ""; - mappedResult.nextCursor = encodeCursor(orderStr, String(lastRow.id)); + if (resolvedOrderField.indexedCustomField) { + if ( + lastOrderValue !== null && + typeof lastOrderValue !== "string" && + typeof lastOrderValue !== "number" + ) { + throw new EmDashValidationError(`Invalid indexed value for order field: ${orderField}`); + } + mappedResult.nextCursor = encodeIndexedFieldCursor( + orderField, + lastOrderValue, + String(lastRow.id), + ); + } else { + const orderStr = + typeof lastOrderValue === "string" || typeof lastOrderValue === "number" + ? String(lastOrderValue) + : ""; + mappedResult.nextCursor = encodeCursor(orderStr, String(lastRow.id)); + } } return mappedResult; @@ -988,6 +1116,15 @@ export class ContentRepository { * Count content items */ async count(type: string, where?: FindManyOptions["where"]): Promise { + const resolvedFieldFilters = await this.resolveFieldFilters(type, where?.fieldFilters); + return this.countWithResolvedFilters(type, where, resolvedFieldFilters); + } + + private async countWithResolvedFilters( + type: string, + where: FindManyOptions["where"] | undefined, + resolvedFieldFilters: ResolvedFieldFilter[], + ): Promise { const tableName = getTableName(type); let query = this.db @@ -1009,6 +1146,7 @@ export class ContentRepository { query = this.applySearchFilter(query, where, type); query = this.applyDateFilter(query, where); + query = this.applyFieldFilters(query, resolvedFieldFilters); const result = await query.executeTakeFirst(); return Number(result?.count || 0); @@ -1684,6 +1822,199 @@ export class ContentRepository { }; } + private normalizeFilterScalar( + field: string, + type: FieldType, + value: unknown, + ): NormalizedFilterScalar { + if (type === "number" || type === "integer") { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new EmDashValidationError(`Filter for field "${field}" must use a finite number`); + } + if (type === "integer" && !Number.isInteger(value)) { + throw new EmDashValidationError(`Filter for field "${field}" must use an integer`); + } + return value; + } + + if (type === "boolean") { + if (typeof value !== "boolean") { + throw new EmDashValidationError(`Filter for field "${field}" must use a boolean`); + } + return value ? 1 : 0; + } + + if (typeof value !== "string") { + throw new EmDashValidationError(`Filter for field "${field}" must use a string`); + } + if (value.length > MAX_FILTER_STRING_LENGTH) { + throw new EmDashValidationError( + `Filter value for field "${field}" exceeds ${MAX_FILTER_STRING_LENGTH} characters`, + ); + } + return value; + } + + private normalizeFieldFilter( + field: string, + type: FieldType, + value: ContentFieldFilterValue, + ): ResolvedFieldFilter { + if (value === null) return { column: field, kind: "null" }; + if (typeof value !== "object") { + return { + column: field, + kind: "exact", + value: this.normalizeFilterScalar(field, type, value), + }; + } + if (Array.isArray(value)) { + throw new EmDashValidationError(`Invalid filter for field "${field}"`); + } + + const record = value as Record; + const keys = Object.keys(record); + if (keys.length === 1 && keys[0] === "in") { + if (!Array.isArray(record.in) || record.in.length === 0) { + throw new EmDashValidationError(`IN filter for field "${field}" must not be empty`); + } + if (record.in.length > MAX_IN_FILTER_VALUES) { + throw new EmDashValidationError( + `IN filter for field "${field}" exceeds ${MAX_IN_FILTER_VALUES} values`, + ); + } + return { + column: field, + kind: "in", + values: record.in.map((entry) => this.normalizeFilterScalar(field, type, entry)), + }; + } + + const rangeKeys = new Set(["gt", "gte", "lt", "lte"]); + if (keys.length === 0 || keys.some((key) => !rangeKeys.has(key))) { + throw new EmDashValidationError(`Invalid filter operator for field "${field}"`); + } + if (type === "boolean") { + throw new EmDashValidationError(`Boolean field "${field}" does not support range filters`); + } + + const bounds: Partial> = {}; + for (const key of keys as Array<"gt" | "gte" | "lt" | "lte">) { + if (record[key] === undefined) continue; + bounds[key] = this.normalizeFilterScalar(field, type, record[key]); + } + if (Object.keys(bounds).length === 0) { + throw new EmDashValidationError(`Range filter for field "${field}" has no bounds`); + } + return { column: field, kind: "range", bounds }; + } + + private async resolveFieldFilters( + type: string, + filters: ContentFieldFilters | undefined, + ): Promise { + const resolvedFilters = filters ?? {}; + const fields = Object.keys(resolvedFilters); + if (fields.length === 0) return []; + if (fields.length > MAX_INDEXED_FIELD_FILTERS) { + throw new EmDashValidationError( + `Content list queries support at most ${MAX_INDEXED_FIELD_FILTERS} indexed field filters`, + ); + } + const rows = await this.db + .selectFrom("_emdash_fields as field") + .innerJoin("_emdash_collections as collection", "collection.id", "field.collection_id") + .where("collection.slug", "=", type) + .where("field.slug", "in", fields) + .where("field.indexed", "=", 1) + .select(["field.slug", "field.type"]) + .execute(); + const metadata = new Map(rows.map((row) => [row.slug, row.type as FieldType])); + + if (metadata.size === 0) { + const collection = await this.db + .selectFrom("_emdash_collections") + .where("slug", "=", type) + .select("id") + .executeTakeFirst(); + if (!collection) return []; + } + + for (const field of fields) { + try { + validateIdentifier(field, "content filter field"); + } catch { + throw new EmDashValidationError(`Invalid content filter field: ${field}`); + } + } + + const normalized = fields.map((field) => { + const fieldType = metadata.get(field); + if (!fieldType || !isIndexableFieldType(fieldType)) { + throw new EmDashValidationError( + `Cannot filter by field "${field}". Custom fields must be indexed before filtering.`, + ); + } + return this.normalizeFieldFilter(field, fieldType, resolvedFilters[field]); + }); + const operandCount = normalized.reduce((total, filter) => { + if (filter.kind === "null") return total; + if (filter.kind === "exact") return total + 1; + if (filter.kind === "in") return total + filter.values.length; + return total + Object.keys(filter.bounds).length; + }, 0); + if (operandCount > SQL_BATCH_SIZE) { + throw new EmDashValidationError( + `Indexed field filters have a total operand budget of ${SQL_BATCH_SIZE}`, + ); + } + return normalized; + } + + private applyFieldFilters unknown) => QB }>( + query: QB, + filters: ResolvedFieldFilter[], + ): QB { + let next = query; + for (const filter of filters) { + const column = sql.ref(filter.column); + const isPresent = sql`${column} IS NOT NULL`; + if (filter.kind === "null") { + next = next.where(() => sql`(${isPresent}) = FALSE AND ${column} IS NULL`); + continue; + } + if (filter.kind === "exact") { + next = next.where( + () => sql`(${isPresent}) = TRUE AND ${column} = ${filter.value}`, + ); + continue; + } + if (filter.kind === "in") { + const values = sql.join( + filter.values.map((value) => sql`${value}`), + sql`, `, + ); + next = next.where(() => sql`(${isPresent}) = TRUE AND ${column} IN (${values})`); + continue; + } + + next = next.where(() => sql`(${isPresent}) = TRUE`); + if (filter.bounds.gt !== undefined) { + next = next.where(() => sql`${column} > ${filter.bounds.gt}`); + } + if (filter.bounds.gte !== undefined) { + next = next.where(() => sql`${column} >= ${filter.bounds.gte}`); + } + if (filter.bounds.lt !== undefined) { + next = next.where(() => sql`${column} < ${filter.bounds.lt}`); + } + if (filter.bounds.lte !== undefined) { + next = next.where(() => sql`${column} <= ${filter.bounds.lte}`); + } + } + return next; + } + /** * Map order field names to database columns. * Only allows known fields to prevent column enumeration via crafted orderBy values. @@ -1708,4 +2039,30 @@ export class ContentRepository { } return mapped; } + + private async resolveOrderField(type: string, field: string): Promise { + try { + return { column: this.mapOrderField(field), indexedCustomField: false }; + } catch (error) { + if (!(error instanceof EmDashValidationError)) throw error; + } + + const customField = await this.db + .selectFrom("_emdash_fields as field") + .innerJoin("_emdash_collections as collection", "collection.id", "field.collection_id") + .where("collection.slug", "=", type) + .where("field.slug", "=", field) + .where("field.indexed", "=", 1) + .select("field.slug") + .executeTakeFirst(); + + if (!customField) { + throw new EmDashValidationError( + `Invalid order field: ${field}. Custom fields must be indexed before sorting.`, + ); + } + + validateIdentifier(customField.slug, "content order field"); + return { column: customField.slug, indexedCustomField: true }; + } } diff --git a/packages/core/src/database/repositories/types.ts b/packages/core/src/database/repositories/types.ts index b98f36ab2a..74b97929f4 100644 --- a/packages/core/src/database/repositories/types.ts +++ b/packages/core/src/database/repositories/types.ts @@ -1,3 +1,4 @@ +import type { ContentFieldFilters } from "../../content-list-query.js"; import type { CustomFieldValue } from "../../schema/types.js"; import { encodeBase64, decodeBase64 } from "../../utils/base64.js"; @@ -168,6 +169,8 @@ export interface FindManyOptions { useFts?: boolean; /** Inclusive date range over a whitelisted timestamp column. */ dateFilter?: ContentDateFilter; + /** AND-combined filters over custom fields explicitly marked as indexed. */ + fieldFilters?: ContentFieldFilters; }; orderBy?: { field: string; diff --git a/packages/core/src/database/types.ts b/packages/core/src/database/types.ts index 0b00056fc9..c0e7237ad6 100644 --- a/packages/core/src/database/types.ts +++ b/packages/core/src/database/types.ts @@ -300,6 +300,7 @@ export interface CollectionTable { label_singular: string | null; description: string | null; icon: string | null; + admin_config: Generated; // JSON: { listColumns?: string[] } supports: string | null; // JSON array source: string | null; search_config: string | null; // JSON: { enabled: boolean, weights: Record } @@ -340,6 +341,7 @@ export interface FieldTable { options: string | null; // JSON sort_order: number; searchable: Generated; // boolean as 0/1, defaults to 0 + indexed: Generated; // boolean as 0/1, defaults to 0 translatable: Generated; // boolean as 0/1, defaults to 1 created_at: Generated; } diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index c50f363211..d4c1c44c2b 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -22,6 +22,7 @@ import type { import type { EmDashManifest, ManifestCollection } from "./astro/types.js"; import { getAuthMode } from "./auth/mode.js"; import { getTrustedProxyHeaders } from "./auth/trusted-proxy.js"; +import type { ContentFieldFilters } from "./content-list-query.js"; import { isSqlite } from "./database/dialect-helpers.js"; import { kyselyLogOption } from "./database/instrumentation.js"; import { @@ -229,6 +230,17 @@ const FIELD_TYPE_TO_KIND: Record = { const DRAFT_ONLY_UPDATE_KEYS = new Set(["data", "slug", "locale", "skipRevision"]); const MAX_DRAFT_STAGE_ATTEMPTS = 32; +const LIST_COLUMN_FIELD_TYPES: ReadonlySet = new Set([ + "string", + "number", + "integer", + "boolean", + "datetime", + "select", + "multiSelect", +]); +const MAX_LIST_COLUMNS = 4; + /** * Sandboxed plugin entry from virtual module */ @@ -2360,12 +2372,34 @@ export class EmDashRuntime { fields[field.slug] = entry; } + const configuredListColumns = collection.admin?.listColumns ?? []; + const fieldTypes = new Map(collection.fields.map((field) => [field.slug, field.type])); + const listColumns: string[] = []; + for (const slug of configuredListColumns) { + if (listColumns.includes(slug)) continue; + const fieldType = fieldTypes.get(slug); + if (!fieldType || !LIST_COLUMN_FIELD_TYPES.has(fieldType)) { + console.warn( + `EmDash: Ignoring unsupported or unknown list column "${slug}" in collection "${collection.slug}".`, + ); + continue; + } + if (listColumns.length >= MAX_LIST_COLUMNS) { + console.warn( + `EmDash: Collection "${collection.slug}" declares more than ${MAX_LIST_COLUMNS} list columns; extra columns are ignored.`, + ); + break; + } + listColumns.push(slug); + } + manifestCollections[collection.slug] = { label: collection.label, labelSingular: collection.labelSingular || collection.label, supports: collection.supports || [], hasSeo: collection.hasSeo, urlPattern: collection.urlPattern, + listColumns: listColumns.length > 0 ? listColumns : undefined, fields, }; } @@ -2614,6 +2648,7 @@ export class EmDashRuntime { dateField?: ContentDateField; dateFrom?: string; dateTo?: string; + fieldFilters?: ContentFieldFilters; }, ) { return handleContentList(this.db, collection, params); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4392244aaa..569412cb10 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -27,6 +27,13 @@ export type { FindManyOptions, FindManyResult, } from "./database/repositories/index.js"; +export type { + ContentFieldFilterScalar, + ContentFieldFilterValue, + ContentFieldFilters, + ContentFieldInFilter, + ContentFieldRangeFilter, +} from "./content-list-query.js"; export type { MediaItem, CreateMediaInput } from "./database/repositories/media.js"; export type { CompoundSelectLimitedAdapter } from "./database/dialect-helpers.js"; diff --git a/packages/core/src/mcp/server.ts b/packages/core/src/mcp/server.ts index e25338de24..b38679cca7 100644 --- a/packages/core/src/mcp/server.ts +++ b/packages/core/src/mcp/server.ts @@ -1805,6 +1805,7 @@ export function createMcpServer( .boolean() .optional() .describe("Include in full-text search index (default false)"), + indexed: z.boolean().optional().describe("Create a physical index for structured sorting"), translatable: z .boolean() .optional() @@ -1831,6 +1832,7 @@ export function createMcpServer( validation: args.validation, options: args.options, searchable: args.searchable, + indexed: args.indexed, translatable: args.translatable, }); return jsonResult(field); diff --git a/packages/core/src/plugins/types.ts b/packages/core/src/plugins/types.ts index 7e3ad2bd9f..a4839248cb 100644 --- a/packages/core/src/plugins/types.ts +++ b/packages/core/src/plugins/types.ts @@ -42,8 +42,17 @@ import type { z } from "astro/zod"; // Core Types // ============================================================================= +import type { ContentFieldFilters } from "../content-list-query.js"; import type { FieldType } from "../schema/types.js"; +export type { + ContentFieldFilterScalar, + ContentFieldFilterValue, + ContentFieldFilters, + ContentFieldInFilter, + ContentFieldRangeFilter, +} from "../content-list-query.js"; + export { CAPABILITY_RENAMES, capabilitiesToDeclaredAccess, @@ -227,6 +236,8 @@ export interface ContentListWhere { status?: string; /** Exact match on `locale` (e.g. `"en"`, `"fr-CA"`). */ locale?: string; + /** AND-combined filters over custom fields explicitly marked as indexed. */ + fieldFilters?: ContentFieldFilters; } /** diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index a2608f45a2..95ac6ba5ea 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -14,6 +14,7 @@ import { FTSManager } from "../search/fts-manager.js"; import { chunks, SQL_BATCH_SIZE } from "../utils/chunks.js"; import { type Collection, + type CollectionAdminConfig, type CollectionSource, type CollectionSupport, type ColumnType, @@ -25,6 +26,7 @@ import { type CollectionWithFields, type FieldType, FIELD_TYPE_TO_COLUMN, + isIndexableFieldType, RESERVED_FIELD_SLUGS, RESERVED_COLLECTION_SLUGS, } from "./types.js"; @@ -35,6 +37,7 @@ const EC_PREFIX_PATTERN = /^ec_/; const SINGLE_QUOTE_PATTERN = /'/g; const UNDERSCORE_PATTERN = /_/g; const WORD_BOUNDARY_PATTERN = /\b\w/g; +const FIELD_ID_PATTERN = /^[0-9A-Z]{26}$/; /** Valid column types for runtime validation */ const COLUMN_TYPES: ReadonlySet = new Set(["TEXT", "REAL", "INTEGER", "JSON"]); @@ -69,10 +72,19 @@ const VALID_COLLECTION_SUPPORTS: ReadonlySet = new Set { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseCollectionAdmin(raw: string | null | undefined): CollectionAdminConfig | undefined { + if (!raw) return undefined; + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed)) return undefined; + const listColumns = parsed.listColumns; + return { + listColumns: Array.isArray(listColumns) + ? listColumns.filter((value): value is string => typeof value === "string") + : undefined, + }; +} + /** * Error thrown when a schema operation fails */ @@ -253,6 +281,7 @@ export class SchemaRegistry { label_singular: input.labelSingular ?? null, description: input.description ?? null, icon: input.icon ?? null, + admin_config: input.admin ? JSON.stringify(input.admin) : null, supports: JSON.stringify(supports), source: input.source ?? "manual", has_seo: hasSeo ? 1 : 0, @@ -300,6 +329,7 @@ export class SchemaRegistry { const fieldSlugs = new Set(); for (const field of fields) { this.validateSlug(field.slug, "field"); + assertIndexableField(field.type, field.indexed, field.slug); if (RESERVED_FIELD_SLUGS.includes(field.slug)) { throw new SchemaError(`Field slug "${field.slug}" is reserved`, "RESERVED_SLUG"); } @@ -335,6 +365,7 @@ export class SchemaRegistry { options: field.options ? JSON.stringify(field.options) : null, sort_order: sortOrder, searchable: field.searchable ? 1 : 0, + indexed: field.indexed ? 1 : 0, translatable: field.translatable === false ? 0 : 1, }; }); @@ -351,6 +382,7 @@ export class SchemaRegistry { label_singular: input.labelSingular ?? null, description: input.description ?? null, icon: input.icon ?? null, + admin_config: input.admin ? JSON.stringify(input.admin) : null, supports: JSON.stringify(supports), source: "seed", has_seo: hasSeo ? 1 : 0, @@ -362,6 +394,12 @@ export class SchemaRegistry { await this.createContentTable(input.slug, trx, fields); + for (const field of fieldRows) { + if (field.indexed === 1) { + await this.createFieldIndex(input.slug, field.id, field.slug, trx); + } + } + for (const fieldBatch of chunks(fieldRows, SEED_FIELD_INSERT_BATCH_SIZE)) { await trx.insertInto("_emdash_fields").values(fieldBatch).execute(); } @@ -408,6 +446,12 @@ export class SchemaRegistry { label_singular: input.labelSingular ?? existing.labelSingular ?? null, description: input.description ?? existing.description ?? null, icon: input.icon ?? existing.icon ?? null, + admin_config: + input.admin !== undefined + ? JSON.stringify(input.admin) + : existing.admin + ? JSON.stringify(existing.admin) + : null, supports: input.supports ? JSON.stringify(input.supports) : JSON.stringify(existing.supports), @@ -571,6 +615,7 @@ export class SchemaRegistry { const id = ulid(); const columnType = FIELD_TYPE_TO_COLUMN[input.type]; + assertIndexableField(input.type, input.indexed, input.slug); // Get max sort order const maxSort = await this.db @@ -603,6 +648,7 @@ export class SchemaRegistry { options: input.options ? JSON.stringify(input.options) : null, sort_order: sortOrder, searchable: input.searchable ? 1 : 0, + indexed: input.indexed ? 1 : 0, translatable: input.translatable === false ? 0 : 1, }) .execute(); @@ -620,6 +666,10 @@ export class SchemaRegistry { trx, ); + if (input.indexed) { + await this.createFieldIndex(collectionSlug, id, input.slug, trx); + } + // Read the created field via trx (not this.db) to avoid connection mutex deadlock const fieldRow = await trx .selectFrom("_emdash_fields") @@ -702,6 +752,9 @@ export class SchemaRegistry { nextColumnType = newColumnType; } + const nextIndexed = input.indexed ?? field.indexed; + assertIndexableField(nextType, nextIndexed, fieldSlug); + let schemaMutated = false; try { const updatedField = await withTransaction(this.db, async (trx) => { @@ -722,6 +775,7 @@ export class SchemaRegistry { : field.searchable ? 1 : 0, + indexed: nextIndexed ? 1 : 0, translatable: input.translatable !== undefined ? input.translatable @@ -749,6 +803,14 @@ export class SchemaRegistry { .execute(); schemaMutated = true; + if (nextIndexed !== field.indexed) { + if (nextIndexed) { + await this.createFieldIndex(collectionSlug, field.id, fieldSlug, trx); + } else { + await this.dropFieldIndex(field.id, trx); + } + } + // Read the updated field via trx (not this.db) to avoid connection mutex deadlock const updatedRow = await trx .selectFrom("_emdash_fields") @@ -856,6 +918,10 @@ export class SchemaRegistry { await this.syncSearchState(collectionSlug, trx); } + if (field.indexed) { + await this.dropFieldIndex(field.id, trx); + } + // Drop column from content table — safe now because FTS triggers are gone await this.dropColumn(collectionSlug, fieldSlug, trx); }); @@ -1025,6 +1091,41 @@ export class SchemaRegistry { `.execute(conn); } + private getFieldIndexName(fieldId: string): string { + if (!FIELD_ID_PATTERN.test(fieldId)) { + throw new SchemaError(`Invalid field id "${fieldId}"`, "INVALID_FIELD_ID"); + } + return `idx_cf_${fieldId.toLowerCase()}`; + } + + private async createFieldIndex( + collectionSlug: string, + fieldId: string, + fieldSlug: string, + db?: Kysely, + ): Promise { + const conn = db ?? this.db; + const tableName = this.getTableName(collectionSlug); + const columnName = this.getColumnName(fieldSlug); + const indexName = this.getFieldIndexName(fieldId); + + await sql` + CREATE INDEX IF NOT EXISTS ${sql.ref(indexName)} + ON ${sql.ref(tableName)} ( + deleted_at, + (${sql.ref(columnName)} IS NOT NULL), + ${sql.ref(columnName)}, + id + ) + WHERE deleted_at IS NULL + `.execute(conn); + } + + private async dropFieldIndex(fieldId: string, db?: Kysely): Promise { + const conn = db ?? this.db; + await sql`DROP INDEX IF EXISTS ${sql.ref(this.getFieldIndexName(fieldId))}`.execute(conn); + } + /** * Add a column to a content table */ @@ -1224,6 +1325,7 @@ export class SchemaRegistry { labelSingular: row.label_singular ?? undefined, description: row.description ?? undefined, icon: row.icon ?? undefined, + admin: parseCollectionAdmin(row.admin_config), supports: parseSupports(row.supports), source: row.source && isCollectionSource(row.source) ? row.source : undefined, hasSeo: row.has_seo === 1, @@ -1259,6 +1361,7 @@ export class SchemaRegistry { options: row.options ? JSON.parse(row.options) : undefined, sortOrder: row.sort_order, searchable: row.searchable === 1, + indexed: row.indexed === 1, translatable: row.translatable !== 0, createdAt: row.created_at, }; diff --git a/packages/core/src/schema/types.ts b/packages/core/src/schema/types.ts index 6d5055e981..8d1049701f 100644 --- a/packages/core/src/schema/types.ts +++ b/packages/core/src/schema/types.ts @@ -48,6 +48,23 @@ export const FIELD_TYPES: readonly FieldType[] = [ "repeater", ] as const; +/** Scalar field types that can be backed by a content-list query index. */ +export const INDEXABLE_FIELD_TYPES: ReadonlySet = new Set([ + "string", + "url", + "number", + "integer", + "boolean", + "datetime", + "select", + "reference", + "slug", +]); + +export function isIndexableFieldType(type: FieldType): boolean { + return INDEXABLE_FIELD_TYPES.has(type); +} + /** * SQLite column types that map from field types */ @@ -155,6 +172,12 @@ export interface FieldWidgetOptions { [key: string]: unknown; } +/** Collection-level admin presentation options. */ +export interface CollectionAdminConfig { + /** Custom field slugs to show in the content list. */ + listColumns?: string[]; +} + /** * A collection definition */ @@ -165,6 +188,7 @@ export interface Collection { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports: CollectionSupport[]; source?: CollectionSource; /** Whether this collection has SEO metadata fields enabled */ @@ -201,6 +225,8 @@ export interface Field { options?: FieldWidgetOptions; sortOrder: number; searchable: boolean; + /** Whether this field has a physical index for structured list queries. */ + indexed: boolean; /** Whether this field is translatable (default true). Non-translatable fields are synced across locales. */ translatable: boolean; createdAt: string; @@ -215,6 +241,7 @@ export interface CreateCollectionInput { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports?: CollectionSupport[]; source?: CollectionSource; urlPattern?: string; @@ -230,6 +257,7 @@ export interface UpdateCollectionInput { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports?: CollectionSupport[]; urlPattern?: string; hasSeo?: boolean; @@ -255,6 +283,8 @@ export interface CreateFieldInput { sortOrder?: number; /** Whether this field should be indexed for search */ searchable?: boolean; + /** Create a physical index for structured sorting. */ + indexed?: boolean; /** Whether this field is translatable (default true). Non-translatable fields are synced across locales. */ translatable?: boolean; } @@ -283,6 +313,8 @@ export interface UpdateFieldInput { sortOrder?: number; /** Whether this field should be indexed for search */ searchable?: boolean; + /** Create or remove the physical index used by structured sorting. */ + indexed?: boolean; /** Whether this field is translatable (default true). Non-translatable fields are synced across locales. */ translatable?: boolean; } diff --git a/packages/core/src/seed/apply.ts b/packages/core/src/seed/apply.ts index efe5ed0060..4b019edb4d 100644 --- a/packages/core/src/seed/apply.ts +++ b/packages/core/src/seed/apply.ts @@ -177,6 +177,7 @@ export async function applySeed( labelSingular: collection.labelSingular, description: collection.description, icon: collection.icon, + admin: collection.admin, supports: collection.supports || [], urlPattern: collection.urlPattern, commentsEnabled: collection.commentsEnabled, @@ -193,6 +194,7 @@ export async function applySeed( required: field.required || false, unique: field.unique || false, searchable: field.searchable || false, + indexed: field.indexed || false, defaultValue: field.defaultValue, validation: field.validation, widget: field.widget, @@ -207,6 +209,7 @@ export async function applySeed( required: field.required || false, unique: field.unique || false, searchable: field.searchable || false, + indexed: field.indexed || false, defaultValue: field.defaultValue, validation: field.validation, widget: field.widget, @@ -231,6 +234,7 @@ export async function applySeed( required: field.required || false, unique: field.unique || false, searchable: field.searchable || false, + indexed: field.indexed || false, defaultValue: field.defaultValue, validation: field.validation, widget: field.widget, @@ -245,6 +249,7 @@ export async function applySeed( labelSingular: collection.labelSingular, description: collection.description, icon: collection.icon, + admin: collection.admin, supports: collection.supports || [], urlPattern: collection.urlPattern, commentsEnabled: collection.commentsEnabled, diff --git a/packages/core/src/seed/types.ts b/packages/core/src/seed/types.ts index a106bfcd51..3bf574fb1a 100644 --- a/packages/core/src/seed/types.ts +++ b/packages/core/src/seed/types.ts @@ -5,7 +5,7 @@ * collections, fields, menus, settings, taxonomies, redirects, widget areas, and optional sample content. */ -import type { FieldType } from "../schema/types.js"; +import type { CollectionAdminConfig, FieldType } from "../schema/types.js"; import type { SiteSettings } from "../settings/types.js"; import type { Storage } from "../storage/types.js"; @@ -72,6 +72,7 @@ export interface SeedCollection { labelSingular?: string; description?: string; icon?: string; + admin?: CollectionAdminConfig; supports?: ("drafts" | "revisions" | "preview" | "scheduling" | "search" | "seo")[]; urlPattern?: string; /** Enable comments on this collection */ @@ -89,6 +90,7 @@ export interface SeedField { required?: boolean; unique?: boolean; searchable?: boolean; + indexed?: boolean; defaultValue?: unknown; validation?: Record; widget?: string; diff --git a/packages/core/src/seed/validate.ts b/packages/core/src/seed/validate.ts index d4396d6c14..55ef5d4a18 100644 --- a/packages/core/src/seed/validate.ts +++ b/packages/core/src/seed/validate.ts @@ -108,6 +108,35 @@ export function validateSeed(data: unknown): ValidationResult { errors.push(`${prefix}: label is required`); } + const declaredFieldSlugs = new Set( + Array.isArray(collection.fields) + ? collection.fields.flatMap((field) => + isRecord(field) && typeof field.slug === "string" ? [field.slug] : [], + ) + : [], + ); + + if (collection.admin !== undefined) { + if (!isRecord(collection.admin)) { + errors.push(`${prefix}.admin: must be an object`); + } else if (collection.admin.listColumns !== undefined) { + if (!Array.isArray(collection.admin.listColumns)) { + errors.push(`${prefix}.admin.listColumns: must be an array`); + } else { + for (let j = 0; j < collection.admin.listColumns.length; j++) { + const slug = collection.admin.listColumns[j]; + if (typeof slug !== "string" || !COLLECTION_FIELD_SLUG_PATTERN.test(slug)) { + errors.push(`${prefix}.admin.listColumns[${j}]: must be a valid field slug`); + } else if (!declaredFieldSlugs.has(slug)) { + errors.push( + `${prefix}.admin.listColumns[${j}]: references unknown field "${slug}"`, + ); + } + } + } + } + } + // Validate fields if (!Array.isArray(collection.fields)) { errors.push(`${prefix}.fields: must be an array`); diff --git a/packages/core/tests/database/migrations.test.ts b/packages/core/tests/database/migrations.test.ts index 3afbfed073..9f2654ca98 100644 --- a/packages/core/tests/database/migrations.test.ts +++ b/packages/core/tests/database/migrations.test.ts @@ -1,7 +1,8 @@ -import type { Kysely } from "kysely"; +import { sql, type Kysely } from "kysely"; import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { createDatabase } from "../../src/database/connection.js"; +import * as indexedContentFields from "../../src/database/migrations/056_indexed_content_fields.js"; import { runMigrations, getMigrationStatus, @@ -77,6 +78,51 @@ describe("Database Migrations", () => { expect(status.applied).toHaveLength(MIGRATION_COUNT); // derived from MIGRATIONS map in runner.ts }); + it("should remove custom field indexes when rolling back indexed field support", async () => { + await runMigrations(db); + + const fieldId = "01J00000000000000000000000"; + const indexName = `idx_cf_${fieldId.toLowerCase()}`; + + await db + .insertInto("_emdash_collections") + .values({ id: "collection-1", slug: "posts", label: "Posts" }) + .execute(); + await db + .insertInto("_emdash_fields") + .values({ + id: fieldId, + collection_id: "collection-1", + slug: "priority", + label: "Priority", + type: "integer", + column_type: "INTEGER", + required: 0, + unique: 0, + default_value: null, + validation: null, + widget: null, + options: null, + sort_order: 0, + indexed: 1, + }) + .execute(); + await sql`CREATE INDEX ${sql.ref(indexName)} ON _emdash_fields (id)`.execute(db); + + await indexedContentFields.down(db); + + const indexes = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'index' AND name = ${indexName} + `.execute(db); + const fieldsTable = (await db.introspection.getTables()).find( + (table) => table.name === "_emdash_fields", + ); + + expect(indexes.rows).toEqual([]); + expect(fieldsTable?.columns.map((column) => column.name)).not.toContain("indexed"); + await expect(indexedContentFields.down(db)).resolves.not.toThrow(); + }); + it("should record migration in tracking table", async () => { await runMigrations(db); @@ -136,6 +182,7 @@ describe("Database Migrations", () => { expect(columns).toContain("label_singular"); expect(columns).toContain("description"); expect(columns).toContain("icon"); + expect(columns).toContain("admin_config"); expect(columns).toContain("supports"); expect(columns).toContain("source"); expect(columns).toContain("created_at"); diff --git a/packages/core/tests/database/repositories/content-order-plan.test.ts b/packages/core/tests/database/repositories/content-order-plan.test.ts new file mode 100644 index 0000000000..07f7762510 --- /dev/null +++ b/packages/core/tests/database/repositories/content-order-plan.test.ts @@ -0,0 +1,133 @@ +import BetterSqlite3 from "better-sqlite3"; +import { Kysely, SqliteDialect } from "kysely"; +import { afterAll, beforeAll, expect, it } from "vitest"; + +import { runMigrations } from "../../../src/database/migrations/runner.js"; +import { ContentRepository } from "../../../src/database/repositories/content.js"; +import type { Database } from "../../../src/database/types.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; + +const sqlite = new BetterSqlite3(":memory:"); +const captured: Array<{ sql: string; parameters: readonly unknown[] }> = []; +const db = new Kysely({ + dialect: new SqliteDialect({ database: sqlite }), + log(event) { + if (event.level === "query") { + captured.push({ sql: event.query.sql, parameters: event.query.parameters }); + } + }, +}); +const repo = new ContentRepository(db); +let indexName: string; + +beforeAll(async () => { + await runMigrations(db); + const registry = new SchemaRegistry(db); + await registry.createCollection({ slug: "post", label: "Posts", labelSingular: "Post" }); + const field = await registry.createField("post", { + slug: "priority", + label: "Priority", + type: "number", + indexed: true, + }); + indexName = `idx_cf_${field.id.toLowerCase()}`; + + const insert = sqlite.prepare('INSERT INTO "ec_post" ("id", "priority") VALUES (?, ?)'); + sqlite.transaction(() => { + for (let i = 0; i < 1_100; i++) { + insert.run(`post-${i.toString().padStart(4, "0")}`, i < 100 ? null : (i - 100) % 20); + } + })(); + sqlite.exec("ANALYZE"); +}); + +afterAll(async () => { + await db.destroy(); +}); + +function getListPlan() { + const listQuery = captured.find( + (query) => + query.sql.includes('from "ec_post"') && + query.sql.includes("order by") && + query.sql.includes("limit"), + ); + expect(listQuery, "expected to capture the content list query").toBeDefined(); + return sqlite + .prepare(`EXPLAIN QUERY PLAN ${listQuery!.sql}`) + .all(...listQuery!.parameters) as Array<{ detail: string }>; +} + +function expectIndexSearch(plan: Array<{ detail: string }>) { + const details = plan.map((row) => row.detail).join("\n"); + expect(details).toContain(`SEARCH ec_post USING INDEX ${indexName}`); + expect(details).not.toContain("SCAN ec_post"); + expect(details).not.toContain("TEMP B-TREE"); +} + +it("uses the custom-field index for ordered cursor pages", async () => { + captured.length = 0; + const firstPage = await repo.findMany("post", { + limit: 3, + orderBy: { field: "priority", direction: "desc" }, + }); + const firstPlan = getListPlan(); + + captured.length = 0; + const secondPage = await repo.findMany("post", { + limit: 3, + cursor: firstPage.nextCursor, + orderBy: { field: "priority", direction: "desc" }, + }); + const secondPlan = getListPlan(); + + expect(firstPage.items.map((item) => item.data.priority)).toEqual([19, 19, 19]); + expect(secondPage.items.map((item) => item.data.priority)).toEqual([19, 19, 19]); + expect(new Set([...firstPage.items, ...secondPage.items].map((item) => item.id)).size).toBe(6); + expect(firstPage.total).toBe(1_100); + expect(secondPage.total).toBe(1_100); + expectIndexSearch(firstPlan); + expectIndexSearch(secondPlan); +}); + +it.each([ + ["an exact", 7, 50], + ["a membership", { in: [3, 7, 11] }, 150], + ["a null", null, 100], +])("uses the custom-field index for %s predicate", async (_label, filter, total) => { + captured.length = 0; + const result = await repo.findMany("post", { + orderBy: { field: "priority", direction: "asc" }, + where: { fieldFilters: { priority: filter } }, + }); + + expect(result.total).toBe(total); + expectIndexSearch(getListPlan()); +}); + +it("uses the custom-field index for range filtering and cursor pages", async () => { + captured.length = 0; + const firstPage = await repo.findMany("post", { + limit: 3, + orderBy: { field: "priority", direction: "asc" }, + where: { fieldFilters: { priority: { gte: 10 } } }, + }); + const firstPlan = getListPlan(); + + captured.length = 0; + const secondPage = await repo.findMany("post", { + limit: 3, + cursor: firstPage.nextCursor, + orderBy: { field: "priority", direction: "asc" }, + where: { fieldFilters: { priority: { gte: 10 } } }, + }); + const secondPlan = getListPlan(); + + expect(firstPage.items.map((item) => item.data.priority)).toEqual([10, 10, 10]); + expect(secondPage.items.map((item) => item.data.priority)).toEqual([10, 10, 10]); + expect(new Set([...firstPage.items, ...secondPage.items].map((item) => item.id)).size).toBe(6); + expect(firstPage.total).toBe(500); + expect(secondPage.total).toBe(500); + expectIndexSearch(firstPlan); + expectIndexSearch(secondPlan); +}); diff --git a/packages/core/tests/database/repositories/content.test.ts b/packages/core/tests/database/repositories/content.test.ts index da04c9e124..c6bcca8836 100644 --- a/packages/core/tests/database/repositories/content.test.ts +++ b/packages/core/tests/database/repositories/content.test.ts @@ -429,6 +429,101 @@ describe("ContentRepository", () => { }); describe("orderBy", () => { + it("paginates indexed custom fields with stable null ordering", async () => { + await registry.createField("post", { + slug: "priority", + label: "Priority", + type: "number", + indexed: true, + }); + + const seeded = await repo.findMany("post"); + const priorities = [null, 2, 1, 2, null]; + for (const [index, item] of seeded.items.entries()) { + await repo.update("post", item.id, { data: { priority: priorities[index] } }); + } + + const collect = async (direction: "asc" | "desc") => { + const items = []; + let cursor: string | undefined; + do { + const page = await repo.findMany("post", { + limit: 2, + cursor, + orderBy: { field: "priority", direction }, + }); + items.push(...page.items); + cursor = page.nextCursor; + } while (cursor); + return items; + }; + + const ascending = await collect("asc"); + const descending = await collect("desc"); + const values = (items: typeof ascending) => items.map((item) => item.data.priority ?? null); + + expect(values(ascending)).toEqual([null, null, 1, 2, 2]); + expect(values(descending)).toEqual([2, 2, 1, null, null]); + expect(new Set(ascending.map((item) => item.id)).size).toBe(5); + expect(new Set(descending.map((item) => item.id)).size).toBe(5); + }); + + it("paginates indexed datetime fields without skipping equal values", async () => { + await registry.createField("post", { + slug: "starts_at", + label: "Starts at", + type: "datetime", + indexed: true, + }); + + const seeded = await repo.findMany("post"); + const startsAt = [ + null, + "2026-08-02T12:00:00.000Z", + "2026-07-01T08:30:00.000Z", + "2026-08-02T12:00:00.000Z", + null, + ]; + for (const [index, item] of seeded.items.entries()) { + await repo.update("post", item.id, { data: { starts_at: startsAt[index] } }); + } + + const items = []; + let cursor: string | undefined; + do { + const page = await repo.findMany("post", { + limit: 2, + cursor, + orderBy: { field: "starts_at", direction: "asc" }, + }); + items.push(...page.items); + cursor = page.nextCursor; + } while (cursor); + + expect(items.map((item) => item.data.starts_at ?? null)).toEqual([ + null, + null, + "2026-07-01T08:30:00.000Z", + "2026-08-02T12:00:00.000Z", + "2026-08-02T12:00:00.000Z", + ]); + expect(new Set(items.map((item) => item.id)).size).toBe(5); + }); + + it("rejects unindexed custom order fields", async () => { + await registry.createField("post", { + slug: "priority", + label: "Priority", + type: "number", + }); + + await expect( + repo.findMany("post", { + orderBy: { field: "priority", direction: "asc" }, + }), + ).rejects.toThrow(EmDashValidationError); + }); + // Regression guard for "table headers aren't sort controls": the // admin now sends orderBy={field,direction} — the repo must accept // the columns the UI wants to expose, not just dates. @@ -457,6 +552,238 @@ describe("ContentRepository", () => { }); }); + describe("indexed field filters", () => { + async function seedIndexedFields() { + await registry.createField("post", { + slug: "score", + label: "Score", + type: "number", + indexed: true, + }); + await registry.createField("post", { + slug: "queue", + label: "Queue", + type: "string", + indexed: true, + }); + await registry.createField("post", { + slug: "resolved", + label: "Resolved", + type: "boolean", + indexed: true, + }); + await registry.createField("post", { + slug: "starts_at", + label: "Starts at", + type: "datetime", + indexed: true, + }); + + const seeded = await repo.findMany("post", { + orderBy: { field: "slug", direction: "asc" }, + }); + const values = [ + { + score: null, + queue: "urgent", + resolved: false, + starts_at: "2026-01-01T00:00:00.000Z", + }, + { + score: 50, + queue: "normal", + resolved: false, + starts_at: "2026-02-01T00:00:00.000Z", + }, + { + score: 80, + queue: "urgent", + resolved: true, + starts_at: "2026-03-01T00:00:00.000Z", + }, + { + score: 90, + queue: "high", + resolved: false, + starts_at: "2026-04-01T00:00:00.000Z", + }, + { score: null, queue: "normal", resolved: false, starts_at: null }, + ]; + for (const [index, item] of seeded.items.entries()) { + await repo.update("post", item.id, { data: values[index] }); + } + } + + it("combines exact, membership, and range filters without changing total semantics", async () => { + await seedIndexedFields(); + + const result = await repo.findMany("post", { + where: { + fieldFilters: { + queue: { in: ["urgent", "high"] }, + score: { gte: 80 }, + resolved: false, + }, + }, + }); + + expect(result.items.map((item) => item.slug)).toEqual(["post-3"]); + expect(result.total).toBe(1); + }); + + it("supports exact boolean and datetime range filters with AND semantics", async () => { + await seedIndexedFields(); + + const result = await repo.findMany("post", { + orderBy: { field: "slug", direction: "asc" }, + where: { + fieldFilters: { + queue: "urgent", + resolved: true, + starts_at: { + gte: "2026-02-01T00:00:00.000Z", + lte: "2026-03-01T00:00:00.000Z", + }, + }, + }, + }); + + expect(result.items.map((item) => item.slug)).toEqual(["post-2"]); + expect(result.total).toBe(1); + }); + + it("paginates null matches while keeping the filtered total stable", async () => { + await seedIndexedFields(); + + const page1 = await repo.findMany("post", { + limit: 1, + orderBy: { field: "slug", direction: "asc" }, + where: { fieldFilters: { score: null } }, + }); + const page2 = await repo.findMany("post", { + limit: 1, + cursor: page1.nextCursor, + orderBy: { field: "slug", direction: "asc" }, + where: { fieldFilters: { score: null } }, + }); + + expect(page1.items.map((item) => item.slug)).toEqual(["post-0"]); + expect(page2.items.map((item) => item.slug)).toEqual(["post-4"]); + expect(page1.total).toBe(2); + expect(page2.total).toBe(2); + }); + + it("combines filtering and indexed sorting across cursor pages without duplicates", async () => { + await seedIndexedFields(); + + const items = []; + const totals = []; + let cursor: string | undefined; + do { + const page = await repo.findMany("post", { + limit: 1, + cursor, + orderBy: { field: "score", direction: "asc" }, + where: { fieldFilters: { resolved: false } }, + }); + items.push(...page.items); + totals.push(page.total); + cursor = page.nextCursor; + } while (cursor); + + expect(items.map((item) => item.data.score ?? null)).toEqual([null, null, 50, 90]); + expect(new Set(items.map((item) => item.id)).size).toBe(4); + expect(totals).toEqual([4, 4, 4, 4]); + expect(await repo.count("post", { fieldFilters: { resolved: false } })).toBe(4); + }); + + it("parameterizes exact filter values", async () => { + await seedIndexedFields(); + const injectedValue = "urgent' OR 1=1 --"; + const target = ( + await repo.findMany("post", { + orderBy: { field: "slug", direction: "asc" }, + }) + ).items[1]!; + await repo.update("post", target.id, { data: { queue: injectedValue } }); + + const result = await repo.findMany("post", { + where: { fieldFilters: { queue: injectedValue } }, + }); + + expect(result.items.map((item) => item.id)).toEqual([target.id]); + expect(result.total).toBe(1); + }); + + it("enforces direct repository filter boundaries", async () => { + await seedIndexedFields(); + const fiftyValues = [ + ...Array.from({ length: 49 }, (_, index) => `queue-${index}`), + "urgent", + ]; + + await expect( + repo.findMany("post", { + where: { fieldFilters: { queue: { in: fiftyValues } } }, + }), + ).resolves.toMatchObject({ total: 2 }); + await expect( + repo.findMany("post", { + where: { fieldFilters: { queue: { in: [...fiftyValues, "overflow"] } } }, + }), + ).rejects.toThrow(/exceeds 50 values/); + await expect( + repo.findMany("post", { + where: { + fieldFilters: { + queue: { in: Array.from({ length: 30 }, (_, index) => `queue-${index}`) }, + resolved: { in: Array.from({ length: 21 }, (_, index) => index % 2 === 0) }, + }, + }, + }), + ).rejects.toThrow(/total operand budget of 50/); + await expect( + repo.findMany("post", { + where: { fieldFilters: { queue: "x".repeat(2049) } }, + }), + ).rejects.toThrow(/exceeds 2048 characters/); + await expect( + repo.findMany("post", { + where: { + fieldFilters: Object.fromEntries( + Array.from({ length: 21 }, (_, index) => [`field_${index}`, index]), + ), + }, + }), + ).rejects.toThrow(/at most 20 indexed field filters/); + await expect( + repo.findMany("post", { + where: { fieldFilters: { "queue;drop": "urgent" } }, + }), + ).rejects.toThrow(/Invalid content filter field/); + }); + + it("rejects unindexed fields and values that do not match the field type", async () => { + await seedIndexedFields(); + await registry.createField("post", { + slug: "internal_note", + label: "Internal note", + type: "string", + }); + + await expect( + repo.findMany("post", { + where: { fieldFilters: { internal_note: "review" } }, + }), + ).rejects.toThrow(/must be indexed/); + await expect( + repo.findMany("post", { + where: { fieldFilters: { score: "high" } }, + }), + ).rejects.toThrow(/finite number/); + }); + }); + describe("total", () => { // Regression guard for the admin "denominator grows as you page // forward" bug: each list response must include the full count so diff --git a/packages/core/tests/integration/content/content-list-filters.test.ts b/packages/core/tests/integration/content/content-list-filters.test.ts index cf053ed40e..33e54e90c8 100644 --- a/packages/core/tests/integration/content/content-list-filters.test.ts +++ b/packages/core/tests/integration/content/content-list-filters.test.ts @@ -6,6 +6,7 @@ import { handleContentList, } from "../../../src/api/handlers/content.js"; import { UserRepository } from "../../../src/database/repositories/user.js"; +import { createContentAccess } from "../../../src/plugins/context.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { describeEachDialect, @@ -28,6 +29,12 @@ describeEachDialect("content list filters (#1288)", (dialect) => { const registry = new SchemaRegistry(ctx.db); await registry.createCollection({ slug: "posts", label: "Posts", labelSingular: "Post" }); await registry.createField("posts", { slug: "title", label: "Title", type: "string" }); + await registry.createField("posts", { + slug: "priority", + label: "Priority", + type: "string", + indexed: true, + }); const users = new UserRepository(ctx.db); const alice = await users.create({ email: "alice@example.com", name: "Alice" }); @@ -37,14 +44,32 @@ describeEachDialect("content list filters (#1288)", (dialect) => { // Three posts across 2023/2024/2025, two by Alice and one by Bob. const seed = [ - { slug: "y2023", title: "Old", authorId: aliceId, createdAt: "2023-06-01T12:00:00.000Z" }, - { slug: "y2024", title: "Mid", authorId: bobId, createdAt: "2024-06-01T12:00:00.000Z" }, - { slug: "y2025", title: "New", authorId: aliceId, createdAt: "2025-06-01T12:00:00.000Z" }, + { + slug: "y2023", + title: "Old", + priority: "normal", + authorId: aliceId, + createdAt: "2023-06-01T12:00:00.000Z", + }, + { + slug: "y2024", + title: "Mid", + priority: "urgent", + authorId: bobId, + createdAt: "2024-06-01T12:00:00.000Z", + }, + { + slug: "y2025", + title: "New", + priority: "high", + authorId: aliceId, + createdAt: "2025-06-01T12:00:00.000Z", + }, ]; for (const s of seed) { const created = await handleContentCreate(ctx.db, "posts", { slug: s.slug, - data: { title: s.title }, + data: { title: s.title, priority: s.priority }, authorId: s.authorId, createdAt: s.createdAt, }); @@ -73,6 +98,59 @@ describeEachDialect("content list filters (#1288)", (dialect) => { expect(result.data.total).toBe(2); }); + it("passes indexed custom-field filters through the list handler", async () => { + const result = await handleContentList(ctx.db, "posts", { + fieldFilters: { priority: { in: ["urgent", "high"] } }, + }); + + expect(slugsOf(result).toSorted()).toEqual(["y2024", "y2025"]); + if (!result.success) throw new Error("list failed"); + expect(result.data.total).toBe(2); + }); + + it("reports a missing collection the same way with and without field filters", async () => { + const withFilters = await handleContentList(ctx.db, "ghosts", { + fieldFilters: { priority: "urgent" }, + }); + const withoutFilters = await handleContentList(ctx.db, "ghosts", {}); + + expect(withFilters.success).toBe(false); + expect(withoutFilters.success).toBe(false); + if (withFilters.success || withoutFilters.success) throw new Error("expected failures"); + expect(withFilters.error.code).toBe("COLLECTION_NOT_FOUND"); + expect(withFilters.error.code).toBe(withoutFilters.error.code); + }); + + it("reports a missing collection ahead of an invalid field filter name", async () => { + const result = await handleContentList(ctx.db, "ghosts", { + fieldFilters: { "not a field": "urgent" }, + }); + + expect(result.success).toBe(false); + if (result.success) throw new Error("expected a failure"); + expect(result.error.code).toBe("COLLECTION_NOT_FOUND"); + }); + + it("rejects an invalid field filter name on a collection that exists", async () => { + const result = await handleContentList(ctx.db, "posts", { + fieldFilters: { "not a field": "urgent" }, + }); + + expect(result.success).toBe(false); + if (result.success) throw new Error("expected a failure"); + expect(result.error.code).toBe("VALIDATION_ERROR"); + }); + + it("passes indexed custom-field filters through plugin content access", async () => { + const content = createContentAccess(ctx.db); + const result = await content.list("posts", { + where: { fieldFilters: { priority: "urgent" } }, + }); + + expect(result.items.map((item) => item.slug)).toEqual(["y2024"]); + expect(result.hasMore).toBe(false); + }); + it("filters by an inclusive createdAt date range", async () => { const result = await handleContentList(ctx.db, "posts", { dateField: "createdAt", diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index ce2e6de4c3..3d51142bb1 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -141,6 +141,8 @@ describe("Database Migrations (Integration)", () => { "052_media_usage_read_index", "053_plugin_mcp_tools", "054_media_upload_attempts", + "055_collection_admin_config", + "056_indexed_content_fields", ]; await db.deleteFrom("_emdash_migrations").where("name", "in", trailing).execute(); diff --git a/packages/core/tests/unit/api/schemas.test.ts b/packages/core/tests/unit/api/schemas.test.ts index 5a8e995860..7a84618c61 100644 --- a/packages/core/tests/unit/api/schemas.test.ts +++ b/packages/core/tests/unit/api/schemas.test.ts @@ -152,6 +152,77 @@ describe("localeCode validator", () => { expect(result.locale).toBe("zh-TW"); }); + it("contentListQuery parses bounded indexed field filters", () => { + const result = contentListQuery.parse({ + fieldFilters: JSON.stringify({ + priority: { in: ["urgent", "high"] }, + score: { gte: 80 }, + resolved: false, + }), + }); + + expect(result.fieldFilters).toEqual({ + priority: { in: ["urgent", "high"] }, + score: { gte: 80 }, + resolved: false, + }); + }); + + it("contentListQuery rejects malformed or unsupported field filters", () => { + expect(() => contentListQuery.parse({ fieldFilters: "not-json" })).toThrow(); + expect(() => + contentListQuery.parse({ fieldFilters: JSON.stringify({ priority: { in: [] } }) }), + ).toThrow(); + expect(() => + contentListQuery.parse({ fieldFilters: JSON.stringify({ "priority;drop": "urgent" }) }), + ).toThrow(); + }); + + it("contentListQuery enforces indexed field filter boundaries", () => { + const twentyFilters = Object.fromEntries( + Array.from({ length: 20 }, (_, index) => [`field_${index}`, index]), + ); + expect( + contentListQuery.parse({ fieldFilters: JSON.stringify(twentyFilters) }).fieldFilters, + ).toEqual(twentyFilters); + expect(() => + contentListQuery.parse({ + fieldFilters: JSON.stringify({ ...twentyFilters, field_20: 20 }), + }), + ).toThrow(); + + const fiftyValues = Array.from({ length: 50 }, (_, index) => index); + expect( + contentListQuery.parse({ + fieldFilters: JSON.stringify({ score: { in: fiftyValues } }), + }).fieldFilters, + ).toEqual({ score: { in: fiftyValues } }); + expect(() => + contentListQuery.parse({ + fieldFilters: JSON.stringify({ score: { in: [...fiftyValues, 50] } }), + }), + ).toThrow(); + expect(() => + contentListQuery.parse({ + fieldFilters: JSON.stringify({ + queue: { in: Array.from({ length: 30 }, (_, index) => `queue-${index}`) }, + resolved: { in: Array.from({ length: 21 }, (_, index) => index % 2 === 0) }, + }), + }), + ).toThrow(); + + expect( + contentListQuery.parse({ + fieldFilters: JSON.stringify({ queue: "x".repeat(2048) }), + }).fieldFilters, + ).toEqual({ queue: "x".repeat(2048) }); + expect(() => + contentListQuery.parse({ + fieldFilters: JSON.stringify({ queue: "x".repeat(2049) }), + }), + ).toThrow(); + }); + it("contentCreateBody keeps the locale casing", () => { const result = contentCreateBody.parse({ data: { title: "Hi" }, locale: "pt-BR" }); expect(result.locale).toBe("pt-BR"); diff --git a/packages/core/tests/unit/astro/content-routes-authz.test.ts b/packages/core/tests/unit/astro/content-routes-authz.test.ts index 419b0e70d6..643d52913a 100644 --- a/packages/core/tests/unit/astro/content-routes-authz.test.ts +++ b/packages/core/tests/unit/astro/content-routes-authz.test.ts @@ -98,12 +98,17 @@ function buildEmdash( compare?: { hasChanges: boolean; live: unknown; draft: unknown }; } = {}, ) { - const handleContentList = vi.fn(async (_collection: string, params: { status?: string }) => { - const items = params.status - ? (opts.listItems ?? []).filter((i) => i.status === params.status) - : (opts.listItems ?? []); - return { success: true as const, data: { items, nextCursor: undefined } }; - }); + const handleContentList = vi.fn( + async ( + _collection: string, + params: { status?: string; fieldFilters?: Record }, + ) => { + const items = params.status + ? (opts.listItems ?? []).filter((i) => i.status === params.status) + : (opts.listItems ?? []); + return { success: true as const, data: { items, nextCursor: undefined } }; + }, + ); const handleContentGet = vi.fn(async (_collection: string, _id: string) => { if (!opts.getItem) { @@ -223,6 +228,28 @@ describe("GET /content/:collection — subscriber drafts leak", () => { const call = emdash.handleContentList.mock.calls[0]?.[1]; expect(call?.status).toBeUndefined(); }); + + it("parses indexed field filters before calling the runtime", async () => { + const emdash = buildEmdash({ listItems: items }); + const fieldFilters = encodeURIComponent( + JSON.stringify({ priority: { in: ["urgent", "high"] }, resolved: false }), + ); + const res = await getList( + ctx({ + user: contributor, + emdash, + url: `http://localhost/?fieldFilters=${fieldFilters}`, + }), + ); + + expect(res.status).toBe(200); + expect(emdash.handleContentList).toHaveBeenCalledWith( + "post", + expect.objectContaining({ + fieldFilters: { priority: { in: ["urgent", "high"] }, resolved: false }, + }), + ); + }); }); // --------------------------------------------------------------------------- diff --git a/packages/core/tests/unit/cli/seed-commands.test.ts b/packages/core/tests/unit/cli/seed-commands.test.ts index 5ac70d9eab..4fca3708eb 100644 --- a/packages/core/tests/unit/cli/seed-commands.test.ts +++ b/packages/core/tests/unit/cli/seed-commands.test.ts @@ -8,6 +8,7 @@ import { join } from "node:path"; import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { exportSeed } from "../../../src/cli/commands/export-seed.js"; import { createDatabase } from "../../../src/database/connection.js"; import { runMigrations } from "../../../src/database/migrations/runner.js"; import { applySeed } from "../../../src/seed/apply.js"; @@ -213,6 +214,39 @@ describe("CLI Seed Commands", () => { }); describe("export-seed output", () => { + it("preserves collection list columns through apply and export", async () => { + const dbPath = join(tempDir, "admin-config.db"); + const db = createDatabase({ url: `file:${dbPath}` }); + + try { + await runMigrations(db); + await applySeed(db, { + version: "1", + collections: [ + { + slug: "tickets", + label: "Tickets", + admin: { listColumns: ["ticket_number", "priority"] }, + fields: [ + { slug: "ticket_number", label: "Ticket number", type: "string" }, + { slug: "priority", label: "Priority", type: "select" }, + ], + }, + ], + }); + + const exported = await exportSeed(db); + const tickets = exported.collections?.find((collection) => collection.slug === "tickets"); + + expect(tickets?.admin).toEqual({ + listColumns: ["ticket_number", "priority"], + }); + expect(validateSeed(exported)).toMatchObject({ valid: true, errors: [] }); + } finally { + await db.destroy(); + } + }); + it("should produce valid seed from exported data", async () => { const dbPath = join(tempDir, "test.db"); const db = createDatabase({ url: `file:${dbPath}` }); diff --git a/packages/core/tests/unit/client/client.test.ts b/packages/core/tests/unit/client/client.test.ts index 9e809ab33e..bd9784fc10 100644 --- a/packages/core/tests/unit/client/client.test.ts +++ b/packages/core/tests/unit/client/client.test.ts @@ -426,6 +426,40 @@ describe("EmDashClient", () => { expect(result.items).toHaveLength(2); expect(result.nextCursor).toBe("cursor123"); }); + + it("serializes indexed field filters", async () => { + let fieldFilters: string | null = null; + const backend = createMockBackend([ + { + method: "GET", + path: "/content/posts", + handler: (request) => { + fieldFilters = new URL(request.url).searchParams.get("fieldFilters"); + return jsonResponse({ items: [] }); + }, + }, + ]); + + const client = new EmDashClient({ + baseUrl: "http://localhost:4321", + token: "test", + interceptors: [backend], + }); + + await client.list("posts", { + fieldFilters: { + priority: { in: ["urgent", "high"] }, + score: { gte: 80 }, + resolved: false, + }, + }); + + expect(JSON.parse(fieldFilters!)).toEqual({ + priority: { in: ["urgent", "high"] }, + score: { gte: 80 }, + resolved: false, + }); + }); }); describe("listAll()", () => { diff --git a/packages/core/tests/unit/runtime/manifest-build.test.ts b/packages/core/tests/unit/runtime/manifest-build.test.ts index ee9e379005..17284d83f8 100644 --- a/packages/core/tests/unit/runtime/manifest-build.test.ts +++ b/packages/core/tests/unit/runtime/manifest-build.test.ts @@ -15,7 +15,7 @@ */ import type { Kysely } from "kysely"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { EmDashConfig } from "../../../src/astro/integration/runtime.js"; import type { Database } from "../../../src/database/types.js"; @@ -72,6 +72,7 @@ describe("EmDashRuntime.getManifest()", () => { }); afterEach(async () => { + vi.restoreAllMocks(); await teardownTestDatabase(db); }); @@ -158,4 +159,66 @@ describe("EmDashRuntime.getManifest()", () => { expect(manifest.collections[`coll_${i}`]?.fields.title?.kind).toBe("string"); } }); + + it("publishes only supported, existing list columns and caps them at four", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const registry = new SchemaRegistry(db); + await registry.createCollection({ + slug: "tickets", + label: "Tickets", + admin: { + listColumns: [ + "ticket_number", + "ticket_number", + "details", + "missing", + "priority", + "urgent", + "queue", + "opened_at", + ], + }, + }); + await registry.createField("tickets", { + slug: "ticket_number", + label: "Ticket number", + type: "string", + }); + await registry.createField("tickets", { + slug: "details", + label: "Details", + type: "json", + }); + await registry.createField("tickets", { + slug: "priority", + label: "Priority", + type: "select", + }); + await registry.createField("tickets", { + slug: "urgent", + label: "Urgent", + type: "boolean", + }); + await registry.createField("tickets", { + slug: "queue", + label: "Queue", + type: "string", + }); + await registry.createField("tickets", { + slug: "opened_at", + label: "Opened", + type: "datetime", + }); + + const runtime = buildRuntime(db); + const manifest = await runtime.getManifest(); + + expect(manifest.collections.tickets?.listColumns).toEqual([ + "ticket_number", + "priority", + "urgent", + "queue", + ]); + expect(warn).toHaveBeenCalledTimes(3); + }); }); diff --git a/packages/core/tests/unit/schema/registry.test.ts b/packages/core/tests/unit/schema/registry.test.ts index 0991d0d9ce..9a0bb8c577 100644 --- a/packages/core/tests/unit/schema/registry.test.ts +++ b/packages/core/tests/unit/schema/registry.test.ts @@ -128,6 +128,19 @@ describe("SchemaRegistry", () => { expect(updated.supports).toEqual(["drafts"]); }); + it("persists collection admin list columns", async () => { + const created = await registry.createCollection({ + slug: "tickets", + label: "Tickets", + admin: { listColumns: ["ticket_number", "priority"] }, + }); + + expect(created.admin?.listColumns).toEqual(["ticket_number", "priority"]); + + const updated = await registry.updateCollection("tickets", { label: "Support tickets" }); + expect(updated.admin?.listColumns).toEqual(["ticket_number", "priority"]); + }); + it("should throw when updating non-existent collection", async () => { await expect(registry.updateCollection("nonexistent", { label: "Test" })).rejects.toThrow( SchemaError, @@ -196,6 +209,125 @@ describe("SchemaRegistry", () => { expect(field.required).toBe(true); }); + it("keeps an indexed field's physical index in sync", async () => { + const listFieldIndexes = async () => + ( + await sql<{ name: string }>` + SELECT name + FROM sqlite_master + WHERE type = 'index' + AND tbl_name = 'ec_posts' + AND name LIKE 'idx_cf_%' + `.execute(db) + ).rows; + + const field = await registry.createField("posts", { + slug: "priority", + label: "Priority", + type: "number", + indexed: true, + }); + + expect(field.indexed).toBe(true); + expect(await listFieldIndexes()).toHaveLength(1); + + await registry.updateField("posts", "priority", { indexed: false }); + expect(await listFieldIndexes()).toHaveLength(0); + + await registry.updateField("posts", "priority", { indexed: true }); + expect(await listFieldIndexes()).toHaveLength(1); + + await registry.deleteField("posts", "priority"); + expect(await listFieldIndexes()).toHaveLength(0); + }); + + it("drops the index when an indexed field moves to a type that cannot carry one", async () => { + await registry.createField("posts", { + slug: "summary", + label: "Summary", + type: "string", + indexed: true, + }); + + await expect( + registry.updateField("posts", "summary", { type: "text" }), + ).rejects.toMatchObject({ code: "FIELD_NOT_INDEXABLE" }); + + const updated = await registry.updateField("posts", "summary", { + type: "text", + indexed: false, + }); + + expect(updated.type).toBe("text"); + expect(updated.indexed).toBe(false); + }); + + it("reuses an existing generated index when enabling indexed metadata", async () => { + const field = await registry.createField("posts", { + slug: "priority", + label: "Priority", + type: "number", + }); + const indexName = `idx_cf_${field.id.toLowerCase()}`; + + await sql` + CREATE INDEX ${sql.ref(indexName)} + ON ec_posts (deleted_at, (priority IS NOT NULL), priority, id) + WHERE deleted_at IS NULL + `.execute(db); + + await expect( + registry.updateField("posts", "priority", { indexed: true }), + ).resolves.toMatchObject({ indexed: true }); + + const indexes = await sql<{ name: string }>` + SELECT name FROM sqlite_master WHERE type = 'index' AND name = ${indexName} + `.execute(db); + expect(indexes.rows).toHaveLength(1); + }); + + it("uses the generated index for indexed custom field ordering", async () => { + const field = await registry.createField("posts", { + slug: "priority", + label: "Priority", + type: "number", + indexed: true, + }); + const indexName = `idx_cf_${field.id.toLowerCase()}`; + + const ascending = await sql<{ detail: string }>` + EXPLAIN QUERY PLAN + SELECT * FROM ec_posts + WHERE deleted_at IS NULL + ORDER BY (priority IS NOT NULL) ASC, priority ASC, id ASC + LIMIT 51 + `.execute(db); + const descending = await sql<{ detail: string }>` + EXPLAIN QUERY PLAN + SELECT * FROM ec_posts + WHERE deleted_at IS NULL + ORDER BY (priority IS NOT NULL) DESC, priority DESC, id DESC + LIMIT 51 + `.execute(db); + + for (const plan of [ascending, descending]) { + const details = plan.rows.map((row) => row.detail).join("\n"); + expect(details).toContain(indexName); + expect(details).not.toContain("USE TEMP B-TREE"); + } + }); + + it("rejects indexes for non-scalar fields", async () => { + await expect( + registry.createField("posts", { + slug: "body", + label: "Body", + type: "portableText", + indexed: true, + }), + ).rejects.toMatchObject({ code: "FIELD_NOT_INDEXABLE" }); + }); + it("should add column to content table when creating field", async () => { await registry.createField("posts", { slug: "title", diff --git a/packages/core/tests/unit/seed/validate.test.ts b/packages/core/tests/unit/seed/validate.test.ts index bca4c0e019..c793879baa 100644 --- a/packages/core/tests/unit/seed/validate.test.ts +++ b/packages/core/tests/unit/seed/validate.test.ts @@ -197,6 +197,25 @@ describe("validateSeed", () => { expect(result.valid).toBe(true); expect(result.errors).toHaveLength(0); }); + + it("should reject list columns that do not reference collection fields", () => { + const result = validateSeed({ + version: "1", + collections: [ + { + slug: "posts", + label: "Posts", + admin: { listColumns: ["priority"] }, + fields: [{ slug: "title", label: "Title", type: "string" }], + }, + ], + }); + + expect(result.valid).toBe(false); + expect(result.errors).toContain( + 'collections[0].admin.listColumns[0]: references unknown field "priority"', + ); + }); }); describe("taxonomy validation", () => {