diff --git a/docs/pipeline/ingest-stage.md b/docs/pipeline/ingest-stage.md new file mode 100644 index 0000000..0931314 --- /dev/null +++ b/docs/pipeline/ingest-stage.md @@ -0,0 +1,106 @@ +# Ingest stage + +The **ingest** stage is the pipeline's format-detecting front door. It turns a +source asset — an InDesign `.idml` package or an exported `.pdf` — into the +canonical [IR `Document`](../../packages/pipeline/src/indesign/ir.js) that the +[token mapper](../../packages/pipeline/src/indesign/map/index.js) and +[theme generator](../../packages/pipeline/src/indesign/generate/index.js) +consume, and enriches it with a derived **content model**. + +``` +source (.idml | .pdf | bytes) + │ + ▼ + ┌───────────┐ detect format by magic bytes (PK+designmap.xml / %PDF-) + │ ingest │ parse → IR → validate (Document.parse) + │ stage │ extract reading-order content model + └───────────┘ + │ + ▼ + artifact JSON ──► map-tokens ──► generate-theme + (IR + content) +``` + +## Why a separate stage + +The per-format parse bins (`flavian-parse-idml`, `flavian-parse-pdf`) each handle +one extension and print to stdout. Ingest adds three things they don't: + +1. **Byte sniffing** — the format is detected from the content, not the + filename, so extensionless files, stdin streams, and upload buffers all work. + IDML is recognized by the Zip signature plus the literal `designmap.xml` + manifest name (Zip stores entry *filenames* uncompressed); PDF by its `%PDF-` + header. +2. **A validated, writable artifact** — the IR is re-checked against the schema + and written as one JSON file (`--out`), ready for the next stage. +3. **A semantic content model** — see below. + +## Content model + +The IR records design and geometry but nothing about editorial structure. The +ingest stage derives a normalized, reading-order outline: + +- Each editorial spread becomes a **section** of ordered **blocks**: + `heading` (with an inferred level 1–6), `paragraph`, or `figure`. +- **Heading levels** are inferred from font size: the smallest paragraph size on + the spreads is the body baseline; larger distinct sizes rank largest → `

`, + capped at `

`. A single size means no headings. +- **Reading order** is top-to-bottom then left-to-right by frame bounds, with a + small row tolerance so two-column rows aren't interleaved. +- Consecutive runs that share a paragraph style merge into one block (a bold + word mid-sentence doesn't fragment the paragraph). +- Master-spread chrome (running header/footer) is excluded — the generator + derives that separately. + +The content model is attached to the artifact under an additive `content` key. +Downstream stages ignore it, so the artifact is still a valid `Document`: +`flavian-map-tokens` and `flavian-generate-theme` accept it directly. + +## Usage + +```bash +# Auto-detect and write the enriched IR +pnpm pipeline:ingest path/to/brochure.idml --out build/brochure.ir.json + +# From stdin, forcing the PDF parser, bare IR (no content model) +cat export.pdf | pnpm pipeline:ingest - --format pdf --no-content > ir.json + +# Pipe straight into theme generation (generate-theme reads the IR from stdin) +pnpm pipeline:ingest brochure.idml --quiet | \ + node packages/pipeline/bin/generate-theme.mjs --out-dir themes/brochure +``` + +| Option | Description | +| --- | --- | +| `--out ` | Write the artifact here (default: stdout) | +| `--format ` | `idml` \| `pdf` \| `auto` (default: `auto`) | +| `--dpi ` | Unit-normalization DPI (default 96) | +| `--name ` | Override the document name | +| `--no-content` | Emit the bare IR without the content model | +| `--quiet` | Suppress the stderr summary and parse warnings | + +## Programmatic API + +```js +import { ingestSource, toArtifact } from '@flavian/pipeline/indesign'; + +const result = await ingestSource('brochure.idml'); +// → { format: 'idml', ir: , content: } + +await fs.writeFile('out.json', JSON.stringify(toArtifact(result), null, 2)); +``` + +`ingestBuffer(bytes, opts)` is the in-memory variant. Both are pure of side +effects beyond reading the input path; the content transform (`extractContent`) +and the detector (`detectFormat`) are exported for direct use and are +unit-tested in `packages/pipeline/tests/indesign/ingest-*.test.mjs`. + +## Tests & fixtures + +Fixtures are generated in code (no binary blobs in git), consistent with the +rest of the pipeline: + +- `ingest-detect.test.mjs` — magic-byte detection on built IDML/PDF/zip buffers. +- `ingest-content.test.mjs` — reading order, heading-level inference, figures. +- `ingest.test.mjs` — full IDML + PDF ingest, determinism, and proof that the + artifact is generator-consumable. diff --git a/package.json b/package.json index 35b5439..57c0611 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "init": "node scripts/init.mjs", "test:init": "node --test \"tests/init/**/*.test.mjs\"", "test:pipeline": "pnpm --filter @flavian/pipeline test", + "pipeline:ingest": "node packages/pipeline/bin/ingest.mjs", "test:canva-e2e": "node --test tests/canva-e2e/*.test.mjs", "test:headless-e2e": "node --test tests/headless-e2e/*.test.mjs", "visual:capture": "node tests/visual/capture.mjs", diff --git a/packages/pipeline/bin/ingest.mjs b/packages/pipeline/bin/ingest.mjs new file mode 100644 index 0000000..974ae86 --- /dev/null +++ b/packages/pipeline/bin/ingest.mjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node +// CLI: ingest a source asset (IDML or PDF), auto-detecting the format, and write +// the canonical IR — enriched with a derived content model — as JSON. +// +// flavian-ingest [options] +// +// Reads a file path or stdin (binary-safe), writes the artifact to --out (or +// stdout), and prints a one-line summary plus parse warnings to stderr. The +// artifact is a valid IR that flavian-map-tokens / flavian-generate-theme accept +// directly. +// +// Options: +// --out Write the artifact here (default: stdout) +// --format idml | pdf | auto (default: auto — sniff the bytes) +// --dpi Unit-normalization DPI (default 96) +// --name Override the document name +// --no-content Emit the bare IR without the content model +// --quiet Suppress the stderr summary and parse warnings +// -h, --help Show this help + +import { promises as fs } from 'node:fs'; + +import { ingestBuffer, toArtifact } from '../src/indesign/ingest/index.js'; + +const args = process.argv.slice(2); +const opts = { format: 'auto', content: true, quiet: false }; +let inputPath; + +let i = 0; +function wantsValue(flag) { + const next = args[i + 1]; + if (next === undefined || next.startsWith('-')) { + process.stderr.write(`${flag} requires a value\n`); + process.exit(2); + } + i += 1; + return next; +} + +for (; i < args.length; i += 1) { + const arg = args[i]; + switch (arg) { + case '--out': opts.out = wantsValue(arg); break; + case '--format': opts.format = wantsValue(arg); break; + case '--dpi': opts.dpi = Number(wantsValue(arg)); break; + case '--name': opts.name = wantsValue(arg); break; + case '--no-content': opts.content = false; break; + case '--quiet': opts.quiet = true; break; + case '-h': case '--help': printUsage(); process.exit(0); break; + default: + // A bare "-" is the explicit stdin sentinel; any other dash-prefixed + // token is an unknown flag. + if (!inputPath && (arg === '-' || !arg.startsWith('-'))) { + inputPath = arg; + } else { + process.stderr.write(`Unknown argument: ${arg}\n`); + printUsage(); + process.exit(2); + } + } +} + +if (!['auto', 'idml', 'pdf'].includes(opts.format)) { + process.stderr.write('--format must be one of: auto, idml, pdf\n'); + process.exit(2); +} +if (opts.dpi !== undefined && (Number.isNaN(opts.dpi) || opts.dpi <= 0)) { + process.stderr.write('--dpi must be a positive number\n'); + process.exit(2); +} + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) chunks.push(chunk); + return Buffer.concat(chunks); +} + +try { + const bytes = inputPath && inputPath !== '-' ? await fs.readFile(inputPath) : await readStdin(); + if (bytes.length === 0) { + process.stderr.write('error: empty input\n'); + process.exit(1); + } + + const result = await ingestBuffer(bytes, { + format: opts.format, + dpi: opts.dpi, + name: opts.name, + content: opts.content, + }); + + const json = `${JSON.stringify(toArtifact(result), null, 2)}\n`; + if (opts.out) await fs.writeFile(opts.out, json); + else process.stdout.write(json); + + if (!opts.quiet) { + const { ir, content } = result; + const warnings = ir.warnings ?? []; + for (const w of warnings) process.stderr.write(`[${w.code}] ${w.message}\n`); + const lines = [`format: ${result.format} spreads: ${ir.spreads.length} warnings: ${warnings.length}`]; + if (content) { + const s = content.stats; + lines.push(`content: ${s.sections} sections, ${s.headings} headings, ${s.paragraphs} paragraphs, ${s.figures} figures`); + } + lines.push(`written: ${opts.out ?? ''}`); + process.stderr.write(`${lines.join('\n')}\n`); + } + + process.exit(0); +} catch (err) { + process.stderr.write(`error: ${err.message}\n`); + process.exit(1); +} + +function printUsage() { + process.stderr.write( + [ + 'Usage: flavian-ingest [options]', + '', + 'Options:', + ' --out Write the artifact here (default: stdout)', + ' --format idml | pdf | auto (default: auto — sniff the bytes)', + ' --dpi Unit-normalization DPI (default 96)', + ' --name Override the document name', + ' --no-content Emit the bare IR without the content model', + ' --quiet Suppress the stderr summary and parse warnings', + ' -h, --help Show this help', + '', + ].join('\n'), + ); +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 7c59f75..f37333f 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -9,6 +9,7 @@ "./indesign": "./src/indesign/index.js" }, "bin": { + "flavian-ingest": "./bin/ingest.mjs", "flavian-parse-idml": "./bin/parse-idml.mjs", "flavian-parse-pdf": "./bin/parse-pdf.mjs", "flavian-map-tokens": "./bin/map-tokens.mjs", diff --git a/packages/pipeline/src/indesign/index.js b/packages/pipeline/src/indesign/index.js index 8c240aa..ba0e963 100644 --- a/packages/pipeline/src/indesign/index.js +++ b/packages/pipeline/src/indesign/index.js @@ -1,5 +1,6 @@ export { parseIdml, parseIdmlBuffer } from './parse-idml.js'; export { parsePdf, parsePdfBuffer } from './parse-pdf.js'; +export { ingestSource, ingestBuffer, toArtifact, detectFormat, extractContent } from './ingest/index.js'; export * as ir from './ir.js'; export { WarningCollector } from './warnings.js'; export { lengthToPx, ptToPx, roundPx } from './units.js'; diff --git a/packages/pipeline/src/indesign/ingest/content.js b/packages/pipeline/src/indesign/ingest/content.js new file mode 100644 index 0000000..6fd2338 --- /dev/null +++ b/packages/pipeline/src/indesign/ingest/content.js @@ -0,0 +1,256 @@ +// Content extraction: the InDesign IR captures *design* (swatches, fonts, +// styles) and *geometry* (spreads → frames), but nothing semantic about the +// editorial content. This transform walks the IR and derives a normalized, +// reading-order content model: each editorial spread becomes a section of +// classified blocks (headings, paragraphs, figures), plus a flat document +// outline and summary stats. +// +// Pure and deterministic: IR in, content object out — no fs, no clock, no +// randomness. The same IR always yields the same model. +// +// Heuristics (necessary because neither IDML nor PDF labels semantics): +// • Heading vs body is inferred from font size. The smallest paragraph size +// used across the spreads is the body baseline; every larger distinct size +// is a heading, ranked largest →

, capped at

. A single size (or +// none) ⇒ no headings, everything is a paragraph. +// • Reading order = top-to-bottom, then left-to-right, by frame bounds, with +// a small row tolerance so a two-column row isn't interleaved by sub-pixel y. +// • Consecutive runs that share a paragraph style merge into one block, so a +// sentence split into character runs (e.g. a bold word) stays one paragraph. +// • Master-spread frames (running header/footer chrome) are excluded — the +// generator derives those separately. + +import { z } from 'zod'; +import { Rect } from '../ir.js'; + +export const ContentBlock = z.discriminatedUnion('type', [ + z.object({ + type: z.literal('heading'), + level: z.number().int().min(1).max(6), + text: z.string(), + styleRef: z.string().optional(), + frameRef: z.string().optional(), + }), + z.object({ + type: z.literal('paragraph'), + text: z.string(), + styleRef: z.string().optional(), + frameRef: z.string().optional(), + }), + z.object({ + type: z.literal('figure'), + href: z.string().optional(), + embedded: z.boolean().default(false), + bounds: Rect.optional(), + frameRef: z.string().optional(), + }), +]); + +export const ContentSection = z.object({ + id: z.string(), + source: z.string().optional(), + // The section's first heading, when it has one — a convenient label. + title: z.string().optional(), + blocks: z.array(ContentBlock), +}); + +export const ContentModel = z.object({ + contentVersion: z.literal(1).default(1), + sections: z.array(ContentSection), + outline: z.array( + z.object({ level: z.number().int().min(1).max(6), text: z.string() }), + ), + stats: z.object({ + sections: z.number().int().nonnegative(), + blocks: z.number().int().nonnegative(), + headings: z.number().int().nonnegative(), + paragraphs: z.number().int().nonnegative(), + figures: z.number().int().nonnegative(), + emptyTextFrames: z.number().int().nonnegative(), + danglingStoryRefs: z.number().int().nonnegative(), + }), +}); + +/** + * @typedef {z.infer} ContentModelData + * @typedef {z.infer} ContentBlockData + */ + +// Frames whose tops fall within this band read as the same row. +const ROW_TOLERANCE_PX = 8; + +/** + * @param {import('../ir.js').DocumentIR} ir + * @returns {ContentModelData} + */ +export function extractContent(ir) { + const storiesById = indexBy(ir.stories ?? [], 'id'); + const stylesById = indexBy(ir.styles ?? [], 'id'); + const spreads = ir.spreads ?? []; + + // Pass 1 — flatten each spread into reading-ordered items. Text classification + // waits until pass 2, once the global size distribution is known. + const raw = spreads.map((spread) => ({ + id: spread.id, + source: spread.source, + items: orderFrames(spread.frames ?? []).flatMap((frame) => + frame.kind === 'image' + ? [imageItem(frame)] + : textItems(frame, storiesById, stylesById), + ), + })); + + // Pass 2 — derive heading levels from the set of text sizes, then build blocks. + const sizeToLevel = buildHeadingScale(raw); + + const stats = { + sections: 0, + blocks: 0, + headings: 0, + paragraphs: 0, + figures: 0, + emptyTextFrames: 0, + danglingStoryRefs: 0, + }; + const outline = []; + const sections = []; + + for (const section of raw) { + const blocks = []; + for (const item of section.items) { + if (item.note === 'empty') { + stats.emptyTextFrames += 1; + continue; + } + if (item.note === 'dangling') { + stats.danglingStoryRefs += 1; + continue; + } + if (item.kind === 'figure') { + blocks.push({ + type: 'figure', + href: item.href, + embedded: item.embedded, + bounds: item.bounds, + frameRef: item.frameRef, + }); + stats.figures += 1; + continue; + } + const level = typeof item.size === 'number' ? sizeToLevel.get(item.size) : undefined; + if (level) { + blocks.push({ type: 'heading', level, text: item.text, styleRef: item.styleRef, frameRef: item.frameRef }); + stats.headings += 1; + outline.push({ level, text: item.text }); + } else { + blocks.push({ type: 'paragraph', text: item.text, styleRef: item.styleRef, frameRef: item.frameRef }); + stats.paragraphs += 1; + } + } + stats.blocks += blocks.length; + const title = blocks.find((b) => b.type === 'heading')?.text; + sections.push({ id: section.id, source: section.source, title, blocks }); + } + stats.sections = sections.length; + + return ContentModel.parse({ contentVersion: 1, sections, outline, stats }); +} + +function indexBy(items, key) { + const map = new Map(); + for (const item of items) map.set(item[key], item); + return map; +} + +/** + * Stable top-to-bottom, then left-to-right ordering. Frame tops within + * ROW_TOLERANCE_PX share a row so multi-column rows aren't interleaved by tiny + * y differences; ties fall back to source order for determinism. + */ +function orderFrames(frames) { + return frames + .map((frame, index) => ({ frame, index })) + .sort((a, b) => { + const rowA = Math.round((a.frame.bounds?.y ?? 0) / ROW_TOLERANCE_PX); + const rowB = Math.round((b.frame.bounds?.y ?? 0) / ROW_TOLERANCE_PX); + if (rowA !== rowB) return rowA - rowB; + const xA = a.frame.bounds?.x ?? 0; + const xB = b.frame.bounds?.x ?? 0; + if (xA !== xB) return xA - xB; + return a.index - b.index; + }) + .map((entry) => entry.frame); +} + +function imageItem(frame) { + return { + kind: 'figure', + href: frame.href, + embedded: frame.embedded ?? false, + bounds: frame.bounds, + frameRef: frame.id, + }; +} + +/** One text frame → zero or more text items (runs merged by paragraph style). */ +function textItems(frame, storiesById, stylesById) { + if (!frame.storyRef) return [{ kind: 'text', note: 'empty', frameRef: frame.id }]; + const story = storiesById.get(frame.storyRef); + if (!story) return [{ kind: 'text', note: 'dangling', frameRef: frame.id }]; + + const groups = groupRuns(story.runs ?? []); + if (groups.length === 0) return [{ kind: 'text', note: 'empty', frameRef: frame.id }]; + + return groups.map((group) => { + const style = group.styleRef ? stylesById.get(group.styleRef) : undefined; + return { + kind: 'text', + text: group.text, + styleRef: group.styleRef, + size: style?.fontSize, + frameRef: frame.id, + }; + }); +} + +/** + * Merge consecutive runs sharing a paragraph style into one block and drop + * whitespace-only fragments (frame padding, stray paragraph breaks). + */ +function groupRuns(runs) { + const groups = []; + for (const run of runs) { + const text = run.text ?? ''; + const styleRef = run.paragraphStyleRef; + const last = groups[groups.length - 1]; + if (last && last.styleRef === styleRef) last.text += text; + else groups.push({ styleRef, text }); + } + return groups + .map((group) => ({ styleRef: group.styleRef, text: group.text.trim() })) + .filter((group) => group.text.length > 0); +} + +/** + * Map each distinct text size to a heading level: the smallest size is the body + * baseline (no level); larger sizes rank largest → 1, capped at 6. Sizes that + * aren't headings are absent from the map (callers treat that as a paragraph). + */ +function buildHeadingScale(raw) { + const sizes = new Set(); + for (const section of raw) { + for (const item of section.items) { + if (item.kind === 'text' && !item.note && typeof item.size === 'number') { + sizes.add(item.size); + } + } + } + const map = new Map(); + const ascending = [...sizes].sort((a, b) => a - b); + if (ascending.length <= 1) return map; // one size (or none) ⇒ all body + + const baseline = ascending[0]; + const headingSizes = ascending.filter((size) => size > baseline).sort((a, b) => b - a); + headingSizes.forEach((size, index) => map.set(size, Math.min(index + 1, 6))); + return map; +} diff --git a/packages/pipeline/src/indesign/ingest/detect.js b/packages/pipeline/src/indesign/ingest/detect.js new file mode 100644 index 0000000..1bc6a74 --- /dev/null +++ b/packages/pipeline/src/indesign/ingest/detect.js @@ -0,0 +1,51 @@ +// Source-format detection for the ingest stage. Sniffs the *bytes*, not the +// filename, so the pipeline can ingest an extensionless file, a stream, or an +// in-memory upload — the per-format parse bins only dispatch on `.idml`/`.pdf`. +// +// IDML is a Zip package (PK signature) whose manifest is always `designmap.xml`. +// A Zip's entry *data* may be deflated, but its entry *filenames* are written +// verbatim into each local file header (and again into the central directory), +// so the manifest name is greppable even in a compressed package — that, not +// the easily-deflated mimetype marker, is what we key on. +// +// PDF starts with "%PDF-", which some exporters emit a few bytes in (after a +// BOM or leading whitespace), so we scan a short prefix rather than only byte 0. + +const ZIP_SIG_0 = 0x50; // 'P' +const ZIP_SIG_1 = 0x4b; // 'K' +const IDML_MANIFEST = 'designmap.xml'; +const PDF_HEADER = '%PDF-'; +const SNIFF_BYTES = 1024; + +/** + * @param {Uint8Array|Buffer} bytes + * @returns {'idml'|'pdf'|'unknown'} + */ +export function detectFormat(bytes) { + const buf = toBuffer(bytes); + if (buf.length < 5) return 'unknown'; + if (isPdf(buf)) return 'pdf'; + if (isIdml(buf)) return 'idml'; + return 'unknown'; +} + +function isPdf(buf) { + const head = buf.subarray(0, SNIFF_BYTES).toString('latin1'); + return head.includes(PDF_HEADER); +} + +function isIdml(buf) { + if (buf[0] !== ZIP_SIG_0 || buf[1] !== ZIP_SIG_1) return false; + // The manifest filename lives in a local file header near the front and in + // the central directory at the tail, so a whole-buffer scan is reliable. + return buf.includes(IDML_MANIFEST, 0, 'latin1'); +} + +/** Normalize any byte source to a Buffer view without copying when possible. */ +function toBuffer(bytes) { + if (Buffer.isBuffer(bytes)) return bytes; + if (bytes instanceof Uint8Array) { + return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + } + throw new TypeError('detectFormat expects a Buffer or Uint8Array'); +} diff --git a/packages/pipeline/src/indesign/ingest/index.js b/packages/pipeline/src/indesign/ingest/index.js new file mode 100644 index 0000000..882a939 --- /dev/null +++ b/packages/pipeline/src/indesign/ingest/index.js @@ -0,0 +1,97 @@ +// Ingest stage: a single, format-detecting front door for the pipeline. It +// turns a source asset (IDML or PDF) into the canonical IR the mapper and +// generator consume, enriched with a derived semantic content model. +// +// Where the per-format parse bins each handle one extension and print to +// stdout, this stage sniffs the bytes (so extensionless files, streams, and +// uploads all work), re-asserts the IR contract, and emits one validated, +// content-bearing artifact that downstream stages accept unchanged. +// +// ingestBuffer(bytes, opts) → { format, ir, content } +// ingestSource(pathOrBytes, opts) → same, reading a file path when given one +// toArtifact(result) → the single JSON object written to disk + +import { promises as fs } from 'node:fs'; +import path from 'node:path'; + +import { Document } from '../ir.js'; +import { parseIdmlBuffer } from '../parse-idml.js'; +import { parsePdfBuffer } from '../parse-pdf.js'; +import { detectFormat } from './detect.js'; +import { extractContent } from './content.js'; + +export { detectFormat } from './detect.js'; +export { extractContent, ContentModel } from './content.js'; + +/** + * @typedef {Object} IngestOptions + * @property {'auto'|'idml'|'pdf'} [format] Force a parser; default 'auto' (sniff bytes). + * @property {number} [dpi] Unit-normalization DPI (default 96). + * @property {string} [name] Override the document name. + * @property {boolean} [content] Derive the content model (default true). + * + * @typedef {Object} IngestResult + * @property {'idml'|'pdf'} format + * @property {import('../ir.js').DocumentIR} ir + * @property {import('./content.js').ContentModelData|null} content + */ + +/** + * Ingest in-memory source bytes. + * + * @param {Uint8Array} bytes + * @param {IngestOptions} [options] + * @returns {Promise} + */ +export async function ingestBuffer(bytes, options = {}) { + const requested = options.format ?? 'auto'; + const format = requested === 'auto' ? detectFormat(bytes) : requested; + + let parsed; + if (format === 'idml') { + parsed = parseIdmlBuffer(bytes, parseOptions(options)); + } else if (format === 'pdf') { + parsed = await parsePdfBuffer(bytes, parseOptions(options)); + } else { + throw new Error('Unrecognized source: expected an IDML package or a PDF document'); + } + + // Re-assert the IR contract before downstream stages build on it. + const ir = Document.parse(parsed); + const content = options.content === false ? null : extractContent(ir); + return { format, ir, content }; +} + +/** + * Ingest from a file path (or delegate to {@link ingestBuffer} for raw bytes). + * + * @param {string|Uint8Array} input + * @param {IngestOptions} [options] + * @returns {Promise} + */ +export async function ingestSource(input, options = {}) { + if (typeof input !== 'string') return ingestBuffer(input, options); + const bytes = await fs.readFile(input); + const name = options.name ?? path.basename(input).replace(/\.(idml|pdf)$/i, ''); + return ingestBuffer(bytes, { ...options, name }); +} + +/** + * Combine an ingest result into the single artifact written to disk: a valid + * `Document` (exactly what the generator reads) plus an additive `content` key + * downstream stages ignore. Keeping content as a sibling key — not folded into + * the IR — means the artifact still round-trips through the existing schema. + * + * @param {IngestResult} result + * @returns {object} + */ +export function toArtifact(result) { + return result.content ? { ...result.ir, content: result.content } : result.ir; +} + +function parseOptions(options) { + const opts = {}; + if (options.dpi !== undefined) opts.dpi = options.dpi; + if (options.name !== undefined) opts.name = options.name; + return opts; +} diff --git a/packages/pipeline/tests/indesign/ingest-content.test.mjs b/packages/pipeline/tests/indesign/ingest-content.test.mjs new file mode 100644 index 0000000..2811f96 --- /dev/null +++ b/packages/pipeline/tests/indesign/ingest-content.test.mjs @@ -0,0 +1,130 @@ +// The content-extraction transform: reading order, multi-level heading +// inference from font sizes, figure extraction, and run grouping — exercised on +// a small in-code IDML fixture whose frames are deliberately out of order. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { parseIdmlBuffer } from '../../src/indesign/parse-idml.js'; +import { extractContent } from '../../src/indesign/ingest/content.js'; +import { buildIdml } from './helpers/build-idml.js'; + +/** Three type sizes (36/24/12pt → 48/32/16px), one image, frames shuffled. */ +function miniIr() { + const bytes = buildIdml({ + name: 'Mini', + fonts: [{ id: 'f', family: 'Helvetica', style: 'Regular' }], + styles: [ + { id: 'h1', name: 'Heading 1', kind: 'paragraph', pointSize: 36, appliedFont: 'f' }, + { id: 'h2', name: 'Heading 2', kind: 'paragraph', pointSize: 24, appliedFont: 'f' }, + { id: 'body', name: 'Body', kind: 'paragraph', pointSize: 12, appliedFont: 'f' }, + ], + stories: [ + { id: 's-h1', runs: [{ text: 'Big Title', paragraphStyle: 'h1' }] }, + { id: 's-h2', runs: [{ text: 'Subhead', paragraphStyle: 'h2' }] }, + { id: 's-body', runs: [{ text: 'Body copy here.', paragraphStyle: 'body' }] }, + ], + spreads: [ + { + id: 'sp1', + pages: [{ id: 'pg1', bounds: [0, 0, 792, 612] }], + // Source order is scrambled on purpose; reading order is by y, then x. + frames: [ + { kind: 'text', id: 'tf-body', bounds: [300, 60, 360, 540], parentStory: 's-body' }, + { kind: 'text', id: 'tf-h1', bounds: [60, 60, 120, 540], parentStory: 's-h1' }, + { kind: 'image', id: 'img', bounds: [130, 60, 280, 540], href: 'file:Links/pic.png' }, + { kind: 'text', id: 'tf-h2', bounds: [285, 60, 320, 540], parentStory: 's-h2' }, + ], + }, + ], + }); + return parseIdmlBuffer(bytes); +} + +test('builds one section per spread with reading-ordered, classified blocks', () => { + const content = extractContent(miniIr()); + + assert.equal(content.sections.length, 1); + const { blocks, title } = content.sections[0]; + assert.equal(title, 'Big Title'); + + assert.deepEqual( + blocks.map((b) => [b.type, b.level ?? b.text]), + [ + ['heading', 1], // Big Title (largest size → h1), top of the page + ['figure', undefined], // image sits below the title + ['heading', 2], // Subhead (next size → h2) + ['paragraph', 'Body copy here.'], // smallest size = body baseline + ], + ); + assert.equal(blocks[0].text, 'Big Title'); + assert.equal(blocks[2].text, 'Subhead'); + assert.equal(blocks[1].href, 'file:Links/pic.png'); +}); + +test('emits a flat heading outline', () => { + const content = extractContent(miniIr()); + assert.deepEqual(content.outline, [ + { level: 1, text: 'Big Title' }, + { level: 2, text: 'Subhead' }, + ]); +}); + +test('reports accurate stats', () => { + const content = extractContent(miniIr()); + assert.deepEqual(content.stats, { + sections: 1, + blocks: 4, + headings: 2, + paragraphs: 1, + figures: 1, + emptyTextFrames: 0, + danglingStoryRefs: 0, + }); +}); + +test('a single type size yields no headings (everything is body)', () => { + const bytes = buildIdml({ + name: 'Flat', + styles: [{ id: 'body', name: 'Body', kind: 'paragraph', pointSize: 12 }], + stories: [ + { id: 's1', runs: [{ text: 'One.', paragraphStyle: 'body' }] }, + { id: 's2', runs: [{ text: 'Two.', paragraphStyle: 'body' }] }, + ], + spreads: [ + { + id: 'sp1', + pages: [{ id: 'pg1', bounds: [0, 0, 792, 612] }], + frames: [ + { kind: 'text', id: 'f1', bounds: [60, 60, 100, 540], parentStory: 's1' }, + { kind: 'text', id: 'f2', bounds: [120, 60, 160, 540], parentStory: 's2' }, + ], + }, + ], + }); + const content = extractContent(parseIdmlBuffer(bytes)); + assert.equal(content.stats.headings, 0); + assert.equal(content.stats.paragraphs, 2); + assert.equal(content.outline.length, 0); +}); + +test('counts a text frame with a dangling story reference', () => { + const bytes = buildIdml({ + name: 'Dangling', + styles: [{ id: 'body', name: 'Body', kind: 'paragraph', pointSize: 12 }], + stories: [{ id: 's-real', runs: [{ text: 'Real.', paragraphStyle: 'body' }] }], + spreads: [ + { + id: 'sp1', + pages: [{ id: 'pg1', bounds: [0, 0, 792, 612] }], + frames: [ + { kind: 'text', id: 'f-real', bounds: [60, 60, 100, 540], parentStory: 's-real' }, + { kind: 'text', id: 'f-ghost', bounds: [120, 60, 160, 540], parentStory: 's-missing' }, + ], + }, + ], + }); + const content = extractContent(parseIdmlBuffer(bytes)); + assert.equal(content.stats.danglingStoryRefs, 1); + assert.equal(content.stats.paragraphs, 1); +}); diff --git a/packages/pipeline/tests/indesign/ingest-detect.test.mjs b/packages/pipeline/tests/indesign/ingest-detect.test.mjs new file mode 100644 index 0000000..dc3851b --- /dev/null +++ b/packages/pipeline/tests/indesign/ingest-detect.test.mjs @@ -0,0 +1,47 @@ +// Byte-sniffing format detection: real IDML/PDF fixtures (built in code) are +// recognized, and lookalikes (a plain zip, garbage, an empty buffer) are not. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { zipSync, strToU8 } from 'fflate'; + +import { detectFormat } from '../../src/indesign/ingest/detect.js'; +import { buildIdml } from './helpers/build-idml.js'; +import { buildPdf } from './helpers/build-pdf.js'; + +test('detects an IDML package by its zip signature + manifest name', () => { + const idml = buildIdml({ + name: 'Mini', + spreads: [{ id: 'sp', pages: [{ id: 'pg', bounds: [0, 0, 100, 100] }], frames: [] }], + }); + assert.equal(detectFormat(idml), 'idml'); +}); + +test('detects a PDF by its header', () => { + const pdf = buildPdf({ + title: 'Mini', + pages: [{ width: 100, height: 100, texts: [{ text: 'hi', x: 10, y: 20, size: 12 }] }], + }); + assert.equal(detectFormat(pdf), 'pdf'); +}); + +test('a non-IDML zip is not mistaken for a source', () => { + const zip = zipSync({ 'readme.txt': strToU8('hello world'), 'data/x.json': strToU8('{}') }); + assert.equal(detectFormat(zip), 'unknown'); +}); + +test('garbage and too-short buffers are unknown', () => { + assert.equal(detectFormat(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])), 'unknown'); + assert.equal(detectFormat(new Uint8Array([0x50, 0x4b])), 'unknown'); // bare "PK", too short + assert.equal(detectFormat(new Uint8Array([])), 'unknown'); +}); + +test('tolerates a PDF header a few bytes in (BOM / leading whitespace)', () => { + const pdf = Buffer.concat([Buffer.from('', 'utf8'), Buffer.from('%PDF-1.7\n1 0 obj\n')]); + assert.equal(detectFormat(pdf), 'pdf'); +}); + +test('accepts a Buffer as well as a Uint8Array', () => { + const idml = Buffer.from(buildIdml({ name: 'B', spreads: [] })); + assert.equal(detectFormat(idml), 'idml'); +}); diff --git a/packages/pipeline/tests/indesign/ingest.test.mjs b/packages/pipeline/tests/indesign/ingest.test.mjs new file mode 100644 index 0000000..a5c499f --- /dev/null +++ b/packages/pipeline/tests/indesign/ingest.test.mjs @@ -0,0 +1,143 @@ +// End-to-end ingest stage: auto-detect → parse → validate → enrich → artifact, +// for both input formats. The emitted artifact must still be consumable by the +// generator (it's a valid IR plus an additive `content` key) and deterministic. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { ingestBuffer, toArtifact } from '../../src/indesign/ingest/index.js'; +import { generateTheme } from '../../src/indesign/generate/index.js'; +import { Document } from '../../src/indesign/ir.js'; +import { buildIdml } from './helpers/build-idml.js'; +import { buildPdf, solidRgbImage } from './helpers/build-pdf.js'; + +function sampleIdml() { + return buildIdml({ + name: 'Spring Mini', + colors: [{ id: 'c-brand', name: 'Brand', space: 'RGB', values: [0, 102, 204] }], + fonts: [{ id: 'f', family: 'Helvetica', style: 'Bold', postScriptName: 'Helvetica-Bold' }], + styles: [ + { id: 'h1', name: 'Heading 1', kind: 'paragraph', pointSize: 36, appliedFont: 'f', fillColor: 'c-brand' }, + { id: 'body', name: 'Body', kind: 'paragraph', pointSize: 12, appliedFont: 'f' }, + ], + stories: [ + { id: 's-title', runs: [{ text: 'Welcome', paragraphStyle: 'h1' }] }, + { id: 's-body', runs: [{ text: 'Browse our spring collection.', paragraphStyle: 'body' }] }, + { id: 's-hero', runs: [{ text: 'On Sale Now', paragraphStyle: 'h1' }] }, + ], + spreads: [ + { + id: 'cover', + pages: [{ id: 'p1', bounds: [0, 0, 792, 612] }], + frames: [ + { kind: 'text', id: 'tf-title', bounds: [60, 60, 150, 540], parentStory: 's-title' }, + { kind: 'text', id: 'tf-body', bounds: [170, 60, 320, 540], parentStory: 's-body' }, + ], + }, + { + id: 'hero', + pages: [{ id: 'p2', bounds: [0, 0, 792, 612] }], + frames: [ + { kind: 'image', id: 'if-hero', bounds: [0, 0, 612, 792], href: 'file:Links/hero.png' }, + { kind: 'text', id: 'tf-hero', bounds: [240, 100, 330, 500], parentStory: 's-hero' }, + ], + }, + ], + }); +} + +function samplePdf() { + const unit = ([r, g, b]) => [r / 255, g / 255, b / 255]; + return buildPdf({ + title: 'Spring Mini', + pages: [ + { + width: 612, + height: 792, + texts: [ + { text: 'Welcome', x: 60, y: 96, size: 36, font: 'Helvetica-Bold', color: unit([0, 102, 204]) }, + { text: 'Browse our spring collection.', x: 60, y: 150, size: 12, font: 'Helvetica', color: unit([0, 0, 0]) }, + ], + }, + { + width: 612, + height: 792, + images: [{ x: 0, y: 0, width: 612, height: 792, rgb: solidRgbImage(16, 16, [0, 102, 204]) }], + texts: [{ text: 'On Sale Now', x: 80, y: 420, size: 36, font: 'Helvetica-Bold', color: unit([255, 255, 255]) }], + }, + ], + }); +} + +test('ingests IDML bytes into a validated, content-bearing IR', async () => { + const result = await ingestBuffer(sampleIdml()); + + assert.equal(result.format, 'idml'); + assert.equal(result.ir.meta.name, 'Spring Mini'); + assert.equal(result.ir.spreads.length, 2); + + // Content model: a heading per spread, a paragraph, and the hero figure. + assert.equal(result.content.stats.sections, 2); + assert.equal(result.content.stats.headings, 2); + assert.equal(result.content.stats.figures, 1); + assert.deepEqual(result.content.outline.map((h) => h.text), ['Welcome', 'On Sale Now']); + + // On the hero spread, the image reads before the overlaid headline. + const heroBlocks = result.content.sections[1].blocks.map((b) => b.type); + assert.deepEqual(heroBlocks, ['figure', 'heading']); +}); + +test('the artifact is a valid IR the generator consumes unchanged', async () => { + const result = await ingestBuffer(sampleIdml()); + const artifact = toArtifact(result); + + // Superset of a valid Document: schema still parses (it ignores `content`). + assert.doesNotThrow(() => Document.parse(artifact)); + assert.ok(artifact.content, 'artifact carries the content model'); + + // The generator accepts the enriched artifact and produces a valid theme. + const theme = generateTheme(artifact); + assert.equal(theme.report.data.valid, true); + const patterns = theme.files.filter((f) => /^patterns\/spread-\d+\.php$/.test(f.path)); + assert.equal(patterns.length, result.ir.spreads.length); +}); + +test('ingest is deterministic — same bytes yield byte-identical JSON', async () => { + const bytes = sampleIdml(); + const a = JSON.stringify(toArtifact(await ingestBuffer(bytes))); + const b = JSON.stringify(toArtifact(await ingestBuffer(bytes))); + assert.equal(a, b); +}); + +test('--no-content emits the bare IR (no content key)', async () => { + const result = await ingestBuffer(sampleIdml(), { content: false }); + assert.equal(result.content, null); + assert.equal(toArtifact(result).content, undefined); +}); + +test('ingests PDF bytes via the lossy fallback path', async () => { + const result = await ingestBuffer(samplePdf()); + + assert.equal(result.format, 'pdf'); + assert.ok(result.ir.spreads.length >= 1); + // PDF reconstruction is always lossy → fidelity warnings are expected. + assert.ok((result.ir.warnings ?? []).length > 0); + + const s = result.content.stats; + assert.ok(s.sections >= 1); + assert.ok(s.blocks >= 1); + assert.equal(s.blocks, s.headings + s.paragraphs + s.figures); +}); + +test('honors an explicit --format override', async () => { + const forced = await ingestBuffer(sampleIdml(), { format: 'idml' }); + assert.equal(forced.format, 'idml'); + assert.equal(forced.ir.spreads.length, 2); +}); + +test('rejects an unrecognized source', async () => { + await assert.rejects( + () => ingestBuffer(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])), + /Unrecognized source/, + ); +});