feat(pipeline): add format-detecting ingest stage with content extraction

docs/pipeline/ingest-stage.md

@@ -0,0 +1,106 @@
+# Ingest stage
+
+The **ingest** stage is the pipeline's format-detecting front door. It turns a
+source asset — an InDesign `.idml` package or an exported `.pdf` — into the
+canonical [IR `Document`](../../packages/pipeline/src/indesign/ir.js) that the
+[token mapper](../../packages/pipeline/src/indesign/map/index.js) and
+[theme generator](../../packages/pipeline/src/indesign/generate/index.js)
+consume, and enriches it with a derived **content model**.
+
+```
+source (.idml | .pdf | bytes)
+        │
+        ▼
+  ┌───────────┐   detect format by magic bytes (PK+designmap.xml / %PDF-)
+  │  ingest   │   parse → IR → validate (Document.parse)
+  │  stage    │   extract reading-order content model
+  └───────────┘
+        │
+        ▼
+  artifact JSON  ──►  map-tokens  ──►  generate-theme
+  (IR + content)
+```
+
+## Why a separate stage
+
+The per-format parse bins (`flavian-parse-idml`, `flavian-parse-pdf`) each handle
+one extension and print to stdout. Ingest adds three things they don't:
+
+1. **Byte sniffing** — the format is detected from the content, not the
+   filename, so extensionless files, stdin streams, and upload buffers all work.
+   IDML is recognized by the Zip signature plus the literal `designmap.xml`
+   manifest name (Zip stores entry *filenames* uncompressed); PDF by its `%PDF-`
+   header.
+2. **A validated, writable artifact** — the IR is re-checked against the schema
+   and written as one JSON file (`--out`), ready for the next stage.
+3. **A semantic content model** — see below.
+
+## Content model
+
+The IR records design and geometry but nothing about editorial structure. The
+ingest stage derives a normalized, reading-order outline:
+
+- Each editorial spread becomes a **section** of ordered **blocks**:
+  `heading` (with an inferred level 1–6), `paragraph`, or `figure`.
+- **Heading levels** are inferred from font size: the smallest paragraph size on
+  the spreads is the body baseline; larger distinct sizes rank largest → `<h1>`,
+  capped at `<h6>`. A single size means no headings.
+- **Reading order** is top-to-bottom then left-to-right by frame bounds, with a
+  small row tolerance so two-column rows aren't interleaved.
+- Consecutive runs that share a paragraph style merge into one block (a bold
+  word mid-sentence doesn't fragment the paragraph).
+- Master-spread chrome (running header/footer) is excluded — the generator
+  derives that separately.
+
+The content model is attached to the artifact under an additive `content` key.
+Downstream stages ignore it, so the artifact is still a valid `Document`:
+`flavian-map-tokens` and `flavian-generate-theme` accept it directly.
+
+## Usage
+
+```bash
+# Auto-detect and write the enriched IR
+pnpm pipeline:ingest path/to/brochure.idml --out build/brochure.ir.json
+
+# From stdin, forcing the PDF parser, bare IR (no content model)
+cat export.pdf | pnpm pipeline:ingest - --format pdf --no-content > ir.json
+
+# Pipe straight into theme generation (generate-theme reads the IR from stdin)
+pnpm pipeline:ingest brochure.idml --quiet | \
+  node packages/pipeline/bin/generate-theme.mjs --out-dir themes/brochure
+```
+
+| Option | Description |
+| --- | --- |
+| `--out <file>` | Write the artifact here (default: stdout) |
+| `--format <fmt>` | `idml` \| `pdf` \| `auto` (default: `auto`) |
+| `--dpi <n>` | Unit-normalization DPI (default 96) |
+| `--name <str>` | Override the document name |
+| `--no-content` | Emit the bare IR without the content model |
+| `--quiet` | Suppress the stderr summary and parse warnings |
+
+## Programmatic API
+
+```js
+import { ingestSource, toArtifact } from '@flavian/pipeline/indesign';
+
+const result = await ingestSource('brochure.idml');
+// → { format: 'idml', ir: <DocumentIR>, content: <ContentModel> }
+
+await fs.writeFile('out.json', JSON.stringify(toArtifact(result), null, 2));
+```
+
+`ingestBuffer(bytes, opts)` is the in-memory variant. Both are pure of side
+effects beyond reading the input path; the content transform (`extractContent`)
+and the detector (`detectFormat`) are exported for direct use and are
+unit-tested in `packages/pipeline/tests/indesign/ingest-*.test.mjs`.
+
+## Tests & fixtures
+
+Fixtures are generated in code (no binary blobs in git), consistent with the
+rest of the pipeline:
+
+- `ingest-detect.test.mjs` — magic-byte detection on built IDML/PDF/zip buffers.
+- `ingest-content.test.mjs` — reading order, heading-level inference, figures.
+- `ingest.test.mjs` — full IDML + PDF ingest, determinism, and proof that the
+  artifact is generator-consumable.

package.json

@@ -25,6 +25,7 @@
     "init": "node scripts/init.mjs",
     "test:init": "node --test \"tests/init/**/*.test.mjs\"",
     "test:pipeline": "pnpm --filter @flavian/pipeline test",
+    "pipeline:ingest": "node packages/pipeline/bin/ingest.mjs",
     "test:canva-e2e": "node --test tests/canva-e2e/*.test.mjs",
     "test:headless-e2e": "node --test tests/headless-e2e/*.test.mjs",
     "visual:capture": "node tests/visual/capture.mjs",

packages/pipeline/bin/ingest.mjs

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+// CLI: ingest a source asset (IDML or PDF), auto-detecting the format, and write
+// the canonical IR — enriched with a derived content model — as JSON.
+//
+//   flavian-ingest <source.idml | source.pdf | -> [options]
+//
+// Reads a file path or stdin (binary-safe), writes the artifact to --out (or
+// stdout), and prints a one-line summary plus parse warnings to stderr. The
+// artifact is a valid IR that flavian-map-tokens / flavian-generate-theme accept
+// directly.
+//
+// Options:
+//   --out <file>     Write the artifact here (default: stdout)
+//   --format <fmt>   idml | pdf | auto (default: auto — sniff the bytes)
+//   --dpi <n>        Unit-normalization DPI (default 96)
+//   --name <str>     Override the document name
+//   --no-content     Emit the bare IR without the content model
+//   --quiet          Suppress the stderr summary and parse warnings
+//   -h, --help       Show this help
+
+import { promises as fs } from 'node:fs';
+
+import { ingestBuffer, toArtifact } from '../src/indesign/ingest/index.js';
+
+const args = process.argv.slice(2);
+const opts = { format: 'auto', content: true, quiet: false };
+let inputPath;
+
+let i = 0;
+function wantsValue(flag) {
+	const next = args[i + 1];
+	if (next === undefined || next.startsWith('-')) {
+		process.stderr.write(`${flag} requires a value\n`);
+		process.exit(2);
+	}
+	i += 1;
+	return next;
+}
+
+for (; i < args.length; i += 1) {
+	const arg = args[i];
+	switch (arg) {
+		case '--out': opts.out = wantsValue(arg); break;
+		case '--format': opts.format = wantsValue(arg); break;
+		case '--dpi': opts.dpi = Number(wantsValue(arg)); break;
+		case '--name': opts.name = wantsValue(arg); break;
+		case '--no-content': opts.content = false; break;
+		case '--quiet': opts.quiet = true; break;
+		case '-h': case '--help': printUsage(); process.exit(0); break;
+		default:
+			// A bare "-" is the explicit stdin sentinel; any other dash-prefixed
+			// token is an unknown flag.
+			if (!inputPath && (arg === '-' || !arg.startsWith('-'))) {
+				inputPath = arg;
+			} else {
+				process.stderr.write(`Unknown argument: ${arg}\n`);
+				printUsage();
+				process.exit(2);
+			}
+	}
+}
+
+if (!['auto', 'idml', 'pdf'].includes(opts.format)) {
+	process.stderr.write('--format must be one of: auto, idml, pdf\n');
+	process.exit(2);
+}
+if (opts.dpi !== undefined && (Number.isNaN(opts.dpi) || opts.dpi <= 0)) {
+	process.stderr.write('--dpi must be a positive number\n');
+	process.exit(2);
+}
+
+async function readStdin() {
+	const chunks = [];
+	for await (const chunk of process.stdin) chunks.push(chunk);
+	return Buffer.concat(chunks);
+}
+
+try {
+	const bytes = inputPath && inputPath !== '-' ? await fs.readFile(inputPath) : await readStdin();
+	if (bytes.length === 0) {
+		process.stderr.write('error: empty input\n');
+		process.exit(1);
+	}
+
+	const result = await ingestBuffer(bytes, {
+		format: opts.format,
+		dpi: opts.dpi,
+		name: opts.name,
+		content: opts.content,
+	});
+
+	const json = `${JSON.stringify(toArtifact(result), null, 2)}\n`;
+	if (opts.out) await fs.writeFile(opts.out, json);
+	else process.stdout.write(json);
+
+	if (!opts.quiet) {
+		const { ir, content } = result;
+		const warnings = ir.warnings ?? [];
+		for (const w of warnings) process.stderr.write(`[${w.code}] ${w.message}\n`);
+		const lines = [`format: ${result.format}  spreads: ${ir.spreads.length}  warnings: ${warnings.length}`];
+		if (content) {
+			const s = content.stats;
+			lines.push(`content: ${s.sections} sections, ${s.headings} headings, ${s.paragraphs} paragraphs, ${s.figures} figures`);
+		}
+		lines.push(`written: ${opts.out ?? '<stdout>'}`);
+		process.stderr.write(`${lines.join('\n')}\n`);
+	}
+
+	process.exit(0);
+} catch (err) {
+	process.stderr.write(`error: ${err.message}\n`);
+	process.exit(1);
+}
+
+function printUsage() {
+	process.stderr.write(
+		[
+			'Usage: flavian-ingest <source.idml | source.pdf | -> [options]',
+			'',
+			'Options:',
+			'  --out <file>     Write the artifact here (default: stdout)',
+			'  --format <fmt>   idml | pdf | auto (default: auto — sniff the bytes)',
+			'  --dpi <n>        Unit-normalization DPI (default 96)',
+			'  --name <str>     Override the document name',
+			'  --no-content     Emit the bare IR without the content model',
+			'  --quiet          Suppress the stderr summary and parse warnings',
+			'  -h, --help       Show this help',
+			'',
+		].join('\n'),
+	);
+}

packages/pipeline/package.json

@@ -9,6 +9,7 @@
     "./indesign": "./src/indesign/index.js"
   },
   "bin": {
+    "flavian-ingest": "./bin/ingest.mjs",
     "flavian-parse-idml": "./bin/parse-idml.mjs",
     "flavian-parse-pdf": "./bin/parse-pdf.mjs",
     "flavian-map-tokens": "./bin/map-tokens.mjs",

packages/pipeline/src/indesign/index.js

@@ -1,5 +1,6 @@
 export { parseIdml, parseIdmlBuffer } from './parse-idml.js';
 export { parsePdf, parsePdfBuffer } from './parse-pdf.js';
+export { ingestSource, ingestBuffer, toArtifact, detectFormat, extractContent } from './ingest/index.js';
 export * as ir from './ir.js';
 export { WarningCollector } from './warnings.js';
 export { lengthToPx, ptToPx, roundPx } from './units.js';