From b69e5494badca5d5ba21f80e7e630715abce6af8 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 12:44:37 +0200 Subject: [PATCH 01/16] feat: export all built-in transformers and isAdminUser filter All 27 transformers and all 18 filter predicates are now part of the public API surface. Users can import and compose them in custom presets. Also saves MCP server implementation plan. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plans/2026-08-03-mcp-server.md | 557 ++++++++++++++++++ src/index.ts | 37 +- 2 files changed, 592 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-03-mcp-server.md diff --git a/docs/superpowers/plans/2026-08-03-mcp-server.md b/docs/superpowers/plans/2026-08-03-mcp-server.md new file mode 100644 index 0000000..131f2c9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-mcp-server.md @@ -0,0 +1,557 @@ +# MCP Server Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build an MCP server inside `@webiny/data-transfer` that serves documentation about all public API components (presets, transformers, processors, scanners, filters, config, pipeline runtime) so AI agents can help users write custom presets, transformers, and pipelines. + +**Architecture:** Two MCP tools (`list_topics` / `get_topic`) serve markdown documentation files from `docs/mcp/`. Entry point at `src/mcp/server.ts`, bin entry `webiny-data-transfer-mcp`, stdio transport via `@modelcontextprotocol/sdk`. Documentation is organized by category (presets, transformers, processors, scanners, guides). All docs ship with the published npm package. + +**Tech Stack:** `@modelcontextprotocol/sdk`, `zod` (already a dependency), `front-matter` for YAML front-matter parsing. + +## Global Constraints + +- MCP server lives inside the data-transfer package at `src/mcp/` +- All tools are read-only (annotations: `readOnlyHint: true`) +- Documentation files use YAML front-matter with `name`, `description`, `category` fields +- Transformer docs are small — name, one-line description, what it does, when to use it +- Existing `docs/guides/` are NOT moved — MCP docs are a separate set at `docs/mcp/` +- `docs/mcp/` MUST be included in the published npm package — `ArtifactCopier.copyAssets()` must copy it to `dist/docs/mcp/` +- Bin entry points to `./dist/mcp/bin.js` (compiled from `src/mcp/bin.ts` by tsc) +- `BuildOrchestrator.ensureShebang()` must handle `dist/mcp/bin.js` in addition to `dist/cli.js` +- Follow existing project patterns: ESM, no reflect-metadata imports, camelCase file names +- Transformers: all 27 are public exports from `@webiny/data-transfer`. Docs show import path for each. +- Filters: all 18 exported in public API (including `isAdminUser`). Docs cover all 18 exported filters. + +--- + +### Task 1: MCP server skeleton + two tools + +**Files:** +- Create: `src/mcp/server.ts` +- Create: `src/mcp/discoverDocs.ts` +- Create: `src/mcp/bin.ts` +- Modify: `package.json` (add bin entry, add `@modelcontextprotocol/sdk` + `front-matter` dependencies) +- Modify: `scripts/features/BuildPackages/ArtifactCopier.ts` (copy `docs/mcp/` to dist) +- Modify: `scripts/features/BuildPackages/BuildOrchestrator.ts` (add shebang to `dist/mcp/bin.js`) + +**Interfaces:** +- Produces: `startMcpServer()` function, `discoverDocs(dirs: string[])` returning `Map`, `buildCatalog(docs: Map)` returning formatted markdown + +- [ ] **Step 1: Install dependencies** + +```bash +yarn add @modelcontextprotocol/sdk front-matter +``` + +- [ ] **Step 2: Create `src/mcp/discoverDocs.ts`** + +Recursively finds all `.md` files in given directories, parses YAML front-matter (`name`, `description`, `category`), returns a `Map`. First-match-wins for duplicate names. + +```typescript +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import fm from "front-matter"; + +interface Doc { + name: string; + description: string; + category: string; + filePath: string; + body: string; +} + +function findMarkdownFiles(dir: string): string[] { + const results: string[] = []; + + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const stat = statSync(full); + + if (stat.isDirectory()) { + results.push(...findMarkdownFiles(full)); + } else if (entry.endsWith(".md")) { + results.push(full); + } + } + + return results; +} + +function parseDoc(filePath: string): Doc | null { + const raw = readFileSync(filePath, "utf-8"); + const parsed = fm<{ name?: string; description?: string; category?: string }>(raw); + + if (!parsed.attributes.name || !parsed.attributes.description) { + return null; + } + + return { + name: parsed.attributes.name, + description: parsed.attributes.description, + category: parsed.attributes.category ?? "general", + filePath, + body: parsed.body + }; +} + +export function discoverDocs(dirs: string[]): Map { + const docs = new Map(); + + for (const dir of dirs) { + for (const file of findMarkdownFiles(dir)) { + const doc = parseDoc(file); + if (doc && !docs.has(doc.name)) { + docs.set(doc.name, doc); + } + } + } + + return docs; +} + +export function buildCatalog(docs: Map): string { + const byCategory = new Map(); + + for (const doc of docs.values()) { + const list = byCategory.get(doc.category) ?? []; + list.push(doc); + byCategory.set(doc.category, list); + } + + const lines: string[] = ["# Available Topics\n"]; + + for (const [category, categoryDocs] of [...byCategory.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + lines.push(`## ${category}\n`); + lines.push("| Topic | Description |"); + lines.push("|-------|-------------|"); + + for (const doc of categoryDocs.sort((a, b) => a.name.localeCompare(b.name))) { + lines.push(`| ${doc.name} | ${doc.description} |`); + } + + lines.push(""); + } + + return lines.join("\n"); +} + +export type { Doc }; +``` + +- [ ] **Step 3: Create `src/mcp/server.ts`** + +Registers `list_topics` and `get_topic` tools, starts stdio transport. The `DEFAULT_DOCS_DIR` resolves relative to compiled output (`dist/docs/mcp` in production, `docs/mcp` in dev). + +```typescript +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { z } from "zod"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { discoverDocs, buildCatalog, type Doc } from "./discoverDocs.ts"; + +const DEFAULT_DOCS_DIR = join(fileURLToPath(new URL(".", import.meta.url)), "../docs/mcp"); + +export async function startMcpServer(docsDirs?: string[]): Promise { + const dirs = docsDirs ?? [DEFAULT_DOCS_DIR]; + let docsCache: Map | null = null; + + function getDocs(): Map { + if (!docsCache) { + docsCache = discoverDocs(dirs); + } + return docsCache; + } + + const server = new McpServer({ name: "webiny-data-transfer", version: "1.0.0" }); + + server.registerTool( + "list_topics", + { + title: "List Data Transfer Topics", + description: + "Returns a catalog of all available @webiny/data-transfer documentation topics. " + + "Call this first to discover what topics are available, then use get_topic to read the full documentation for a specific topic. " + + "Topics cover: presets, transformers, processors, scanners, filters, config, pipeline runtime, and how to write custom components.", + inputSchema: {}, + annotations: { readOnlyHint: true } + }, + async () => ({ + content: [{ type: "text" as const, text: buildCatalog(getDocs()) }] + }) + ); + + server.registerTool( + "get_topic", + { + title: "Get Data Transfer Topic", + description: + "Returns the full documentation for a specific @webiny/data-transfer topic. " + + "Use exact topic names from list_topics.", + inputSchema: { + topic: z.string().describe("Topic name — use exact names from list_topics") + }, + annotations: { readOnlyHint: true } + }, + async ({ topic }) => { + const docs = getDocs(); + const doc = docs.get(topic); + + if (!doc) { + const available = [...docs.keys()].sort().join(", "); + return { + content: [{ type: "text" as const, text: `Topic "${topic}" not found. Available topics: ${available}` }] + }; + } + + return { + content: [{ type: "text" as const, text: doc.body }] + }; + } + ); + + const transport = new StdioServerTransport(); + await server.connect(transport); +} +``` + +- [ ] **Step 4: Create `src/mcp/bin.ts`** + +```typescript +import { startMcpServer } from "./server.ts"; + +await startMcpServer(); +``` + +- [ ] **Step 5: Add bin entry + dependencies to `package.json`** + +Add to existing `bin` field: +```json +"bin": { + "webiny-data-transfer": "./dist/cli.js", + "webiny-data-transfer-mcp": "./dist/mcp/bin.js" +} +``` + +Add to `dependencies`: +```json +"@modelcontextprotocol/sdk": "^1.30.0", +"front-matter": "^4.0.2" +``` + +- [ ] **Step 6: Update `ArtifactCopier.copyAssets()` to copy `docs/mcp/`** + +In `scripts/features/BuildPackages/ArtifactCopier.ts`, add `"docs/mcp"` to the asset directories copied to dist: + +```typescript +public copyAssets(sourceDir: string, distAbsDir: string): void { + for (const dir of ["templates", "projects", "docs/mcp"]) { + const src = join(sourceDir, dir); + if (existsSync(src)) { + cpSync(src, join(distAbsDir, dir), { recursive: true }); + } + } +} +``` + +- [ ] **Step 7: Update `BuildOrchestrator` to add shebang to MCP bin** + +In `scripts/features/BuildPackages/BuildOrchestrator.ts`, extend the shebang logic to handle all bin entries (not just `cli.js`): + +```typescript +for (const binPath of ["dist/cli.js", "dist/mcp/bin.js"]) { + const fullPath = join(rootDir, binPath); + if (!existsSync(fullPath)) { + continue; + } + const content = readFileSync(fullPath, "utf-8"); + if (!content.startsWith("#!")) { + writeFileSync(fullPath, "#!/usr/bin/env node\n" + content); + } +} +``` + +- [ ] **Step 8: Create a test doc file and verify server starts** + +Create `docs/mcp/test.md`: +```markdown +--- +name: test-topic +description: Test topic for verifying MCP server +category: test +--- + +This is a test topic. +``` + +Run: `tsx src/mcp/bin.ts` — should start without errors (ctrl+c to exit). + +- [ ] **Step 9: Commit** + +```bash +git add src/mcp/ package.json yarn.lock docs/mcp/test.md scripts/features/BuildPackages/ArtifactCopier.ts scripts/features/BuildPackages/BuildOrchestrator.ts +git commit -m "feat: add MCP server skeleton with list_topics and get_topic tools" +``` + +--- + +### Task 2: Preset documentation (5 files) + +**Files:** +- Create: `docs/mcp/presets/copy-ddb.md` +- Create: `docs/mcp/presets/copy-os.md` +- Create: `docs/mcp/presets/copy-files.md` +- Create: `docs/mcp/presets/v5-to-v6-ddb.md` +- Create: `docs/mcp/presets/v5-to-v6-os.md` + +**Interfaces:** +- Consumes: reads preset source files in `src/presets/` for accuracy +- Produces: 5 markdown files with front-matter, each documenting one built-in preset + +Each file follows this template: +```markdown +--- +name: +description: +category: Presets +--- + +# + +**Use when:** + +**What it does:** + + +**Pipelines registered:** + + +**Transformers applied:** + + +**Example usage in a custom preset:** + +``` + +- [ ] **Step 1: Read each preset source file** + +Read `src/presets/copy-ddb.ts`, `copy-os.ts`, `copy-files.ts`, `v5-to-v6-ddb.ts`, `v5-to-v6-os.ts` to understand exactly what each registers. + +- [ ] **Step 2: Write all 5 preset doc files** + +Follow the template above. Extract pipeline names, scanner/processor combos, and transformer chains from the source. + +- [ ] **Step 3: Delete test doc file** + +Remove `docs/mcp/test.md`. + +- [ ] **Step 4: Commit** + +```bash +git add docs/mcp/presets/ && git rm docs/mcp/test.md +git commit -m "docs: add MCP documentation for all 5 built-in presets" +``` + +--- + +### Task 3: Processor and scanner documentation (6 files) + +**Files:** +- Create: `docs/mcp/processors/ddbProcessor.md` +- Create: `docs/mcp/processors/osProcessor.md` +- Create: `docs/mcp/processors/s3Processor.md` +- Create: `docs/mcp/processors/auditLogProcessor.md` +- Create: `docs/mcp/scanners/ddbScanner.md` +- Create: `docs/mcp/scanners/osScanner.md` + +**Interfaces:** +- Consumes: reads processor/scanner source files for accuracy +- Produces: 6 markdown files documenting processors and scanners + +Each processor doc follows this template: +```markdown +--- +name: +description: +category: Processors +--- + +# + +**Import:** `import { } from "@webiny/data-transfer";` + +**What it does:** + +**Context slice it adds:** + +**Commands it handles:** + +**`onEnd` hook behavior:** + +**Usage in pipelineBuilderFactory.create():** + +``` + +Scanner docs follow a similar pattern with scan behavior, record shape, segment support. + +- [ ] **Step 1: Read processor/scanner source files** + +- [ ] **Step 2: Write all 6 doc files** + +- [ ] **Step 3: Commit** + +```bash +git add docs/mcp/processors/ docs/mcp/scanners/ +git commit -m "docs: add MCP documentation for processors and scanners" +``` + +--- + +### Task 4: Transformer documentation (27 files) + +**Files:** +- Create: `docs/mcp/transformers/.md` for each of the 27 built-in transformers + +**Interfaces:** +- Consumes: reads transformer source files for accuracy +- Produces: 27 small markdown files + +All 27 transformers are public exports from `@webiny/data-transfer`. + +Each transformer doc is intentionally small: +```markdown +--- +name: +description: +category: Transformers +--- + +# + +**Import:** `import { } from "@webiny/data-transfer";` + +**Category:** + +**What it does:** <2-3 sentences max> + +**Record types it targets:** + +**Context type required:** `` +``` + +The 27 transformers: +`addGsiTenant`, `addLiveField`, `addTransferTimestamp`, `copyFileToTarget`, `coreFieldsTransformer`, `createMetadata`, `dataFieldsTransformer`, `extractImageMetadata`, `fixBrokenStorageKeys`, `fixCmePk`, `groupsToRoles`, `migrateFileManagerSettings`, `migrateMailerSettings`, `removeAttributes`, `removeFolderRevision`, `removeLocale`, `removeTenant`, `renameFieldAttributes`, `replaceFileUrls`, `storageShapeTransformer`, `transformModelGroup`, `transformPermissions`, `transformRichText`, `updateFlpIds`, `updateModelIds`, `updateOsIndex`, `wrapInData` + +- [ ] **Step 1: Read all transformer source files** + +- [ ] **Step 2: Write all 27 doc files** + +Batch by category (cms, security, global, file-manager, folders, mailer, auditLogs). + +- [ ] **Step 3: Commit** + +```bash +git add docs/mcp/transformers/ +git commit -m "docs: add MCP documentation for all 27 built-in transformers" +``` + +--- + +### Task 5: Filter and pipeline guide documentation (4 files) + +**Files:** +- Create: `docs/mcp/guides/filters.md` +- Create: `docs/mcp/guides/writingPresets.md` +- Create: `docs/mcp/guides/writingTransformers.md` +- Create: `docs/mcp/guides/configReference.md` + +**Interfaces:** +- Consumes: existing `docs/guides/` files as source material, `src/domain/transform/filters.ts` for exact filter signatures +- Produces: 4 markdown files with front-matter + +These are MCP-specific versions of the existing guides, optimized for AI agent consumption (more structured, more code examples, less prose). They cover: + +1. **filters** — all 18 exported filter predicates (`byType`, `byTypePrefix`, `isCmsGroup`, `isCmsModel`, `isCmsEntry`, `byIncludesModelId`, `isAcoSearchRecord`, `isAdminUser`, `isBackgroundTask`, `isFmFile`, `isFlpRecord`, `isBuiltInSecurityRole`, `isSecurityTeam`, `isOsBackgroundTask`, `isOsMailerSettings`, `isAuditLogEntry`, `isMigrationRecord`, `isFormBuilderRecord`) with signatures and examples +2. **writingPresets** — how to write a custom preset from scratch, `createTransferPreset` shape, `pipelineBuilderFactory.create()`, `runner.register()`, filter/use/hook composition +3. **writingTransformers** — `createDdbTransformer`/`createOsTransformer`/`createTransformer` factories, context types, processor slices, working with `ctx.record`, `ctx.putRecord()`, `ctx.blackhole()` +4. **configReference** — `createConfig` shape, `fromEnv`/`numberFromEnv`, credentials (`fromAwsProfile`, `fromAwsCredentialChain`, literal), `register` callback, tuning, debug/snapshot + +- [ ] **Step 1: Read existing guides and `filters.ts` for reference** + +- [ ] **Step 2: Write all 4 doc files** + +- [ ] **Step 3: Commit** + +```bash +git add docs/mcp/guides/ +git commit -m "docs: add MCP guide documentation for filters, presets, transformers, and config" +``` + +--- + +### Task 6: Pipeline runtime and public API docs (2 files) + +**Files:** +- Create: `docs/mcp/guides/pipelineRuntime.md` +- Create: `docs/mcp/guides/publicApi.md` + +**Interfaces:** +- Produces: 2 markdown files covering pipeline runtime semantics and full public API surface + +1. **pipelineRuntime** — merge groups, first-match-wins dispatch, unmatched record drops, `onEnd` hooks, `flushEvery`, parallelism (segments/shards), hook ordering +2. **publicApi** — every export from `src/index.ts` with import path, type, one-line description. Organized by category (config, credentials, transformer factories, built-in transformers, filters, scanners, processors, service clients, context types, lifecycle hooks, customization). Mark which are values vs types. + +- [ ] **Step 1: Write both doc files** + +- [ ] **Step 2: Commit** + +```bash +git add docs/mcp/guides/ +git commit -m "docs: add MCP documentation for pipeline runtime and public API surface" +``` + +--- + +### Task 7: Integration — verify, run `yarn full`, final commit + +**Files:** +- Possibly modify: `src/mcp/server.ts` (fix any issues found) +- Possibly modify: various doc files (fix any issues found) + +- [ ] **Step 1: Verify MCP server discovers all docs** + +```bash +tsx -e " +import { discoverDocs } from './src/mcp/discoverDocs.ts'; +const docs = discoverDocs(['./docs/mcp']); +console.log('Total topics:', docs.size); +for (const [name, doc] of docs) { + console.log(' ', doc.category, '/', name, '—', doc.description); +} +" +``` + +Expect: ~44 topics (5 presets + 27 transformers + 6 processors/scanners + 6 guides). + +- [ ] **Step 2: Verify build includes MCP docs and bin** + +```bash +yarn build +ls dist/mcp/bin.js # must exist with shebang +ls dist/docs/mcp/ # must contain all doc files +head -1 dist/mcp/bin.js # must start with #!/usr/bin/env node +``` + +- [ ] **Step 3: Run `yarn full`** + +```bash +yarn full +``` + +Fix any format/lint/typecheck issues. + +- [ ] **Step 4: Commit any fixes** + +```bash +git commit -m "fix: address issues found during MCP integration verification" +``` diff --git a/src/index.ts b/src/index.ts index be9572d..4762f34 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,9 +43,41 @@ export { createTransformer } from "./transformers/createTransformer.ts"; export { createDdbTransformer } from "./transformers/createDdbTransformer.ts"; export { createOsTransformer } from "./transformers/createOsTransformer.ts"; -// Built-in transformers — ready-made for common patterns in custom presets. -export { copyFileToTarget } from "./transformers/file-manager/copyFileToTarget.ts"; +// Built-in transformers — ready-made for use in custom presets. +// CMS +export { addLiveField } from "./transformers/cms/addLiveField.ts"; +export { fixBrokenStorageKeys } from "./transformers/cms/fixBrokenStorageKeys.ts"; +export { fixCmePk } from "./transformers/cms/fixCmePk.ts"; +export { removeFolderRevision } from "./transformers/cms/removeFolderRevision.ts"; +export { renameFieldAttributes } from "./transformers/cms/renameFieldAttributes.ts"; export { replaceFileUrls } from "./transformers/cms/replaceFileUrls.ts"; +export { transformModelGroup } from "./transformers/cms/transformModelGroup.ts"; +export { transformRichText } from "./transformers/cms/transformRichText.ts"; +export { updateModelIds } from "./transformers/cms/updateModelIds.ts"; +export { updateOsIndex } from "./transformers/cms/updateOsIndex.ts"; +// File manager +export { copyFileToTarget } from "./transformers/file-manager/copyFileToTarget.ts"; +export { createMetadata } from "./transformers/file-manager/createMetadata.ts"; +export { extractImageMetadata } from "./transformers/file-manager/extractImageMetadata.ts"; +export { migrateFileManagerSettings } from "./transformers/file-manager/migrateFileManagerSettings.ts"; +// Folders +export { updateFlpIds } from "./transformers/folders/updateFlpIds.ts"; +// Global +export { addGsiTenant } from "./transformers/global/addGsiTenant.ts"; +export { addTransferTimestamp } from "./transformers/global/addTransferTimestamp.ts"; +export { removeAttributes } from "./transformers/global/removeAttributes.ts"; +export { removeLocale } from "./transformers/global/removeLocale.ts"; +export { wrapInData } from "./transformers/global/wrapInData.ts"; +// Security +export { groupsToRoles } from "./transformers/security/groupsToRoles.ts"; +export { removeTenant } from "./transformers/security/removeTenant.ts"; +export { transformPermissions } from "./transformers/security/transformPermissions.ts"; +// Mailer +export { migrateMailerSettings } from "./transformers/mailer/migrateMailerSettings.ts"; +// Audit logs +export { coreFieldsTransformer } from "./transformers/auditLogs/coreFieldsTransformer.ts"; +export { dataFieldsTransformer } from "./transformers/auditLogs/dataFieldsTransformer.ts"; +export { storageShapeTransformer } from "./transformers/auditLogs/storageShapeTransformer.ts"; // Pipeline factories export { createFilter, type Filter } from "./domain/pipeline/Filter.ts"; @@ -59,6 +91,7 @@ export { isCmsEntry, byIncludesModelId, isAcoSearchRecord, + isAdminUser, isBackgroundTask, isFmFile, isFlpRecord, From e9eb20de237e8980a6aec6440b18fcb973b9e7c6 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 12:51:30 +0200 Subject: [PATCH 02/16] feat: add MCP server skeleton with list_topics and get_topic tools Adds a stdio-based MCP server (webiny-data-transfer-mcp bin) exposing list_topics and get_topic tools that serve markdown docs from docs/mcp/. discoverDocs parses YAML front-matter (name/description/category) and builds a per-category catalog; first-match-wins on duplicate names. Build system: ArtifactCopier now copies docs/mcp/ to dist, and BuildOrchestrator's shebang step covers dist/mcp/bin.js in addition to dist/cli.js. Includes a local ambient-module override for front-matter's types: its shipped index.d.ts uses `export default` but the real runtime export is plain CommonJS, which is unusable under moduleResolution: nodenext given this package's own "type": "module". The override restates the same shape with `export =`, which interops correctly. --- package.json | 5 +- .../features/BuildPackages/ArtifactCopier.ts | 2 +- .../BuildPackages/BuildOrchestrator.ts | 16 +- src/mcp/bin.ts | 3 + src/mcp/discoverDocs.ts | 90 +++++ src/mcp/front-matter.d.ts | 26 ++ src/mcp/server.ts | 75 ++++ yarn.lock | 380 ++++++++++++++++-- 8 files changed, 558 insertions(+), 39 deletions(-) create mode 100644 src/mcp/bin.ts create mode 100644 src/mcp/discoverDocs.ts create mode 100644 src/mcp/front-matter.d.ts create mode 100644 src/mcp/server.ts diff --git a/package.json b/package.json index 9d2c21a..c251cf2 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "bin": { - "webiny-data-transfer": "./dist/cli.js" + "webiny-data-transfer": "./dist/cli.js", + "webiny-data-transfer-mcp": "./dist/mcp/bin.js" }, "exports": { ".": { @@ -53,6 +54,7 @@ "@aws-sdk/credential-providers": "^3.1100.0", "@inquirer/core": "^11.2.1", "@inquirer/prompts": "^8.5.2", + "@modelcontextprotocol/sdk": "^1.30.0", "@opensearch-project/opensearch": "3.6.0", "@types/node": "^24.13.3", "@webiny/api-headless-cms-ddb-es": "^6.4.5", @@ -65,6 +67,7 @@ "dotenv": "^17.4.2", "execa": "^10.0.1", "exifreader": "^4.41.3", + "front-matter": "^4.0.2", "jsdom": "^30.0.1", "pino": "^10.3.1", "pino-pretty": "^13.1.3", diff --git a/scripts/features/BuildPackages/ArtifactCopier.ts b/scripts/features/BuildPackages/ArtifactCopier.ts index 5d5a286..c2c2c82 100644 --- a/scripts/features/BuildPackages/ArtifactCopier.ts +++ b/scripts/features/BuildPackages/ArtifactCopier.ts @@ -73,7 +73,7 @@ class ArtifactCopierImpl implements ArtifactCopierAbstraction.Interface { public copyAssets(sourceDir: string, distAbsDir: string): void { // presets live in src/presets/ and are compiled by tsc — not copied here - for (const dir of ["templates", "projects"]) { + for (const dir of ["templates", "projects", "docs/mcp"]) { const src = join(sourceDir, dir); if (existsSync(src)) { cpSync(src, join(distAbsDir, dir), { recursive: true }); diff --git a/scripts/features/BuildPackages/BuildOrchestrator.ts b/scripts/features/BuildPackages/BuildOrchestrator.ts index ff27b66..853b1d1 100644 --- a/scripts/features/BuildPackages/BuildOrchestrator.ts +++ b/scripts/features/BuildPackages/BuildOrchestrator.ts @@ -48,13 +48,15 @@ class BuildOrchestratorImpl implements BuildOrchestratorAbstraction.Interface { } private ensureShebang(rootDir: string): void { - const cliPath = join(rootDir, "dist", "cli.js"); - if (!existsSync(cliPath)) { - return; - } - const content = readFileSync(cliPath, "utf-8"); - if (!content.startsWith("#!")) { - writeFileSync(cliPath, "#!/usr/bin/env node\n" + content); + for (const binPath of ["dist/cli.js", "dist/mcp/bin.js"]) { + const fullPath = join(rootDir, binPath); + if (!existsSync(fullPath)) { + continue; + } + const content = readFileSync(fullPath, "utf-8"); + if (!content.startsWith("#!")) { + writeFileSync(fullPath, "#!/usr/bin/env node\n" + content); + } } } } diff --git a/src/mcp/bin.ts b/src/mcp/bin.ts new file mode 100644 index 0000000..321bd1e --- /dev/null +++ b/src/mcp/bin.ts @@ -0,0 +1,3 @@ +import { startMcpServer } from "./server.ts"; + +await startMcpServer(); diff --git a/src/mcp/discoverDocs.ts b/src/mcp/discoverDocs.ts new file mode 100644 index 0000000..a123288 --- /dev/null +++ b/src/mcp/discoverDocs.ts @@ -0,0 +1,90 @@ +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import fm from "front-matter"; + +interface Doc { + name: string; + description: string; + category: string; + filePath: string; + body: string; +} + +function findMarkdownFiles(dir: string): string[] { + const results: string[] = []; + + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const stat = statSync(full); + + if (stat.isDirectory()) { + results.push(...findMarkdownFiles(full)); + } else if (entry.endsWith(".md")) { + results.push(full); + } + } + + return results; +} + +function parseDoc(filePath: string): Doc | null { + const raw = readFileSync(filePath, "utf-8"); + const parsed = fm<{ name?: string; description?: string; category?: string }>(raw); + + if (!parsed.attributes.name || !parsed.attributes.description) { + return null; + } + + return { + name: parsed.attributes.name, + description: parsed.attributes.description, + category: parsed.attributes.category ?? "general", + filePath, + body: parsed.body + }; +} + +export function discoverDocs(dirs: string[]): Map { + const docs = new Map(); + + for (const dir of dirs) { + for (const file of findMarkdownFiles(dir)) { + const doc = parseDoc(file); + if (doc && !docs.has(doc.name)) { + docs.set(doc.name, doc); + } + } + } + + return docs; +} + +export function buildCatalog(docs: Map): string { + const byCategory = new Map(); + + for (const doc of docs.values()) { + const list = byCategory.get(doc.category) ?? []; + list.push(doc); + byCategory.set(doc.category, list); + } + + const lines: string[] = ["# Available Topics\n"]; + + for (const [category, categoryDocs] of [...byCategory.entries()].sort((a, b) => + a[0].localeCompare(b[0]) + )) { + lines.push(`## ${category}\n`); + lines.push("| Topic | Description |"); + lines.push("|-------|-------------|"); + + for (const doc of categoryDocs.sort((a, b) => a.name.localeCompare(b.name))) { + lines.push(`| ${doc.name} | ${doc.description} |`); + } + + lines.push(""); + } + + return lines.join("\n"); +} + +export type { Doc }; diff --git a/src/mcp/front-matter.d.ts b/src/mcp/front-matter.d.ts new file mode 100644 index 0000000..8a906ee --- /dev/null +++ b/src/mcp/front-matter.d.ts @@ -0,0 +1,26 @@ +// The upstream `front-matter` package ships an `export default` ambient +// declaration, but its actual runtime export (`module.exports = fm`) is a +// plain CommonJS export. Under `moduleResolution: nodenext` combined with +// this package's own `type: module`, that mismatch makes the shipped types +// unusable (`fm` resolves to the module namespace instead of the callable +// function). This override restates the same shape using `export =`, which +// matches the real CJS export and interops correctly via `esModuleInterop`. +declare module "front-matter" { + interface FrontMatterResult { + readonly attributes: T; + readonly body: string; + readonly bodyBegin: number; + readonly frontmatter?: string; + } + + interface FrontMatterOptions { + allowUnsafe?: boolean; + } + + function fm(file: string, options?: FrontMatterOptions): FrontMatterResult; + namespace fm { + function test(file: string): boolean; + } + + export = fm; +} diff --git a/src/mcp/server.ts b/src/mcp/server.ts new file mode 100644 index 0000000..5927c8f --- /dev/null +++ b/src/mcp/server.ts @@ -0,0 +1,75 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { z } from "zod"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { discoverDocs, buildCatalog, type Doc } from "./discoverDocs.ts"; + +const DEFAULT_DOCS_DIR = join(fileURLToPath(new URL(".", import.meta.url)), "../docs/mcp"); + +export async function startMcpServer(docsDirs?: string[]): Promise { + const dirs = docsDirs ?? [DEFAULT_DOCS_DIR]; + let docsCache: Map | null = null; + + function getDocs(): Map { + if (!docsCache) { + docsCache = discoverDocs(dirs); + } + return docsCache; + } + + const server = new McpServer({ name: "webiny-data-transfer", version: "1.0.0" }); + + server.registerTool( + "list_topics", + { + title: "List Data Transfer Topics", + description: + "Returns a catalog of all available @webiny/data-transfer documentation topics. " + + "Call this first to discover what topics are available, then use get_topic to read the full documentation for a specific topic. " + + "Topics cover: presets, transformers, processors, scanners, filters, config, pipeline runtime, and how to write custom components.", + inputSchema: {}, + annotations: { readOnlyHint: true } + }, + async () => ({ + content: [{ type: "text" as const, text: buildCatalog(getDocs()) }] + }) + ); + + server.registerTool( + "get_topic", + { + title: "Get Data Transfer Topic", + description: + "Returns the full documentation for a specific @webiny/data-transfer topic. " + + "Use exact topic names from list_topics.", + inputSchema: { + topic: z.string().describe("Topic name — use exact names from list_topics") + }, + annotations: { readOnlyHint: true } + }, + async ({ topic }) => { + const docs = getDocs(); + const doc = docs.get(topic); + + if (!doc) { + const available = [...docs.keys()].sort().join(", "); + return { + content: [ + { + type: "text" as const, + text: `Topic "${topic}" not found. Available topics: ${available}` + } + ] + }; + } + + return { + content: [{ type: "text" as const, text: doc.body }] + }; + } + ); + + const transport = new StdioServerTransport(); + await server.connect(transport); +} diff --git a/yarn.lock b/yarn.lock index d2e0b7c..1f7f7a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3442,6 +3442,15 @@ __metadata: languageName: node linkType: hard +"@hono/node-server@npm:^1.19.9 || ^2.0.5": + version: 2.0.12 + resolution: "@hono/node-server@npm:2.0.12" + peerDependencies: + hono: ^4 + checksum: 10/7900669baa2c62c97a25c2cd9dc4ba9e56bba572834fc392e8ce9d3d1aac7b78fdad6b72d2cb47bb668295253a964763a6098e546f0484cf9b96edb1d38ccdf3 + languageName: node + linkType: hard + "@iconify/json@npm:2.2.498": version: 2.2.498 resolution: "@iconify/json@npm:2.2.498" @@ -4791,6 +4800,39 @@ __metadata: languageName: node linkType: hard +"@modelcontextprotocol/sdk@npm:^1.30.0": + version: 1.30.0 + resolution: "@modelcontextprotocol/sdk@npm:1.30.0" + dependencies: + "@hono/node-server": "npm:^1.19.9 || ^2.0.5" + ajv: "npm:^8.17.1" + ajv-formats: "npm:^3.0.1" + content-type: "npm:^1.0.5" + cors: "npm:^2.8.5" + cross-spawn: "npm:^7.0.5" + eventsource: "npm:^3.0.2" + eventsource-parser: "npm:^3.0.0" + express: "npm:^5.2.1" + express-rate-limit: "npm:^8.2.1" + hono: "npm:^4.11.4" + jose: "npm:^6.1.3" + json-schema-typed: "npm:^8.0.2" + pkce-challenge: "npm:^5.0.0" + raw-body: "npm:^3.0.0" + zod: "npm:^3.25 || ^4.0" + zod-to-json-schema: "npm:^3.25.1" + peerDependencies: + "@cfworker/json-schema": ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + "@cfworker/json-schema": + optional: true + zod: + optional: false + checksum: 10/369723a62ce230e55d4fc98549fb080a51916a2004f3907bfca899f5602de8791d9b252038abc6cf120a9d1c75845d2459a471b678c9b9cfc7bae5febf112b63 + languageName: node + linkType: hard + "@monaco-editor/loader@npm:^1.5.0": version: 1.7.0 resolution: "@monaco-editor/loader@npm:1.7.0" @@ -10167,6 +10209,7 @@ __metadata: "@faker-js/faker": "npm:^10.5.0" "@inquirer/core": "npm:^11.2.1" "@inquirer/prompts": "npm:^8.5.2" + "@modelcontextprotocol/sdk": "npm:^1.30.0" "@opensearch-project/opensearch": "npm:3.6.0" "@smithy/util-stream": "npm:^4.7.16" "@types/jsdom": "npm:^28.0.3" @@ -10186,6 +10229,7 @@ __metadata: dynalite: "npm:^4.0.0" execa: "npm:^10.0.1" exifreader: "npm:^4.41.3" + front-matter: "npm:^4.0.2" jsdom: "npm:^30.0.1" oxfmt: "npm:^0.61.0" oxlint: "npm:^1.76.0" @@ -10761,6 +10805,16 @@ __metadata: languageName: node linkType: hard +"accepts@npm:^2.0.0": + version: 2.0.0 + resolution: "accepts@npm:2.0.0" + dependencies: + mime-types: "npm:^3.0.0" + negotiator: "npm:^1.0.0" + checksum: 10/ea1343992b40b2bfb3a3113fa9c3c2f918ba0f9197ae565c48d3f84d44b174f6b1d5cd9989decd7655963eb03a272abc36968cc439c2907f999bd5ef8653d5a7 + languageName: node + linkType: hard + "accepts@npm:~1.3.4, accepts@npm:~1.3.8": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -10902,7 +10956,7 @@ __metadata: languageName: node linkType: hard -"ajv@npm:^8.0.0, ajv@npm:^8.12.0": +"ajv@npm:^8.0.0, ajv@npm:^8.12.0, ajv@npm:^8.17.1": version: 8.20.0 resolution: "ajv@npm:8.20.0" dependencies: @@ -11467,6 +11521,23 @@ __metadata: languageName: node linkType: hard +"body-parser@npm:^2.2.1": + version: 2.3.0 + resolution: "body-parser@npm:2.3.0" + dependencies: + bytes: "npm:^3.1.2" + content-type: "npm:^2.0.0" + debug: "npm:^4.4.3" + http-errors: "npm:^2.0.1" + iconv-lite: "npm:^0.7.2" + on-finished: "npm:^2.4.1" + qs: "npm:^6.15.2" + raw-body: "npm:^3.0.2" + type-is: "npm:^2.1.0" + checksum: 10/94f741ff525358dd0d47f0a042936dc61b9d2e348ff1228c5ffd79f1902c673a317bbb9495b3f7d1db4483befe6ffed1c42bc4cdd8da8f41690530810cfe4dad + languageName: node + linkType: hard + "body-parser@npm:~1.20.3, body-parser@npm:~1.20.5": version: 1.20.6 resolution: "body-parser@npm:1.20.6" @@ -11701,7 +11772,7 @@ __metadata: languageName: node linkType: hard -"bytes@npm:3.1.2, bytes@npm:~3.1.2": +"bytes@npm:3.1.2, bytes@npm:^3.1.2, bytes@npm:~3.1.2": version: 3.1.2 resolution: "bytes@npm:3.1.2" checksum: 10/a10abf2ba70c784471d6b4f58778c0beeb2b5d405148e66affa91f23a9f13d07603d0a0354667310ae1d6dc141474ffd44e2a074be0f6e2254edb8fc21445388 @@ -12169,6 +12240,13 @@ __metadata: languageName: node linkType: hard +"content-disposition@npm:^1.0.0": + version: 1.1.0 + resolution: "content-disposition@npm:1.1.0" + checksum: 10/c4f65e3c001a4a8eb87d0d24c0f112abb139836fb13b8ea67276715e7dce09570ef666ba7848ee8b660d467e6588d030c8ed7e8d0128db6ca78a0800dcd8c7a8 + languageName: node + linkType: hard + "content-disposition@npm:~0.5.4": version: 0.5.4 resolution: "content-disposition@npm:0.5.4" @@ -12178,13 +12256,20 @@ __metadata: languageName: node linkType: hard -"content-type@npm:~1.0.4, content-type@npm:~1.0.5": +"content-type@npm:^1.0.5, content-type@npm:~1.0.4, content-type@npm:~1.0.5": version: 1.0.5 resolution: "content-type@npm:1.0.5" checksum: 10/585847d98dc7fb8035c02ae2cb76c7a9bd7b25f84c447e5ed55c45c2175e83617c8813871b4ee22f368126af6b2b167df655829007b21aa10302873ea9c62662 languageName: node linkType: hard +"content-type@npm:^2.0.0": + version: 2.0.0 + resolution: "content-type@npm:2.0.0" + checksum: 10/0bbb276b790ba7e86c479c7d69fae1861b2e908ff3ce2cb01975b516f93eede2216d242902ed6c5b15cd554014611ce9dfdcf51cd35b16569e45a979e50d0cef + languageName: node + linkType: hard + "convert-source-map@npm:^1.5.0": version: 1.9.0 resolution: "convert-source-map@npm:1.9.0" @@ -12199,6 +12284,13 @@ __metadata: languageName: node linkType: hard +"cookie-signature@npm:^1.2.1": + version: 1.2.2 + resolution: "cookie-signature@npm:1.2.2" + checksum: 10/be44a3c9a56f3771aea3a8bd8ad8f0a8e2679bcb967478267f41a510b4eb5ec55085386ba79c706c4ac21605ca76f4251973444b90283e0eb3eeafe8a92c7708 + languageName: node + linkType: hard + "cookie-signature@npm:~1.0.6": version: 1.0.7 resolution: "cookie-signature@npm:1.0.7" @@ -12206,6 +12298,13 @@ __metadata: languageName: node linkType: hard +"cookie@npm:^0.7.1, cookie@npm:~0.7.1, cookie@npm:~0.7.2": + version: 0.7.2 + resolution: "cookie@npm:0.7.2" + checksum: 10/24b286c556420d4ba4e9bc09120c9d3db7d28ace2bd0f8ccee82422ce42322f73c8312441271e5eefafbead725980e5996cc02766dbb89a90ac7f5636ede608f + languageName: node + linkType: hard + "cookie@npm:^1.0.0, cookie@npm:^1.0.1": version: 1.1.1 resolution: "cookie@npm:1.1.1" @@ -12213,13 +12312,6 @@ __metadata: languageName: node linkType: hard -"cookie@npm:~0.7.1, cookie@npm:~0.7.2": - version: 0.7.2 - resolution: "cookie@npm:0.7.2" - checksum: 10/24b286c556420d4ba4e9bc09120c9d3db7d28ace2bd0f8ccee82422ce42322f73c8312441271e5eefafbead725980e5996cc02766dbb89a90ac7f5636ede608f - languageName: node - linkType: hard - "core-js-compat@npm:^3.48.0": version: 3.49.0 resolution: "core-js-compat@npm:3.49.0" @@ -12243,7 +12335,7 @@ __metadata: languageName: node linkType: hard -"cors@npm:2.8.6, cors@npm:~2.8.5": +"cors@npm:2.8.6, cors@npm:^2.8.5, cors@npm:~2.8.5": version: 2.8.6 resolution: "cors@npm:2.8.6" dependencies: @@ -12482,7 +12574,7 @@ __metadata: languageName: node linkType: hard -"debug@npm:4, debug@npm:4.4.3, debug@npm:^4.1.0, debug@npm:^4.3.1, debug@npm:^4.3.4, debug@npm:^4.3.5, debug@npm:^4.4.1, debug@npm:^4.4.3, debug@npm:~4.4.1": +"debug@npm:4, debug@npm:4.4.3, debug@npm:^4.1.0, debug@npm:^4.3.1, debug@npm:^4.3.4, debug@npm:^4.3.5, debug@npm:^4.4.0, debug@npm:^4.4.1, debug@npm:^4.4.3, debug@npm:~4.4.1": version: 4.4.3 resolution: "debug@npm:4.4.3" dependencies: @@ -12663,7 +12755,7 @@ __metadata: languageName: node linkType: hard -"depd@npm:2.0.0, depd@npm:~2.0.0": +"depd@npm:2.0.0, depd@npm:^2.0.0, depd@npm:~2.0.0": version: 2.0.0 resolution: "depd@npm:2.0.0" checksum: 10/c0c8ff36079ce5ada64f46cc9d6fd47ebcf38241105b6e0c98f412e8ad91f084bcf906ff644cc3a4bd876ca27a62accb8b0fff72ea6ed1a414b89d8506f4a5ca @@ -12958,7 +13050,7 @@ __metadata: languageName: node linkType: hard -"encodeurl@npm:~2.0.0": +"encodeurl@npm:^2.0.0, encodeurl@npm:~2.0.0": version: 2.0.0 resolution: "encodeurl@npm:2.0.0" checksum: 10/abf5cd51b78082cf8af7be6785813c33b6df2068ce5191a40ca8b1afe6a86f9230af9a9ce694a5ce4665955e5c1120871826df9c128a642e09c58d592e2807fe @@ -13251,7 +13343,7 @@ __metadata: languageName: node linkType: hard -"escape-html@npm:~1.0.3": +"escape-html@npm:^1.0.3, escape-html@npm:~1.0.3": version: 1.0.3 resolution: "escape-html@npm:1.0.3" checksum: 10/6213ca9ae00d0ab8bccb6d8d4e0a98e76237b2410302cf7df70aaa6591d509a2a37ce8998008cbecae8fc8ffaadf3fb0229535e6a145f3ce0b211d060decbb24 @@ -13298,7 +13390,7 @@ __metadata: languageName: node linkType: hard -"etag@npm:~1.8.1": +"etag@npm:^1.8.1, etag@npm:~1.8.1": version: 1.8.1 resolution: "etag@npm:1.8.1" checksum: 10/571aeb3dbe0f2bbd4e4fadbdb44f325fc75335cd5f6f6b6a091e6a06a9f25ed5392f0863c5442acb0646787446e816f13cbfc6edce5b07658541dff573cab1ff @@ -13335,13 +13427,22 @@ __metadata: languageName: node linkType: hard -"eventsource-parser@npm:^3.0.8": +"eventsource-parser@npm:^3.0.0, eventsource-parser@npm:^3.0.1, eventsource-parser@npm:^3.0.8": version: 3.1.0 resolution: "eventsource-parser@npm:3.1.0" checksum: 10/6aa03b4d6e3450935690fd9cca6e47b9877287c9419dba9705b85a73e741c7dfbc22b4ebeca25adf05c9549e33c4491e3ca71f80ddfac585e469b7da91f76e20 languageName: node linkType: hard +"eventsource@npm:^3.0.2": + version: 3.0.7 + resolution: "eventsource@npm:3.0.7" + dependencies: + eventsource-parser: "npm:^3.0.1" + checksum: 10/e034915bc97068d1d38617951afd798e6776d6a3a78e36a7569c235b177c7afc2625c9fe82656f7341ab72c7eeecb3fd507b7f88e9328f2448872ff9c4742bb6 + languageName: node + linkType: hard + "execa@npm:5.1.1, execa@npm:^5.1.0": version: 5.1.1 resolution: "execa@npm:5.1.1" @@ -13421,6 +13522,18 @@ __metadata: languageName: node linkType: hard +"express-rate-limit@npm:^8.2.1": + version: 8.6.1 + resolution: "express-rate-limit@npm:8.6.1" + dependencies: + debug: "npm:^4.4.3" + ip-address: "npm:^10.2.0" + peerDependencies: + express: ">= 4.11" + checksum: 10/55e40d693fcb9b7dc1adb48e0d83b3552e902982d80ec1812ba3b60deaffda8b0c12392253018b1c70772df4c0fd40c040dad85dc18349e1e025aa3c8fc70aa2 + languageName: node + linkType: hard + "express@npm:4.22.1": version: 4.22.1 resolution: "express@npm:4.22.1" @@ -13499,6 +13612,42 @@ __metadata: languageName: node linkType: hard +"express@npm:^5.2.1": + version: 5.2.1 + resolution: "express@npm:5.2.1" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.1" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10/4aa545d89702ac83f645c77abda1b57bcabe288f0b380fb5580fac4e323ea0eb533005c8e666b4e19152fb16d4abf11ba87b22aa9a10857a0485cd86b94639bd + languageName: node + linkType: hard + "extend@npm:~3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -13781,6 +13930,20 @@ __metadata: languageName: node linkType: hard +"finalhandler@npm:^2.1.0": + version: 2.1.1 + resolution: "finalhandler@npm:2.1.1" + dependencies: + debug: "npm:^4.4.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + on-finished: "npm:^2.4.1" + parseurl: "npm:^1.3.3" + statuses: "npm:^2.0.1" + checksum: 10/f4ba75c23408d8f9d393c3e875b9452e84d68c925411a6e67b7efa678b0bed5075ef33def4bb65ed8e0dd37c92a3ea354bcbde07303cd4dc2550e12b95885067 + languageName: node + linkType: hard + "finalhandler@npm:~1.3.1": version: 1.3.2 resolution: "finalhandler@npm:1.3.2" @@ -13899,6 +14062,13 @@ __metadata: languageName: node linkType: hard +"fresh@npm:^2.0.0": + version: 2.0.0 + resolution: "fresh@npm:2.0.0" + checksum: 10/44e1468488363074641991c1340d2a10c5a6f6d7c353d89fd161c49d120c58ebf9890720f7584f509058385836e3ce50ddb60e9f017315a4ba8c6c3461813bfc + languageName: node + linkType: hard + "fresh@npm:~0.5.2": version: 0.5.2 resolution: "fresh@npm:0.5.2" @@ -13906,6 +14076,15 @@ __metadata: languageName: node linkType: hard +"front-matter@npm:^4.0.2": + version: 4.0.2 + resolution: "front-matter@npm:4.0.2" + dependencies: + js-yaml: "npm:^3.13.1" + checksum: 10/8897a831a82c5d35413b02b806ed421e793068ad8bf75e864163ec07b7f0cfd87e2fcce0893e8ceccc8f6c63a46e953a6c01208e573627626867a8b86cf6abb9 + languageName: node + linkType: hard + "fs-constants@npm:^1.0.0": version: 1.0.0 resolution: "fs-constants@npm:1.0.0" @@ -14416,6 +14595,13 @@ __metadata: languageName: node linkType: hard +"hono@npm:^4.11.4": + version: 4.12.33 + resolution: "hono@npm:4.12.33" + checksum: 10/1a4d8c2f05731d28785b19488d53ed629dd83f10dce79042da75dbbc9581a7205c20a840a6657d79c50c0d638057cb42adc4656ee6e8a73bde6ab0d0c4f68719 + languageName: node + linkType: hard + "hosted-git-info@npm:^7.0.0": version: 7.0.2 resolution: "hosted-git-info@npm:7.0.2" @@ -14488,7 +14674,7 @@ __metadata: languageName: node linkType: hard -"http-errors@npm:2.0.1, http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": +"http-errors@npm:2.0.1, http-errors@npm:^2.0.0, http-errors@npm:^2.0.1, http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": version: 2.0.1 resolution: "http-errors@npm:2.0.1" dependencies: @@ -14605,7 +14791,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2": +"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": version: 0.7.3 resolution: "iconv-lite@npm:0.7.3" dependencies: @@ -14748,7 +14934,7 @@ __metadata: languageName: node linkType: hard -"ip-address@npm:^10.1.1": +"ip-address@npm:^10.1.1, ip-address@npm:^10.2.0": version: 10.4.0 resolution: "ip-address@npm:10.4.0" checksum: 10/8a286dd112a88c372b10c706caca14dcf5ed029a11c3911062ad8937ca0f951aa513ecacf82e15ffcf6343749ae6c079dd9741282fc94033a06ff1b7bd6ce69a @@ -14974,6 +15160,13 @@ __metadata: languageName: node linkType: hard +"is-promise@npm:^4.0.0": + version: 4.0.0 + resolution: "is-promise@npm:4.0.0" + checksum: 10/0b46517ad47b00b6358fd6553c83ec1f6ba9acd7ffb3d30a0bf519c5c69e7147c132430452351b8a9fc198f8dd6c4f76f8e6f5a7f100f8c77d57d9e0f4261a8a + languageName: node + linkType: hard + "is-regex@npm:^1.1.4, is-regex@npm:^1.2.1": version: 1.2.1 resolution: "is-regex@npm:1.2.1" @@ -15186,6 +15379,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:^6.1.3": + version: 6.2.5 + resolution: "jose@npm:6.2.5" + checksum: 10/d0e06fd0e0c3b3ea949049fc15db6724c60c1fa1fcb51ca830b2d513f6bbcaf70c4f80cc47cc463fed5991905846da6de323215413cdde057dc113cb694fc416 + languageName: node + linkType: hard + "joycon@npm:^3.1.1": version: 3.1.1 resolution: "joycon@npm:3.1.1" @@ -15225,7 +15425,7 @@ __metadata: languageName: node linkType: hard -"js-yaml@npm:^3.14.1, js-yaml@npm:^3.6.1": +"js-yaml@npm:^3.13.1, js-yaml@npm:^3.14.1, js-yaml@npm:^3.6.1": version: 3.15.0 resolution: "js-yaml@npm:3.15.0" dependencies: @@ -15369,6 +15569,13 @@ __metadata: languageName: node linkType: hard +"json-schema-typed@npm:^8.0.2": + version: 8.0.2 + resolution: "json-schema-typed@npm:8.0.2" + checksum: 10/fa866d1fe91e3a94aa4fe007861475cd03dcaf47b719861cab171ef2f8598478007c634d29ae45de94ee34ddff4e13414c63ea5ff06c5b868b613142c699d511 + languageName: node + linkType: hard + "json-schema@npm:0.4.0, json-schema@npm:^0.4.0": version: 0.4.0 resolution: "json-schema@npm:0.4.0" @@ -16222,6 +16429,13 @@ __metadata: languageName: node linkType: hard +"media-typer@npm:^1.1.0": + version: 1.1.1 + resolution: "media-typer@npm:1.1.1" + checksum: 10/ceef972e43912621b846f816e8937c200dcd83d0bdd4fdd19f28be62a9334f4e6605d0c952fe804bfab7f63a07d14c0d1bea1d9357c35ad6d3b04b184f09e397 + languageName: node + linkType: hard + "memfs@npm:^4.64.0": version: 4.64.0 resolution: "memfs@npm:4.64.0" @@ -16264,6 +16478,13 @@ __metadata: languageName: node linkType: hard +"merge-descriptors@npm:^2.0.0": + version: 2.0.0 + resolution: "merge-descriptors@npm:2.0.0" + checksum: 10/e383332e700a94682d0125a36c8be761142a1320fc9feeb18e6e36647c9edf064271645f5669b2c21cf352116e561914fd8aa831b651f34db15ef4038c86696a + languageName: node + linkType: hard + "merge-stream@npm:^2.0.0": version: 2.0.0 resolution: "merge-stream@npm:2.0.0" @@ -16302,7 +16523,7 @@ __metadata: languageName: node linkType: hard -"mime-db@npm:>= 1.43.0 < 2, mime-db@npm:^1.52.0": +"mime-db@npm:>= 1.43.0 < 2, mime-db@npm:^1.52.0, mime-db@npm:^1.54.0": version: 1.54.0 resolution: "mime-db@npm:1.54.0" checksum: 10/9e7834be3d66ae7f10eaa69215732c6d389692b194f876198dca79b2b90cbf96688d9d5d05ef7987b20f749b769b11c01766564264ea5f919c88b32a29011311 @@ -16318,6 +16539,15 @@ __metadata: languageName: node linkType: hard +"mime-types@npm:^3.0.0, mime-types@npm:^3.0.2": + version: 3.0.2 + resolution: "mime-types@npm:3.0.2" + dependencies: + mime-db: "npm:^1.54.0" + checksum: 10/9db0ad31f5eff10ee8f848130779b7f2d056ddfdb6bda696cb69be68d486d33a3457b4f3f9bdeb60d0736edb471bd5a7c0a384375c011c51c889fd0d5c3b893e + languageName: node + linkType: hard + "mime@npm:1.6.0": version: 1.6.0 resolution: "mime@npm:1.6.0" @@ -17064,7 +17294,7 @@ __metadata: languageName: node linkType: hard -"on-finished@npm:~2.4.1": +"on-finished@npm:^2.4.1, on-finished@npm:~2.4.1": version: 2.4.1 resolution: "on-finished@npm:2.4.1" dependencies: @@ -17552,7 +17782,7 @@ __metadata: languageName: node linkType: hard -"parseurl@npm:~1.3.3": +"parseurl@npm:^1.3.3, parseurl@npm:~1.3.3": version: 1.3.3 resolution: "parseurl@npm:1.3.3" checksum: 10/407cee8e0a3a4c5cd472559bca8b6a45b82c124e9a4703302326e9ab60fc1081442ada4e02628efef1eb16197ddc7f8822f5a91fd7d7c86b51f530aedb17dfa2 @@ -17611,7 +17841,7 @@ __metadata: languageName: node linkType: hard -"path-to-regexp@npm:^8.3.0": +"path-to-regexp@npm:^8.0.0, path-to-regexp@npm:^8.3.0": version: 8.4.2 resolution: "path-to-regexp@npm:8.4.2" checksum: 10/70fd2cbce0b962cbcf4d312af07818bfce2bae11c09cf3bd86be99c0e30168238a1a7b02b18b452e73f075897df04597d30d63e56da7be41eecfc37998693389 @@ -17831,6 +18061,13 @@ __metadata: languageName: node linkType: hard +"pkce-challenge@npm:^5.0.0": + version: 5.0.1 + resolution: "pkce-challenge@npm:5.0.1" + checksum: 10/51d11f68d5a78617cfb2e9c2706dadcc2cbe55ffb55b21d42a6ed848ac5159db2657bf6c966a5a414119aa839ceb64240afea35e9e1c06946b57606ed0b43789 + languageName: node + linkType: hard + "pluralize@npm:8.0.0": version: 8.0.0 resolution: "pluralize@npm:8.0.0" @@ -18069,7 +18306,7 @@ __metadata: languageName: node linkType: hard -"proxy-addr@npm:~2.0.7": +"proxy-addr@npm:^2.0.7, proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" dependencies: @@ -18124,7 +18361,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.15.2, qs@npm:~6.15.1": +"qs@npm:^6.14.0, qs@npm:^6.15.2, qs@npm:~6.15.1": version: 6.15.3 resolution: "qs@npm:6.15.3" dependencies: @@ -18260,6 +18497,13 @@ __metadata: languageName: node linkType: hard +"range-parser@npm:^1.2.1": + version: 1.3.0 + resolution: "range-parser@npm:1.3.0" + checksum: 10/f0b7a34de67333b80da0a1098cc02579bd5f74da717fdc0807caf87f043af6641d98d94c64a60477a8f88ba7c214aede965fd0e0c3036ea35c9fe89e1149923d + languageName: node + linkType: hard + "range-parser@npm:~1.2.1": version: 1.2.1 resolution: "range-parser@npm:1.2.1" @@ -18267,6 +18511,18 @@ __metadata: languageName: node linkType: hard +"raw-body@npm:^3.0.0, raw-body@npm:^3.0.2": + version: 3.0.2 + resolution: "raw-body@npm:3.0.2" + dependencies: + bytes: "npm:~3.1.2" + http-errors: "npm:~2.0.1" + iconv-lite: "npm:~0.7.0" + unpipe: "npm:~1.0.0" + checksum: 10/4168c82157bd69175d5bd960e59b74e253e237b358213694946a427a6f750a18b8e150f036fed3421b3e83294b071a4e2bb01037a79ccacdac05360c63d3ebba + languageName: node + linkType: hard + "raw-body@npm:~2.5.3": version: 2.5.3 resolution: "raw-body@npm:2.5.3" @@ -18917,6 +19173,19 @@ __metadata: languageName: node linkType: hard +"router@npm:^2.2.0": + version: 2.2.0 + resolution: "router@npm:2.2.0" + dependencies: + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + is-promise: "npm:^4.0.0" + parseurl: "npm:^1.3.3" + path-to-regexp: "npm:^8.0.0" + checksum: 10/8949bd1d3da5403cc024e2989fee58d7fda0f3ffe9f2dc5b8a192f295f400b3cde307b0b554f7d44851077640f36962ca469a766b3d57410d7d96245a7ba6c91 + languageName: node + linkType: hard + "rsbuild-plugin-dts@npm:0.23.2": version: 0.23.2 resolution: "rsbuild-plugin-dts@npm:0.23.2" @@ -19375,6 +19644,25 @@ __metadata: languageName: node linkType: hard +"send@npm:^1.1.0, send@npm:^1.2.0": + version: 1.2.1 + resolution: "send@npm:1.2.1" + dependencies: + debug: "npm:^4.4.3" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.1" + mime-types: "npm:^3.0.2" + ms: "npm:^2.1.3" + on-finished: "npm:^2.4.1" + range-parser: "npm:^1.2.1" + statuses: "npm:^2.0.2" + checksum: 10/274f842d69ccfa49d4940a85598c6825da58dee6cb8ea33b08d5bd3988e6a82267c4d7c32b23d0e4706aad076ee95b1edfa13f859877db9b589829019397e355 + languageName: node + linkType: hard + "send@npm:~0.19.0, send@npm:~0.19.1": version: 0.19.2 resolution: "send@npm:0.19.2" @@ -19406,6 +19694,18 @@ __metadata: languageName: node linkType: hard +"serve-static@npm:^2.2.0": + version: 2.2.1 + resolution: "serve-static@npm:2.2.1" + dependencies: + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + parseurl: "npm:^1.3.3" + send: "npm:^1.2.0" + checksum: 10/71500fe80cc7163fec04e4297de7591ad1cb682d137fc030e7a53e57040fda5187e8082a9c1b2ef37f1d3f9c27c9a94d4ba61806ebc28938ba4a7c8947c9f71e + languageName: node + linkType: hard + "serve-static@npm:~1.16.2": version: 1.16.3 resolution: "serve-static@npm:1.16.3" @@ -19952,7 +20252,7 @@ __metadata: languageName: node linkType: hard -"statuses@npm:~2.0.1, statuses@npm:~2.0.2": +"statuses@npm:^2.0.1, statuses@npm:^2.0.2, statuses@npm:~2.0.1, statuses@npm:~2.0.2": version: 2.0.2 resolution: "statuses@npm:2.0.2" checksum: 10/6927feb50c2a75b2a4caab2c565491f7a93ad3d8dbad7b1398d52359e9243a20e2ebe35e33726dee945125ef7a515e9097d8a1b910ba2bbd818265a2f6c39879 @@ -20723,6 +21023,17 @@ __metadata: languageName: node linkType: hard +"type-is@npm:^2.0.1, type-is@npm:^2.1.0": + version: 2.1.0 + resolution: "type-is@npm:2.1.0" + dependencies: + content-type: "npm:^2.0.0" + media-typer: "npm:^1.1.0" + mime-types: "npm:^3.0.0" + checksum: 10/f011885fefb6c6882f36fab89cc596596b96eab1d208a3774628537cad7ce1f91232a6d758115e1a6ed03f0f8c21e9654bb6d280a5c6827ff72a35f6df0fa182 + languageName: node + linkType: hard + "type-is@npm:~1.6.18": version: 1.6.18 resolution: "type-is@npm:1.6.18" @@ -21260,7 +21571,7 @@ __metadata: languageName: node linkType: hard -"vary@npm:^1, vary@npm:~1.1.2": +"vary@npm:^1, vary@npm:^1.1.2, vary@npm:~1.1.2": version: 1.1.2 resolution: "vary@npm:1.1.2" checksum: 10/31389debef15a480849b8331b220782230b9815a8e0dbb7b9a8369559aed2e9a7800cd904d4371ea74f4c3527db456dc8e7ac5befce5f0d289014dbdf47b2242 @@ -21976,7 +22287,16 @@ __metadata: languageName: node linkType: hard -"zod@npm:4.4.3, zod@npm:^4.4.3": +"zod-to-json-schema@npm:^3.25.1": + version: 3.25.2 + resolution: "zod-to-json-schema@npm:3.25.2" + peerDependencies: + zod: ^3.25.28 || ^4 + checksum: 10/7035328654113f1a0b8e4c2d34a06f918c93650ef8a50d4fb30ad8f22e47d5762c163af9c82494756b34776bae3c41c26cfc6945105b0eee7dceb528cc07e665 + languageName: node + linkType: hard + +"zod@npm:4.4.3, zod@npm:^3.25 || ^4.0, zod@npm:^4.4.3": version: 4.4.3 resolution: "zod@npm:4.4.3" checksum: 10/804b9a42aa8f35f2b3c5a8dff906291cb749115f83ee2afe3576d70b5b5c53c965365c7f4967690647a9c54af9838ff232a85ff9577a0a36c44b68bc6cdefe36 From 83e5a38e480b997b882edb0cc95739300a913fb2 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 12:56:25 +0200 Subject: [PATCH 03/16] fix: resolve DEFAULT_DOCS_DIR correctly in dev mode DEFAULT_DOCS_DIR previously always resolved to `/../docs/mcp`, which is correct in production (dist/mcp/server.js -> dist/docs/mcp) but resolves to the nonexistent src/docs/mcp when run in dev via tsx (src/mcp/server.ts). Now tries both candidates and picks whichever exists, falling back to the production path if neither does. --- src/mcp/server.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 5927c8f..1ba9f6a 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -1,11 +1,17 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { z } from "zod"; +import { existsSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { discoverDocs, buildCatalog, type Doc } from "./discoverDocs.ts"; -const DEFAULT_DOCS_DIR = join(fileURLToPath(new URL(".", import.meta.url)), "../docs/mcp"); +const thisDir = fileURLToPath(new URL(".", import.meta.url)); +const DEFAULT_DOCS_DIR = + [ + join(thisDir, "../docs/mcp"), // production (dist/mcp/ → dist/docs/mcp) + join(thisDir, "../../docs/mcp") // dev (src/mcp/ → docs/mcp) + ].find(existsSync) ?? join(thisDir, "../docs/mcp"); export async function startMcpServer(docsDirs?: string[]): Promise { const dirs = docsDirs ?? [DEFAULT_DOCS_DIR]; From 91a462ea5c3163afa186ca25821a32a83bfff9d0 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 13:00:18 +0200 Subject: [PATCH 04/16] docs: add MCP documentation for all 5 built-in presets --- docs/mcp/presets/copy-ddb.md | 53 +++++++++++++++++ docs/mcp/presets/copy-files.md | 59 +++++++++++++++++++ docs/mcp/presets/copy-os.md | 53 +++++++++++++++++ docs/mcp/presets/v5-to-v6-ddb.md | 91 +++++++++++++++++++++++++++++ docs/mcp/presets/v5-to-v6-os.md | 98 ++++++++++++++++++++++++++++++++ 5 files changed, 354 insertions(+) create mode 100644 docs/mcp/presets/copy-ddb.md create mode 100644 docs/mcp/presets/copy-files.md create mode 100644 docs/mcp/presets/copy-os.md create mode 100644 docs/mcp/presets/v5-to-v6-ddb.md create mode 100644 docs/mcp/presets/v5-to-v6-os.md diff --git a/docs/mcp/presets/copy-ddb.md b/docs/mcp/presets/copy-ddb.md new file mode 100644 index 0000000..8ac2b8b --- /dev/null +++ b/docs/mcp/presets/copy-ddb.md @@ -0,0 +1,53 @@ +--- +name: copy-ddb +description: Verbatim copy of a regular DynamoDB table — no transformations. +category: Presets +--- + +# copy-ddb + +**Use when:** you need to copy a Webiny primary DynamoDB table from one environment to another (e.g. prod → dev) with the data left completely untouched — no migration, no reshaping, no filtering. + +**What it does:** + +- Scans every item in the source DynamoDB table. +- Writes every item to the target DynamoDB table unchanged. +- No records are filtered, transformed, or blackholed — everything that is scanned is written. + +**Pipelines registered:** + +| Pipeline | Scanner | Processors | Filter | Transformers | +| ---------------------------------- | ------------ | -------------- | ------ | ------------ | +| `Regular DynamoDB Table Data` | `DdbScanner` | `[DdbProcessor]` | none | none | + +**Transformers applied:** + +None — pure copy. `DdbProcessor.onEnd` emits a `PutRecord` for the scanned record as-is. + +**Example usage in a custom preset:** + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-ddb-copy", + description: "Copy the DynamoDB table, same as copy-ddb.", + configure({ runner, pipelineBuilderFactory }) { + const everything = pipelineBuilderFactory + .create({ + name: "Regular DynamoDB Table Data", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .build(); // no .filter, no .use → verbatim copy + + runner.register(everything); + } +}); +``` + +Select it directly with `--preset=copy-ddb` (or pick it from the wizard) instead of reimplementing it if you just need a plain copy. diff --git a/docs/mcp/presets/copy-files.md b/docs/mcp/presets/copy-files.md new file mode 100644 index 0000000..e34f287 --- /dev/null +++ b/docs/mcp/presets/copy-files.md @@ -0,0 +1,59 @@ +--- +name: copy-files +description: Copy File Manager S3 files (and their DynamoDB records) verbatim — no transformations. +category: Presets +--- + +# copy-files + +**Use when:** you only need to copy File Manager files — the DynamoDB file record plus the underlying S3 object bytes — without touching any other table data and without any migration reshaping. + +**What it does:** + +- Scans the primary DynamoDB table via `DdbScanner`. +- Keeps only records that pass the `isFmFile` filter (File Manager file records). +- Writes the matched DynamoDB record verbatim via `DdbProcessor`. +- Copies the associated S3 object from source bucket to target bucket via `S3Processor`. +- Everything else scanned from the table is skipped by this pipeline (it's the only pipeline in the preset, so non-matching records are simply not written by this preset). + +**Pipelines registered:** + +| Pipeline | Scanner | Processors | Filter | Transformers | +| ------------ | ------------ | ------------------------------ | --------------------- | ------------ | +| `S3 Files` | `DdbScanner` | `[DdbProcessor, S3Processor]` | `createFilter(isFmFile)` | none | + +**Transformers applied:** + +None — pure copy. Both the DynamoDB record and the S3 file bytes are copied as-is; only a filter (`isFmFile`) narrows which records this pipeline claims. + +**Example usage in a custom preset:** + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + S3Processor, + createFilter, + isFmFile +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-files-copy", + description: "Copy File Manager files, same as copy-files.", + configure({ runner, pipelineBuilderFactory }) { + const s3Files = pipelineBuilderFactory + .create({ + name: "S3 Files", + scanner: DdbScanner, + processors: [DdbProcessor, S3Processor] + }) + .filter(createFilter(isFmFile)) + .build(); // no .use → verbatim copy of matched records + + runner.register(s3Files); + } +}); +``` + +Select it directly with `--preset=copy-files`, or combine it with `copy-ddb`/`copy-os` (as separate runs) if you need the rest of the data too — `copy-ddb` already includes File Manager records since it copies the whole table, so `copy-files` is mainly useful when you want *only* the files, or when composing a custom preset that needs the S3 copy step alongside other pipelines. diff --git a/docs/mcp/presets/copy-os.md b/docs/mcp/presets/copy-os.md new file mode 100644 index 0000000..91e980b --- /dev/null +++ b/docs/mcp/presets/copy-os.md @@ -0,0 +1,53 @@ +--- +name: copy-os +description: Verbatim copy of the OpenSearch companion DynamoDB table — no transformations. +category: Presets +--- + +# copy-os + +**Use when:** you need to copy Webiny's OpenSearch companion DynamoDB table (the "OS table" that indexes CMS entries for search) from one environment to another with no transformation — pairs with `copy-ddb` when you also want the primary table copied. + +**What it does:** + +- Scans every item in the source OpenSearch DynamoDB table via `OsScanner` (which decompresses/normalizes the OS record shape). +- Writes every item to the target OS table unchanged via `OsProcessor`. +- No records are filtered, transformed, or blackholed. + +**Pipelines registered:** + +| Pipeline | Scanner | Processors | Filter | Transformers | +| -------------------------------------- | ----------- | --------------- | ------ | ------------ | +| `OpenSearch DynamoDB Table Data` | `OsScanner` | `[OsProcessor]` | none | none | + +**Transformers applied:** + +None — pure copy. `OsProcessor.onEnd` emits a put for the scanned record as-is. + +**Example usage in a custom preset:** + +```typescript +import { + createTransferPreset, + OsScanner, + OsProcessor +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-os-copy", + description: "Copy the OpenSearch DDB table, same as copy-os.", + configure({ runner, pipelineBuilderFactory }) { + const everything = pipelineBuilderFactory + .create({ + name: "OpenSearch DynamoDB Table Data", + scanner: OsScanner, + processors: [OsProcessor] + }) + .build(); // no .filter, no .use → verbatim copy + + runner.register(everything); + } +}); +``` + +Only relevant if your target config has `target.opensearch` configured — the OS scanner/processor pair are registered conditionally on that setting. Select the built-in directly with `--preset=copy-os` instead of hand-rolling it. diff --git a/docs/mcp/presets/v5-to-v6-ddb.md b/docs/mcp/presets/v5-to-v6-ddb.md new file mode 100644 index 0000000..c38989b --- /dev/null +++ b/docs/mcp/presets/v5-to-v6-ddb.md @@ -0,0 +1,91 @@ +--- +name: v5-to-v6-ddb +description: Full Webiny v5 → v6 migration of the primary DynamoDB table (CMS, File Manager, Security, Mailer, Folders, Audit Logs). +category: Presets +--- + +# v5-to-v6-ddb + +**Use when:** migrating a Webiny v5 project's primary DynamoDB table to v6. This is the flagship preset — it applies all the domain-specific reshaping v6 requires (CMS entry/model shape changes, security groups→roles, file manager settings, mailer settings, folder permissions, audit logs) in one pass. Run `v5-to-v6-os` afterward if the project also has an OpenSearch companion table. + +**What it does:** + +- Scans the primary DynamoDB table via `DdbScanner`; runs 15 first-match-wins pipelines over it (registration order below — order is load-bearing because several record shapes overlap, e.g. File Manager files and Form Builder forms are also CMS entries). +- Blackholes (drops) migration-tracking records, ACO search cache records, background tasks, and Form Builder records (no v6 migration path yet for Form Builder). +- Audit logs are blackholed only if `target.auditLog.dynamodb.tableName` is not configured; otherwise they're transformed and written to the configured audit log table. +- Reshapes CMS groups, CMS models, CMS entries, File Manager settings/files, mailer settings, security groups (→ roles) and teams, and folder permissions (FLP records) into their v6 storage format. +- Copies admin user records verbatim (no transformer, still passes through the catch-all `DdbProcessor.onEnd` put). + +**Pipelines registered** (in registration order — first match wins): + +| # | Pipeline | Scanner + Processors | Filter | Notes | +| - | ---------------------- | ------------------------------------------------- | ---------------------------------------------------------------- | ----- | +| 1 | `MigrationRecords` | `DdbScanner` + `[DdbProcessor]` | `isMigrationRecord` | `.blackhole()` — always dropped | +| 2 | `AuditLogs` | `DdbScanner` + `[AuditLogProcessor]` | `isAuditLogEntry` | Must run before `AcoSearchRecordsPage`/`CmsEntries` (shares the `acoSearchRecord` modelId prefix). `.blackhole()` conditionally — only when `target.auditLog?.dynamodb?.tableName` is unset | +| 3 | `AcoSearchRecordsPage` | `DdbScanner` + `[DdbProcessor]` | `isAcoSearchRecord` | `.blackhole()` — always dropped | +| 4 | `ContentModelGroups` | `DdbScanner` + `[DdbProcessor]` | `isCmsGroup` | | +| 5 | `BackgroundTasks` | `DdbScanner` + `[DdbProcessor]` | `isBackgroundTask` | `.blackhole()` — always dropped | +| 6 | `FileManagerSettings` | `DdbScanner` + `[DdbProcessor]` | `byType("fm.settings")` | | +| 7 | `FileManagerFiles` | `DdbScanner` + `[DdbProcessor, S3Processor]` | `isFmFile` | Must run before `CmsEntries` (fm files are also CMS entries) | +| 8 | `MailerSettings` | `DdbScanner` + `[DdbProcessor]` | inline: `record.SK === "L" && record.modelId === "mailerSettings"` | | +| 9 | `SecurityGroups` | `DdbScanner` + `[DdbProcessor]` | inline: `record.TYPE === "security.group" && !isBuiltInSecurityRole(record)` | | +| 10 | `SecurityTeams` | `DdbScanner` + `[DdbProcessor]` | `isSecurityTeam` | | +| 11 | `CmsModels` | `DdbScanner` + `[DdbProcessor]` | `isCmsModel` | | +| 12 | `FolderPermissions` | `DdbScanner` + `[DdbProcessor]` | `isFlpRecord` | | +| 13 | `CmsEntries` | `DdbScanner` + `[DdbProcessor]` | `isCmsEntry` | Catch-all for remaining CMS entries; must run after `FileManagerFiles` | +| 14 | `AdminUsers` | `DdbScanner` + `[DdbProcessor]` | `isAdminUser` | No transformers — verbatim copy | +| 15 | `FormBuilderRecords` | `DdbScanner` + `[DdbProcessor]` | `isFormBuilderRecord` | `.blackhole()` — no v6 migration path yet; must run after `CmsEntries` (FB forms are CMS entries and would otherwise be claimed there) | + +**Transformers applied** (in pipeline order): + +- `AuditLogs`: `coreFieldsTransformer` → `dataFieldsTransformer` → `storageShapeTransformer` (the `auditLogTransformers` bundle) +- `ContentModelGroups`: `wrapInData` → `addGsiTenant` → `removeLocale` → `removeAttributes` +- `FileManagerSettings`: `wrapInData` → `migrateFileManagerSettings` → `removeAttributes` +- `FileManagerFiles`: `cmsEntryTransformers` bundle (`wrapInData` → `addGsiTenant` → `removeLocale` → `fixCmePk` → `fixBrokenStorageKeys` → `transformRichText` → `updateModelIds` → `removeFolderRevision` → `removeAttributes`) → `createMetadata` → `extractImageMetadata` +- `MailerSettings`: `wrapInData` → `migrateMailerSettings` → `removeAttributes` +- `SecurityGroups`: `wrapInData` → `addGsiTenant` → `groupsToRoles` → `transformPermissions` → `removeAttributes` +- `SecurityTeams`: `wrapInData` → `addGsiTenant` → `removeAttributes` +- `CmsModels`: `wrapInData` → `addGsiTenant` → `removeLocale` → `transformModelGroup` → `renameFieldAttributes` → `removeAttributes` +- `FolderPermissions`: `wrapInData` → `addGsiTenant` → `removeLocale` → `removeAttributes` → `updateFlpIds` +- `CmsEntries`: `cmsEntryTransformers` bundle → `addLiveField` → `replaceFileUrls(config)` +- `AdminUsers`: none — pure copy +- Blackholed pipelines (`MigrationRecords`, `AcoSearchRecordsPage`, `BackgroundTasks`, `FormBuilderRecords`, conditionally `AuditLogs`): no transformers run against their output because everything they emit is discarded, but filters still evaluate. + +**Example usage in a custom preset:** + +Extend or override one pipeline from this preset rather than rewriting all 15 — e.g. add a project-specific transformer to the CMS entries catch-all: + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + createFilter, + isCmsEntry, + cmsEntryTransformers, // not currently a public export — copy the chain manually if unavailable + addLiveField, + replaceFileUrls +} from "@webiny/data-transfer"; +import { stampMigratedAt } from "./transformers/stampMigratedAt.ts"; + +export default createTransferPreset({ + name: "my-v5-to-v6-ddb", + description: "v5-to-v6-ddb plus a custom stamp on every CMS entry.", + configure({ runner, pipelineBuilderFactory, container }) { + const cmsEntries = pipelineBuilderFactory + .create({ name: "CmsEntries", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .use(cmsEntryTransformers) + .use(addLiveField) + .use(replaceFileUrls(container.resolve(/* MigrationConfig */))) + .use(stampMigratedAt) + .build(); + + runner.register(cmsEntries); + // ...register the remaining 14 pipelines from this preset, in the same order, + // or use PipelineCustomizer to patch the built-in preset instead of copying it. + } +}); +``` + +In practice, prefer `PipelineCustomizer` (see `pipeline-customizer.md`) to patch a single pipeline of this built-in preset instead of re-registering all 15 by hand. diff --git a/docs/mcp/presets/v5-to-v6-os.md b/docs/mcp/presets/v5-to-v6-os.md new file mode 100644 index 0000000..4506ac3 --- /dev/null +++ b/docs/mcp/presets/v5-to-v6-os.md @@ -0,0 +1,98 @@ +--- +name: v5-to-v6-os +description: Webiny v5 → v6 migration of the OpenSearch companion DynamoDB table. +category: Presets +--- + +# v5-to-v6-os + +**Use when:** migrating a Webiny v5 project's OpenSearch companion DynamoDB table (the table that backs CMS search indexing) to v6. Run this **after** `v5-to-v6-ddb` — it only makes sense once the primary table has already been migrated. Only relevant if the project has `target.opensearch` configured. + +**What it does:** + +- Scans the OpenSearch DynamoDB table via `OsScanner` (which decompresses/normalizes the OS record shape); runs 5 first-match-wins pipelines over it via `OsProcessor`. +- Blackholes (drops) ACO search records, background tasks, and mailer settings — none of these have a v6 target in the OS table (mailer settings migrate via the DDB preset into the KV store instead). +- Reshapes File Manager file records and all remaining CMS entries into v6 storage format, mirroring the equivalent DDB-preset transformer chain but adapted for the OS record shape (`data` is already populated by `OsScanner`, so `wrapInData` is not needed). + +**Pipelines registered** (in registration order — first match wins): + +| # | Pipeline | Scanner + Processors | Filter | Notes | +| - | -------------------- | -------------------------------- | ---------------------- | ----- | +| 1 | `AcoSearchRecords` | `OsScanner` + `[OsProcessor]` | `isAcoSearchRecord` | `.blackhole()` — always dropped | +| 2 | `BackgroundTasks` | `OsScanner` + `[OsProcessor]` | `isOsBackgroundTask` | `.blackhole()` — must run before `CmsEntries` (background tasks are CMS entries in the OS table) | +| 3 | `MailerSettings` | `OsScanner` + `[OsProcessor]` | `isOsMailerSettings` | `.blackhole()` — v6 stores mailer settings in the KV store, migrated by the DDB preset; must run before `CmsEntries` | +| 4 | `FileManagerFiles` | `OsScanner` + `[OsProcessor]` | `isFmFile` | Must run before `CmsEntries` (fm files satisfy `isCmsEntry` via TYPE prefix) | +| 5 | `CmsEntries` | `OsScanner` + `[OsProcessor]` | `isCmsEntry` | Catch-all for remaining CMS entries | + +**Transformers applied** (in pipeline order): + +- `AcoSearchRecords`, `BackgroundTasks`, `MailerSettings`: none — everything they emit is discarded via `.blackhole()`, but filters still evaluate. +- `FileManagerFiles`: `osCmsEntryTransformers` bundle (`addGsiTenant` → `removeLocale` → `fixCmePk` → `fixBrokenStorageKeys` → `transformRichText` → `updateModelIds` → `updateOsIndex` → `removeFolderRevision` → `removeAttributes` → `addTransferTimestamp`) +- `CmsEntries`: `osCmsEntryTransformers` bundle → `addLiveField` → `replaceFileUrls(config)` + +**Example usage in a custom preset:** + +Extend the catch-all `CmsEntries` pipeline with a project-specific transformer, keeping the same registration order so the blackholed pipelines still claim their records first: + +```typescript +import { + createTransferPreset, + OsScanner, + OsProcessor, + createFilter, + isAcoSearchRecord, + isOsBackgroundTask, + isOsMailerSettings, + isFmFile, + isCmsEntry, + addLiveField, + replaceFileUrls +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-v5-to-v6-os", + description: "v5-to-v6-os plus a custom transformer on the CMS entries catch-all.", + configure({ runner, pipelineBuilderFactory, container }) { + const acoSearchRecords = pipelineBuilderFactory + .create({ name: "AcoSearchRecords", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isAcoSearchRecord)) + .blackhole() + .build(); + + const backgroundTasks = pipelineBuilderFactory + .create({ name: "BackgroundTasks", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isOsBackgroundTask)) + .blackhole() + .build(); + + const mailerSettings = pipelineBuilderFactory + .create({ name: "MailerSettings", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isOsMailerSettings)) + .blackhole() + .build(); + + const fileManagerFiles = pipelineBuilderFactory + .create({ name: "FileManagerFiles", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isFmFile)) + // osCmsEntryTransformers is not currently a public export — build the equivalent + // chain yourself, or import it from this preset's source as a reference. + .build(); + + const cmsEntries = pipelineBuilderFactory + .create({ name: "CmsEntries", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isCmsEntry)) + .use(addLiveField) + .use(replaceFileUrls(container.resolve(/* MigrationConfig */))) + .build(); + + runner + .register(acoSearchRecords) + .register(backgroundTasks) + .register(mailerSettings) + .register(fileManagerFiles) + .register(cmsEntries); + } +}); +``` + +In practice, prefer `PipelineCustomizer` (see `pipeline-customizer.md`) to patch a single pipeline of this built-in preset instead of re-registering all 5 by hand. From ef198e8752fe34f7ad709b123783482d4bbdbeb7 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 13:04:52 +0200 Subject: [PATCH 05/16] docs: add MCP documentation for processors and scanners Documents DdbProcessor, OsProcessor, S3Processor, AuditLogProcessor, DdbScanner, and OsScanner: context slices, commands handled, onEnd behavior, and usage in pipelineBuilderFactory.create(). --- docs/mcp/processors/auditLogProcessor.md | 60 ++++++++++++++++++++++++ docs/mcp/processors/ddbProcessor.md | 45 ++++++++++++++++++ docs/mcp/processors/osProcessor.md | 49 +++++++++++++++++++ docs/mcp/processors/s3Processor.md | 52 ++++++++++++++++++++ docs/mcp/scanners/ddbScanner.md | 39 +++++++++++++++ docs/mcp/scanners/osScanner.md | 39 +++++++++++++++ 6 files changed, 284 insertions(+) create mode 100644 docs/mcp/processors/auditLogProcessor.md create mode 100644 docs/mcp/processors/ddbProcessor.md create mode 100644 docs/mcp/processors/osProcessor.md create mode 100644 docs/mcp/processors/s3Processor.md create mode 100644 docs/mcp/scanners/ddbScanner.md create mode 100644 docs/mcp/scanners/osScanner.md diff --git a/docs/mcp/processors/auditLogProcessor.md b/docs/mcp/processors/auditLogProcessor.md new file mode 100644 index 0000000..764361c --- /dev/null +++ b/docs/mcp/processors/auditLogProcessor.md @@ -0,0 +1,60 @@ +--- +name: AuditLogProcessor +description: Writes audit-log entries scanned from the source table to a dedicated target audit-log table. +category: Processors +--- + +# AuditLogProcessor + +**Import:** `import { AuditLogProcessor } from "@webiny/data-transfer";` + +**What it does:** a narrow, opt-in persistence processor for Webiny's audit log records. It reads `config.target.auditLog?.dynamodb?.tableName` (nullable — audit log transfer is optional) and, when configured, queues writes to that dedicated table separately from the main `DdbProcessor` target table. `checkAccess()` returns `[]` (no check) when no audit log table is configured; otherwise it `describeTable`s the target audit log table the same way `DdbProcessor` does for its own table. + +**Context slice it adds:** + +- `ctx.putAuditLog(record)` — queues an `AuditLogPutRecord` command (`{ table: auditLogTableName, record }`), but only if **both** conditions hold: an audit log table name is configured, and `record.TYPE === "auditLog.log"`. If either check fails, the call is a silent no-op — this makes it safe to call unconditionally from `onEnd` without every transformer having to check record type or config first. + +**Commands it handles:** `AuditLogPutRecord` (key `AUDIT_LOG_PUT_RECORD`). `execute()` returns immediately if no audit log table is configured; otherwise it maps each `AuditLogPutRecord` to a `PutRecord` (same `table`/`record`, just re-wrapped as the command type `DdbExecutor` understands) and forwards the batch to `DdbExecutor.execute()`. Note this processor does **not** respect `transferContext.dryRun` directly — the guard is purely the missing-table-name check (this differs from `DdbProcessor`/`OsProcessor`/`S3Processor`, all of which check `dryRun` explicitly). + +**`onEnd` hook behavior:** automatically calls `ctx.putAuditLog(ctx.record)` after the pipeline's transformers run. Combined with the slice guard above, this means a pipeline using `AuditLogProcessor` only writes records that are still typed `auditLog.log` by the time `onEnd` fires and only if the target has an audit log table configured — otherwise the pipeline's `.blackhole(() => !config.target.auditLog?.dynamodb?.tableName)` pattern (see usage below) is the idiomatic way to make the intent explicit and avoid the runner's unclaimed-command warning when the table is absent. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { + createTransferPreset, + DdbScanner, + AuditLogProcessor, + MigrationConfig, + createFilter, + isAuditLogEntry, + coreFieldsTransformer, + dataFieldsTransformer, + storageShapeTransformer +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-preset-with-audit-logs", + description: "Audit log transfer, gated on target.auditLog being configured.", + async configure({ runner, pipelineBuilderFactory, container }) { + const config = container.resolve(MigrationConfig); + + const auditLogs = await pipelineBuilderFactory + .create({ + name: "AuditLogs", + scanner: DdbScanner, + processors: [AuditLogProcessor] + }) + .filter(createFilter(isAuditLogEntry)) + .use(coreFieldsTransformer) + .use(dataFieldsTransformer) + .use(storageShapeTransformer) + .blackhole(() => !config.target.auditLog?.dynamodb?.tableName) + .build(); + + runner.register(auditLogs); + } +}); +``` + +This mirrors the pattern used internally in the `v5-to-v6-ddb` preset (which composes the same three transformers as a single internal `auditLogTransformers` array, not part of the public API — the individual transformer functions above are the public equivalent). It must be registered **before** the `AcoSearchRecordsPage` and `CmsEntries` pipelines because audit log records share the same `acoSearchRecord` modelId prefix and would otherwise be claimed by those pipelines first (first-match-wins dispatch). diff --git a/docs/mcp/processors/ddbProcessor.md b/docs/mcp/processors/ddbProcessor.md new file mode 100644 index 0000000..8445413 --- /dev/null +++ b/docs/mcp/processors/ddbProcessor.md @@ -0,0 +1,45 @@ +--- +name: DdbProcessor +description: Writes scanned/transformed records to the target DynamoDB table. +category: Processors +--- + +# DdbProcessor + +**Import:** `import { DdbProcessor } from "@webiny/data-transfer";` + +**What it does:** the default persistence processor for regular DynamoDB table pipelines. It reads `config.source.dynamodb.tableName` / `config.target.dynamodb.tableName` from `MigrationConfig`, exposes helpers on the transform context for queuing writes and looking up records on either side, and drains queued `PutRecord` commands into the target table via `DdbExecutor` at flush time. Its `checkAccess()` probes both the source and target tables with `describeTable` and reports `ok` / `denied` / `missing` / `unknown` so the orchestrator can abort before spawning workers if credentials or table names are wrong. + +**Context slice it adds:** + +- `ctx.putRecord(record)` — queues a `PutRecord` command (`{ table: targetTable, record }`) into the pending buffer via `ctx.addCommand`. Does not write immediately — commands accumulate and are drained in batches (`tuning.flushEvery`, default 500). +- `ctx.querySourceRecord(pk, sk?)` — queries the **source** DynamoDB table directly (bypasses the pipeline/scanner) and returns the first matching item or `null`. Useful when a transformer needs to look up a related record that isn't the one currently being processed. +- `ctx.queryTargetRecord(pk, sk?)` — same as above but against the **target** table. Useful for checking whether something was already migrated. + +**Commands it handles:** `PutRecord` (key `PUT_RECORD`). `execute()` calls `commands.get(PutRecord.key)` — this marks the key as "claimed" so the runner's unclaimed-command warning doesn't fire — and forwards every collected `PutRecord` to `DdbExecutor.execute()`, which performs the actual batched `BatchWriteItem` calls against the target table. If `transferContext.dryRun` is set, `execute()` returns immediately without writing anything (commands are still collected and shown in dry-run reporting, just never sent to AWS). + +**`onEnd` hook behavior:** automatically calls `ctx.putRecord(ctx.record)` — i.e. by default, **the record scanned by `DdbScanner` is written to the target table verbatim**, after any transformers in the pipeline have mutated `ctx.record` in place. This is what gives "zero transformers required" pipelines (like `copy-ddb`) their behavior: a pipeline with `processors: [DdbProcessor]` and no `.use(...)` calls is a pure 1:1 copy. To suppress the write for a pipeline (e.g. to blackhole/drop records), call `.blackhole()` on the pipeline builder — that discards all commands collected during the pipeline, `onEnd` included — rather than trying to prevent `onEnd` from running. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, DdbScanner, DdbProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-ddb-copy", + description: "Copy the DynamoDB table, same as copy-ddb.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "Regular DynamoDB Table Data", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .build(); // no .filter, no .use → verbatim copy of every scanned record + + runner.register(everything); + } +}); +``` + +`DdbProcessor` is also combined with other processors in the same pipeline when a record needs more than one side effect — e.g. `processors: [DdbProcessor, S3Processor]` in the `FileManagerFiles` pipeline of `v5-to-v6-ddb`, where the DDB record is written and an associated S3 file may be copied in the same pass. Multiple processors in one array run sequentially (both `onEnd` and `execute`, in array order) and their context slices are merged, so a single processor's `putRecord`/`copyFile` calls don't collide as long as their slice keys are disjoint. diff --git a/docs/mcp/processors/osProcessor.md b/docs/mcp/processors/osProcessor.md new file mode 100644 index 0000000..560e090 --- /dev/null +++ b/docs/mcp/processors/osProcessor.md @@ -0,0 +1,49 @@ +--- +name: OsProcessor +description: Writes scanned/transformed records to the target OpenSearch-backed DynamoDB table, managing index lifecycle and gzip compression. +category: Processors +--- + +# OsProcessor + +**Import:** `import { OsProcessor } from "@webiny/data-transfer";` + +**What it does:** the persistence processor for OpenSearch-table pipelines (the DynamoDB table that backs a Webiny OpenSearch index, not the search cluster itself). It reads `config.source.opensearch.tableName` / `config.target.opensearch.tableName`, exposes the same put/query helper shape as `DdbProcessor`, and at flush time gzip-compresses each record's `data` field, ensures the target OpenSearch index exists (creating it with resolved mappings/settings if missing, or temporarily disabling `refresh_interval` on an existing one for faster bulk writes), then writes through `DdbExecutor`. `checkAccess()` calls `osClient.listIndexes()` against the target cluster and classifies HTTP 401/403 as `denied`, 404 as `missing`. `afterShard()` persists the list of touched indexes (and their original `refresh_interval`) to a per-segment state file under `.transfer//` so a later orchestrator-side hook can restore refresh settings once all shards finish. + +**Context slice it adds:** + +- `ctx.putRecord(record)` — queues a `PutRecord` (`{ table: targetTable, record }`). Throws at pipeline setup (`extendContext`) if `config.target.opensearch` or `config.source.opensearch` is missing — this fails fast rather than silently writing nowhere. +- `ctx.querySourceRecord(pk, sk?)` — queries the source OpenSearch-table directly by PK/SK. +- `ctx.queryTargetRecord(pk, sk?)` — same, against the target OpenSearch-table. + +**Commands it handles:** `PutRecord` (key `PUT_RECORD`). `execute()`: +1. Returns immediately if `transferContext.dryRun` or there are no queued puts. +2. Gzip-compresses `record.data` on every put, in batches sized by `config.tuning.os.gzipConcurrency` (default 16). +3. Collects the distinct `record.index` values across the batch and calls `ensureIndex()` for each — creating the index or disabling refresh on an existing one, with retry (`config.tuning.os.retryScheduleMs`, default `[5000, 10000, 20000, 30000, 30000]` ms) on retryable AWS errors. +4. Forwards the gzipped puts to `DdbExecutor.execute()`. + +**`onEnd` hook behavior:** automatically calls `ctx.putRecord(ctx.record)` — the scanned/transformed OS record is written to the target table by default, same "zero transformers required" contract as `DdbProcessor`. Use `.blackhole()` on the pipeline builder to suppress the write for a whole pipeline (e.g. `AcoSearchRecords`, `BackgroundTasks`, `MailerSettings` in `v5-to-v6-os` are all blackholed). + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, OsScanner, OsProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-os-copy", + description: "Copy the OpenSearch-backed table, same as copy-os.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "OpenSearch DynamoDB Table Data", + scanner: OsScanner, + processors: [OsProcessor] + }) + .build(); // no .filter, no .use → verbatim copy of every scanned record + + runner.register(everything); + } +}); +``` + +`OsProcessor` requires `OsScanner` as its paired scanner (both operate on the OpenSearch-table record shape, `{ index, data, ...BaseRecord }`) and is only registered by `bootstrap.ts` when `config.target.opensearch != null`. The related `OsIndexPrefixHook` (a `BeforeTransferHook`, not a `Processor`) sets `process.env.OPENSEARCH_INDEX_PREFIX` from `config.target.opensearch.indexPrefix` once before the transfer starts, so index names resolved during `ensureIndex()` carry the right prefix. diff --git a/docs/mcp/processors/s3Processor.md b/docs/mcp/processors/s3Processor.md new file mode 100644 index 0000000..29c15a2 --- /dev/null +++ b/docs/mcp/processors/s3Processor.md @@ -0,0 +1,52 @@ +--- +name: S3Processor +description: Copies S3 objects (e.g. File Manager files) from the source bucket to the target bucket. +category: Processors +--- + +# S3Processor + +**Import:** `import { S3Processor } from "@webiny/data-transfer";` + +**What it does:** handles S3-side side effects for pipelines that also touch DynamoDB records referencing files (typically File Manager entries). It reads `config.source.s3.bucket` / `config.target.s3.bucket`, exposes helpers to read a source object or queue a copy, and at flush time issues batched `CopyObject` calls via `targetS3.batchCopy()`. `checkAccess()` runs `headBucket` against both source and target buckets, and — when source and target accounts differ (cross-account transfer) — adds an extra check that probes the source bucket using the **target** account's credentials (since `CopyObject` executes with target credentials), returning a `hint` explaining the bucket policy needed if that probe is denied. + +**Context slice it adds:** + +- `ctx.copyFile(sourceKey, targetKey)` — queues an `S3Copy` command (`{ sourceBucket, sourceKey, targetBucket, targetKey }`). No file bytes move at call time; the actual copy happens in `execute()`. +- `ctx.getFile(key)` — reads an object from the **source** bucket directly, returning its body as a `Buffer` or `null` if absent. Useful for transformers that need to inspect file content (e.g. `extractImageMetadata`) before deciding what to queue. + +**Commands it handles:** `S3Copy` (key `S3_COPY`). `execute()` returns immediately under `dryRun` or if there are no queued copies; otherwise it maps each `S3Copy` command to a `{ sourceBucket, sourceKey, targetBucket, targetKey }` tuple and calls `targetS3.batchCopy(...)` once for the whole batch. + +**`onEnd` hook behavior:** **none — `S3Processor` has no `onEnd` hook.** Unlike `DdbProcessor`/`OsProcessor`, there is no sensible per-record default for "copy this file" (not every record has an associated file, and the same record's transformers may not want the file copied at all). Transformers must explicitly call `ctx.copyFile(sourceKey, targetKey)` when a copy is wanted; if none do, nothing is queued and `execute()` is a no-op for that pipeline. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + S3Processor, + createFilter, + isFmFile +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "copy-files", + description: "Copy all the S3 files loaded via DynamoDB regular table - pure copy.", + async configure({ runner, pipelineBuilderFactory }) { + const s3Files = await pipelineBuilderFactory + .create({ + name: "S3 Files", + scanner: DdbScanner, + processors: [DdbProcessor, S3Processor] + }) + .filter(createFilter(isFmFile)) + .build(); + + runner.register(s3Files); + } +}); +``` + +`S3Processor` is always paired with a DDB-side processor (`DdbProcessor`) in the same pipeline — it never appears alone, since it scans records via `DdbScanner` (the File Manager entry) and only conditionally acts on the file itself. In `copy-files` it runs after a plain `isFmFile` filter with no transformers (pure copy of the FM record plus, if a transformer queued one, the underlying file). In `v5-to-v6-ddb`'s `FileManagerFiles` pipeline it runs alongside `cmsEntryTransformers`, `createMetadata`, and `extractImageMetadata`, though that pipeline currently has file copying commented out (`// .blackhole()` left as a TODO marker) rather than removed. diff --git a/docs/mcp/scanners/ddbScanner.md b/docs/mcp/scanners/ddbScanner.md new file mode 100644 index 0000000..345863f --- /dev/null +++ b/docs/mcp/scanners/ddbScanner.md @@ -0,0 +1,39 @@ +--- +name: DdbScanner +description: Scans every item in the source DynamoDB table, segment by segment. +category: Scanners +--- + +# DdbScanner + +**Import:** `import { DdbScanner } from "@webiny/data-transfer";` + +**Scan behavior:** reads `config.source.dynamodb.tableName` and performs a parallel `Scan` against it via `SourceDynamoDbClient`, using DynamoDB's native segment/totalSegments parallel-scan support. `scan(shard)` is an async generator that yields raw table items one at a time as they page in — there is no buffering of the whole table in memory, and no transformation or filtering happens at the scanner level (that's the pipeline's job downstream). + +**Record shape:** yields `BaseRecord` — `{ PK, SK, _et, _ct, _md, TYPE, [key: string]: unknown }`. This is the raw DynamoDB item exactly as stored, with only the four Webiny bookkeeping fields (`_et`, `_ct`, `_md`, `TYPE`) guaranteed present alongside `PK`/`SK`; everything else is whatever attributes the record happens to have. No decompression or decoding is applied — that distinguishes it from `OsScanner`, which decompresses a `data` payload. + +**Segment support:** `listShards()` returns `total = config.pipeline?.segments ?? 1` shards, each `{ segment: i, total }`. Each shard is handed to a separate worker process (per the runtime model — one worker per shard), and `scan(shard)` passes `{ segment: shard.segment, totalSegments: shard.total }` straight through to the underlying DynamoDB `Scan` call's native `Segment`/`TotalSegments` parameters, so increasing `pipeline.segments` in config directly increases scan parallelism against the source table (and correspondingly the number of worker processes spawned). + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, DdbScanner, DdbProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-ddb-copy", + description: "Copy the DynamoDB table, same as copy-ddb.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "Regular DynamoDB Table Data", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .build(); + + runner.register(everything); + } +}); +``` + +`DdbScanner` is the scanner for every pipeline in `copy-ddb`, `copy-files`, and `v5-to-v6-ddb` (it's paired with `DdbProcessor` and/or `S3Processor`/`AuditLogProcessor`, never with `OsProcessor`). All pipelines that share a scanner within one preset form a "merge group" — records are scanned once and dispatched first-match-wins across the pipelines registered against that scanner, in registration order. diff --git a/docs/mcp/scanners/osScanner.md b/docs/mcp/scanners/osScanner.md new file mode 100644 index 0000000..e089599 --- /dev/null +++ b/docs/mcp/scanners/osScanner.md @@ -0,0 +1,39 @@ +--- +name: OsScanner +description: Scans the source OpenSearch-backed DynamoDB table, decompressing each record's payload. +category: Scanners +--- + +# OsScanner + +**Import:** `import { OsScanner } from "@webiny/data-transfer";` + +**Scan behavior:** reads `config.source.opensearch.tableName` (throws if `config.source.opensearch` isn't configured) and performs a parallel `Scan` against that table via `SourceDynamoDbClient` — same underlying scan mechanism as `DdbScanner`, just against the OpenSearch-table rather than the regular table. For each raw item, it skips anything with no `index` field (not a valid OS record), then runs the item through `OsRecordDecompressor.decompress()` to inflate the gzip-compressed `data` payload that OS-table records store. If decompression yields nothing, a debug log records the record's PK/SK and `data` is yielded as `{}` rather than throwing — so downstream transformers always get an object, never `null`/`undefined`, for `record.data`. + +**Record shape:** yields `OsRecord`, which extends `BaseRecord` (`PK`, `SK`, `_et`, `_ct`, `_md`, `TYPE`, plus arbitrary attributes) with two additions: `index: string` (the target OpenSearch index name this record belongs to) and `data: Record` (the decompressed document body — this is the field `OsProcessor.execute()` re-compresses before writing to the target). + +**Segment support:** `listShards()` returns `total = config.pipeline?.segments ?? 1` shards as `{ segment, total }`, identical to `DdbScanner`. `scan(shard)` forwards `{ segment: shard.segment, totalSegments: shard.total }` to the underlying parallel `Scan`, so `pipeline.segments` scales OS-table scan parallelism the same way it does for the regular table. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, OsScanner, OsProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-os-copy", + description: "Copy the OpenSearch-backed table, same as copy-os.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "OpenSearch DynamoDB Table Data", + scanner: OsScanner, + processors: [OsProcessor] + }) + .build(); + + runner.register(everything); + } +}); +``` + +`OsScanner` is always paired with `OsProcessor` (the only processor that understands the `{ index, data }` record shape) and is used across every pipeline in `copy-os` and `v5-to-v6-os`. It's only registered — and its preset only selectable — when `config.target.opensearch != null`; `bootstrap.ts` skips OS feature registration entirely otherwise. From 086970d2d4e6e00c7866cb263d2ac4df4f62f3de Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 13:08:01 +0200 Subject: [PATCH 06/16] docs: add MCP documentation for all 27 built-in transformers --- docs/mcp/transformers/addGsiTenant.md | 17 +++++++++++++++++ docs/mcp/transformers/addLiveField.md | 17 +++++++++++++++++ docs/mcp/transformers/addTransferTimestamp.md | 17 +++++++++++++++++ docs/mcp/transformers/copyFileToTarget.md | 17 +++++++++++++++++ docs/mcp/transformers/coreFieldsTransformer.md | 17 +++++++++++++++++ docs/mcp/transformers/createMetadata.md | 17 +++++++++++++++++ docs/mcp/transformers/dataFieldsTransformer.md | 17 +++++++++++++++++ docs/mcp/transformers/extractImageMetadata.md | 17 +++++++++++++++++ docs/mcp/transformers/fixBrokenStorageKeys.md | 17 +++++++++++++++++ docs/mcp/transformers/fixCmePk.md | 17 +++++++++++++++++ docs/mcp/transformers/groupsToRoles.md | 17 +++++++++++++++++ .../transformers/migrateFileManagerSettings.md | 17 +++++++++++++++++ docs/mcp/transformers/migrateMailerSettings.md | 17 +++++++++++++++++ docs/mcp/transformers/removeAttributes.md | 17 +++++++++++++++++ docs/mcp/transformers/removeFolderRevision.md | 17 +++++++++++++++++ docs/mcp/transformers/removeLocale.md | 17 +++++++++++++++++ docs/mcp/transformers/removeTenant.md | 17 +++++++++++++++++ docs/mcp/transformers/renameFieldAttributes.md | 17 +++++++++++++++++ docs/mcp/transformers/replaceFileUrls.md | 17 +++++++++++++++++ .../mcp/transformers/storageShapeTransformer.md | 17 +++++++++++++++++ docs/mcp/transformers/transformModelGroup.md | 17 +++++++++++++++++ docs/mcp/transformers/transformPermissions.md | 17 +++++++++++++++++ docs/mcp/transformers/transformRichText.md | 17 +++++++++++++++++ docs/mcp/transformers/updateFlpIds.md | 17 +++++++++++++++++ docs/mcp/transformers/updateModelIds.md | 17 +++++++++++++++++ docs/mcp/transformers/updateOsIndex.md | 17 +++++++++++++++++ docs/mcp/transformers/wrapInData.md | 17 +++++++++++++++++ 27 files changed, 459 insertions(+) create mode 100644 docs/mcp/transformers/addGsiTenant.md create mode 100644 docs/mcp/transformers/addLiveField.md create mode 100644 docs/mcp/transformers/addTransferTimestamp.md create mode 100644 docs/mcp/transformers/copyFileToTarget.md create mode 100644 docs/mcp/transformers/coreFieldsTransformer.md create mode 100644 docs/mcp/transformers/createMetadata.md create mode 100644 docs/mcp/transformers/dataFieldsTransformer.md create mode 100644 docs/mcp/transformers/extractImageMetadata.md create mode 100644 docs/mcp/transformers/fixBrokenStorageKeys.md create mode 100644 docs/mcp/transformers/fixCmePk.md create mode 100644 docs/mcp/transformers/groupsToRoles.md create mode 100644 docs/mcp/transformers/migrateFileManagerSettings.md create mode 100644 docs/mcp/transformers/migrateMailerSettings.md create mode 100644 docs/mcp/transformers/removeAttributes.md create mode 100644 docs/mcp/transformers/removeFolderRevision.md create mode 100644 docs/mcp/transformers/removeLocale.md create mode 100644 docs/mcp/transformers/removeTenant.md create mode 100644 docs/mcp/transformers/renameFieldAttributes.md create mode 100644 docs/mcp/transformers/replaceFileUrls.md create mode 100644 docs/mcp/transformers/storageShapeTransformer.md create mode 100644 docs/mcp/transformers/transformModelGroup.md create mode 100644 docs/mcp/transformers/transformPermissions.md create mode 100644 docs/mcp/transformers/transformRichText.md create mode 100644 docs/mcp/transformers/updateFlpIds.md create mode 100644 docs/mcp/transformers/updateModelIds.md create mode 100644 docs/mcp/transformers/updateOsIndex.md create mode 100644 docs/mcp/transformers/wrapInData.md diff --git a/docs/mcp/transformers/addGsiTenant.md b/docs/mcp/transformers/addGsiTenant.md new file mode 100644 index 0000000..b423674 --- /dev/null +++ b/docs/mcp/transformers/addGsiTenant.md @@ -0,0 +1,17 @@ +--- +name: addGsiTenant +description: Populates the GSI_TENANT attribute from the record's PK or data.tenant. +category: Transformers +--- + +# addGsiTenant + +**Import:** `import { addGsiTenant } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Skips records that already have `GSI_TENANT`. Otherwise extracts the tenant from a `T##...` PK prefix; if that pattern isn't present, falls back to `data.tenant` (requires `wrapInData` to have run first). + +**Record types it targets:** Any record — applied broadly to backfill the `GSI_TENANT` GSI attribute for tenant-scoped queries. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/addLiveField.md b/docs/mcp/transformers/addLiveField.md new file mode 100644 index 0000000..adf394e --- /dev/null +++ b/docs/mcp/transformers/addLiveField.md @@ -0,0 +1,17 @@ +--- +name: addLiveField +description: Computes and attaches the `live` pointer (published revision version) to CMS entry records. +category: Transformers +--- + +# addLiveField + +**Import:** `import { addLiveField } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Resolves the published revision version for a CMS entry and sets `data.live = { version }` (or `null` if none is published). The published (`P`) record and a published latest (`L` with `status: "published"`) record already know their own version; any other revision queries the source `P` record via `ctx.querySourceRecord`. Results are cached per entry PK via `ctx.cache` to avoid repeat queries. Skips internal models (`fmfile`, `wbyfmfile`). + +**Record types it targets:** CMS entry records (`data.modelId` present), keyed like `T##L##CMS#CME#` with `SK` of `P` or a revision number/`L`. + +**Context type required:** `DdbCoreTransformContext` diff --git a/docs/mcp/transformers/addTransferTimestamp.md b/docs/mcp/transformers/addTransferTimestamp.md new file mode 100644 index 0000000..05ff3e8 --- /dev/null +++ b/docs/mcp/transformers/addTransferTimestamp.md @@ -0,0 +1,17 @@ +--- +name: addTransferTimestamp +description: Stamps every record with the transfer time as `_tt`. +category: Transformers +--- + +# addTransferTimestamp + +**Import:** `import { addTransferTimestamp } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Sets `record._tt = Date.now()` on the record, unconditionally. Useful as a generic audit/debug marker for when a record passed through the pipeline. + +**Record types it targets:** Any record. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/copyFileToTarget.md b/docs/mcp/transformers/copyFileToTarget.md new file mode 100644 index 0000000..2f60c85 --- /dev/null +++ b/docs/mcp/transformers/copyFileToTarget.md @@ -0,0 +1,17 @@ +--- +name: copyFileToTarget +description: Emits a verbatim S3 copy for a file-manager record, source key equal to target key. +category: Transformers +--- + +# copyFileToTarget + +**Import:** `import { copyFileToTarget } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** Reads the S3 key from `values["text@key"]`, checking both the raw v5 shape (`record.values`) and the post-`wrapInData` shape (`record.data.values`), and calls `ctx.copyFile(key, key)` to queue an S3 copy at the same key path. Use this when the file's storage key does not need to change; for the v5→v6 key-path migration use `createMetadata` instead. + +**Record types it targets:** File-manager file records with a `text@key` value. + +**Context type required:** `DdbTransformContext` (pipeline must include `S3Processor`) diff --git a/docs/mcp/transformers/coreFieldsTransformer.md b/docs/mcp/transformers/coreFieldsTransformer.md new file mode 100644 index 0000000..8d88ff5 --- /dev/null +++ b/docs/mcp/transformers/coreFieldsTransformer.md @@ -0,0 +1,17 @@ +--- +name: coreFieldsTransformer +description: Resolves an audit-log record's creator identity and creation time, and stamps a fresh id and TTL expiry. +category: Transformers +--- + +# coreFieldsTransformer + +**Import:** `import { coreFieldsTransformer } from "@webiny/data-transfer";` + +**Category:** auditLogs + +**What it does:** Tries root-level fields first (`revisionCreatedBy`/`createdBy`/`savedBy`/`revisionSavedBy` and their `*On` counterparts, in priority order); if unavailable, decompresses the legacy `values["object@data"]["text@data"]` envelope and looks for creator info in the payload or its `before`/`after` sub-objects. Sets `record.id` (new `mdbid()`), `record.createdBy`, `record.createdOn`, and `record.expiresAt` (now + 60 days, ISO string). If no creator can be resolved, logs a warning and leaves the record untouched — downstream pipeline logic is expected to drop such records. + +**Record types it targets:** Audit log source records (v5 `cms.entry`-shaped audit log entries). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/createMetadata.md b/docs/mcp/transformers/createMetadata.md new file mode 100644 index 0000000..fd9ec12 --- /dev/null +++ b/docs/mcp/transformers/createMetadata.md @@ -0,0 +1,17 @@ +--- +name: createMetadata +description: Creates a KeyValueStore file-metadata record and copies the underlying S3 object to its new tenant-scoped path. +category: Transformers +--- + +# createMetadata + +**Import:** `import { createMetadata } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** For `cms.entry.l` file records, computes the new S3 key (`tenants//files/`), emits an S3 copy from the old key to the new key when they differ, and emits a new `KeyValueStore` record (`ctx.putRecord`) at `KV#global:FileManager/File//Metadata` holding `bucketKey`, `contentType`, `id`, `size`, and `tenant`. The file ID has its revision suffix (`#0001`) stripped. + +**Record types it targets:** File-manager file entry records (`TYPE === "cms.entry.l"`) with `text@key`/`text@name` values. + +**Context type required:** `DdbTransformContext` diff --git a/docs/mcp/transformers/dataFieldsTransformer.md b/docs/mcp/transformers/dataFieldsTransformer.md new file mode 100644 index 0000000..3db46a1 --- /dev/null +++ b/docs/mcp/transformers/dataFieldsTransformer.md @@ -0,0 +1,17 @@ +--- +name: dataFieldsTransformer +description: Lifts audit-log content fields (app, action, message, entity, tags, content) out of the legacy values envelope onto the record root. +category: Transformers +--- + +# dataFieldsTransformer + +**Import:** `import { dataFieldsTransformer } from "@webiny/data-transfer";` + +**Category:** auditLogs + +**What it does:** Reads `values["object@data"]` and copies `text@app` → `record.app`, `text@action` → `record.action`, `text@message` → `record.message`, `text@entity` → `record.entity`, and `text@data` → `record.content`; sets `record.entityId` from `record.entryId`; sets `record.tags` from `values["text@tags"]` (defaulting to `[]`). Intended to run before `storageShapeTransformer`, which consumes these root-level fields. + +**Record types it targets:** Audit log source records with `values["object@data"]`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/extractImageMetadata.md b/docs/mcp/transformers/extractImageMetadata.md new file mode 100644 index 0000000..b830bff --- /dev/null +++ b/docs/mcp/transformers/extractImageMetadata.md @@ -0,0 +1,17 @@ +--- +name: extractImageMetadata +description: Extracts image dimensions, EXIF, and IPTC metadata from raster image files and renames the legacy meta field. +category: Transformers +--- + +# extractImageMetadata + +**Import:** `import { extractImageMetadata } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** Deletes the legacy `values["object@meta"]` field. For non-raster or non-image types it sets `values["object@metadata"] = {}` and returns. For raster images, it resolves the file's S3 key (preferring an existing KV metadata record's `bucketKey`, falling back to `text@key`), reads the file via `ctx.getFile`, and uses `sharp` for dimensions/format/orientation plus `exifreader` for EXIF/IPTC tags, writing the result to `values["object@metadata"]`. Results are cached per file ID via `ctx.cache` so each file is only fetched/processed once across records. + +**Record types it targets:** File-manager file records with `data.values["text@type"]` set (image files get full extraction; others get an empty metadata object). + +**Context type required:** `DdbTransformContext` diff --git a/docs/mcp/transformers/fixBrokenStorageKeys.md b/docs/mcp/transformers/fixBrokenStorageKeys.md new file mode 100644 index 0000000..fb84cf9 --- /dev/null +++ b/docs/mcp/transformers/fixBrokenStorageKeys.md @@ -0,0 +1,17 @@ +--- +name: fixBrokenStorageKeys +description: Corrects mismatched field storage keys in CMS entry values against the model's declared storageId. +category: Transformers +--- + +# fixBrokenStorageKeys + +**Import:** `import { fixBrokenStorageKeys } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Walks a CMS entry's `data.values` against its model definition (via `ctx.modelProvider`) and, for each field, moves the value found under a wrong key (the declared `storageId` or `fieldId`) to the correct storage key computed by `getCorrectStorageId`. Logs and skips models it cannot find (warning once per missing model). Skips internal models (`fmfile`, `wbyfmfile`) and fragment-uuid fields. + +**Record types it targets:** CMS entry records with `data.modelId` and `data.values`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/fixCmePk.md b/docs/mcp/transformers/fixCmePk.md new file mode 100644 index 0000000..3f74834 --- /dev/null +++ b/docs/mcp/transformers/fixCmePk.md @@ -0,0 +1,17 @@ +--- +name: fixCmePk +description: Removes a duplicated #CME#CME# segment from a record's PK. +category: Transformers +--- + +# fixCmePk + +**Import:** `import { fixCmePk } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Fixes a known PK corruption where `#CME#` appears twice in a row (e.g. `T#root#L#en-US#CMS#CME#CME#` → `T#root#CMS#CME#`) by replacing the first occurrence of `#CME#CME#` with `#CME#`. No-op if the pattern isn't present. + +**Record types it targets:** Any record whose `PK` contains `#CME#CME#` (CMS entry records). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/groupsToRoles.md b/docs/mcp/transformers/groupsToRoles.md new file mode 100644 index 0000000..fb13fbf --- /dev/null +++ b/docs/mcp/transformers/groupsToRoles.md @@ -0,0 +1,17 @@ +--- +name: groupsToRoles +description: Renames security "group" records and their GROUP/GROUPS key segments to the v6 "role" terminology. +category: Transformers +--- + +# groupsToRoles + +**Import:** `import { groupsToRoles } from "@webiny/data-transfer";` + +**Category:** security + +**What it does:** For records with `TYPE === "security.group"`, sets `TYPE` to `security.role`, updates `_et` from `SecurityGroup` to `SecurityRole` if set, and rewrites `GROUPS`→`ROLES` and `GROUP`→`ROLE` segments (in that order, to avoid partial matches) across `PK`, `SK`, `GSI1_PK/SK`, `GSI2_PK/SK`. + +**Record types it targets:** Security group records (`TYPE === "security.group"`). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/migrateFileManagerSettings.md b/docs/mcp/transformers/migrateFileManagerSettings.md new file mode 100644 index 0000000..0096a74 --- /dev/null +++ b/docs/mcp/transformers/migrateFileManagerSettings.md @@ -0,0 +1,17 @@ +--- +name: migrateFileManagerSettings +description: Converts a legacy File Manager settings record into the v6 KeyValueStore format. +category: Transformers +--- + +# migrateFileManagerSettings + +**Import:** `import { migrateFileManagerSettings } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** For records with `original.TYPE === "fm.settings"`, replaces the record wholesale (`ctx.replace`) with a `KeyValueStore` shape: `PK: KV#:FileManager/General`, `SK: A`, `data.value` holding all settings fields except `tenant`. Expects `wrapInData` to have run first so settings live under `record.data`. + +**Record types it targets:** File Manager settings records (`TYPE === "fm.settings"`). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/migrateMailerSettings.md b/docs/mcp/transformers/migrateMailerSettings.md new file mode 100644 index 0000000..1cbeb78 --- /dev/null +++ b/docs/mcp/transformers/migrateMailerSettings.md @@ -0,0 +1,17 @@ +--- +name: migrateMailerSettings +description: Converts a legacy Mailer settings record into the v6 KeyValueStore format. +category: Transformers +--- + +# migrateMailerSettings + +**Import:** `import { migrateMailerSettings } from "@webiny/data-transfer";` + +**Category:** mailer + +**What it does:** For records identified by `original.SK === "L"` and `original.modelId === "mailerSettings"`, replaces the record wholesale (`ctx.replace`) with a `KeyValueStore` shape: `PK: KV#:Mailer/Settings/Transport`, `SK: A`, `data.value` holding `data.values`. Expects `wrapInData` to have run first so values live under `record.data.values`. + +**Record types it targets:** Mailer settings records (`SK === "L"`, `modelId === "mailerSettings"`). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeAttributes.md b/docs/mcp/transformers/removeAttributes.md new file mode 100644 index 0000000..05228ee --- /dev/null +++ b/docs/mcp/transformers/removeAttributes.md @@ -0,0 +1,17 @@ +--- +name: removeAttributes +description: Deletes deprecated top-level attributes (currently webinyVersion) from the data envelope. +category: Transformers +--- + +# removeAttributes + +**Import:** `import { removeAttributes } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Deletes `data.webinyVersion` if present — it's no longer needed in v6 (the `tenant` attribute is handled separately, now derived via `GSI_TENANT` from keys). Expects `wrapInData` to have run first so attributes live under `data`. + +**Record types it targets:** Any record with a `data` envelope containing `webinyVersion`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeFolderRevision.md b/docs/mcp/transformers/removeFolderRevision.md new file mode 100644 index 0000000..3162c27 --- /dev/null +++ b/docs/mcp/transformers/removeFolderRevision.md @@ -0,0 +1,17 @@ +--- +name: removeFolderRevision +description: Strips the #0001 revision suffix from folder location IDs and cleans up legacy folder location fields. +category: Transformers +--- + +# removeFolderRevision + +**Import:** `import { removeFolderRevision } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Removes the trailing `#0001` revision marker from `data.location.folderId`, deletes the legacy `data.values["object@location"]` field (location now lives at `data.location`), and for `wbyAcoFolder` records strips a trailing revision number from `data.values["text@parentId"]`. Expects `wrapInData` to have run first so fields are under `data`. + +**Record types it targets:** Folder-related CMS entries, notably `wbyAcoFolder` model records. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeLocale.md b/docs/mcp/transformers/removeLocale.md new file mode 100644 index 0000000..cca1e08 --- /dev/null +++ b/docs/mcp/transformers/removeLocale.md @@ -0,0 +1,17 @@ +--- +name: removeLocale +description: Strips locale segments (e.g. #L#en-US#) from a record's keys and deletes the locale field. +category: Transformers +--- + +# removeLocale + +**Import:** `import { removeLocale } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Regex-strips `#L##` segments from `PK`, `SK`, `GSI1_PK/SK`, `GSI2_PK/SK`, and deletes the top-level `locale` field plus `data.locale` if present. Used broadly during v5→v6 migration since v6 is single-locale-per-tenant at the storage layer. + +**Record types it targets:** Any record whose keys contain a `#L##` segment. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeTenant.md b/docs/mcp/transformers/removeTenant.md new file mode 100644 index 0000000..c670a0d --- /dev/null +++ b/docs/mcp/transformers/removeTenant.md @@ -0,0 +1,17 @@ +--- +name: removeTenant +description: Deletes the top-level tenant attribute from security role records. +category: Transformers +--- + +# removeTenant + +**Import:** `import { removeTenant } from "@webiny/data-transfer";` + +**Category:** security + +**What it does:** Deletes `record.tenant` unconditionally. Tenant scoping is derived from keys (`GSI_TENANT`) rather than a plain attribute in v6. + +**Record types it targets:** Security role records carrying a legacy top-level `tenant` field. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/renameFieldAttributes.md b/docs/mcp/transformers/renameFieldAttributes.md new file mode 100644 index 0000000..3d1b866 --- /dev/null +++ b/docs/mcp/transformers/renameFieldAttributes.md @@ -0,0 +1,17 @@ +--- +name: renameFieldAttributes +description: Renames legacy CMS model field attributes (helpText, placeholderText, multipleValues) to their v6 equivalents. +category: Transformers +--- + +# renameFieldAttributes + +**Import:** `import { renameFieldAttributes } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Recursively walks a CMS model's `data.fields` (including nested object fields and dynamic-zone template fields) renaming `helpText` → `note`, `placeholderText` → `placeholder`, and `multipleValues` → `list`. Only renames when the target attribute doesn't already exist, and always deletes the old attribute. + +**Record types it targets:** CMS model definition records (`data.fields` array present). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/replaceFileUrls.md b/docs/mcp/transformers/replaceFileUrls.md new file mode 100644 index 0000000..94e0fda --- /dev/null +++ b/docs/mcp/transformers/replaceFileUrls.md @@ -0,0 +1,17 @@ +--- +name: replaceFileUrls +description: Rewrites file-manager URLs embedded in CMS "file" and "rich-text" field values from a source domain to a target domain. +category: Transformers +--- + +# replaceFileUrls + +**Import:** `import { replaceFileUrls } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** A factory — call `replaceFileUrls(config)` with the resolved `MigrationConfig` to get the transformer. It looks up the CMS entry's model, walks `data.values` via the shared field visitor, and for `file`-type fields does a plain string replace of `config.fileUrls.source` with `config.fileUrls.target` (including array values); for `rich-text` fields it decompresses the value, replaces the URL substring inside `state`/`html`, and re-compresses. No-ops entirely if `config.fileUrls.source`/`target` aren't both set. + +**Record types it targets:** CMS entry records with `data.modelId` and `data.values` containing `file` or `rich-text` fields. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/storageShapeTransformer.md b/docs/mcp/transformers/storageShapeTransformer.md new file mode 100644 index 0000000..76dbabc --- /dev/null +++ b/docs/mcp/transformers/storageShapeTransformer.md @@ -0,0 +1,17 @@ +--- +name: storageShapeTransformer +description: Builds the final v6 audit-log storage record — nine GSI key sets plus the data envelope and TTL expiry. +category: Transformers +--- + +# storageShapeTransformer + +**Import:** `import { storageShapeTransformer } from "@webiny/data-transfer";` + +**Category:** auditLogs + +**What it does:** Requires `createdBy`/`createdOn` (and the other fields populated by `coreFieldsTransformer` and `dataFieldsTransformer`) already present on the record — logs a warning and skips otherwise. Replaces the record wholesale (`ctx.replace`) with `PK: T##AUDIT_LOG`, `SK: `, `TYPE: auditLog.log`, and nine `GSI_PK/SK` pairs indexing by app, createdBy, entity, entityId, and action in various combinations, plus a `data` envelope mirroring the fields and a root-level `expiresAt` as Unix-seconds TTL (DynamoDB reads this directly). Must run last in the audit-log transformer chain. + +**Record types it targets:** Audit log records that have already passed through `coreFieldsTransformer` and `dataFieldsTransformer`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/transformModelGroup.md b/docs/mcp/transformers/transformModelGroup.md new file mode 100644 index 0000000..8120885 --- /dev/null +++ b/docs/mcp/transformers/transformModelGroup.md @@ -0,0 +1,17 @@ +--- +name: transformModelGroup +description: Resolves a CMS model's group ID reference to its slug string. +category: Transformers +--- + +# transformModelGroup + +**Import:** `import { transformModelGroup } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Replaces `data.group` (an object with `id`/`name`) with a plain slug string by querying the source group record (`T##L##CMS#CMG` / `group.id`). Falls back to a slugified `group.name` (or `"ungrouped"`) if the group record isn't found, logging a warning. Expects `wrapInData` to have run first so `group` lives at `data.group`. + +**Record types it targets:** CMS model definition records with an object-shaped `data.group`. + +**Context type required:** `DdbCoreTransformContext` diff --git a/docs/mcp/transformers/transformPermissions.md b/docs/mcp/transformers/transformPermissions.md new file mode 100644 index 0000000..d8069f9 --- /dev/null +++ b/docs/mcp/transformers/transformPermissions.md @@ -0,0 +1,17 @@ +--- +name: transformPermissions +description: Migrates security role permissions to v6 shape — drops content.i18n, flattens per-locale model lists, and resolves group IDs to slugs. +category: Transformers +--- + +# transformPermissions + +**Import:** `import { transformPermissions } from "@webiny/data-transfer";` + +**Category:** security + +**What it does:** Walks `data.permissions`: drops any `content.i18n` permission entirely; for `cms.contentModel`, flattens a per-locale `models` object (e.g. `{ "en-US": [...] }`) down to the default locale's array; for `cms.contentModelGroup`, resolves each per-locale group ID to its slug by querying `T##GROUP#` and replaces `groups` with the resolved slug array. Default locale is parsed out of the record's own `PK` (`#L##`). Expects `wrapInData` to have run first. + +**Record types it targets:** Security role records with an array `data.permissions`. + +**Context type required:** `DdbCoreTransformContext` diff --git a/docs/mcp/transformers/transformRichText.md b/docs/mcp/transformers/transformRichText.md new file mode 100644 index 0000000..b67e744 --- /dev/null +++ b/docs/mcp/transformers/transformRichText.md @@ -0,0 +1,17 @@ +--- +name: transformRichText +description: Converts legacy Slate-based rich-text field values into the Lexical state + rendered HTML format. +category: Transformers +--- + +# transformRichText + +**Import:** `import { transformRichText } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** For every `rich-text` field in a CMS entry's `data.values`, decompresses the stored value, and if it has a lexical `root` shape, re-renders it: empty `root.children` gets replaced with `generateInitialLexicalValue()`, then the value is re-compressed as `{ state, html }` (state = JSON string, html = rendered via the internal `LexicalRenderer`). Logs and skips a field on transform failure rather than throwing. + +**Record types it targets:** CMS entry records with `data.modelId`/`data.values` containing compressed `rich-text` field values. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/updateFlpIds.md b/docs/mcp/transformers/updateFlpIds.md new file mode 100644 index 0000000..a1f3717 --- /dev/null +++ b/docs/mcp/transformers/updateFlpIds.md @@ -0,0 +1,17 @@ +--- +name: updateFlpIds +description: Strips the #0001 revision suffix from folder-level-page id and parentId fields. +category: Transformers +--- + +# updateFlpIds + +**Import:** `import { updateFlpIds } from "@webiny/data-transfer";` + +**Category:** folders + +**What it does:** Removes a trailing `#0001` revision marker from `data.id` and `data.parentId` on FLP (folder-level-page) records. These records already carry a `data` envelope natively, so `wrapInData` does not re-wrap them. + +**Record types it targets:** FLP records with `data.id`/`data.parentId`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/updateModelIds.md b/docs/mcp/transformers/updateModelIds.md new file mode 100644 index 0000000..e9bca01 --- /dev/null +++ b/docs/mcp/transformers/updateModelIds.md @@ -0,0 +1,17 @@ +--- +name: updateModelIds +description: Renames legacy system model IDs (fmFile, acoFolder, etc.) to their v6 wby-prefixed equivalents in keys and data.modelId. +category: Transformers +--- + +# updateModelIds + +**Import:** `import { updateModelIds } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Applies a fixed rename map (`fmFile`→`wbyFmFile`, `acoFolder`→`wbyAcoFolder`, `acoFilter`→`wbyAcoFilter`, `webinyTask`→`wbyTask`, `webinyTaskLog`→`wbyTaskLog`, `wby_recordLocking`→`wbyRecordLock`) to every `##`/`#` occurrence in `PK`, `SK`, `GSI1_PK/SK`, `GSI2_PK/SK`, and to `data.modelId` directly. Expects `wrapInData` to have run first. + +**Record types it targets:** Records whose keys or `data.modelId` reference any of the renamed system model IDs. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/updateOsIndex.md b/docs/mcp/transformers/updateOsIndex.md new file mode 100644 index 0000000..787b400 --- /dev/null +++ b/docs/mcp/transformers/updateOsIndex.md @@ -0,0 +1,17 @@ +--- +name: updateOsIndex +description: Recomputes an OpenSearch record's target index name from its modelId and tenant. +category: Transformers +--- + +# updateOsIndex + +**Import:** `import { updateOsIndex } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Reads `record.data.modelId` and `record.data.tenant`, builds a minimal model shape, and calls `@webiny/api-headless-cms-ddb-es`'s `configurations.es()` to derive the correct index name, then sets `record.index` to it. Logs a warning and skips the record if `modelId` or `tenant` is missing. + +**Record types it targets:** OpenSearch CMS entry records (`data.modelId`, `data.tenant` present). + +**Context type required:** `OsTransformContext` (bound via `createOsTransformer`) diff --git a/docs/mcp/transformers/wrapInData.md b/docs/mcp/transformers/wrapInData.md new file mode 100644 index 0000000..9dea14a --- /dev/null +++ b/docs/mcp/transformers/wrapInData.md @@ -0,0 +1,17 @@ +--- +name: wrapInData +description: Wraps all non-reserved top-level attributes of a record into a `data` envelope. +category: Transformers +--- + +# wrapInData + +**Import:** `import { wrapInData } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Moves every attribute not in the reserved set (`PK`, `SK`, `GSI_TENANT`, `GSI1_PK/SK`, `GSI2_PK/SK`, `TYPE`, `data`, `expiresAt`, `_ct`, `_et`, `_md`) into a new `data` object, then replaces the record (`ctx.replace`) with the reserved attributes plus this `data` envelope. No-op if `record.data` already exists. Many other transformers (`transformModelGroup`, `updateModelIds`, `addGsiTenant`, `removeAttributes`, etc.) document that they expect `wrapInData` to run first in the pipeline. + +**Record types it targets:** Any v5-shaped record without an existing `data` envelope. + +**Context type required:** `BaseTransformContext` From 35ceca82b4c9fa7c09101455973cca41e27d5b56 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 13:15:26 +0200 Subject: [PATCH 07/16] docs: add MCP guide documentation for filters, presets, transformers, and config --- docs/mcp/guides/configReference.md | 301 +++++++++++++++++++++++++ docs/mcp/guides/filters.md | 163 +++++++++++++ docs/mcp/guides/writingPresets.md | 260 +++++++++++++++++++++ docs/mcp/guides/writingTransformers.md | 250 ++++++++++++++++++++ 4 files changed, 974 insertions(+) create mode 100644 docs/mcp/guides/configReference.md create mode 100644 docs/mcp/guides/filters.md create mode 100644 docs/mcp/guides/writingPresets.md create mode 100644 docs/mcp/guides/writingTransformers.md diff --git a/docs/mcp/guides/configReference.md b/docs/mcp/guides/configReference.md new file mode 100644 index 0000000..398cf95 --- /dev/null +++ b/docs/mcp/guides/configReference.md @@ -0,0 +1,301 @@ +--- +name: configReference +description: createConfig shape — fromEnv/numberFromEnv, credentials (fromAwsProfile, fromAwsCredentialChain, literal), register callback, pipeline settings, tuning, debug/snapshot. +category: Guides +--- + +# Config reference + +`createConfig(input)` validates a Zod schema (`unifiedTransferInputSchema`) and returns the parsed `MigrationConfiguration`. One `config.ts` file covers DynamoDB, S3, and optional OpenSearch for both source and target. Source: `src/features/MigrationConfig/createConfig.ts`, `src/features/MigrationConfig/schemas/unified.schema.ts`, `src/features/MigrationConfig/schemas/shared.schema.ts`. + +```typescript +function createConfig(input: z.input): MigrationConfiguration; +``` + +## Full shape + +```typescript +import { + loadEnv, + createConfig, + fromAwsProfile, + fromEnv, + numberFromEnv +} from "@webiny/data-transfer"; + +loadEnv(import.meta.url); + +export default createConfig({ + source: { + region: fromEnv("SOURCE_REGION", "eu-central-1"), + credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), + dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, + s3: { bucket: fromEnv("SOURCE_S3_BUCKET") }, + opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } // omit/null if no source OS + }, + target: { + region: fromEnv("TARGET_REGION", "eu-central-1"), + credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), + dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } }, // null/omit to skip + opensearch: { + endpoint: fromEnv("TARGET_OS_ENDPOINT"), + tableName: fromEnv("TARGET_OS_TABLE"), + service: "opensearch", // or "opensearch-serverless" + indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + } + }, + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" // optional — custom preset files, listed alongside built-ins + }, + fileUrls: { source: "https://old-cdn.example.com", target: "https://new-cdn.example.com" }, // optional + register: async container => { /* ... */ }, // optional + tuning: { /* ... */ }, // optional + debug: { /* ... */ } // optional +}); +``` + +`loadEnv(import.meta.url)` loads the `.env` file sitting next to this config file — keep one `.env` per project so credentials stay isolated. + +## `source` / `target` field reference + +| Field | Required | Notes | +| --- | --- | --- | +| `region` | yes | Non-empty, trimmed string. | +| `credentials` | yes | See Credentials below. | +| `accountId` | no | Optional; used by the wizard's cross-account S3 warning. | +| `dynamodb.tableName` | yes | Non-empty, trimmed string. | +| `s3.bucket` | yes | Non-empty, trimmed string. | +| `opensearch` | no | `{ tableName }` on source, `{ endpoint, tableName, service, indexPrefix }` on target. Omit or `null` on **both** sides if unused — mismatched presence (one side set, the other not) fails validation. | +| `target.auditLog.dynamodb.tableName` | no | Omit, or set to `null`, to skip the audit log table — matching records are blackholed instead of written. | + +### Validation guardrails (`superRefine` checks in `unified.schema.ts`) + +`createConfig(...)` throws at call time if any of these hold: + +- `source.s3.bucket === target.s3.bucket` — would overwrite source files. +- `source.region === target.region && source.dynamodb.tableName === target.dynamodb.tableName` — same table on both sides in the same region. +- `target.auditLog.dynamodb.tableName === target.dynamodb.tableName` — audit log table must differ from the main target table. +- `(source.opensearch != null) !== (target.opensearch != null)` — OS must be configured on both sides or neither. +- OS present on both sides AND `source.region === target.region && source.opensearch.tableName === target.opensearch.tableName` — same OS companion table on both sides in the same region. + +These exist to catch copy-paste config mistakes before any data moves — cross-account setups that legitimately reuse a name/region should differentiate one side (rename or use a different region) to signal intent. + +## Env helpers + +Source: `src/utils/fromEnv.ts`. + +```typescript +function fromEnv(name: string): string; // throws if unset/empty +function fromEnv(name: string, defaultValue: string): string; // falls back to defaultValue +function fromEnv(name: string, defaultValue: null): string | null; // returns null instead of throwing + +function numberFromEnv(name: string): number; // throws if unset +function numberFromEnv(name: string, defaultValue: number): number; +``` + +- **`fromEnv(name)`** — required; throws `Environment variable "" is not set and no default was provided.` if unset **or empty** (`KEY=` in `.env` counts as unset — treated as a forgotten value, not an intentional empty override). +- **`fromEnv(name, default)`** — returns `default` when unset/empty. +- **`fromEnv(name, null)`** — returns `string | null`; use for genuinely optional config sections, e.g. `fromEnv("SOURCE_OS_TABLE", null)`. +- **`numberFromEnv(name, default?)`** — parses via `Number(...)`; throws `Environment variable "" is not a valid number (got "").` on parse failure (e.g. `SEGMENTS=four`) so typos surface immediately instead of becoming `NaN` downstream. + +## Credentials + +Both `source.credentials` and `target.credentials` accept one of three shapes (`credentialsOrProviderSchema` in `shared.schema.ts`): + +```typescript +import { fromAwsProfile, fromAwsCredentialChain } from "@webiny/data-transfer"; + +credentials: fromAwsProfile({ profile: "prod-reader" }) +// or +credentials: fromAwsCredentialChain() +// or — literal, for temporary STS credentials +credentials: { accessKeyId: "...", secretAccessKey: "...", sessionToken: "..." /* optional */ } +``` + +| Shape | Re-exported from | Behavior | +| --- | --- | --- | +| `fromAwsProfile({ profile })` | `@aws-sdk/credential-providers`'s `fromIni` | Reads `~/.aws/credentials`. Explicit account selection — best for local dev with multiple profiles; no risk of a stray `AWS_ACCESS_KEY_ID` env var silently hijacking the wrong account. | +| `fromAwsCredentialChain()` | `@aws-sdk/credential-providers`'s `fromNodeProviderChain` | AWS SDK default chain: env vars → shared credentials file → SSO/web-identity → EC2/ECS IAM role. Best for CI / cloud runs where the same config must work without code changes. | +| Literal `{ accessKeyId, secretAccessKey, sessionToken? }` | n/a — validated directly | Explicit strings, e.g. short-lived STS credentials. `sessionToken` optional. | + +Both `fromAwsProfile`/`fromAwsCredentialChain` return an `AwsCredentialsProvider` (`() => Promise`) — the schema's union validates either a literal object or any function, so a hand-written custom provider function also satisfies it. + +## `register` callback (optional) + +Runs **before** preset loading; wires custom DI bindings the preset's `configure()` can later resolve via `container`: + +```typescript +import { createConfig, SourceDynamoDbClient, PipelineCustomizer } from "@webiny/data-transfer"; + +export default createConfig({ + // ...source, target, pipeline... + register: async container => { + const sourceDb = container.resolve(SourceDynamoDbClient); // direct AWS access, e.g. pre-flight checks + + container.register(MyCustomProcessorImpl); // custom Processor.createImplementation(...) + container.register(MyCustomizerImpl); // PipelineCustomizer.createImplementation(...) + } +}); +``` + +Signature: `type RegisterFn = (container: Container) => void | Promise`. + +**Service client abstractions** resolvable from `container`: `SourceDynamoDbClient`, `TargetDynamoDbClient`, `OpenSearchClient`, `SourceS3Client`, `TargetS3Client`. + +**Lifecycle hooks** can also be registered here — `BeforeTransferHook` / `AfterTransferHook` (`execute(): Promise`, run once per whole transfer) and `BeforeLoadPresetHook` / `AfterLoadPresetHook` (`execute(config, preset?): Promise`, run around preset loading). These abstractions use `{ multiple: true }` — registering one adds to the list rather than replacing a default. + +### Customizing OpenSearch index configuration + +Override `IndexConfigurationProvider` to control index creation — mappings, settings, per-index overrides: + +```typescript +import { createConfig, IndexConfigurationProvider } from "@webiny/data-transfer"; + +class CustomIndexConfig implements IndexConfigurationProvider.Interface { + public getConfiguration(indexName: string, base: IndexConfigurationProvider.Configuration) { + const settings = { ...base.settings, number_of_replicas: 2 }; + if (indexName.includes("articles")) { + return { + settings, + mappings: { + ...base.mappings, + properties: { ...base.mappings?.properties, title: { type: "text", analyzer: "english" } } + } + }; + } + return { ...base, settings }; + } +} + +const CustomIndexConfigImpl = IndexConfigurationProvider.createImplementation({ + implementation: CustomIndexConfig, + dependencies: [] +}); + +export default createConfig({ + // ...source, target, pipeline... + register: container => { container.register(CustomIndexConfigImpl); } +}); +``` + +The tool disables `refresh_interval` just-in-time on first write to each index and restores the original value after transfer completes. Missing indexes are created with the Webiny base mapping; only touched indexes are affected. + +## `pipeline` settings + +```typescript +pipeline: { + segments: numberFromEnv("SEGMENTS", 4), // parallel worker processes / DDB scan segments + modelsDir: fromEnv("MODELS_DIR", "./models"), // CMS model JSON directory + presetsDir: "./presets" // optional — custom preset files +} +``` + +All three fields are optional in the schema (`pipelineSettingsSchema`). `modelsDir` is required in practice by the OS preset and by rich-text / field-key transformers — point it at a directory of exported CMS model definitions: + +``` +models/ + single-model.json # { "modelId": "...", "fields": [...], ... } + array-of-models.json # [{ "modelId": "...", "fields": [...] }, ...] + webiny-export.json # { "groups": [...], "models": [...] } ← Webiny admin export +``` + +Three JSON shapes can be mixed in the same directory; JSON models override DB-loaded models when both exist for the same `modelId`. + +## `tuning` (optional) + +```typescript +tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500), // records per shard flush — bounds peak memory + ddb: { maxRetries: 3, initialBackoffMs: 100, requestTimeoutMs: 5000 }, + s3: { concurrency: 10, maxRetries: 3, initialBackoffMs: 100, requestTimeoutMs: 10000 }, + os: { maxRetries: 3, retryScheduleMs: [5000, 10000, 20000], gzipConcurrency: 16 } +} +``` + +All fields optional (`tuningSchema`); absent = built-in defaults. + +- **`flushEvery`** (default 500) — the runner calls each processor's `execute()` every N records and resets the command buffer, bounding peak memory to `flushEvery × avg_record_size` (≈ 5 MB at a 10 KB average, default). Lower to `100` for tables with very large records. +- **DynamoDB `BatchWriteItem` batch size is NOT tunable** — AWS hard-caps it at 25 items per call. +- DDB and S3 clients run in AWS SDK `adaptive` retry mode; `tuning.{ddb,s3}.maxRetries` caps the **outer** retry envelope on top of the SDK's own self-tuning backoff — it doesn't replace SDK retry logic. +- `tuning.os.retryScheduleMs` is an explicit backoff schedule (array of delays in ms) rather than a `maxRetries` + exponential-backoff pair. + +## `debug` (optional) + +```typescript +debug: { + logLevel: "debug", // "debug" | "info" | "warn" | "error" (default "info"); overridable via --log-level CLI flag + snapshot: true, // or: { dir: "./my-snapshot", compress: false } + logFile: true // or: "./my-transfer.log" +} +``` + +### `debug.snapshot` + +Dumps every record the pipeline touches to local JSONL files (default dir `.transfer//snapshot`): + +``` +.transfer// +├── snapshot/ +│ ├── / +│ │ ├── segment-0.source.jsonl.gz ← post-filter, pre-transform +│ │ ├── segment-0.post-transform.jsonl.gz ← after the whole transformer chain +│ │ └── segment-0.commands.jsonl.gz ← PutRecord + S3Copy + etc. +│ └── dropped/ +│ └── segment-0.jsonl.gz ← records matching no pipeline +├── segment-0-blackholed.log +└── segment-0-unmatched.log +``` + +```bash +zcat .transfer//snapshot/cmsEntries/segment-0.source.jsonl.gz | jq 'select(.PK=="T#tenant#CME#abc")' +``` + +Set `compress: false` to `grep` directly without `zcat`. Best-effort — write errors log `warn` but never abort the transfer. + +### `debug.logFile` + +Captures the full runner log to disk. `true` → each process writes to `.transfer//logs/.log` (one file per process, no interleaving under parallelism). String → all processes write to that path instead. Content is raw pino JSONL: + +```bash +cat .transfer//logs/*.log | pino-pretty +``` + +## `fileUrls` (optional) + +Required only if you use the `replaceFileUrls` transformer factory (see `writingTransformers.md`): + +```typescript +fileUrls: { + source: "https://old-cdn.example.com", + target: "https://new-cdn.example.com" +} +``` + +## Required IAM permissions + +The tool runs a pre-flight access check (`HeadBucket` for S3, `DescribeTable` for DynamoDB) before any data moves; a failing check aborts the run with the specific resource/credential set that failed. + +**Source credentials:** + +| Service | Actions | Resource | +| --- | --- | --- | +| DynamoDB | `Scan`, `Query`, `DescribeTable` | Source primary table | +| S3 | `GetObject`, `ListBucket` | Source bucket | +| DynamoDB | `Scan`, `Query`, `DescribeTable` | Source OS companion table (if using OpenSearch) | + +**Target credentials:** + +| Service | Actions | Resource | +| --- | --- | --- | +| DynamoDB | `BatchWriteItem`, `Query`, `DescribeTable` | Target primary table | +| S3 | `PutObject`, `ListBucket` | Target bucket | +| S3 | `GetObject` on the **source** bucket | Cross-account note below | +| DynamoDB | `BatchWriteItem`, `Query`, `DescribeTable` | Target OS companion table (if using OpenSearch) | +| OpenSearch | `ESHttpGet`, `ESHttpPut`, `ESHttpPost` | Target OpenSearch domain (if using OpenSearch) | +| DynamoDB | `BatchWriteItem`, `DescribeTable` | Target audit log table (if configured) | + +**S3 cross-account access:** `CopyObjectCommand` runs with **target credentials**. When source and target are in different AWS accounts, the target account needs read access to the source bucket — either a bucket policy on the source granting `s3:GetObject` to the target account, or a cross-account IAM role. Without this, S3 file copies fail with `AccessDenied`; the wizard warns on detected account-ID mismatch, and the pre-flight check verifies target credentials can reach the source bucket. diff --git a/docs/mcp/guides/filters.md b/docs/mcp/guides/filters.md new file mode 100644 index 0000000..9403795 --- /dev/null +++ b/docs/mcp/guides/filters.md @@ -0,0 +1,163 @@ +--- +name: filters +description: All 18 built-in filter predicates plus createFilter — signatures, matching rules, and usage examples for pipeline .filter() calls. +category: Guides +--- + +# Filters + +A `Filter` is `{ kind: "filter", check: (record) => boolean | Promise }`. Build one with `createFilter(predicate)` and attach it to a pipeline with `.filter(...)`. Multiple `.filter()` calls on the same builder AND-compose — a record must pass every filter to be claimed by that pipeline. + +```typescript +import { createFilter, isCmsEntry } from "@webiny/data-transfer"; + +pipelineBuilderFactory + .create({ name: "cms-entries", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .build(); +``` + +## `createFilter` + +Source: `src/domain/pipeline/Filter.ts`. + +```typescript +export interface Filter { + readonly kind: "filter"; + readonly check: (record: TRecord) => boolean | Promise; +} + +function createFilter( + predicate: (record: TRecord) => boolean | Promise +): Filter; +``` + +Wraps any predicate — sync or async, built-in or inline — into the typed shape the pipeline builder expects. + +```typescript +// Inline predicate — no built-in needed for one-off conditions +.filter(createFilter(r => r.TYPE === "cms.entry" && r.modelId === "article")) + +// Async predicate is supported (check() may return Promise) +.filter(createFilter(async r => (await lookupSomething(r)) != null)) +``` + +## Import + +All 18 predicates are exported directly from the package root, alongside `createFilter`: + +```typescript +import { + createFilter, + byType, + byTypePrefix, + isCmsGroup, + isCmsModel, + isCmsEntry, + byIncludesModelId, + isAcoSearchRecord, + isAdminUser, + isBackgroundTask, + isFmFile, + isFlpRecord, + isBuiltInSecurityRole, + isSecurityTeam, + isOsBackgroundTask, + isOsMailerSettings, + isAuditLogEntry, + isMigrationRecord, + isFormBuilderRecord +} from "@webiny/data-transfer"; +``` + +Source: `src/domain/transform/filters.ts`. + +## Predicate reference + +| Predicate | Signature | Matches | Notes | +| --- | --- | --- | --- | +| `byType` | `(type: string) => (record) => boolean` | `record.TYPE === type` exactly | Factory — call with the exact type string, e.g. `byType("cms.model")` | +| `byTypePrefix` | `(prefix: string) => (record: BaseRecord) => boolean` | `record.TYPE` starts with `prefix` | Factory — for TYPE families like `"cms.entry"` | +| `isCmsGroup` | `(record: BaseRecord) => boolean` | `TYPE === "cms.group"` OR `PK` includes `"#CMS#CMG"` | Handles both raw v5 and reshaped record forms | +| `isCmsModel` | `(record: BaseRecord) => boolean` | `= byType("cms.model")` | Direct alias | +| `isCmsEntry` | `(input: BaseRecord) => boolean` | `TYPE` prefixed `"cms.entry"` OR `PK` includes `"#CMS#CME#"` | Catch-all for CMS entries regardless of shape | +| `byIncludesModelId` | `(target: string) => (record: BaseRecord) => boolean` | `record.index` or `record.modelId` (also checked under `record.data`) contains `target`, case-insensitive | Factory — used to build `isAcoSearchRecord` | +| `isAcoSearchRecord` | `(record: BaseRecord) => boolean` | `= byIncludesModelId("acoSearchRecord")` | ACO search cache records | +| `isAdminUser` | `(record: BaseRecord) => boolean` | `PK` includes `"#SECURITY#USER#"` AND `GSI1_PK === "securityRole#full-access"` | Full-access admin user records | +| `isBackgroundTask` | `(item: BaseRecord) => boolean` | `modelId === "webinyTask"` / `"webinyTaskLog"`, or `GSI1_PK` includes either string | DDB-side background task records | +| `isFmFile` | `(record: BaseRecord) => boolean` | `modelId` (top-level or `record.data.modelId`) is `"fmFile"` or `"wbyFmFile"` | File Manager file records | +| `isFlpRecord` | `(record: Record) => boolean` | `PK` (string) includes `"#FLP#"` | Folder location permission records | +| `isBuiltInSecurityRole` | `(record: Record) => boolean` | `slug` or `GSI1_SK` is `"full-access"` or `"anonymous"` | Use to exclude built-in roles from a custom `SecurityGroups` filter | +| `isSecurityTeam` | `(record: BaseRecord) => boolean` | `= byType("security.team")` | Direct alias | +| `isOsBackgroundTask` | `(record: Record) => boolean` | `record.data.modelId` is `"webinyTask"` or `"webinyTaskLog"` | OS-side equivalent of `isBackgroundTask` — reads from the decompressed `data` payload | +| `isOsMailerSettings` | `(record: Record) => boolean` | `record.data.modelId === "mailerSettings"` | OS-side mailer settings | +| `isAuditLogEntry` | `(record: BaseRecord) => boolean` | `modelId` (top-level or `.data`) lowercases to `"acosearchrecord-auditlogs"` AND `SK === "L"` | Must be filtered for before `isAcoSearchRecord`/`isCmsEntry` — shares the modelId prefix | +| `isMigrationRecord` | `(record: BaseRecord) => boolean` | `PK` starts with `"MIGRATION"` | v5 migration-tracking records; typically blackholed | +| `isFormBuilderRecord` | `(record: BaseRecord) => boolean` | `PK` includes `"#FB#"`, OR `TYPE` starts with `"fb.form."` / `"fb.formSubmission"` | Form Builder forms + submissions; no v6 migration path yet | + +## Predicates that read both raw and reshaped record shapes + +Several predicates check a helper that falls back from a top-level property to the same property nested under `record.data` — this lets one filter match records both **before** and **after** a `wrapInData`-style transformer has run: + +```typescript +// Internal helper used by byIncludesModelId / isAuditLogEntry +function getPropertyFromRecord(record, propertyName: string): T | undefined { + const value = record[propertyName]; + if (value !== undefined) return value; + return record.data?.[propertyName]; +} +``` + +`isFmFile`, `isOsBackgroundTask`, and `isOsMailerSettings` apply the same pattern directly for `modelId`. + +## Ordering rules when composing filters + +Filters only decide whether a pipeline **claims** a record — they don't decide overall precedence across pipelines by themselves. Combine filter choice with **registration order** (first-match-wins across a merge group): + +```typescript +import { + createFilter, + isAuditLogEntry, + isAcoSearchRecord, + isCmsEntry, + isFmFile +} from "@webiny/data-transfer"; + +// 1. Audit logs FIRST — isAuditLogEntry and isAcoSearchRecord both match +// modelId "acoSearchRecord-AuditLogs" style prefixes; audit logs must win. +runner.register(auditLogsPipeline); + +// 2. Generic ACO search cache records +runner.register(acoSearchPipeline); + +// 3. File manager files BEFORE the CMS-entry catch-all — fm files are +// also CMS entries and would otherwise be claimed by #4. +runner.register(fileManagerFilesPipeline); + +// 4. Everything else that looks like a CMS entry +runner.register(cmsEntriesPipeline); +``` + +See `writingPresets.md` for the full first-match-wins model and `pipeline-runtime.md` for merge-group semantics. + +## AND-composing multiple filters on one pipeline + +```typescript +import { createFilter, isCmsEntry, byIncludesModelId } from "@webiny/data-transfer"; + +pipelineBuilderFactory + .create({ name: "articles", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .filter(createFilter(byIncludesModelId("article"))) // AND — must ALSO be modelId "article" + .build(); +``` + +## Zero-filter catch-all + +Omitting `.filter(...)` entirely makes a pipeline accept every record its merge group offers it — used for verbatim-copy presets or as a final catch-all registered last: + +```typescript +const everything = pipelineBuilderFactory + .create({ name: "everything", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); // no .filter() → matches all +``` diff --git a/docs/mcp/guides/writingPresets.md b/docs/mcp/guides/writingPresets.md new file mode 100644 index 0000000..f8317cd --- /dev/null +++ b/docs/mcp/guides/writingPresets.md @@ -0,0 +1,260 @@ +--- +name: writingPresets +description: How to write a custom preset — createTransferPreset shape, pipelineBuilderFactory.create(), builder methods, filter/use/hook composition, first-match-wins dispatch, built-in presets. +category: Guides +--- + +# Writing presets + +A preset is a file exporting `default: MigrationPreset` — `{ name, description, configure }`. `configure(ctx)` receives `{ runner, pipelineBuilderFactory, container }`, builds one or more `Pipeline` objects, and registers them on `runner`. The wizard/CLI selects a preset by `name` at runtime (`--preset=` or `presetsDir` for custom files). + +Source: `src/utils/createTransferPreset.ts`, `src/domain/transform/Preset.ts`, `src/domain/pipeline/PipelineBuilder.ts`, `src/features/PipelineBuilderFactory/`. + +## Minimal preset + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + S3Processor, + createFilter +} from "@webiny/data-transfer"; +import { stampMigratedAt } from "./transformers/stampMigratedAt.ts"; + +export default createTransferPreset({ + name: "my-preset", + description: "One-line description shown in CLI output.", + async configure({ runner, pipelineBuilderFactory }) { + const pipeline = await pipelineBuilderFactory + .create({ name: "my-pipeline", scanner: DdbScanner, processors: [DdbProcessor, S3Processor] }) + .filter(createFilter(r => r.TYPE === "cms.entry")) + .use(stampMigratedAt) + .build(); + + runner.register(pipeline); + } +}); +``` + +Drop the file in your `pipeline.presetsDir` (see `configReference.md`). The wizard offers it by name alongside the 5 built-ins. + +**`createTransferPreset(preset)`** is an identity function — it exists purely so `configure({...})` gets typed inference without you importing and annotating `MigrationPreset` yourself. Pair it with `export default`; the loader looks for `default` first. + +## `PresetConfigureContext` — what `configure` receives + +```typescript +interface PresetConfigureContext { + runner: PipelineRunner.Interface; + pipelineBuilderFactory: PipelineBuilderFactory.Interface; + container: Container; // @webiny/di container +} +``` + +- **`runner`** — call `runner.register(...pipelines)` (variadic, chainable) once your pipelines are built. +- **`pipelineBuilderFactory`** — call `.create({ name, scanner, processors })` to start a new `PipelineBuilder`. +- **`container`** — resolve DI-registered services, e.g. `container.resolve(MigrationConfig)` to read the parsed config at preset-build time (used by built-in presets to decide whether to blackhole audit logs — see below). + +`configure` may be `sync` or `async` — declare it `async` whenever you `await` a `.build()` call (you always will, since `.build()` is async — see below). + +## `pipelineBuilderFactory.create({ name, scanner, processors })` + +```typescript +factory.create>(input: { + name: string; + scanner: ScannerImpl; + processors: TProcessors; // non-empty tuple, disjoint slice keys enforced at compile time +}): PipelineBuilder +``` + +- **`name`** — unique across the run; `PipelineBuilder`'s constructor throws if empty/whitespace-only, and the runner throws on duplicate names. +- **`scanner`** — `DdbScanner` or `OsScanner` (both exported from `@webiny/data-transfer`). Determines which table is scanned and the `TRecord` shape flowing through filters/transformers. +- **`processors`** — a non-empty array of processor implementation classes (`DdbProcessor`, `S3Processor`, `OsProcessor`, `AuditLogProcessor`). Each contributes a **slice** of helpers onto the effective transformer context (`ctx.putRecord`, `ctx.copyFile`, etc. — see `writingTransformers.md`). TypeScript rejects: + - an empty `processors` array, and + - combinations whose slices share a key (e.g. `[DdbProcessor, OsProcessor]` — both contribute `putRecord`/`querySourceRecord`/`queryTargetRecord`). + +## Builder methods + +All methods return `this` (chainable) except `.build()`. + +| Method | Signature | Behavior | +| --- | --- | --- | +| `.filter(filter)` | `(filter: Filter) => this` | Adds a filter. Multiple calls AND-compose — order across calls doesn't matter, all must pass. | +| `.use(transformer)` | `(t: Transformer \| readonly Transformer[]) => this` | Adds one transformer or an array (spread in order). Execution order = registration order; arrays and single calls can mix freely. | +| `.blackhole(condition?)` | `(condition?: () => boolean) => this` | Observe-only mode: filters/transformers/`onEnd` still run, but every emitted command for this pipeline is discarded — nothing lands on the target. `condition` is evaluated **immediately, synchronously**, at call time (not per-record); omit it to always blackhole. | +| `.beforeExecuteCommands(token)` | `(token: Abstraction) => this` | Registers a DI-resolved hook to run once per merge group **before** any shard runs. | +| `.afterExecuteCommands(token)` | `(token: Abstraction) => this` | Registers a DI-resolved hook to run once **after** all shards in the merge group succeed. Skipped if any shard fails. | +| `.build()` | `() => Promise>` | **Async** — snapshots the builder into an immutable `Pipeline`. Always `await` it before passing to `runner.register(...)`. | + +### `.build()` is async — always `await` it + +Every built-in preset does `await factory.create({...}).build()` inside an `async configure(...)`. This is easy to miss since older example code sometimes omits `await` — don't copy that pattern: + +```typescript +async configure({ runner, pipelineBuilderFactory: factory }) { + const everything = await factory + .create({ name: "Regular DynamoDB Table Data", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); + + runner.register(everything); +} +``` + +`.build()` is async because it resolves and applies any `PipelineCustomizer`s registered against this pipeline's `name` before finalizing it (see `pipeline-customizer.md`). + +### `runner.register(...pipelines)` + +```typescript +register(...pipelines: Pipeline[]): this +``` + +Variadic and chainable — heterogeneous pipelines (different scanners/processors) can be registered in one call: `runner.register(p1, p2, p3)`. + +## Filters + +Use `createFilter` with a built-in predicate or an inline function: + +```typescript +import { createFilter, isFmFile, isCmsEntry, byType } from "@webiny/data-transfer"; + +.filter(createFilter(isFmFile)) +.filter(createFilter(isCmsEntry)) +.filter(createFilter(byType("cms.model"))) +.filter(createFilter(r => r.TYPE === "cms.entry" && r.modelId === "article")) // inline +``` + +Full predicate reference (all 18 built-ins): see `filters.md`. + +## First-match-wins dispatch + +Pipelines sharing the same `scanner` type run as a **merge group**. The scanner scans once; each record is offered to every pipeline in the group **in registration order**. The first pipeline whose filters all pass claims the record — no other pipeline in the group sees it. + +```typescript +async configure({ runner, pipelineBuilderFactory: factory }) { + const articles = await factory + .create({ name: "articles", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(r => r.TYPE === "cms.entry" && r.modelId === "article")) + .use(migrateArticle) + .build(); + + const rest = await factory + .create({ name: "rest", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); // no filter → catches everything articles doesn't + + runner.register(articles, rest); // order is load-bearing: articles MUST come first +} +``` + +Swap the registration order and `articles` never fires — `rest` (unfiltered) claims everything first. + +**Records matching no pipeline in any merge group are silently dropped** (not written to target) — see `pipeline-runtime.md` for the unmatched-record logging. + +## Hooks (`beforeExecuteCommands` / `afterExecuteCommands`) + +These accept a **DI abstraction token**, not an inline callback — the runner resolves the token from the container and calls `.run({ runId, mergeGroupId })` on the resolved instance: + +```typescript +interface Hook { + run(params: { runId: string; mergeGroupId: string }): Promise; +} +``` + +To use one, define an implementation and register it via `Abstraction.createImplementation({ implementation, dependencies })` (the same pattern used throughout the codebase for `BeforeTransferHook`/`AfterTransferHook`), register the resulting token in your `config.register` callback, then pass the abstraction token itself to the builder: + +```typescript +.beforeExecuteCommands(MyHookToken) +.afterExecuteCommands(MyHookToken) +``` + +**Caveat:** the base `Hook` abstraction (`src/domain/pipeline/abstractions/Hook.ts`) that these methods are typed against is **not currently re-exported from the public `@webiny/data-transfer` package root** — only the higher-level `BeforeTransferHook`/`AfterTransferHook` (whole-transfer lifecycle, registered via `config.register`) and `BeforeLoadPresetHook`/`AfterLoadPresetHook` are public. In practice, reach for those transfer-lifecycle hooks (documented in `configReference.md`) for cross-cutting setup/teardown; treat per-pipeline `.beforeExecuteCommands()`/`.afterExecuteCommands()` as an advanced/internal extension point until `Hook` is added to the public surface. + +## Zero-transformer preset (pure data copy) + +No transformer is required — a pipeline with only a scanner + processor(s) copies records verbatim: + +```typescript +import { createTransferPreset, DdbScanner, DdbProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "copy", + description: "Copy every record verbatim.", + async configure({ runner, pipelineBuilderFactory: factory }) { + const copyAll = await factory + .create({ name: "copy-all", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); // no .filter → accepts all; no .use → no transformations + + runner.register(copyAll); + } +}); +``` + +`DdbProcessor` (and `OsProcessor`) auto-emit a `PutRecord` for `ctx.record` in their `onEnd` hook — pure-passthrough pipelines still produce writes without you calling `ctx.putRecord()` explicitly. `S3Processor` has no `onEnd`; file copies must be emitted explicitly via a transformer calling `ctx.copyFile(...)` (see `copyFileToTarget` in `writingTransformers.md`). + +## Real example: extending a built-in pipeline's transformer chain + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + createFilter, + isCmsEntry, + addLiveField, + replaceFileUrls, + MigrationConfig +} from "@webiny/data-transfer"; +import { stampMigratedAt } from "./transformers/stampMigratedAt.ts"; + +export default createTransferPreset({ + name: "my-v5-to-v6-ddb", + description: "v5-to-v6-ddb plus a custom stamp on every CMS entry.", + async configure({ runner, pipelineBuilderFactory, container }) { + const config = container.resolve(MigrationConfig); + const cmsEntries = await pipelineBuilderFactory + .create({ name: "CmsEntries", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .use(addLiveField) + .use(replaceFileUrls(config)) // factory — takes the resolved MigrationConfig, needs config.fileUrls set + .use(stampMigratedAt) + .build(); + + runner.register(cmsEntries); + // ...register any additional pipelines this preset needs. + } +}); +``` + +Prefer `PipelineCustomizer` (see `pipeline-customizer.md`) to patch a single pipeline of a built-in preset instead of re-registering all of its pipelines by hand — it's the supported extension point for "built-in preset + a few extra transformers." + +## Conditional blackholing using the resolved config + +Built-in presets use `container.resolve(MigrationConfig)` plus `.blackhole(condition)` to make a pipeline's disposition depend on the parsed config, evaluated once at `configure()` time: + +```typescript +async configure({ runner, pipelineBuilderFactory: factory, container }) { + const config = container.resolve(MigrationConfig); + + const auditLogs = await factory + .create({ name: "AuditLogs", scanner: DdbScanner, processors: [AuditLogProcessor] }) + .filter(createFilter(isAuditLogEntry)) + .use(auditLogTransformers) + .blackhole(() => !config.target.auditLog?.dynamodb?.tableName) + .build(); + + runner.register(auditLogs); +} +``` + +## Built-in presets + +Select by `name` when the wizard asks "Which preset do you want to run?" (or pass `--preset=`): + +| Name | Description | +| --- | --- | +| `v5-to-v6-ddb` | Full Webiny v5 → v6 migration of the primary DynamoDB table (CMS, File Manager, Security, Mailer, Folders, Audit Logs) | +| `v5-to-v6-os` | Migration of the OpenSearch companion DynamoDB table. Run **after** `v5-to-v6-ddb` | +| `copy-ddb` | Verbatim DynamoDB + S3 copy, no transformations | +| `copy-os` | Verbatim OpenSearch companion table copy, no transformations | +| `copy-files` | S3-only file copy | + +Custom presets placed in `pipeline.presetsDir` are listed alongside these five. Source: `src/presets/`. diff --git a/docs/mcp/guides/writingTransformers.md b/docs/mcp/guides/writingTransformers.md new file mode 100644 index 0000000..e2f41c5 --- /dev/null +++ b/docs/mcp/guides/writingTransformers.md @@ -0,0 +1,250 @@ +--- +name: writingTransformers +description: How to write custom transformers — createDdbTransformer/createOsTransformer/createTransformer factories, context types, processor slices, ctx.record, ctx.putRecord(), ctx.blackhole(). +category: Guides +--- + +# Writing transformers + +A transformer is a named function `(ctx) => void | Promise` that mutates `ctx.record`. Register it on a pipeline builder with `.use(...)`; transformers run in registration order, each seeing the mutations of the ones before it. + +Source: `docs/guides/writing-transformers.md`, `src/transformers/createTransformer.ts`, `src/transformers/createDdbTransformer.ts`, `src/transformers/createOsTransformer.ts`, `src/features/TransformContext/abstractions/`. + +## Factories + +```typescript +// src/transformers/createTransformer.ts +function createTransformer( + name: string, + fn: Transformer.Interface +): Transformer.Interface; + +// src/transformers/createDdbTransformer.ts +function createDdbTransformer( + name: string, + fn: Transformer.Interface +): Transformer.Interface; + +// src/transformers/createOsTransformer.ts +function createOsTransformer( + name: string, + fn: Transformer.Interface> +): Transformer.Interface>; +``` + +All three do the same thing at runtime — stamp a non-enumerable `transformerName` property onto `fn` for logging/debugging — and differ only in which context type they bind `fn`'s parameter to: + +- **`createTransformer(name, fn)`** — generic over any context type. Use for `BaseTransformContext` or `DdbCoreTransformContext` transformers, or anything the two convenience factories don't fit. +- **`createDdbTransformer(name, fn)`** — binds `DdbTransformContext.Interface` (Base + `DdbProcessor` slice + `S3Processor` slice). Default choice for v5-to-v6 DDB transformers. +- **`createOsTransformer(name, fn)`** — binds `OsTransformContext.Interface` (Base + `OsProcessor` slice). + +```typescript +import { createDdbTransformer } from "@webiny/data-transfer"; +import type { DdbTransformContext } from "@webiny/data-transfer"; + +export const stampMigratedAt = createDdbTransformer( + "stampMigratedAt", + (ctx: DdbTransformContext.Interface) => { + ctx.record.migratedAt = new Date().toISOString(); + } +); +``` + +**Compile-time contract, not just naming:** whichever context type you bind the transformer to must match the processors actually registered on any pipeline that uses it. A `createDdbTransformer` transformer calling `ctx.copyFile(...)` will only type-check on a pipeline whose `processors` includes `S3Processor`. + +## Context type aliases + +Use the narrowest type that covers what your transformer needs. Source: `src/features/TransformContext/abstractions/contextAliases.ts`. + +| Type | Processors required in pipeline | When to use | +| --- | --- | --- | +| `BaseTransformContext.Interface` | any | Only touches `ctx.record`, `ctx.cache`, `ctx.logger`, etc. — no processor-specific helpers. | +| `DdbCoreTransformContext.Interface` | `DdbProcessor` only | Needs `querySourceRecord` / `queryTargetRecord` / `putRecord` but not S3 helpers. | +| `DdbTransformContext.Interface` | `DdbProcessor` + `S3Processor` | Default for v5-to-v6 DDB transformers that may call `ctx.copyFile` / `ctx.getFile`. | +| `OsTransformContext.Interface` | `OsProcessor` | OS transformers. `ctx.record.data` is the decompressed payload — always present. | + +All four are generic over `TRecord`, defaulting to `BaseRecord`. Import from `@webiny/data-transfer`. + +## Base context API + +Available on every transformer context regardless of pipeline configuration (`src/features/TransformContext/abstractions/BaseTransformContext.ts`): + +```typescript +interface BaseTransformContext { + record: TRecord; + readonly original: Readonly; + readonly modelProvider: ModelProvider.Interface; + readonly cache: Cache.Interface; + readonly logger: Logger.Interface; + readonly compressionHandler: CompressionHandler.Interface; + replace(newRecord: TRecord): void; + addCommand(cmd: Command): void; + blackhole(): void; + readonly isBlackholed: boolean; +} +``` + +| Member | Description | +| --- | --- | +| `ctx.record` | Mutable record. Transformers mutate this directly. | +| `ctx.original` | Frozen, deep-cloned pre-transform snapshot — **always present**, never touched by earlier transformers in the chain. Use for gate-checks or audit comparisons; never modify it. | +| `ctx.replace(newRecord)` | Replace `ctx.record` wholesale (rather than mutating field by field). | +| `ctx.addCommand(cmd)` | Push a raw command onto the command bag. Processor slice helpers (`putRecord`, `copyFile`, `putAuditLog`) are sugar over this — reach for it directly only when emitting a command type no slice helper covers. | +| `ctx.modelProvider` | Loaded CMS models (from DB + `pipeline.modelsDir` JSON files if configured). `ctx.modelProvider.getModel(modelId)`. | +| `ctx.cache` | Shared `Map`-like cache, persists across records **within a shard** (not across shards/workers). Useful for dedup or memoizing lookups. | +| `ctx.logger` | Logger bound to the current worker — use instead of `console.*`; respects `debug.logLevel`. | +| `ctx.compressionHandler` | Gzip compression utility (used internally by OS record handling). Rarely needed directly. | +| `ctx.blackhole()` | Per-record blackholing — suppresses all writes for **this record only**. Remaining transformers and each processor's `onEnd` still run; the runner discards the accumulated commands before forwarding to processors. Irreversible for the record's lifetime. | +| `ctx.isBlackholed` | Read-only flag; `true` after `ctx.blackhole()` has been called for this record. | + +## Processor slices + +Each processor class in a pipeline's `processors` array contributes additional helpers onto the effective context — types intersect via `MergeSlices`, so the context your transformer receives is `BaseTransformContext & `. + +### `DdbProcessor` slice + +Present on `DdbCoreTransformContext` and `DdbTransformContext`. + +```typescript +interface DdbProcessorSlice { + putRecord(record: Record): void; + querySourceRecord>(pk: string, sk?: string): Promise; + queryTargetRecord>(pk: string, sk?: string): Promise; +} +``` + +| Member | Description | +| --- | --- | +| `ctx.putRecord(record)` | Emit an **extra** `PutRecord` to the DDB target, beyond the automatic put at chain end (see Auto-put below). Use for transformers that need to write a second/related record (e.g. a denormalized index entry). | +| `ctx.querySourceRecord(pk, sk?)` | Query the source DDB primary table directly. Returns `null` if not found. | +| `ctx.queryTargetRecord(pk, sk?)` | Query the target DDB primary table directly. Returns `null` if not found. | + +### `S3Processor` slice + +Present on `DdbTransformContext` only (not `DdbCoreTransformContext`). + +```typescript +interface S3ProcessorSlice { + copyFile(sourceKey: string, targetKey: string): void; + getFile(key: string): Promise; +} +``` + +| Member | Description | +| --- | --- | +| `ctx.copyFile(sourceKey, targetKey)` | Emit an S3 copy command (source → target bucket, per config). Keys may differ — e.g. reshaping the storage path during migration. | +| `ctx.getFile(key)` | Read a file from the **source** bucket. Returns `Buffer \| null`. | + +### `OsProcessor` slice + +Present on `OsTransformContext`. Same member names as the DDB slice but reading/writing the OS companion DDB table: + +```typescript +interface OsProcessorSlice { + putRecord(record: Record): void; + querySourceRecord>(pk: string, sk?: string): Promise; + queryTargetRecord>(pk: string, sk?: string): Promise; +} +``` + +Because the key names collide with `DdbProcessorSlice`, `[DdbProcessor, OsProcessor]` cannot be registered together on the same pipeline — the factory's disjoint-keys check rejects it at compile time. + +### `AuditLogProcessor` slice + +Not one of the four context type aliases (no dedicated `AuditLogTransformContext` export), but usable via `createTransformer` with an inline intersection, or accessible implicitly when writing the audit log transformer chain: + +```typescript +interface AuditLogProcessorSlice { + putAuditLog(record: Record): void; +} +``` + +`ctx.putAuditLog(record)` emits a put to the configured audit log table. No-op if `target.auditLog` is not configured. + +## Auto-put behavior + +`DdbProcessor`, `OsProcessor`, and `AuditLogProcessor` each register an `onEnd` hook that fires once per record, after all transformers have run: it calls `putRecord(ctx.record)` (or `putAuditLog(ctx.record)`) automatically. This is why a **zero-transformer pipeline still writes records** — see the "Zero-transformer preset" example in `writingPresets.md`. + +`S3Processor` has **no** `onEnd` — nothing is copied unless a transformer explicitly calls `ctx.copyFile(...)`. + +`onEnd` hooks run **sequentially in processor array order** when a pipeline has multiple processors (e.g. `[DdbProcessor, S3Processor]` → DDB auto-put runs, then S3Processor's `onEnd` runs — which is a no-op since S3Processor doesn't define one). + +## Built-in transformer factories that take config + +Not every built-in transformer is a bare `(ctx) => void` — some are factory functions that must be **called with a config argument** before `.use(...)`: + +```typescript +import { replaceFileUrls, MigrationConfig } from "@webiny/data-transfer"; + +// replaceFileUrls(config: MigrationConfig.Interface) => Transformer> +async configure({ runner, pipelineBuilderFactory: factory, container }) { + const config = container.resolve(MigrationConfig); + + const pipeline = await factory + .create({ name: "cms-entries", scanner: DdbScanner, processors: [DdbProcessor] }) + .use(replaceFileUrls(config)) // NOTE: called, not passed bare + .build(); + + runner.register(pipeline); +} +``` + +`replaceFileUrls` requires a `fileUrls: { source, target }` block at the config root (see `configReference.md`) — it's a no-op transformer if that block is absent. + +Most other built-ins (`wrapInData`, `addGsiTenant`, `removeLocale`, `groupsToRoles`, etc.) are plain transformers — pass them bare to `.use(...)`. Check each transformer's doc under `docs/mcp/transformers/` for its exact call shape before wiring it in. + +## Full example: DDB transformer with a source lookup + +```typescript +import { createDdbTransformer } from "@webiny/data-transfer"; +import type { DdbTransformContext } from "@webiny/data-transfer"; + +export const enrichFromSource = createDdbTransformer( + "enrichFromSource", + async (ctx: DdbTransformContext.Interface) => { + if (ctx.record.TYPE !== "cms.entry") { + return; + } + + const related = await ctx.querySourceRecord<{ title: string }>( + `T#${ctx.record.tenant}#L#en-US#CMS#CMG#category`, + ctx.record.categoryId as string + ); + + if (!related) { + ctx.logger.warn(`enrichFromSource: category not found for ${ctx.record.PK}`); + return; + } + + ctx.record.categoryTitle = related.title; + } +); +``` + +## Full example: OS transformer + +```typescript +import { createOsTransformer } from "@webiny/data-transfer"; +import type { OsTransformContext } from "@webiny/data-transfer"; + +export const dropInternalField = createOsTransformer( + "dropInternalField", + (ctx: OsTransformContext.Interface) => { + const data = ctx.record.data as Record | undefined; + if (data) { + delete data.internalDebugFlag; + } + } +); +``` + +## Built-in processors reference + +| Processor | Slice helpers | Auto-put (`onEnd`) | Notes | +| --- | --- | --- | --- | +| `DdbProcessor` | `putRecord`, `querySourceRecord`, `queryTargetRecord` | Yes | Primary DDB table. | +| `S3Processor` | `copyFile`, `getFile` | No | S3 bucket — emit copies explicitly via `ctx.copyFile`. | +| `OsProcessor` | `putRecord`, `querySourceRecord`, `queryTargetRecord` | Yes | OS companion DDB table. Gzips on write, ensures the target index exists. | +| `AuditLogProcessor` | `putAuditLog` | Yes | Writes to the audit log table. No-op when `target.auditLog` is null/unset. | + +For built-in ready-made transformers (e.g. `copyFileToTarget`), see `docs/mcp/transformers/`. From 6a48189ffa3db71e1c45549d33a71643d984f64b Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 13:21:56 +0200 Subject: [PATCH 08/16] docs: add MCP documentation for pipeline runtime and public API surface --- docs/mcp/guides/pipelineRuntime.md | 149 +++++++++++++++ docs/mcp/guides/publicApi.md | 296 +++++++++++++++++++++++++++++ 2 files changed, 445 insertions(+) create mode 100644 docs/mcp/guides/pipelineRuntime.md create mode 100644 docs/mcp/guides/publicApi.md diff --git a/docs/mcp/guides/pipelineRuntime.md b/docs/mcp/guides/pipelineRuntime.md new file mode 100644 index 0000000..6130574 --- /dev/null +++ b/docs/mcp/guides/pipelineRuntime.md @@ -0,0 +1,149 @@ +--- +name: pipelineRuntime +description: How the pipeline runtime dispatches records — merge groups, first-match-wins, unmatched-record drops, onEnd/afterShard hook ordering, flushEvery buffering, and segment/worker parallelism. +category: Guides +--- + +# Pipeline runtime + +How records flow through the transfer pipeline at runtime, and the exact ordering of every hook the runner invokes. Source: `docs/guides/pipeline-runtime.md`, `src/features/PipelineRunner/PipelineRunner.ts`, `src/commands/run/handler.ts`, `src/commands/processSegment/handler.ts`. + +## Merge groups (keyed by scanner) + +`PipelineRunner` keys pipelines by their `scanner` instance: + +```typescript +private mergeGroups: Map, AnyPipeline[]> = new Map(); +``` + +`runner.register(...)` pushes each pipeline onto the array for its `pipeline.scanner`. All pipelines built with `scanner: DdbScanner` land in one merge group; all pipelines built with `scanner: OsScanner` land in another. Each merge group's scanner runs its scan **once**; every record it yields is offered to that group's pipelines in **registration order** — the order `runner.register(...)` was called (or the order pipelines appear within one variadic call). + +## First-match-wins dispatch + +Inside a shard's record loop, the runner walks the merge group's pipeline list and stops at the first one that accepts the record: + +```typescript +for (const pipeline of pipelines) { + if (!(await pipeline.accepts(record))) { + continue; + } + matched = true; + // ...run filters already passed; run transformers, onEnd, buffer commands... + break; // no other pipeline in this merge group sees the record +} +``` + +`pipeline.accepts(record)` is true when every filter attached via `.filter(...)` passes (AND-composed). Once one pipeline claims a record, the loop `break`s — later pipelines in the same merge group never see it, regardless of whether their filters would also have matched. This makes **registration order semantically significant**: put more specific pipelines first, catch-alls last. + +## Unmatched records are dropped + +If no pipeline in the merge group accepts a record, `matched` stays `false` and the record is **not written anywhere** — dropped, not an error: + +```typescript +if (!matched) { + const { PK, SK, TYPE } = record as any; + const typeKey = TYPE && TYPE !== "unknown" ? TYPE : `${PK}:${SK}`; + unmatchedByType.set(typeKey, (unmatchedByType.get(typeKey) ?? 0) + 1); + this.logger.warn(`unmatched record — TYPE=${typeKey} PK=${PK} SK=${SK}`); + // ... snapshot to dropped/segment-N.jsonl if debug.snapshot is on ... + this.droppedLog.add(record, new RecordDisposition.Unmatched()); +} +``` + +Observability for unmatched records: + +- A `warn` log line per unmatched record (`unmatched record — TYPE=... PK=... SK=...`; falls back to `PK:SK` when `TYPE` is absent/`"unknown"`). +- An `info`-level shard summary line: `[ shard N/M] scanned ..., transferred ..., blackholed ..., unmatched 14 (pb.page=4, ...)`. +- A per-worker dropped-record log flushed at shard end (`this.droppedLog.flush(shardCtx.segment)`), and, if `debug.snapshot` is enabled, a `dropped/segment-N.jsonl(.gz)` file with the full record. + +To transfer **every** record, register a zero-filter catch-all pipeline last in the merge group — see `writingPresets.md` and `filters.md`. + +## Record processing: filters → transformers → onEnd + +For each record a pipeline claims, `runRecord()` runs, in this exact order: + +1. **Slice merge** — each processor's `extendContext(ctx)` (if defined) is `Object.assign`-ed onto the shared `ctx`, contributing helpers like `ctx.putRecord`/`ctx.copyFile`. +2. **Transformers** — `pipeline.transformerFns`, in `.use(...)` registration order, each mutating `ctx.record` in place. +3. **`onEnd` hooks** — each processor's `onEnd(ctx)` (if defined) runs **sequentially, in the pipeline's `processors` array order** — e.g. `processors: [DdbProcessor, S3Processor]` runs `DdbProcessor.onEnd` (auto-`putRecord`) then `S3Processor.onEnd` (a no-op — `S3Processor` defines none). This is what gives zero-transformer pipelines their "verbatim copy" behavior: `DdbProcessor`/`OsProcessor`/`AuditLogProcessor` each auto-emit a put in `onEnd`; `S3Processor` never does. +4. **Blackhole check** — if `pipeline.isBlackhole` or `ctx.isBlackholed` (set via `ctx.blackhole()` inside a transformer), every command this record emitted is discarded here; nothing reaches the shard buffer. +5. **Fold into shard buffer** — otherwise, every command in the record's local `commands` bag is added to the shared `shardCommands` buffer for later flushing. + +## `flushEvery` — bounded peak memory + +Commands don't hit the target after every record — they accumulate in a shared `Commands` buffer for the whole shard, and that buffer is drained periodically: + +```typescript +const flushEvery = this.config.tuning?.flushEvery ?? 500; // tuning.flushEvery, default 500 + +// ...inside the per-record loop... +recordCount++; +if (recordCount % flushEvery === 0) { + await this.flushShard(pendingCommands, processorOrder); + pendingCommands = new Commands(); + periodicFlushCount++; +} + +// ...after the loop, a final flush for any remainder... +if (pendingCommands.size() > 0 || periodicFlushCount === 0) { + await this.flushShard(pendingCommands, processorOrder); +} +``` + +`flushShard` calls `processor.execute(commands)` for **every distinct processor across the merge group's pipelines, in the order they were first encountered** (`collectProcessorOrder`) — not just the processors of the pipeline that produced the commands. Each processor's `execute()` drains only the command keys it owns (`commands.get(key)`, which marks that key "claimed"); any key nobody claims surfaces via `Commands.unclaimedKeys()` and triggers a one-time-per-key runner warning. + +Net effect: peak memory is bounded to roughly `flushEvery × average_record_size` (≈ 5 MB at the 500-record default with a 10 KB average record), not the whole shard's worth of pending writes. Lower `flushEvery` (e.g. to 100) for tables with unusually large records; see `configReference.md` for the `tuning` block. + +## Parallelism: segments, shards, and worker processes + +`pipeline.segments` (optional in the schema; the orchestrator falls back to `1` if unset — `config.pipeline?.segments || 1` in `src/commands/run/handler.ts`. The example config in `configReference.md` sets `numberFromEnv("SEGMENTS", 4)`, but that `4` is a user-chosen convention, not a schema default) sets **both**: + +- how many shards each scanner's `listShards()` reports (`{ segment: i, total: segments }`, passed straight through to DynamoDB's native parallel-`Scan` `Segment`/`TotalSegments` parameters), and +- how many **child worker processes** the orchestrator spawns. + +The orchestrator (`src/commands/run/handler.ts`) resolves `segmentsToRun` (all segments, or a filtered subset via `--segments=1,3`), then spawns one worker per segment **concurrently**: + +```typescript +const workers = segmentsToRun.map(segment => + spawnWorker(segment, segments, runId, configPath, presetName, logLevel, dryRun) +); +const results = await Promise.allSettled(workers); +``` + +Each worker is a separate `node bin.js process-segment --segment N --total M ...` child process (via `execa`) — workers share nothing except the target table/bucket/index they write to. Inside a worker (`src/commands/processSegment/handler.ts`), `runner.run({ segment, totalSegments })` runs exactly **one shard** for exactly **one merge group** (a worker only handles one preset/merge-group at a time; `PipelineRunner.run({...})` throws if more than one merge group is registered when shard options are passed). + +A partial failure can be re-run without rescanning everything: `--segments=1,3` reruns only those two workers; each still receives the original `totalSegments`, so it scans the identical slice of the table it would have scanned in a full run. + +## Hook ordering + +There are four independent hook layers, each with its own scope and ordering: + +| Layer | Registered via | Scope | Runs in | Order | +| --- | --- | --- | --- | --- | +| Transfer lifecycle | `BeforeTransferHook` / `AfterTransferHook` (`config.register`) | Whole transfer, once | Orchestrator process, around spawning all workers | Registration order (`{ multiple: true }` abstraction) | +| Preset lifecycle | `BeforeLoadPresetHook` / `AfterLoadPresetHook` (`config.register`) | Once per worker, around preset loading | Each worker process, before/after `preset.configure(...)` | Registration order | +| Pipeline-level | `.beforeExecuteCommands(token)` / `.afterExecuteCommands(token)` on the builder | Once per merge group | Only in `PipelineRunner.runMergeGroup` — see caveat below | Before-hooks: pipeline/registration order (deduped). After-hooks: **reverse** of that order | +| Per-record / per-shard | Processor `onEnd(ctx)` / `afterShard(ctx)` | Every record / once per shard | Inside `runRecord()` / end of `runShard()` | Sequential, `processors` array order (not reversed) | + +**Before-hooks run forward, after-hooks run in reverse:** + +```typescript +// before: forward order +for (const hookToken of beforeHookTokens) { + await this.container.resolve(hookToken).run(hookParams); +} +// ...shards run... +// after: REVERSE order +for (let i = afterHookTokens.length - 1; i >= 0; i--) { + await this.container.resolve(afterHookTokens[i]!).run(hookParams); +} +``` + +This mirrors a typical setup/teardown stack: the last pipeline's `afterExecuteCommands` hook runs first on the way out. + +**Caveat — pipeline-level hooks only fire without shard options:** `.beforeExecuteCommands`/`.afterExecuteCommands` are wired into `runMergeGroup`, which only executes when `PipelineRunner.run()` is called with **no** `{ segment, totalSegments }` argument. The real segmented CLI flow always calls `runner.run({ segment, totalSegments })` from each worker (`runSingleShard` path), which never calls `runMergeGroup` and therefore never invokes these hooks. In practice, reach for the whole-transfer `BeforeTransferHook`/`AfterTransferHook` (documented in `configReference.md`) for cross-cutting setup/teardown instead; treat `.beforeExecuteCommands()`/`.afterExecuteCommands()` as an advanced/internal extension point until this is reconciled (see the same caveat in `writingPresets.md`). + +**Processor `onEnd` vs `afterShard`:** `onEnd` is per-record (step 3 of the record pipeline above); `afterShard` is per-shard, called once at the very end of `runShard()` — after the final flush — for processors that persist shard-level side-effect state (e.g. `OsProcessor` recording which indexes it touched, for a later orchestrator-side hook to restore `refresh_interval` on). Both run sequentially in the same `processors` array order. + +## Blackholing + +`.blackhole(condition?)` on the pipeline builder, or `ctx.blackhole()` per-record inside a transformer, suppress writes without skipping the pipeline: filters, transformers, and `onEnd` all still run — only the final "fold commands into the shard buffer" step is skipped. Combine with `debug.snapshot` to inspect what *would* have been written. See `writingPresets.md` and `writingTransformers.md` for usage examples. diff --git a/docs/mcp/guides/publicApi.md b/docs/mcp/guides/publicApi.md new file mode 100644 index 0000000..9e3abdc --- /dev/null +++ b/docs/mcp/guides/publicApi.md @@ -0,0 +1,296 @@ +--- +name: publicApi +description: Every export from src/index.ts, organized by category, with import path, value-vs-type marker, and a one-line description for each. +category: Guides +--- + +# Public API surface + +Everything a config/preset/transformer author can import from `@webiny/data-transfer` is re-exported from one file: `src/index.ts`. This is a complete inventory — every export, in file order within each category, marked **value** (importable without `type`, usable at runtime) or **type** (erased at compile time; import with `import type` or inline `type` specifiers). + +Rule of thumb from `AGENTS.md`: anything added here must be something a user writing their own config/transformers/presets genuinely needs — domain-specific migration transformers stay internal. + +## Config & Env + +```typescript +import { + createConfig, migrationConfigSchema, loadEnv, fromEnv, numberFromEnv, + initDataTransfer, findPackageRoot, MigrationConfig +} from "@webiny/data-transfer"; +import type { MigrationConfiguration, InitDataTransferContext } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createConfig` | value | Validates a config object against `unifiedTransferInputSchema` (Zod) and returns the parsed `MigrationConfiguration`. The one function every `config.ts` calls and default-exports. | +| `migrationConfigSchema` | value | The raw Zod schema `createConfig` validates against — for advanced users who want to validate/parse config data themselves without going through `createConfig`. | +| `MigrationConfiguration` | type | The parsed, validated shape `createConfig(...)` returns (and `MigrationConfig`'s DI token resolves to). | +| `loadEnv` | value | `loadEnv(import.meta.url)` loads the `.env` file sitting next to the calling config file, so env vars are available before `fromEnv`/`numberFromEnv` read them. | +| `fromEnv` | value | Reads a required (or defaulted, or nullable) env var as a string; throws a descriptive error on a missing/empty required var instead of silently producing `undefined`. | +| `numberFromEnv` | value | Same as `fromEnv` but parses to `number`; throws if the raw value isn't a valid number (catches typos like `SEGMENTS=four`). | +| `initDataTransfer` | value | Typed identity helper for a project's optional `setup.ts` — wraps `(ctx: { container }) => void \| Promise` so the callback gets container typing without a separate import/annotation. | +| `InitDataTransferContext` | type | The `{ container: Container }` shape passed into the `initDataTransfer(...)` callback. | +| `findPackageRoot` | value | Walks up from a directory to locate `@webiny/data-transfer`'s own `package.json` root — works across source, compiled, and installed-via-npm contexts. Used internally by `WorkerSpawner`; exported for advanced tooling that needs the same resolution. | +| `MigrationConfig` | value | DI abstraction token for the **resolved** config. `container.resolve(MigrationConfig)` inside `register`/`configure`/transformer code returns the same `MigrationConfiguration` object `createConfig(...)` produced — read table names, regions, credentials, tuning, etc. at runtime. | + +## Credentials + +Re-exported directly from `@aws-sdk/credential-providers` (under friendlier names) so config authors don't need that package as a separate dependency: + +```typescript +import { fromAwsProfile, fromAwsCredentialChain } from "@webiny/data-transfer"; +``` + +| Export | Kind | Aliases | Description | +| --- | --- | --- | --- | +| `fromAwsProfile` | value | `fromIni` | Reads credentials for a named profile from `~/.aws/credentials`. Best for local dev with multiple accounts — no risk of a stray env var silently hijacking the wrong one. | +| `fromAwsCredentialChain` | value | `fromNodeProviderChain` | The AWS SDK's default resolution chain: env vars → shared credentials file → SSO/web-identity → EC2/ECS IAM role. Best for CI/cloud runs that must work without code changes. | + +Both return an `AwsCredentialsProvider` (`() => Promise`); a literal `{ accessKeyId, secretAccessKey, sessionToken? }` object is also accepted directly by `source.credentials`/`target.credentials` without importing anything. + +## Transformer Factories + +```typescript +import { createTransformer, createDdbTransformer, createOsTransformer } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createTransformer` | value | Generic transformer factory — `createTransformer(name, fn)`. Stamps a `transformerName` property onto `fn` for logging; use for `BaseTransformContext`/`DdbCoreTransformContext` transformers or anything the two convenience factories below don't fit. | +| `createDdbTransformer` | value | Same runtime behavior as `createTransformer`, but binds `fn`'s parameter to `DdbTransformContext.Interface` (Base + `DdbProcessor` slice + `S3Processor` slice). Default choice for v5-to-v6 DDB transformers. | +| `createOsTransformer` | value | Same, but binds `fn`'s parameter to `OsTransformContext.Interface` (Base + `OsProcessor` slice). | + +## Built-in Transformers (27) + +Ready-made transformers for use in custom presets via `.use(...)`. Source: `src/transformers/**`. Full per-transformer detail (signatures, config args, edge cases) lives under `docs/mcp/transformers/`. + +```typescript +import { + // CMS + addLiveField, fixBrokenStorageKeys, fixCmePk, removeFolderRevision, + renameFieldAttributes, replaceFileUrls, transformModelGroup, transformRichText, + updateModelIds, updateOsIndex, + // File manager + copyFileToTarget, createMetadata, extractImageMetadata, migrateFileManagerSettings, + // Folders + updateFlpIds, + // Global + addGsiTenant, addTransferTimestamp, removeAttributes, removeLocale, wrapInData, + // Security + groupsToRoles, removeTenant, transformPermissions, + // Mailer + migrateMailerSettings, + // Audit logs + coreFieldsTransformer, dataFieldsTransformer, storageShapeTransformer +} from "@webiny/data-transfer"; +``` + +All 27 are **value** exports (plain functions, or factory functions in `replaceFileUrls`'s case — see `writingTransformers.md`). + +### CMS (10) + +| Export | Description | +| --- | --- | +| `addLiveField` | Computes and attaches the `live` pointer (published revision version) to CMS entry records. | +| `fixBrokenStorageKeys` | Corrects mismatched field storage keys in CMS entry values against the model's declared `storageId`. | +| `fixCmePk` | Removes a duplicated `#CME#CME#` segment from a record's `PK`. | +| `removeFolderRevision` | Strips the `#0001` revision suffix from folder location IDs and cleans up legacy folder location fields. | +| `renameFieldAttributes` | Renames legacy CMS model field attributes (`helpText`, `placeholderText`, `multipleValues`) to their v6 equivalents. | +| `replaceFileUrls` | Factory — `replaceFileUrls(config)` rewrites file-manager URLs embedded in CMS `file`/`rich-text` field values from `fileUrls.source` to `fileUrls.target`. | +| `transformModelGroup` | Resolves a CMS model's group ID reference to its slug string. | +| `transformRichText` | Converts legacy Slate-based rich-text field values into the Lexical state + rendered HTML format. | +| `updateModelIds` | Renames legacy system model IDs (`fmFile`, `acoFolder`, etc.) to their v6 `wby`-prefixed equivalents in keys and `data.modelId`. | +| `updateOsIndex` | Recomputes an OpenSearch record's target index name from its `modelId` and tenant. | + +### File Manager (4) + +| Export | Description | +| --- | --- | +| `copyFileToTarget` | Emits a verbatim S3 copy for a file-manager record, source key equal to target key. | +| `createMetadata` | Creates a KeyValueStore file-metadata record and copies the underlying S3 object to its new tenant-scoped path. | +| `extractImageMetadata` | Extracts image dimensions, EXIF, and IPTC metadata from raster image files and renames the legacy meta field. | +| `migrateFileManagerSettings` | Converts a legacy File Manager settings record into the v6 KeyValueStore format. | + +### Folders (1) + +| Export | Description | +| --- | --- | +| `updateFlpIds` | Strips the `#0001` revision suffix from folder-level-page `id` and `parentId` fields. | + +### Global (5) + +| Export | Description | +| --- | --- | +| `addGsiTenant` | Populates the `GSI_TENANT` attribute from the record's `PK` or `data.tenant`. | +| `addTransferTimestamp` | Stamps every record with the transfer time as `_tt`. | +| `removeAttributes` | Deletes deprecated top-level attributes (currently `webinyVersion`) from the data envelope. | +| `removeLocale` | Strips locale segments (e.g. `#L#en-US#`) from a record's keys and deletes the locale field. | +| `wrapInData` | Wraps all non-reserved top-level attributes of a record into a `data` envelope. | + +### Security (3) + +| Export | Description | +| --- | --- | +| `groupsToRoles` | Renames security "group" records and their `GROUP`/`GROUPS` key segments to the v6 "role" terminology. | +| `removeTenant` | Deletes the top-level `tenant` attribute from security role records. | +| `transformPermissions` | Migrates security role permissions to v6 shape — drops `content.i18n`, flattens per-locale model lists, and resolves group IDs to slugs. | + +### Mailer (1) + +| Export | Description | +| --- | --- | +| `migrateMailerSettings` | Converts a legacy Mailer settings record into the v6 KeyValueStore format. | + +### Audit Logs (3) + +| Export | Description | +| --- | --- | +| `coreFieldsTransformer` | Resolves an audit-log record's creator identity and creation time, and stamps a fresh id and TTL expiry. | +| `dataFieldsTransformer` | Lifts audit-log content fields (`app`, `action`, `message`, `entity`, `tags`, `content`) out of the legacy `values` envelope onto the record root. | +| `storageShapeTransformer` | Builds the final v6 audit-log storage record — nine GSI key sets plus the data envelope and TTL expiry. | + +## Filters (18 predicates + `createFilter`) + +```typescript +import { + createFilter, + byType, byTypePrefix, isCmsGroup, isCmsModel, isCmsEntry, byIncludesModelId, + isAcoSearchRecord, isAdminUser, isBackgroundTask, isFmFile, isFlpRecord, + isBuiltInSecurityRole, isSecurityTeam, isOsBackgroundTask, isOsMailerSettings, + isAuditLogEntry, isMigrationRecord, isFormBuilderRecord +} from "@webiny/data-transfer"; +import type { Filter } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createFilter` | value | Wraps any predicate (sync or async) into the `{ kind: "filter", check }` shape a pipeline builder's `.filter(...)` expects. | +| `Filter` | type | `{ readonly kind: "filter"; readonly check: (record: TRecord) => boolean \| Promise }` — the shape `createFilter` produces. | +| `byType` | value | Factory — `byType(type)` matches `record.TYPE === type` exactly. | +| `byTypePrefix` | value | Factory — `byTypePrefix(prefix)` matches when `record.TYPE` starts with `prefix`. | +| `isCmsGroup` | value | Matches `TYPE === "cms.group"` or `PK` including `"#CMS#CMG"`. | +| `isCmsModel` | value | Alias for `byType("cms.model")`. | +| `isCmsEntry` | value | Matches CMS entries by `TYPE` prefix `"cms.entry"` or `PK` including `"#CMS#CME#"`, regardless of raw-vs-reshaped record form. | +| `byIncludesModelId` | value | Factory — `byIncludesModelId(target)` matches when `record.index`/`record.modelId` (also checked under `record.data`) contains `target`, case-insensitive. | +| `isAcoSearchRecord` | value | Alias for `byIncludesModelId("acoSearchRecord")`. | +| `isAdminUser` | value | Matches full-access admin user records (`PK` includes `"#SECURITY#USER#"` and `GSI1_PK === "securityRole#full-access"`). | +| `isBackgroundTask` | value | Matches DDB-side background task records (`modelId`/`GSI1_PK` referencing `webinyTask`/`webinyTaskLog`). | +| `isFmFile` | value | Matches File Manager file records (`modelId` `"fmFile"` or `"wbyFmFile"`, top-level or under `data`). | +| `isFlpRecord` | value | Matches folder location permission records (`PK` includes `"#FLP#"`). | +| `isBuiltInSecurityRole` | value | Matches the two built-in security roles (`slug`/`GSI1_SK` is `"full-access"` or `"anonymous"`) — use to exclude these from a custom roles filter. | +| `isSecurityTeam` | value | Alias for `byType("security.team")`. | +| `isOsBackgroundTask` | value | OS-side equivalent of `isBackgroundTask`, reading from the decompressed `data` payload. | +| `isOsMailerSettings` | value | Matches `record.data.modelId === "mailerSettings"` on OS-table records. | +| `isAuditLogEntry` | value | Matches audit log entries (`modelId` lowercases to `"acosearchrecord-auditlogs"` and `SK === "L"`) — must be filtered for **before** `isAcoSearchRecord`/`isCmsEntry` since they share a `modelId` prefix. | +| `isMigrationRecord` | value | Matches v5 migration-tracking records (`PK` starts with `"MIGRATION"`); typically blackholed. | +| `isFormBuilderRecord` | value | Matches Form Builder forms and submissions (`PK` includes `"#FB#"`, or `TYPE` prefixed `"fb.form."`/`"fb.formSubmission"`); no v6 migration path yet. | + +## Scanners + +```typescript +import { DdbScanner, OsScanner } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `DdbScanner` | value | Scans every item in the source DynamoDB table, segment by segment, yielding raw `BaseRecord` items with no transformation. | +| `OsScanner` | value | Scans the source OpenSearch companion DynamoDB table, decompressing each record's gzip `data` payload; yields `{ index, data, ...BaseRecord }`. Only registered when `config.target.opensearch != null`. | + +## Processors + +```typescript +import { DdbProcessor, S3Processor, AuditLogProcessor, OsProcessor, Processor } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `DdbProcessor` | value | Writes scanned/transformed records to the target DynamoDB table; auto-`putRecord`s in `onEnd` (zero-transformer copy behavior). | +| `S3Processor` | value | Copies S3 objects (source → target bucket) queued via `ctx.copyFile(...)`; no `onEnd` — never writes anything unless a transformer explicitly asks. | +| `AuditLogProcessor` | value | Writes audit-log entries to a dedicated target table, gated on both a configured `target.auditLog` table and `record.TYPE === "auditLog.log"`. | +| `OsProcessor` | value | Writes records to the target OS companion table, gzip-compressing `data` and managing target-index lifecycle (create / disable-refresh); auto-`putRecord`s in `onEnd`. | +| `Processor` | value | Base DI abstraction token every processor implementation shares. Reach for this when declaring a custom processor via `Processor.createImplementation({...})`; its namespace also carries the `Processor.Interface`, `Processor.Context`, and `Processor.SliceOf

` **types** used when typing a custom implementation. | + +## Service Clients + +DI abstractions for direct AWS access — resolve from `container` (e.g. inside `config.register` or a preset's `configure`) for pre-flight checks or custom side-effect code outside the transformer/processor pipeline. + +```typescript +import { + SourceDynamoDbClient, TargetDynamoDbClient, OpenSearchClient, SourceS3Client, TargetS3Client +} from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `SourceDynamoDbClient` | value | DI token for the source DynamoDB client (`scan`/`query`/`get`, no writes) — bound to `source.region`/`source.credentials`. | +| `TargetDynamoDbClient` | value | DI token for the target DynamoDB client (adds `batchWrite`) — bound to `target.region`/`target.credentials`. | +| `OpenSearchClient` | value | DI token for the OpenSearch client used for index lifecycle (`indexExists`, `createIndex`, `listIndexes`, `putIndexSettings`, `getIndexSettings`) — bound to `target.opensearch.endpoint`. | +| `SourceS3Client` | value | DI token for the source S3 client (`getObject`) — bound to `source.s3.bucket`/`source.region`. | +| `TargetS3Client` | value | DI token for the target S3 client (`copy`, `batchCopy`, `getObject`) — bound to `target.s3.bucket`/`target.region`. `copy`/`batchCopy` run with **target** credentials even when copying from the source bucket (cross-account implications — see `configReference.md`). | + +## Context Types + +Types used to annotate custom transformer functions. Import with `import type`. See `writingTransformers.md` for the full base-context API and processor-slice reference. + +```typescript +import type { + BaseTransformContext, DdbCoreTransformContext, DdbTransformContext, OsTransformContext, Transformer +} from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `BaseTransformContext` | type | The context every transformer gets regardless of pipeline processors: `record`, `original`, `modelProvider`, `cache`, `logger`, `compressionHandler`, `replace()`, `addCommand()`, `blackhole()`, `isBlackholed`. | +| `DdbCoreTransformContext` | type | `BaseTransformContext` + `DdbProcessor` slice (`putRecord`, `querySourceRecord`, `queryTargetRecord`) — no S3 helpers. For pipelines registering `DdbProcessor` only. | +| `DdbTransformContext` | type | `BaseTransformContext` + `DdbProcessor` slice + `S3Processor` slice (`copyFile`, `getFile`). Default context for v5-to-v6 DDB transformers; requires `processors: [DdbProcessor, S3Processor]`. | +| `OsTransformContext` | type | `BaseTransformContext` + `OsProcessor` slice. For OS-mode pipelines registering `OsProcessor`. | +| `Transformer` | type | `Transformer.Interface = (ctx: TContext) => void \| Promise` — the function shape every transformer (built-in or custom) satisfies. | + +## Lifecycle Hooks + +DI abstractions registered via `config.register`; all four use `{ multiple: true }`, so registering one **adds** to the list rather than replacing a default. See `configReference.md` and `pipelineRuntime.md` for exactly when each runs. + +```typescript +import { + BeforeTransferHook, AfterTransferHook, BeforeLoadPresetHook, AfterLoadPresetHook +} from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `BeforeTransferHook` | value | `{ execute(): Promise }` — runs once in the orchestrator process, before any worker is spawned. | +| `AfterTransferHook` | value | `{ execute(): Promise }` — runs once in the orchestrator process, after all workers finish (best-effort; a thrown error here is logged, not fatal). | +| `BeforeLoadPresetHook` | value | `{ execute(config): Promise }` — runs once per **worker** process, before the preset is loaded/`configure()`d. | +| `AfterLoadPresetHook` | value | `{ execute(config, preset): Promise }` — runs once per worker process, after `preset.configure(...)` completes. | + +## Customization + +Extension points for advanced config authors — override via `config.register`. + +```typescript +import { IndexConfigurationProvider, ModelProvider, PipelineCustomizer } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `IndexConfigurationProvider` | value | DI abstraction — override `getConfiguration(indexName, base)` to customize OpenSearch index mappings/settings per index before `OsProcessor` creates/updates it. | +| `ModelProvider` | value | DI abstraction — override `preloadModels`/`getModel`/`getModelIds` to customize how CMS model definitions are loaded (default: DB + `pipeline.modelsDir` JSON files). | +| `PipelineCustomizer` | value | DI abstraction — implement `{ name, canUse(pipelineName), configure(builder) }` to extend a built-in preset's pipeline (by name) from `setup.ts`/`config.register` without re-registering the whole preset. See `pipeline-customizer.md`. | + +## Presets & Pipeline Construction + +Core building blocks for writing a custom preset file. See `writingPresets.md` for the full walkthrough. + +```typescript +import { createTransferPreset } from "@webiny/data-transfer"; +import type { MigrationPreset, PresetConfigureContext, NonEmptyArray } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createTransferPreset` | value | Identity function — `createTransferPreset(preset)` returns `preset` unchanged; exists purely so a preset file's `configure({...})` gets typed inference without a separate `MigrationPreset` annotation. | +| `MigrationPreset` | type | The shape a preset file's `default` export must satisfy: `{ name: string; description: string; configure(ctx): void \| Promise }`. | +| `PresetConfigureContext` | type | The `{ runner, pipelineBuilderFactory, container }` argument bag passed to `configure(...)`. | +| `NonEmptyArray` | type | Tuple-length helper used to type `pipelineBuilderFactory.create({ processors })` — enforces at compile time that `processors` has at least one element. | + +Not (yet) exported from the package root: the base `Hook` abstraction that `.beforeExecuteCommands()`/`.afterExecuteCommands()` are typed against — see the caveat in `writingPresets.md` and `pipelineRuntime.md`. From 9effc14c48d6b73e616fdb8c7906d59da1555a29 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:02:29 +0200 Subject: [PATCH 09/16] chore: update lockfile for MCP server bin entry Co-Authored-By: Claude Opus 4.6 (1M context) --- yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn.lock b/yarn.lock index 1f7f7a3..8b4a518 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10244,6 +10244,7 @@ __metadata: zod: "npm:^4.4.3" bin: webiny-data-transfer: ./dist/cli.js + webiny-data-transfer-mcp: ./dist/mcp/bin.js languageName: unknown linkType: soft From 7432c97c114911d1f7e5f75993e9afd2e400708b Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:12:53 +0200 Subject: [PATCH 10/16] chore: clean up consumed changesets Co-Authored-By: Claude Opus 4.6 (1M context) --- .changeset/early-suns-rule.md | 5 ----- .changeset/pre.json | 2 +- .changeset/witty-glasses-cheat.md | 5 ----- 3 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 .changeset/early-suns-rule.md delete mode 100644 .changeset/witty-glasses-cheat.md diff --git a/.changeset/early-suns-rule.md b/.changeset/early-suns-rule.md deleted file mode 100644 index 0e693d9..0000000 --- a/.changeset/early-suns-rule.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@webiny/data-transfer": patch ---- - -Consolidate CI workflows, update all GitHub Actions to latest versions, add register callback example to scaffolded config template, fix scaffold yarn install in CI environments. diff --git a/.changeset/pre.json b/.changeset/pre.json index 1161fe9..85dbec2 100644 --- a/.changeset/pre.json +++ b/.changeset/pre.json @@ -4,5 +4,5 @@ "initialVersions": { "@webiny/data-transfer": "0.0.0" }, - "changesets": ["early-suns-rule", "witty-glasses-cheat"] + "changesets": [] } diff --git a/.changeset/witty-glasses-cheat.md b/.changeset/witty-glasses-cheat.md deleted file mode 100644 index 2cfdeca..0000000 --- a/.changeset/witty-glasses-cheat.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@webiny/data-transfer": patch ---- - -Initial alpha release of the standalone data-transfer package. Includes CLI with guided wizard, DynamoDB/OpenSearch/S3 transfer support, built-in presets (v5-to-v6, copy), pipeline framework with customizable transformers and filters, and project scaffolding via `npx @webiny/data-transfer`. From f8d8b308f0bb49192770d2002a712dec9f7d5dc0 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:18:15 +0200 Subject: [PATCH 11/16] chore: add changeset for alpha patch Co-Authored-By: Claude Opus 4.6 (1M context) --- .changeset/tall-dancers-care.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tall-dancers-care.md diff --git a/.changeset/tall-dancers-care.md b/.changeset/tall-dancers-care.md new file mode 100644 index 0000000..0935bda --- /dev/null +++ b/.changeset/tall-dancers-care.md @@ -0,0 +1,5 @@ +--- +"@webiny/data-transfer": patch +--- + +Add MCP server (`webiny-data-transfer-mcp`) with `list_topics` and `get_topic` tools serving 44 documentation topics. Export all 27 built-in transformers and all 18 filter predicates as public API. Consolidate CI workflows and update all GitHub Actions to latest versions. From d6c899914932fd4199abb4d3a5d4437b0b428d72 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:27:54 +0200 Subject: [PATCH 12/16] feat: add .mcp.json to scaffolded projects for AI agent discovery Scaffolded projects now include .mcp.json that auto-configures the webiny-data-transfer MCP server. Agents supporting MCP discovery (Claude, Cursor, Kiro, Copilot, etc.) get access to all 44 docs topics with zero setup. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 1 + .gitignore | 1 + templates/.mcp.json | 8 ++++++++ templates/README.md | 19 +++++++++++++++++++ 4 files changed, 29 insertions(+) create mode 100644 templates/.mcp.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd2d57c..8a4d99d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,7 @@ jobs: test -f projects/example/config.ts test -f package.json test -f .yarnrc.yml + test -f .mcp.json test -d presets test -d transformers diff --git a/.gitignore b/.gitignore index da850df..6aa4e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ __tests__/fixtures/es-table-migrated.json __tests__/fixtures/os-table-migrated.json .codegraph .mcp.json +!templates/.mcp.json diff --git a/templates/.mcp.json b/templates/.mcp.json new file mode 100644 index 0000000..f66ccb6 --- /dev/null +++ b/templates/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "webiny-data-transfer": { + "command": "npx", + "args": ["webiny-data-transfer-mcp"] + } + } +} diff --git a/templates/README.md b/templates/README.md index f4230db..cdc4b9b 100644 --- a/templates/README.md +++ b/templates/README.md @@ -67,6 +67,25 @@ For v5 to v6 migration, run `v5-to-v6-ddb` first, then `v5-to-v6-os`. Custom presets in `presets/` are listed alongside built-ins. See `presets/example.ts` for a starting point. +## AI agent support (MCP) + +This project includes an MCP server that gives AI agents (Claude, Cursor, Kiro, Copilot, etc.) access to all data-transfer documentation — presets, transformers, processors, config reference, and how-to guides. + +The `.mcp.json` in the project root auto-configures it for agents that support MCP discovery. No setup needed for new scaffolded projects. + +For existing projects, create `.mcp.json` in your project root: + +```json +{ + "mcpServers": { + "webiny-data-transfer": { + "command": "npx", + "args": ["webiny-data-transfer-mcp"] + } + } +} +``` + ## Documentation - [Config reference](https://github.com/webiny/data-transfer/blob/main/docs/guides/config-reference.md) — config.ts setup, env helpers, credentials, IAM, tuning From ec86726a2373b7b693e951cf041d62cd79d79981 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:31:50 +0200 Subject: [PATCH 13/16] style: format .mcp.json template Co-Authored-By: Claude Opus 4.6 (1M context) --- templates/.mcp.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/.mcp.json b/templates/.mcp.json index f66ccb6..35887cb 100644 --- a/templates/.mcp.json +++ b/templates/.mcp.json @@ -1,8 +1,8 @@ { - "mcpServers": { - "webiny-data-transfer": { - "command": "npx", - "args": ["webiny-data-transfer-mcp"] + "mcpServers": { + "webiny-data-transfer": { + "command": "npx", + "args": ["webiny-data-transfer-mcp"] + } } - } } From 9d38e1947c74405d38ba01fa1ce53d35764f432e Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:38:50 +0200 Subject: [PATCH 14/16] fix(ci): remove --loglevel=silent from scaffold npx to surface errors Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a4d99d..d0ce8e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: - name: Scaffold user project run: | cd /tmp - npx --yes --loglevel=silent --registry http://localhost:4873 "@webiny/data-transfer@${PKG_VERSION}" smoke-test + npx --yes --registry http://localhost:4873 "@webiny/data-transfer@${PKG_VERSION}" smoke-test env: npm_config_registry: http://localhost:4873 PKG_VERSION: ${{ steps.version.outputs.version }} From 34671eec09d9e6496d6021b23098a611f459fcc1 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:44:01 +0200 Subject: [PATCH 15/16] fix: add data-transfer bin alias for npx resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With multiple bin entries, npx @webiny/data-transfer can't auto-select which to run — it looks for a bin matching the unscoped package name (data-transfer). Add the alias so npx resolves correctly. Co-Authored-By: Claude Opus 4.6 (1M context) --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index c251cf2..3897395 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "bin": { + "data-transfer": "./dist/cli.js", "webiny-data-transfer": "./dist/cli.js", "webiny-data-transfer-mcp": "./dist/mcp/bin.js" }, From 30b5e48c42a199201e16f66214ceaa3943ce75e0 Mon Sep 17 00:00:00 2001 From: brunozoric Date: Mon, 3 Aug 2026 14:46:22 +0200 Subject: [PATCH 16/16] fix: update lockfile for data-transfer bin alias Co-Authored-By: Claude Opus 4.6 (1M context) --- yarn.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn.lock b/yarn.lock index 8b4a518..32a1e4d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10243,6 +10243,7 @@ __metadata: yargs: "npm:^18.1.0" zod: "npm:^4.4.3" bin: + data-transfer: ./dist/cli.js webiny-data-transfer: ./dist/cli.js webiny-data-transfer-mcp: ./dist/mcp/bin.js languageName: unknown