diff --git a/.changeset/early-suns-rule.md b/.changeset/early-suns-rule.md deleted file mode 100644 index 0e693d9..0000000 --- a/.changeset/early-suns-rule.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@webiny/data-transfer": patch ---- - -Consolidate CI workflows, update all GitHub Actions to latest versions, add register callback example to scaffolded config template, fix scaffold yarn install in CI environments. diff --git a/.changeset/pre.json b/.changeset/pre.json index 1161fe9..85dbec2 100644 --- a/.changeset/pre.json +++ b/.changeset/pre.json @@ -4,5 +4,5 @@ "initialVersions": { "@webiny/data-transfer": "0.0.0" }, - "changesets": ["early-suns-rule", "witty-glasses-cheat"] + "changesets": [] } diff --git a/.changeset/tall-dancers-care.md b/.changeset/tall-dancers-care.md new file mode 100644 index 0000000..0935bda --- /dev/null +++ b/.changeset/tall-dancers-care.md @@ -0,0 +1,5 @@ +--- +"@webiny/data-transfer": patch +--- + +Add MCP server (`webiny-data-transfer-mcp`) with `list_topics` and `get_topic` tools serving 44 documentation topics. Export all 27 built-in transformers and all 18 filter predicates as public API. Consolidate CI workflows and update all GitHub Actions to latest versions. diff --git a/.changeset/witty-glasses-cheat.md b/.changeset/witty-glasses-cheat.md deleted file mode 100644 index 2cfdeca..0000000 --- a/.changeset/witty-glasses-cheat.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@webiny/data-transfer": patch ---- - -Initial alpha release of the standalone data-transfer package. Includes CLI with guided wizard, DynamoDB/OpenSearch/S3 transfer support, built-in presets (v5-to-v6, copy), pipeline framework with customizable transformers and filters, and project scaffolding via `npx @webiny/data-transfer`. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd2d57c..d0ce8e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: - name: Scaffold user project run: | cd /tmp - npx --yes --loglevel=silent --registry http://localhost:4873 "@webiny/data-transfer@${PKG_VERSION}" smoke-test + npx --yes --registry http://localhost:4873 "@webiny/data-transfer@${PKG_VERSION}" smoke-test env: npm_config_registry: http://localhost:4873 PKG_VERSION: ${{ steps.version.outputs.version }} @@ -108,6 +108,7 @@ jobs: test -f projects/example/config.ts test -f package.json test -f .yarnrc.yml + test -f .mcp.json test -d presets test -d transformers diff --git a/.gitignore b/.gitignore index da850df..6aa4e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ __tests__/fixtures/es-table-migrated.json __tests__/fixtures/os-table-migrated.json .codegraph .mcp.json +!templates/.mcp.json diff --git a/docs/mcp/guides/configReference.md b/docs/mcp/guides/configReference.md new file mode 100644 index 0000000..398cf95 --- /dev/null +++ b/docs/mcp/guides/configReference.md @@ -0,0 +1,301 @@ +--- +name: configReference +description: createConfig shape — fromEnv/numberFromEnv, credentials (fromAwsProfile, fromAwsCredentialChain, literal), register callback, pipeline settings, tuning, debug/snapshot. +category: Guides +--- + +# Config reference + +`createConfig(input)` validates a Zod schema (`unifiedTransferInputSchema`) and returns the parsed `MigrationConfiguration`. One `config.ts` file covers DynamoDB, S3, and optional OpenSearch for both source and target. Source: `src/features/MigrationConfig/createConfig.ts`, `src/features/MigrationConfig/schemas/unified.schema.ts`, `src/features/MigrationConfig/schemas/shared.schema.ts`. + +```typescript +function createConfig(input: z.input): MigrationConfiguration; +``` + +## Full shape + +```typescript +import { + loadEnv, + createConfig, + fromAwsProfile, + fromEnv, + numberFromEnv +} from "@webiny/data-transfer"; + +loadEnv(import.meta.url); + +export default createConfig({ + source: { + region: fromEnv("SOURCE_REGION", "eu-central-1"), + credentials: fromAwsProfile({ profile: fromEnv("SOURCE_PROFILE", "default") }), + dynamodb: { tableName: fromEnv("SOURCE_DDB_TABLE") }, + s3: { bucket: fromEnv("SOURCE_S3_BUCKET") }, + opensearch: { tableName: fromEnv("SOURCE_OS_TABLE") } // omit/null if no source OS + }, + target: { + region: fromEnv("TARGET_REGION", "eu-central-1"), + credentials: fromAwsProfile({ profile: fromEnv("TARGET_PROFILE", "default") }), + dynamodb: { tableName: fromEnv("TARGET_DDB_TABLE") }, + s3: { bucket: fromEnv("TARGET_S3_BUCKET") }, + auditLog: { dynamodb: { tableName: fromEnv("TARGET_AUDIT_LOGS_TABLE") } }, // null/omit to skip + opensearch: { + endpoint: fromEnv("TARGET_OS_ENDPOINT"), + tableName: fromEnv("TARGET_OS_TABLE"), + service: "opensearch", // or "opensearch-serverless" + indexPrefix: fromEnv("TARGET_OS_INDEX_PREFIX", "") + } + }, + pipeline: { + segments: numberFromEnv("SEGMENTS", 4), + modelsDir: fromEnv("MODELS_DIR", "./models"), + presetsDir: "./presets" // optional — custom preset files, listed alongside built-ins + }, + fileUrls: { source: "https://old-cdn.example.com", target: "https://new-cdn.example.com" }, // optional + register: async container => { /* ... */ }, // optional + tuning: { /* ... */ }, // optional + debug: { /* ... */ } // optional +}); +``` + +`loadEnv(import.meta.url)` loads the `.env` file sitting next to this config file — keep one `.env` per project so credentials stay isolated. + +## `source` / `target` field reference + +| Field | Required | Notes | +| --- | --- | --- | +| `region` | yes | Non-empty, trimmed string. | +| `credentials` | yes | See Credentials below. | +| `accountId` | no | Optional; used by the wizard's cross-account S3 warning. | +| `dynamodb.tableName` | yes | Non-empty, trimmed string. | +| `s3.bucket` | yes | Non-empty, trimmed string. | +| `opensearch` | no | `{ tableName }` on source, `{ endpoint, tableName, service, indexPrefix }` on target. Omit or `null` on **both** sides if unused — mismatched presence (one side set, the other not) fails validation. | +| `target.auditLog.dynamodb.tableName` | no | Omit, or set to `null`, to skip the audit log table — matching records are blackholed instead of written. | + +### Validation guardrails (`superRefine` checks in `unified.schema.ts`) + +`createConfig(...)` throws at call time if any of these hold: + +- `source.s3.bucket === target.s3.bucket` — would overwrite source files. +- `source.region === target.region && source.dynamodb.tableName === target.dynamodb.tableName` — same table on both sides in the same region. +- `target.auditLog.dynamodb.tableName === target.dynamodb.tableName` — audit log table must differ from the main target table. +- `(source.opensearch != null) !== (target.opensearch != null)` — OS must be configured on both sides or neither. +- OS present on both sides AND `source.region === target.region && source.opensearch.tableName === target.opensearch.tableName` — same OS companion table on both sides in the same region. + +These exist to catch copy-paste config mistakes before any data moves — cross-account setups that legitimately reuse a name/region should differentiate one side (rename or use a different region) to signal intent. + +## Env helpers + +Source: `src/utils/fromEnv.ts`. + +```typescript +function fromEnv(name: string): string; // throws if unset/empty +function fromEnv(name: string, defaultValue: string): string; // falls back to defaultValue +function fromEnv(name: string, defaultValue: null): string | null; // returns null instead of throwing + +function numberFromEnv(name: string): number; // throws if unset +function numberFromEnv(name: string, defaultValue: number): number; +``` + +- **`fromEnv(name)`** — required; throws `Environment variable "" is not set and no default was provided.` if unset **or empty** (`KEY=` in `.env` counts as unset — treated as a forgotten value, not an intentional empty override). +- **`fromEnv(name, default)`** — returns `default` when unset/empty. +- **`fromEnv(name, null)`** — returns `string | null`; use for genuinely optional config sections, e.g. `fromEnv("SOURCE_OS_TABLE", null)`. +- **`numberFromEnv(name, default?)`** — parses via `Number(...)`; throws `Environment variable "" is not a valid number (got "").` on parse failure (e.g. `SEGMENTS=four`) so typos surface immediately instead of becoming `NaN` downstream. + +## Credentials + +Both `source.credentials` and `target.credentials` accept one of three shapes (`credentialsOrProviderSchema` in `shared.schema.ts`): + +```typescript +import { fromAwsProfile, fromAwsCredentialChain } from "@webiny/data-transfer"; + +credentials: fromAwsProfile({ profile: "prod-reader" }) +// or +credentials: fromAwsCredentialChain() +// or — literal, for temporary STS credentials +credentials: { accessKeyId: "...", secretAccessKey: "...", sessionToken: "..." /* optional */ } +``` + +| Shape | Re-exported from | Behavior | +| --- | --- | --- | +| `fromAwsProfile({ profile })` | `@aws-sdk/credential-providers`'s `fromIni` | Reads `~/.aws/credentials`. Explicit account selection — best for local dev with multiple profiles; no risk of a stray `AWS_ACCESS_KEY_ID` env var silently hijacking the wrong account. | +| `fromAwsCredentialChain()` | `@aws-sdk/credential-providers`'s `fromNodeProviderChain` | AWS SDK default chain: env vars → shared credentials file → SSO/web-identity → EC2/ECS IAM role. Best for CI / cloud runs where the same config must work without code changes. | +| Literal `{ accessKeyId, secretAccessKey, sessionToken? }` | n/a — validated directly | Explicit strings, e.g. short-lived STS credentials. `sessionToken` optional. | + +Both `fromAwsProfile`/`fromAwsCredentialChain` return an `AwsCredentialsProvider` (`() => Promise`) — the schema's union validates either a literal object or any function, so a hand-written custom provider function also satisfies it. + +## `register` callback (optional) + +Runs **before** preset loading; wires custom DI bindings the preset's `configure()` can later resolve via `container`: + +```typescript +import { createConfig, SourceDynamoDbClient, PipelineCustomizer } from "@webiny/data-transfer"; + +export default createConfig({ + // ...source, target, pipeline... + register: async container => { + const sourceDb = container.resolve(SourceDynamoDbClient); // direct AWS access, e.g. pre-flight checks + + container.register(MyCustomProcessorImpl); // custom Processor.createImplementation(...) + container.register(MyCustomizerImpl); // PipelineCustomizer.createImplementation(...) + } +}); +``` + +Signature: `type RegisterFn = (container: Container) => void | Promise`. + +**Service client abstractions** resolvable from `container`: `SourceDynamoDbClient`, `TargetDynamoDbClient`, `OpenSearchClient`, `SourceS3Client`, `TargetS3Client`. + +**Lifecycle hooks** can also be registered here — `BeforeTransferHook` / `AfterTransferHook` (`execute(): Promise`, run once per whole transfer) and `BeforeLoadPresetHook` / `AfterLoadPresetHook` (`execute(config, preset?): Promise`, run around preset loading). These abstractions use `{ multiple: true }` — registering one adds to the list rather than replacing a default. + +### Customizing OpenSearch index configuration + +Override `IndexConfigurationProvider` to control index creation — mappings, settings, per-index overrides: + +```typescript +import { createConfig, IndexConfigurationProvider } from "@webiny/data-transfer"; + +class CustomIndexConfig implements IndexConfigurationProvider.Interface { + public getConfiguration(indexName: string, base: IndexConfigurationProvider.Configuration) { + const settings = { ...base.settings, number_of_replicas: 2 }; + if (indexName.includes("articles")) { + return { + settings, + mappings: { + ...base.mappings, + properties: { ...base.mappings?.properties, title: { type: "text", analyzer: "english" } } + } + }; + } + return { ...base, settings }; + } +} + +const CustomIndexConfigImpl = IndexConfigurationProvider.createImplementation({ + implementation: CustomIndexConfig, + dependencies: [] +}); + +export default createConfig({ + // ...source, target, pipeline... + register: container => { container.register(CustomIndexConfigImpl); } +}); +``` + +The tool disables `refresh_interval` just-in-time on first write to each index and restores the original value after transfer completes. Missing indexes are created with the Webiny base mapping; only touched indexes are affected. + +## `pipeline` settings + +```typescript +pipeline: { + segments: numberFromEnv("SEGMENTS", 4), // parallel worker processes / DDB scan segments + modelsDir: fromEnv("MODELS_DIR", "./models"), // CMS model JSON directory + presetsDir: "./presets" // optional — custom preset files +} +``` + +All three fields are optional in the schema (`pipelineSettingsSchema`). `modelsDir` is required in practice by the OS preset and by rich-text / field-key transformers — point it at a directory of exported CMS model definitions: + +``` +models/ + single-model.json # { "modelId": "...", "fields": [...], ... } + array-of-models.json # [{ "modelId": "...", "fields": [...] }, ...] + webiny-export.json # { "groups": [...], "models": [...] } ← Webiny admin export +``` + +Three JSON shapes can be mixed in the same directory; JSON models override DB-loaded models when both exist for the same `modelId`. + +## `tuning` (optional) + +```typescript +tuning: { + flushEvery: numberFromEnv("FLUSH_EVERY", 500), // records per shard flush — bounds peak memory + ddb: { maxRetries: 3, initialBackoffMs: 100, requestTimeoutMs: 5000 }, + s3: { concurrency: 10, maxRetries: 3, initialBackoffMs: 100, requestTimeoutMs: 10000 }, + os: { maxRetries: 3, retryScheduleMs: [5000, 10000, 20000], gzipConcurrency: 16 } +} +``` + +All fields optional (`tuningSchema`); absent = built-in defaults. + +- **`flushEvery`** (default 500) — the runner calls each processor's `execute()` every N records and resets the command buffer, bounding peak memory to `flushEvery × avg_record_size` (≈ 5 MB at a 10 KB average, default). Lower to `100` for tables with very large records. +- **DynamoDB `BatchWriteItem` batch size is NOT tunable** — AWS hard-caps it at 25 items per call. +- DDB and S3 clients run in AWS SDK `adaptive` retry mode; `tuning.{ddb,s3}.maxRetries` caps the **outer** retry envelope on top of the SDK's own self-tuning backoff — it doesn't replace SDK retry logic. +- `tuning.os.retryScheduleMs` is an explicit backoff schedule (array of delays in ms) rather than a `maxRetries` + exponential-backoff pair. + +## `debug` (optional) + +```typescript +debug: { + logLevel: "debug", // "debug" | "info" | "warn" | "error" (default "info"); overridable via --log-level CLI flag + snapshot: true, // or: { dir: "./my-snapshot", compress: false } + logFile: true // or: "./my-transfer.log" +} +``` + +### `debug.snapshot` + +Dumps every record the pipeline touches to local JSONL files (default dir `.transfer//snapshot`): + +``` +.transfer// +├── snapshot/ +│ ├── / +│ │ ├── segment-0.source.jsonl.gz ← post-filter, pre-transform +│ │ ├── segment-0.post-transform.jsonl.gz ← after the whole transformer chain +│ │ └── segment-0.commands.jsonl.gz ← PutRecord + S3Copy + etc. +│ └── dropped/ +│ └── segment-0.jsonl.gz ← records matching no pipeline +├── segment-0-blackholed.log +└── segment-0-unmatched.log +``` + +```bash +zcat .transfer//snapshot/cmsEntries/segment-0.source.jsonl.gz | jq 'select(.PK=="T#tenant#CME#abc")' +``` + +Set `compress: false` to `grep` directly without `zcat`. Best-effort — write errors log `warn` but never abort the transfer. + +### `debug.logFile` + +Captures the full runner log to disk. `true` → each process writes to `.transfer//logs/.log` (one file per process, no interleaving under parallelism). String → all processes write to that path instead. Content is raw pino JSONL: + +```bash +cat .transfer//logs/*.log | pino-pretty +``` + +## `fileUrls` (optional) + +Required only if you use the `replaceFileUrls` transformer factory (see `writingTransformers.md`): + +```typescript +fileUrls: { + source: "https://old-cdn.example.com", + target: "https://new-cdn.example.com" +} +``` + +## Required IAM permissions + +The tool runs a pre-flight access check (`HeadBucket` for S3, `DescribeTable` for DynamoDB) before any data moves; a failing check aborts the run with the specific resource/credential set that failed. + +**Source credentials:** + +| Service | Actions | Resource | +| --- | --- | --- | +| DynamoDB | `Scan`, `Query`, `DescribeTable` | Source primary table | +| S3 | `GetObject`, `ListBucket` | Source bucket | +| DynamoDB | `Scan`, `Query`, `DescribeTable` | Source OS companion table (if using OpenSearch) | + +**Target credentials:** + +| Service | Actions | Resource | +| --- | --- | --- | +| DynamoDB | `BatchWriteItem`, `Query`, `DescribeTable` | Target primary table | +| S3 | `PutObject`, `ListBucket` | Target bucket | +| S3 | `GetObject` on the **source** bucket | Cross-account note below | +| DynamoDB | `BatchWriteItem`, `Query`, `DescribeTable` | Target OS companion table (if using OpenSearch) | +| OpenSearch | `ESHttpGet`, `ESHttpPut`, `ESHttpPost` | Target OpenSearch domain (if using OpenSearch) | +| DynamoDB | `BatchWriteItem`, `DescribeTable` | Target audit log table (if configured) | + +**S3 cross-account access:** `CopyObjectCommand` runs with **target credentials**. When source and target are in different AWS accounts, the target account needs read access to the source bucket — either a bucket policy on the source granting `s3:GetObject` to the target account, or a cross-account IAM role. Without this, S3 file copies fail with `AccessDenied`; the wizard warns on detected account-ID mismatch, and the pre-flight check verifies target credentials can reach the source bucket. diff --git a/docs/mcp/guides/filters.md b/docs/mcp/guides/filters.md new file mode 100644 index 0000000..9403795 --- /dev/null +++ b/docs/mcp/guides/filters.md @@ -0,0 +1,163 @@ +--- +name: filters +description: All 18 built-in filter predicates plus createFilter — signatures, matching rules, and usage examples for pipeline .filter() calls. +category: Guides +--- + +# Filters + +A `Filter` is `{ kind: "filter", check: (record) => boolean | Promise }`. Build one with `createFilter(predicate)` and attach it to a pipeline with `.filter(...)`. Multiple `.filter()` calls on the same builder AND-compose — a record must pass every filter to be claimed by that pipeline. + +```typescript +import { createFilter, isCmsEntry } from "@webiny/data-transfer"; + +pipelineBuilderFactory + .create({ name: "cms-entries", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .build(); +``` + +## `createFilter` + +Source: `src/domain/pipeline/Filter.ts`. + +```typescript +export interface Filter { + readonly kind: "filter"; + readonly check: (record: TRecord) => boolean | Promise; +} + +function createFilter( + predicate: (record: TRecord) => boolean | Promise +): Filter; +``` + +Wraps any predicate — sync or async, built-in or inline — into the typed shape the pipeline builder expects. + +```typescript +// Inline predicate — no built-in needed for one-off conditions +.filter(createFilter(r => r.TYPE === "cms.entry" && r.modelId === "article")) + +// Async predicate is supported (check() may return Promise) +.filter(createFilter(async r => (await lookupSomething(r)) != null)) +``` + +## Import + +All 18 predicates are exported directly from the package root, alongside `createFilter`: + +```typescript +import { + createFilter, + byType, + byTypePrefix, + isCmsGroup, + isCmsModel, + isCmsEntry, + byIncludesModelId, + isAcoSearchRecord, + isAdminUser, + isBackgroundTask, + isFmFile, + isFlpRecord, + isBuiltInSecurityRole, + isSecurityTeam, + isOsBackgroundTask, + isOsMailerSettings, + isAuditLogEntry, + isMigrationRecord, + isFormBuilderRecord +} from "@webiny/data-transfer"; +``` + +Source: `src/domain/transform/filters.ts`. + +## Predicate reference + +| Predicate | Signature | Matches | Notes | +| --- | --- | --- | --- | +| `byType` | `(type: string) => (record) => boolean` | `record.TYPE === type` exactly | Factory — call with the exact type string, e.g. `byType("cms.model")` | +| `byTypePrefix` | `(prefix: string) => (record: BaseRecord) => boolean` | `record.TYPE` starts with `prefix` | Factory — for TYPE families like `"cms.entry"` | +| `isCmsGroup` | `(record: BaseRecord) => boolean` | `TYPE === "cms.group"` OR `PK` includes `"#CMS#CMG"` | Handles both raw v5 and reshaped record forms | +| `isCmsModel` | `(record: BaseRecord) => boolean` | `= byType("cms.model")` | Direct alias | +| `isCmsEntry` | `(input: BaseRecord) => boolean` | `TYPE` prefixed `"cms.entry"` OR `PK` includes `"#CMS#CME#"` | Catch-all for CMS entries regardless of shape | +| `byIncludesModelId` | `(target: string) => (record: BaseRecord) => boolean` | `record.index` or `record.modelId` (also checked under `record.data`) contains `target`, case-insensitive | Factory — used to build `isAcoSearchRecord` | +| `isAcoSearchRecord` | `(record: BaseRecord) => boolean` | `= byIncludesModelId("acoSearchRecord")` | ACO search cache records | +| `isAdminUser` | `(record: BaseRecord) => boolean` | `PK` includes `"#SECURITY#USER#"` AND `GSI1_PK === "securityRole#full-access"` | Full-access admin user records | +| `isBackgroundTask` | `(item: BaseRecord) => boolean` | `modelId === "webinyTask"` / `"webinyTaskLog"`, or `GSI1_PK` includes either string | DDB-side background task records | +| `isFmFile` | `(record: BaseRecord) => boolean` | `modelId` (top-level or `record.data.modelId`) is `"fmFile"` or `"wbyFmFile"` | File Manager file records | +| `isFlpRecord` | `(record: Record) => boolean` | `PK` (string) includes `"#FLP#"` | Folder location permission records | +| `isBuiltInSecurityRole` | `(record: Record) => boolean` | `slug` or `GSI1_SK` is `"full-access"` or `"anonymous"` | Use to exclude built-in roles from a custom `SecurityGroups` filter | +| `isSecurityTeam` | `(record: BaseRecord) => boolean` | `= byType("security.team")` | Direct alias | +| `isOsBackgroundTask` | `(record: Record) => boolean` | `record.data.modelId` is `"webinyTask"` or `"webinyTaskLog"` | OS-side equivalent of `isBackgroundTask` — reads from the decompressed `data` payload | +| `isOsMailerSettings` | `(record: Record) => boolean` | `record.data.modelId === "mailerSettings"` | OS-side mailer settings | +| `isAuditLogEntry` | `(record: BaseRecord) => boolean` | `modelId` (top-level or `.data`) lowercases to `"acosearchrecord-auditlogs"` AND `SK === "L"` | Must be filtered for before `isAcoSearchRecord`/`isCmsEntry` — shares the modelId prefix | +| `isMigrationRecord` | `(record: BaseRecord) => boolean` | `PK` starts with `"MIGRATION"` | v5 migration-tracking records; typically blackholed | +| `isFormBuilderRecord` | `(record: BaseRecord) => boolean` | `PK` includes `"#FB#"`, OR `TYPE` starts with `"fb.form."` / `"fb.formSubmission"` | Form Builder forms + submissions; no v6 migration path yet | + +## Predicates that read both raw and reshaped record shapes + +Several predicates check a helper that falls back from a top-level property to the same property nested under `record.data` — this lets one filter match records both **before** and **after** a `wrapInData`-style transformer has run: + +```typescript +// Internal helper used by byIncludesModelId / isAuditLogEntry +function getPropertyFromRecord(record, propertyName: string): T | undefined { + const value = record[propertyName]; + if (value !== undefined) return value; + return record.data?.[propertyName]; +} +``` + +`isFmFile`, `isOsBackgroundTask`, and `isOsMailerSettings` apply the same pattern directly for `modelId`. + +## Ordering rules when composing filters + +Filters only decide whether a pipeline **claims** a record — they don't decide overall precedence across pipelines by themselves. Combine filter choice with **registration order** (first-match-wins across a merge group): + +```typescript +import { + createFilter, + isAuditLogEntry, + isAcoSearchRecord, + isCmsEntry, + isFmFile +} from "@webiny/data-transfer"; + +// 1. Audit logs FIRST — isAuditLogEntry and isAcoSearchRecord both match +// modelId "acoSearchRecord-AuditLogs" style prefixes; audit logs must win. +runner.register(auditLogsPipeline); + +// 2. Generic ACO search cache records +runner.register(acoSearchPipeline); + +// 3. File manager files BEFORE the CMS-entry catch-all — fm files are +// also CMS entries and would otherwise be claimed by #4. +runner.register(fileManagerFilesPipeline); + +// 4. Everything else that looks like a CMS entry +runner.register(cmsEntriesPipeline); +``` + +See `writingPresets.md` for the full first-match-wins model and `pipeline-runtime.md` for merge-group semantics. + +## AND-composing multiple filters on one pipeline + +```typescript +import { createFilter, isCmsEntry, byIncludesModelId } from "@webiny/data-transfer"; + +pipelineBuilderFactory + .create({ name: "articles", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .filter(createFilter(byIncludesModelId("article"))) // AND — must ALSO be modelId "article" + .build(); +``` + +## Zero-filter catch-all + +Omitting `.filter(...)` entirely makes a pipeline accept every record its merge group offers it — used for verbatim-copy presets or as a final catch-all registered last: + +```typescript +const everything = pipelineBuilderFactory + .create({ name: "everything", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); // no .filter() → matches all +``` diff --git a/docs/mcp/guides/pipelineRuntime.md b/docs/mcp/guides/pipelineRuntime.md new file mode 100644 index 0000000..6130574 --- /dev/null +++ b/docs/mcp/guides/pipelineRuntime.md @@ -0,0 +1,149 @@ +--- +name: pipelineRuntime +description: How the pipeline runtime dispatches records — merge groups, first-match-wins, unmatched-record drops, onEnd/afterShard hook ordering, flushEvery buffering, and segment/worker parallelism. +category: Guides +--- + +# Pipeline runtime + +How records flow through the transfer pipeline at runtime, and the exact ordering of every hook the runner invokes. Source: `docs/guides/pipeline-runtime.md`, `src/features/PipelineRunner/PipelineRunner.ts`, `src/commands/run/handler.ts`, `src/commands/processSegment/handler.ts`. + +## Merge groups (keyed by scanner) + +`PipelineRunner` keys pipelines by their `scanner` instance: + +```typescript +private mergeGroups: Map, AnyPipeline[]> = new Map(); +``` + +`runner.register(...)` pushes each pipeline onto the array for its `pipeline.scanner`. All pipelines built with `scanner: DdbScanner` land in one merge group; all pipelines built with `scanner: OsScanner` land in another. Each merge group's scanner runs its scan **once**; every record it yields is offered to that group's pipelines in **registration order** — the order `runner.register(...)` was called (or the order pipelines appear within one variadic call). + +## First-match-wins dispatch + +Inside a shard's record loop, the runner walks the merge group's pipeline list and stops at the first one that accepts the record: + +```typescript +for (const pipeline of pipelines) { + if (!(await pipeline.accepts(record))) { + continue; + } + matched = true; + // ...run filters already passed; run transformers, onEnd, buffer commands... + break; // no other pipeline in this merge group sees the record +} +``` + +`pipeline.accepts(record)` is true when every filter attached via `.filter(...)` passes (AND-composed). Once one pipeline claims a record, the loop `break`s — later pipelines in the same merge group never see it, regardless of whether their filters would also have matched. This makes **registration order semantically significant**: put more specific pipelines first, catch-alls last. + +## Unmatched records are dropped + +If no pipeline in the merge group accepts a record, `matched` stays `false` and the record is **not written anywhere** — dropped, not an error: + +```typescript +if (!matched) { + const { PK, SK, TYPE } = record as any; + const typeKey = TYPE && TYPE !== "unknown" ? TYPE : `${PK}:${SK}`; + unmatchedByType.set(typeKey, (unmatchedByType.get(typeKey) ?? 0) + 1); + this.logger.warn(`unmatched record — TYPE=${typeKey} PK=${PK} SK=${SK}`); + // ... snapshot to dropped/segment-N.jsonl if debug.snapshot is on ... + this.droppedLog.add(record, new RecordDisposition.Unmatched()); +} +``` + +Observability for unmatched records: + +- A `warn` log line per unmatched record (`unmatched record — TYPE=... PK=... SK=...`; falls back to `PK:SK` when `TYPE` is absent/`"unknown"`). +- An `info`-level shard summary line: `[ shard N/M] scanned ..., transferred ..., blackholed ..., unmatched 14 (pb.page=4, ...)`. +- A per-worker dropped-record log flushed at shard end (`this.droppedLog.flush(shardCtx.segment)`), and, if `debug.snapshot` is enabled, a `dropped/segment-N.jsonl(.gz)` file with the full record. + +To transfer **every** record, register a zero-filter catch-all pipeline last in the merge group — see `writingPresets.md` and `filters.md`. + +## Record processing: filters → transformers → onEnd + +For each record a pipeline claims, `runRecord()` runs, in this exact order: + +1. **Slice merge** — each processor's `extendContext(ctx)` (if defined) is `Object.assign`-ed onto the shared `ctx`, contributing helpers like `ctx.putRecord`/`ctx.copyFile`. +2. **Transformers** — `pipeline.transformerFns`, in `.use(...)` registration order, each mutating `ctx.record` in place. +3. **`onEnd` hooks** — each processor's `onEnd(ctx)` (if defined) runs **sequentially, in the pipeline's `processors` array order** — e.g. `processors: [DdbProcessor, S3Processor]` runs `DdbProcessor.onEnd` (auto-`putRecord`) then `S3Processor.onEnd` (a no-op — `S3Processor` defines none). This is what gives zero-transformer pipelines their "verbatim copy" behavior: `DdbProcessor`/`OsProcessor`/`AuditLogProcessor` each auto-emit a put in `onEnd`; `S3Processor` never does. +4. **Blackhole check** — if `pipeline.isBlackhole` or `ctx.isBlackholed` (set via `ctx.blackhole()` inside a transformer), every command this record emitted is discarded here; nothing reaches the shard buffer. +5. **Fold into shard buffer** — otherwise, every command in the record's local `commands` bag is added to the shared `shardCommands` buffer for later flushing. + +## `flushEvery` — bounded peak memory + +Commands don't hit the target after every record — they accumulate in a shared `Commands` buffer for the whole shard, and that buffer is drained periodically: + +```typescript +const flushEvery = this.config.tuning?.flushEvery ?? 500; // tuning.flushEvery, default 500 + +// ...inside the per-record loop... +recordCount++; +if (recordCount % flushEvery === 0) { + await this.flushShard(pendingCommands, processorOrder); + pendingCommands = new Commands(); + periodicFlushCount++; +} + +// ...after the loop, a final flush for any remainder... +if (pendingCommands.size() > 0 || periodicFlushCount === 0) { + await this.flushShard(pendingCommands, processorOrder); +} +``` + +`flushShard` calls `processor.execute(commands)` for **every distinct processor across the merge group's pipelines, in the order they were first encountered** (`collectProcessorOrder`) — not just the processors of the pipeline that produced the commands. Each processor's `execute()` drains only the command keys it owns (`commands.get(key)`, which marks that key "claimed"); any key nobody claims surfaces via `Commands.unclaimedKeys()` and triggers a one-time-per-key runner warning. + +Net effect: peak memory is bounded to roughly `flushEvery × average_record_size` (≈ 5 MB at the 500-record default with a 10 KB average record), not the whole shard's worth of pending writes. Lower `flushEvery` (e.g. to 100) for tables with unusually large records; see `configReference.md` for the `tuning` block. + +## Parallelism: segments, shards, and worker processes + +`pipeline.segments` (optional in the schema; the orchestrator falls back to `1` if unset — `config.pipeline?.segments || 1` in `src/commands/run/handler.ts`. The example config in `configReference.md` sets `numberFromEnv("SEGMENTS", 4)`, but that `4` is a user-chosen convention, not a schema default) sets **both**: + +- how many shards each scanner's `listShards()` reports (`{ segment: i, total: segments }`, passed straight through to DynamoDB's native parallel-`Scan` `Segment`/`TotalSegments` parameters), and +- how many **child worker processes** the orchestrator spawns. + +The orchestrator (`src/commands/run/handler.ts`) resolves `segmentsToRun` (all segments, or a filtered subset via `--segments=1,3`), then spawns one worker per segment **concurrently**: + +```typescript +const workers = segmentsToRun.map(segment => + spawnWorker(segment, segments, runId, configPath, presetName, logLevel, dryRun) +); +const results = await Promise.allSettled(workers); +``` + +Each worker is a separate `node bin.js process-segment --segment N --total M ...` child process (via `execa`) — workers share nothing except the target table/bucket/index they write to. Inside a worker (`src/commands/processSegment/handler.ts`), `runner.run({ segment, totalSegments })` runs exactly **one shard** for exactly **one merge group** (a worker only handles one preset/merge-group at a time; `PipelineRunner.run({...})` throws if more than one merge group is registered when shard options are passed). + +A partial failure can be re-run without rescanning everything: `--segments=1,3` reruns only those two workers; each still receives the original `totalSegments`, so it scans the identical slice of the table it would have scanned in a full run. + +## Hook ordering + +There are four independent hook layers, each with its own scope and ordering: + +| Layer | Registered via | Scope | Runs in | Order | +| --- | --- | --- | --- | --- | +| Transfer lifecycle | `BeforeTransferHook` / `AfterTransferHook` (`config.register`) | Whole transfer, once | Orchestrator process, around spawning all workers | Registration order (`{ multiple: true }` abstraction) | +| Preset lifecycle | `BeforeLoadPresetHook` / `AfterLoadPresetHook` (`config.register`) | Once per worker, around preset loading | Each worker process, before/after `preset.configure(...)` | Registration order | +| Pipeline-level | `.beforeExecuteCommands(token)` / `.afterExecuteCommands(token)` on the builder | Once per merge group | Only in `PipelineRunner.runMergeGroup` — see caveat below | Before-hooks: pipeline/registration order (deduped). After-hooks: **reverse** of that order | +| Per-record / per-shard | Processor `onEnd(ctx)` / `afterShard(ctx)` | Every record / once per shard | Inside `runRecord()` / end of `runShard()` | Sequential, `processors` array order (not reversed) | + +**Before-hooks run forward, after-hooks run in reverse:** + +```typescript +// before: forward order +for (const hookToken of beforeHookTokens) { + await this.container.resolve(hookToken).run(hookParams); +} +// ...shards run... +// after: REVERSE order +for (let i = afterHookTokens.length - 1; i >= 0; i--) { + await this.container.resolve(afterHookTokens[i]!).run(hookParams); +} +``` + +This mirrors a typical setup/teardown stack: the last pipeline's `afterExecuteCommands` hook runs first on the way out. + +**Caveat — pipeline-level hooks only fire without shard options:** `.beforeExecuteCommands`/`.afterExecuteCommands` are wired into `runMergeGroup`, which only executes when `PipelineRunner.run()` is called with **no** `{ segment, totalSegments }` argument. The real segmented CLI flow always calls `runner.run({ segment, totalSegments })` from each worker (`runSingleShard` path), which never calls `runMergeGroup` and therefore never invokes these hooks. In practice, reach for the whole-transfer `BeforeTransferHook`/`AfterTransferHook` (documented in `configReference.md`) for cross-cutting setup/teardown instead; treat `.beforeExecuteCommands()`/`.afterExecuteCommands()` as an advanced/internal extension point until this is reconciled (see the same caveat in `writingPresets.md`). + +**Processor `onEnd` vs `afterShard`:** `onEnd` is per-record (step 3 of the record pipeline above); `afterShard` is per-shard, called once at the very end of `runShard()` — after the final flush — for processors that persist shard-level side-effect state (e.g. `OsProcessor` recording which indexes it touched, for a later orchestrator-side hook to restore `refresh_interval` on). Both run sequentially in the same `processors` array order. + +## Blackholing + +`.blackhole(condition?)` on the pipeline builder, or `ctx.blackhole()` per-record inside a transformer, suppress writes without skipping the pipeline: filters, transformers, and `onEnd` all still run — only the final "fold commands into the shard buffer" step is skipped. Combine with `debug.snapshot` to inspect what *would* have been written. See `writingPresets.md` and `writingTransformers.md` for usage examples. diff --git a/docs/mcp/guides/publicApi.md b/docs/mcp/guides/publicApi.md new file mode 100644 index 0000000..9e3abdc --- /dev/null +++ b/docs/mcp/guides/publicApi.md @@ -0,0 +1,296 @@ +--- +name: publicApi +description: Every export from src/index.ts, organized by category, with import path, value-vs-type marker, and a one-line description for each. +category: Guides +--- + +# Public API surface + +Everything a config/preset/transformer author can import from `@webiny/data-transfer` is re-exported from one file: `src/index.ts`. This is a complete inventory — every export, in file order within each category, marked **value** (importable without `type`, usable at runtime) or **type** (erased at compile time; import with `import type` or inline `type` specifiers). + +Rule of thumb from `AGENTS.md`: anything added here must be something a user writing their own config/transformers/presets genuinely needs — domain-specific migration transformers stay internal. + +## Config & Env + +```typescript +import { + createConfig, migrationConfigSchema, loadEnv, fromEnv, numberFromEnv, + initDataTransfer, findPackageRoot, MigrationConfig +} from "@webiny/data-transfer"; +import type { MigrationConfiguration, InitDataTransferContext } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createConfig` | value | Validates a config object against `unifiedTransferInputSchema` (Zod) and returns the parsed `MigrationConfiguration`. The one function every `config.ts` calls and default-exports. | +| `migrationConfigSchema` | value | The raw Zod schema `createConfig` validates against — for advanced users who want to validate/parse config data themselves without going through `createConfig`. | +| `MigrationConfiguration` | type | The parsed, validated shape `createConfig(...)` returns (and `MigrationConfig`'s DI token resolves to). | +| `loadEnv` | value | `loadEnv(import.meta.url)` loads the `.env` file sitting next to the calling config file, so env vars are available before `fromEnv`/`numberFromEnv` read them. | +| `fromEnv` | value | Reads a required (or defaulted, or nullable) env var as a string; throws a descriptive error on a missing/empty required var instead of silently producing `undefined`. | +| `numberFromEnv` | value | Same as `fromEnv` but parses to `number`; throws if the raw value isn't a valid number (catches typos like `SEGMENTS=four`). | +| `initDataTransfer` | value | Typed identity helper for a project's optional `setup.ts` — wraps `(ctx: { container }) => void \| Promise` so the callback gets container typing without a separate import/annotation. | +| `InitDataTransferContext` | type | The `{ container: Container }` shape passed into the `initDataTransfer(...)` callback. | +| `findPackageRoot` | value | Walks up from a directory to locate `@webiny/data-transfer`'s own `package.json` root — works across source, compiled, and installed-via-npm contexts. Used internally by `WorkerSpawner`; exported for advanced tooling that needs the same resolution. | +| `MigrationConfig` | value | DI abstraction token for the **resolved** config. `container.resolve(MigrationConfig)` inside `register`/`configure`/transformer code returns the same `MigrationConfiguration` object `createConfig(...)` produced — read table names, regions, credentials, tuning, etc. at runtime. | + +## Credentials + +Re-exported directly from `@aws-sdk/credential-providers` (under friendlier names) so config authors don't need that package as a separate dependency: + +```typescript +import { fromAwsProfile, fromAwsCredentialChain } from "@webiny/data-transfer"; +``` + +| Export | Kind | Aliases | Description | +| --- | --- | --- | --- | +| `fromAwsProfile` | value | `fromIni` | Reads credentials for a named profile from `~/.aws/credentials`. Best for local dev with multiple accounts — no risk of a stray env var silently hijacking the wrong one. | +| `fromAwsCredentialChain` | value | `fromNodeProviderChain` | The AWS SDK's default resolution chain: env vars → shared credentials file → SSO/web-identity → EC2/ECS IAM role. Best for CI/cloud runs that must work without code changes. | + +Both return an `AwsCredentialsProvider` (`() => Promise`); a literal `{ accessKeyId, secretAccessKey, sessionToken? }` object is also accepted directly by `source.credentials`/`target.credentials` without importing anything. + +## Transformer Factories + +```typescript +import { createTransformer, createDdbTransformer, createOsTransformer } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createTransformer` | value | Generic transformer factory — `createTransformer(name, fn)`. Stamps a `transformerName` property onto `fn` for logging; use for `BaseTransformContext`/`DdbCoreTransformContext` transformers or anything the two convenience factories below don't fit. | +| `createDdbTransformer` | value | Same runtime behavior as `createTransformer`, but binds `fn`'s parameter to `DdbTransformContext.Interface` (Base + `DdbProcessor` slice + `S3Processor` slice). Default choice for v5-to-v6 DDB transformers. | +| `createOsTransformer` | value | Same, but binds `fn`'s parameter to `OsTransformContext.Interface` (Base + `OsProcessor` slice). | + +## Built-in Transformers (27) + +Ready-made transformers for use in custom presets via `.use(...)`. Source: `src/transformers/**`. Full per-transformer detail (signatures, config args, edge cases) lives under `docs/mcp/transformers/`. + +```typescript +import { + // CMS + addLiveField, fixBrokenStorageKeys, fixCmePk, removeFolderRevision, + renameFieldAttributes, replaceFileUrls, transformModelGroup, transformRichText, + updateModelIds, updateOsIndex, + // File manager + copyFileToTarget, createMetadata, extractImageMetadata, migrateFileManagerSettings, + // Folders + updateFlpIds, + // Global + addGsiTenant, addTransferTimestamp, removeAttributes, removeLocale, wrapInData, + // Security + groupsToRoles, removeTenant, transformPermissions, + // Mailer + migrateMailerSettings, + // Audit logs + coreFieldsTransformer, dataFieldsTransformer, storageShapeTransformer +} from "@webiny/data-transfer"; +``` + +All 27 are **value** exports (plain functions, or factory functions in `replaceFileUrls`'s case — see `writingTransformers.md`). + +### CMS (10) + +| Export | Description | +| --- | --- | +| `addLiveField` | Computes and attaches the `live` pointer (published revision version) to CMS entry records. | +| `fixBrokenStorageKeys` | Corrects mismatched field storage keys in CMS entry values against the model's declared `storageId`. | +| `fixCmePk` | Removes a duplicated `#CME#CME#` segment from a record's `PK`. | +| `removeFolderRevision` | Strips the `#0001` revision suffix from folder location IDs and cleans up legacy folder location fields. | +| `renameFieldAttributes` | Renames legacy CMS model field attributes (`helpText`, `placeholderText`, `multipleValues`) to their v6 equivalents. | +| `replaceFileUrls` | Factory — `replaceFileUrls(config)` rewrites file-manager URLs embedded in CMS `file`/`rich-text` field values from `fileUrls.source` to `fileUrls.target`. | +| `transformModelGroup` | Resolves a CMS model's group ID reference to its slug string. | +| `transformRichText` | Converts legacy Slate-based rich-text field values into the Lexical state + rendered HTML format. | +| `updateModelIds` | Renames legacy system model IDs (`fmFile`, `acoFolder`, etc.) to their v6 `wby`-prefixed equivalents in keys and `data.modelId`. | +| `updateOsIndex` | Recomputes an OpenSearch record's target index name from its `modelId` and tenant. | + +### File Manager (4) + +| Export | Description | +| --- | --- | +| `copyFileToTarget` | Emits a verbatim S3 copy for a file-manager record, source key equal to target key. | +| `createMetadata` | Creates a KeyValueStore file-metadata record and copies the underlying S3 object to its new tenant-scoped path. | +| `extractImageMetadata` | Extracts image dimensions, EXIF, and IPTC metadata from raster image files and renames the legacy meta field. | +| `migrateFileManagerSettings` | Converts a legacy File Manager settings record into the v6 KeyValueStore format. | + +### Folders (1) + +| Export | Description | +| --- | --- | +| `updateFlpIds` | Strips the `#0001` revision suffix from folder-level-page `id` and `parentId` fields. | + +### Global (5) + +| Export | Description | +| --- | --- | +| `addGsiTenant` | Populates the `GSI_TENANT` attribute from the record's `PK` or `data.tenant`. | +| `addTransferTimestamp` | Stamps every record with the transfer time as `_tt`. | +| `removeAttributes` | Deletes deprecated top-level attributes (currently `webinyVersion`) from the data envelope. | +| `removeLocale` | Strips locale segments (e.g. `#L#en-US#`) from a record's keys and deletes the locale field. | +| `wrapInData` | Wraps all non-reserved top-level attributes of a record into a `data` envelope. | + +### Security (3) + +| Export | Description | +| --- | --- | +| `groupsToRoles` | Renames security "group" records and their `GROUP`/`GROUPS` key segments to the v6 "role" terminology. | +| `removeTenant` | Deletes the top-level `tenant` attribute from security role records. | +| `transformPermissions` | Migrates security role permissions to v6 shape — drops `content.i18n`, flattens per-locale model lists, and resolves group IDs to slugs. | + +### Mailer (1) + +| Export | Description | +| --- | --- | +| `migrateMailerSettings` | Converts a legacy Mailer settings record into the v6 KeyValueStore format. | + +### Audit Logs (3) + +| Export | Description | +| --- | --- | +| `coreFieldsTransformer` | Resolves an audit-log record's creator identity and creation time, and stamps a fresh id and TTL expiry. | +| `dataFieldsTransformer` | Lifts audit-log content fields (`app`, `action`, `message`, `entity`, `tags`, `content`) out of the legacy `values` envelope onto the record root. | +| `storageShapeTransformer` | Builds the final v6 audit-log storage record — nine GSI key sets plus the data envelope and TTL expiry. | + +## Filters (18 predicates + `createFilter`) + +```typescript +import { + createFilter, + byType, byTypePrefix, isCmsGroup, isCmsModel, isCmsEntry, byIncludesModelId, + isAcoSearchRecord, isAdminUser, isBackgroundTask, isFmFile, isFlpRecord, + isBuiltInSecurityRole, isSecurityTeam, isOsBackgroundTask, isOsMailerSettings, + isAuditLogEntry, isMigrationRecord, isFormBuilderRecord +} from "@webiny/data-transfer"; +import type { Filter } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createFilter` | value | Wraps any predicate (sync or async) into the `{ kind: "filter", check }` shape a pipeline builder's `.filter(...)` expects. | +| `Filter` | type | `{ readonly kind: "filter"; readonly check: (record: TRecord) => boolean \| Promise }` — the shape `createFilter` produces. | +| `byType` | value | Factory — `byType(type)` matches `record.TYPE === type` exactly. | +| `byTypePrefix` | value | Factory — `byTypePrefix(prefix)` matches when `record.TYPE` starts with `prefix`. | +| `isCmsGroup` | value | Matches `TYPE === "cms.group"` or `PK` including `"#CMS#CMG"`. | +| `isCmsModel` | value | Alias for `byType("cms.model")`. | +| `isCmsEntry` | value | Matches CMS entries by `TYPE` prefix `"cms.entry"` or `PK` including `"#CMS#CME#"`, regardless of raw-vs-reshaped record form. | +| `byIncludesModelId` | value | Factory — `byIncludesModelId(target)` matches when `record.index`/`record.modelId` (also checked under `record.data`) contains `target`, case-insensitive. | +| `isAcoSearchRecord` | value | Alias for `byIncludesModelId("acoSearchRecord")`. | +| `isAdminUser` | value | Matches full-access admin user records (`PK` includes `"#SECURITY#USER#"` and `GSI1_PK === "securityRole#full-access"`). | +| `isBackgroundTask` | value | Matches DDB-side background task records (`modelId`/`GSI1_PK` referencing `webinyTask`/`webinyTaskLog`). | +| `isFmFile` | value | Matches File Manager file records (`modelId` `"fmFile"` or `"wbyFmFile"`, top-level or under `data`). | +| `isFlpRecord` | value | Matches folder location permission records (`PK` includes `"#FLP#"`). | +| `isBuiltInSecurityRole` | value | Matches the two built-in security roles (`slug`/`GSI1_SK` is `"full-access"` or `"anonymous"`) — use to exclude these from a custom roles filter. | +| `isSecurityTeam` | value | Alias for `byType("security.team")`. | +| `isOsBackgroundTask` | value | OS-side equivalent of `isBackgroundTask`, reading from the decompressed `data` payload. | +| `isOsMailerSettings` | value | Matches `record.data.modelId === "mailerSettings"` on OS-table records. | +| `isAuditLogEntry` | value | Matches audit log entries (`modelId` lowercases to `"acosearchrecord-auditlogs"` and `SK === "L"`) — must be filtered for **before** `isAcoSearchRecord`/`isCmsEntry` since they share a `modelId` prefix. | +| `isMigrationRecord` | value | Matches v5 migration-tracking records (`PK` starts with `"MIGRATION"`); typically blackholed. | +| `isFormBuilderRecord` | value | Matches Form Builder forms and submissions (`PK` includes `"#FB#"`, or `TYPE` prefixed `"fb.form."`/`"fb.formSubmission"`); no v6 migration path yet. | + +## Scanners + +```typescript +import { DdbScanner, OsScanner } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `DdbScanner` | value | Scans every item in the source DynamoDB table, segment by segment, yielding raw `BaseRecord` items with no transformation. | +| `OsScanner` | value | Scans the source OpenSearch companion DynamoDB table, decompressing each record's gzip `data` payload; yields `{ index, data, ...BaseRecord }`. Only registered when `config.target.opensearch != null`. | + +## Processors + +```typescript +import { DdbProcessor, S3Processor, AuditLogProcessor, OsProcessor, Processor } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `DdbProcessor` | value | Writes scanned/transformed records to the target DynamoDB table; auto-`putRecord`s in `onEnd` (zero-transformer copy behavior). | +| `S3Processor` | value | Copies S3 objects (source → target bucket) queued via `ctx.copyFile(...)`; no `onEnd` — never writes anything unless a transformer explicitly asks. | +| `AuditLogProcessor` | value | Writes audit-log entries to a dedicated target table, gated on both a configured `target.auditLog` table and `record.TYPE === "auditLog.log"`. | +| `OsProcessor` | value | Writes records to the target OS companion table, gzip-compressing `data` and managing target-index lifecycle (create / disable-refresh); auto-`putRecord`s in `onEnd`. | +| `Processor` | value | Base DI abstraction token every processor implementation shares. Reach for this when declaring a custom processor via `Processor.createImplementation({...})`; its namespace also carries the `Processor.Interface`, `Processor.Context`, and `Processor.SliceOf

` **types** used when typing a custom implementation. | + +## Service Clients + +DI abstractions for direct AWS access — resolve from `container` (e.g. inside `config.register` or a preset's `configure`) for pre-flight checks or custom side-effect code outside the transformer/processor pipeline. + +```typescript +import { + SourceDynamoDbClient, TargetDynamoDbClient, OpenSearchClient, SourceS3Client, TargetS3Client +} from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `SourceDynamoDbClient` | value | DI token for the source DynamoDB client (`scan`/`query`/`get`, no writes) — bound to `source.region`/`source.credentials`. | +| `TargetDynamoDbClient` | value | DI token for the target DynamoDB client (adds `batchWrite`) — bound to `target.region`/`target.credentials`. | +| `OpenSearchClient` | value | DI token for the OpenSearch client used for index lifecycle (`indexExists`, `createIndex`, `listIndexes`, `putIndexSettings`, `getIndexSettings`) — bound to `target.opensearch.endpoint`. | +| `SourceS3Client` | value | DI token for the source S3 client (`getObject`) — bound to `source.s3.bucket`/`source.region`. | +| `TargetS3Client` | value | DI token for the target S3 client (`copy`, `batchCopy`, `getObject`) — bound to `target.s3.bucket`/`target.region`. `copy`/`batchCopy` run with **target** credentials even when copying from the source bucket (cross-account implications — see `configReference.md`). | + +## Context Types + +Types used to annotate custom transformer functions. Import with `import type`. See `writingTransformers.md` for the full base-context API and processor-slice reference. + +```typescript +import type { + BaseTransformContext, DdbCoreTransformContext, DdbTransformContext, OsTransformContext, Transformer +} from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `BaseTransformContext` | type | The context every transformer gets regardless of pipeline processors: `record`, `original`, `modelProvider`, `cache`, `logger`, `compressionHandler`, `replace()`, `addCommand()`, `blackhole()`, `isBlackholed`. | +| `DdbCoreTransformContext` | type | `BaseTransformContext` + `DdbProcessor` slice (`putRecord`, `querySourceRecord`, `queryTargetRecord`) — no S3 helpers. For pipelines registering `DdbProcessor` only. | +| `DdbTransformContext` | type | `BaseTransformContext` + `DdbProcessor` slice + `S3Processor` slice (`copyFile`, `getFile`). Default context for v5-to-v6 DDB transformers; requires `processors: [DdbProcessor, S3Processor]`. | +| `OsTransformContext` | type | `BaseTransformContext` + `OsProcessor` slice. For OS-mode pipelines registering `OsProcessor`. | +| `Transformer` | type | `Transformer.Interface = (ctx: TContext) => void \| Promise` — the function shape every transformer (built-in or custom) satisfies. | + +## Lifecycle Hooks + +DI abstractions registered via `config.register`; all four use `{ multiple: true }`, so registering one **adds** to the list rather than replacing a default. See `configReference.md` and `pipelineRuntime.md` for exactly when each runs. + +```typescript +import { + BeforeTransferHook, AfterTransferHook, BeforeLoadPresetHook, AfterLoadPresetHook +} from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `BeforeTransferHook` | value | `{ execute(): Promise }` — runs once in the orchestrator process, before any worker is spawned. | +| `AfterTransferHook` | value | `{ execute(): Promise }` — runs once in the orchestrator process, after all workers finish (best-effort; a thrown error here is logged, not fatal). | +| `BeforeLoadPresetHook` | value | `{ execute(config): Promise }` — runs once per **worker** process, before the preset is loaded/`configure()`d. | +| `AfterLoadPresetHook` | value | `{ execute(config, preset): Promise }` — runs once per worker process, after `preset.configure(...)` completes. | + +## Customization + +Extension points for advanced config authors — override via `config.register`. + +```typescript +import { IndexConfigurationProvider, ModelProvider, PipelineCustomizer } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `IndexConfigurationProvider` | value | DI abstraction — override `getConfiguration(indexName, base)` to customize OpenSearch index mappings/settings per index before `OsProcessor` creates/updates it. | +| `ModelProvider` | value | DI abstraction — override `preloadModels`/`getModel`/`getModelIds` to customize how CMS model definitions are loaded (default: DB + `pipeline.modelsDir` JSON files). | +| `PipelineCustomizer` | value | DI abstraction — implement `{ name, canUse(pipelineName), configure(builder) }` to extend a built-in preset's pipeline (by name) from `setup.ts`/`config.register` without re-registering the whole preset. See `pipeline-customizer.md`. | + +## Presets & Pipeline Construction + +Core building blocks for writing a custom preset file. See `writingPresets.md` for the full walkthrough. + +```typescript +import { createTransferPreset } from "@webiny/data-transfer"; +import type { MigrationPreset, PresetConfigureContext, NonEmptyArray } from "@webiny/data-transfer"; +``` + +| Export | Kind | Description | +| --- | --- | --- | +| `createTransferPreset` | value | Identity function — `createTransferPreset(preset)` returns `preset` unchanged; exists purely so a preset file's `configure({...})` gets typed inference without a separate `MigrationPreset` annotation. | +| `MigrationPreset` | type | The shape a preset file's `default` export must satisfy: `{ name: string; description: string; configure(ctx): void \| Promise }`. | +| `PresetConfigureContext` | type | The `{ runner, pipelineBuilderFactory, container }` argument bag passed to `configure(...)`. | +| `NonEmptyArray` | type | Tuple-length helper used to type `pipelineBuilderFactory.create({ processors })` — enforces at compile time that `processors` has at least one element. | + +Not (yet) exported from the package root: the base `Hook` abstraction that `.beforeExecuteCommands()`/`.afterExecuteCommands()` are typed against — see the caveat in `writingPresets.md` and `pipelineRuntime.md`. diff --git a/docs/mcp/guides/writingPresets.md b/docs/mcp/guides/writingPresets.md new file mode 100644 index 0000000..f8317cd --- /dev/null +++ b/docs/mcp/guides/writingPresets.md @@ -0,0 +1,260 @@ +--- +name: writingPresets +description: How to write a custom preset — createTransferPreset shape, pipelineBuilderFactory.create(), builder methods, filter/use/hook composition, first-match-wins dispatch, built-in presets. +category: Guides +--- + +# Writing presets + +A preset is a file exporting `default: MigrationPreset` — `{ name, description, configure }`. `configure(ctx)` receives `{ runner, pipelineBuilderFactory, container }`, builds one or more `Pipeline` objects, and registers them on `runner`. The wizard/CLI selects a preset by `name` at runtime (`--preset=` or `presetsDir` for custom files). + +Source: `src/utils/createTransferPreset.ts`, `src/domain/transform/Preset.ts`, `src/domain/pipeline/PipelineBuilder.ts`, `src/features/PipelineBuilderFactory/`. + +## Minimal preset + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + S3Processor, + createFilter +} from "@webiny/data-transfer"; +import { stampMigratedAt } from "./transformers/stampMigratedAt.ts"; + +export default createTransferPreset({ + name: "my-preset", + description: "One-line description shown in CLI output.", + async configure({ runner, pipelineBuilderFactory }) { + const pipeline = await pipelineBuilderFactory + .create({ name: "my-pipeline", scanner: DdbScanner, processors: [DdbProcessor, S3Processor] }) + .filter(createFilter(r => r.TYPE === "cms.entry")) + .use(stampMigratedAt) + .build(); + + runner.register(pipeline); + } +}); +``` + +Drop the file in your `pipeline.presetsDir` (see `configReference.md`). The wizard offers it by name alongside the 5 built-ins. + +**`createTransferPreset(preset)`** is an identity function — it exists purely so `configure({...})` gets typed inference without you importing and annotating `MigrationPreset` yourself. Pair it with `export default`; the loader looks for `default` first. + +## `PresetConfigureContext` — what `configure` receives + +```typescript +interface PresetConfigureContext { + runner: PipelineRunner.Interface; + pipelineBuilderFactory: PipelineBuilderFactory.Interface; + container: Container; // @webiny/di container +} +``` + +- **`runner`** — call `runner.register(...pipelines)` (variadic, chainable) once your pipelines are built. +- **`pipelineBuilderFactory`** — call `.create({ name, scanner, processors })` to start a new `PipelineBuilder`. +- **`container`** — resolve DI-registered services, e.g. `container.resolve(MigrationConfig)` to read the parsed config at preset-build time (used by built-in presets to decide whether to blackhole audit logs — see below). + +`configure` may be `sync` or `async` — declare it `async` whenever you `await` a `.build()` call (you always will, since `.build()` is async — see below). + +## `pipelineBuilderFactory.create({ name, scanner, processors })` + +```typescript +factory.create>(input: { + name: string; + scanner: ScannerImpl; + processors: TProcessors; // non-empty tuple, disjoint slice keys enforced at compile time +}): PipelineBuilder +``` + +- **`name`** — unique across the run; `PipelineBuilder`'s constructor throws if empty/whitespace-only, and the runner throws on duplicate names. +- **`scanner`** — `DdbScanner` or `OsScanner` (both exported from `@webiny/data-transfer`). Determines which table is scanned and the `TRecord` shape flowing through filters/transformers. +- **`processors`** — a non-empty array of processor implementation classes (`DdbProcessor`, `S3Processor`, `OsProcessor`, `AuditLogProcessor`). Each contributes a **slice** of helpers onto the effective transformer context (`ctx.putRecord`, `ctx.copyFile`, etc. — see `writingTransformers.md`). TypeScript rejects: + - an empty `processors` array, and + - combinations whose slices share a key (e.g. `[DdbProcessor, OsProcessor]` — both contribute `putRecord`/`querySourceRecord`/`queryTargetRecord`). + +## Builder methods + +All methods return `this` (chainable) except `.build()`. + +| Method | Signature | Behavior | +| --- | --- | --- | +| `.filter(filter)` | `(filter: Filter) => this` | Adds a filter. Multiple calls AND-compose — order across calls doesn't matter, all must pass. | +| `.use(transformer)` | `(t: Transformer \| readonly Transformer[]) => this` | Adds one transformer or an array (spread in order). Execution order = registration order; arrays and single calls can mix freely. | +| `.blackhole(condition?)` | `(condition?: () => boolean) => this` | Observe-only mode: filters/transformers/`onEnd` still run, but every emitted command for this pipeline is discarded — nothing lands on the target. `condition` is evaluated **immediately, synchronously**, at call time (not per-record); omit it to always blackhole. | +| `.beforeExecuteCommands(token)` | `(token: Abstraction) => this` | Registers a DI-resolved hook to run once per merge group **before** any shard runs. | +| `.afterExecuteCommands(token)` | `(token: Abstraction) => this` | Registers a DI-resolved hook to run once **after** all shards in the merge group succeed. Skipped if any shard fails. | +| `.build()` | `() => Promise>` | **Async** — snapshots the builder into an immutable `Pipeline`. Always `await` it before passing to `runner.register(...)`. | + +### `.build()` is async — always `await` it + +Every built-in preset does `await factory.create({...}).build()` inside an `async configure(...)`. This is easy to miss since older example code sometimes omits `await` — don't copy that pattern: + +```typescript +async configure({ runner, pipelineBuilderFactory: factory }) { + const everything = await factory + .create({ name: "Regular DynamoDB Table Data", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); + + runner.register(everything); +} +``` + +`.build()` is async because it resolves and applies any `PipelineCustomizer`s registered against this pipeline's `name` before finalizing it (see `pipeline-customizer.md`). + +### `runner.register(...pipelines)` + +```typescript +register(...pipelines: Pipeline[]): this +``` + +Variadic and chainable — heterogeneous pipelines (different scanners/processors) can be registered in one call: `runner.register(p1, p2, p3)`. + +## Filters + +Use `createFilter` with a built-in predicate or an inline function: + +```typescript +import { createFilter, isFmFile, isCmsEntry, byType } from "@webiny/data-transfer"; + +.filter(createFilter(isFmFile)) +.filter(createFilter(isCmsEntry)) +.filter(createFilter(byType("cms.model"))) +.filter(createFilter(r => r.TYPE === "cms.entry" && r.modelId === "article")) // inline +``` + +Full predicate reference (all 18 built-ins): see `filters.md`. + +## First-match-wins dispatch + +Pipelines sharing the same `scanner` type run as a **merge group**. The scanner scans once; each record is offered to every pipeline in the group **in registration order**. The first pipeline whose filters all pass claims the record — no other pipeline in the group sees it. + +```typescript +async configure({ runner, pipelineBuilderFactory: factory }) { + const articles = await factory + .create({ name: "articles", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(r => r.TYPE === "cms.entry" && r.modelId === "article")) + .use(migrateArticle) + .build(); + + const rest = await factory + .create({ name: "rest", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); // no filter → catches everything articles doesn't + + runner.register(articles, rest); // order is load-bearing: articles MUST come first +} +``` + +Swap the registration order and `articles` never fires — `rest` (unfiltered) claims everything first. + +**Records matching no pipeline in any merge group are silently dropped** (not written to target) — see `pipeline-runtime.md` for the unmatched-record logging. + +## Hooks (`beforeExecuteCommands` / `afterExecuteCommands`) + +These accept a **DI abstraction token**, not an inline callback — the runner resolves the token from the container and calls `.run({ runId, mergeGroupId })` on the resolved instance: + +```typescript +interface Hook { + run(params: { runId: string; mergeGroupId: string }): Promise; +} +``` + +To use one, define an implementation and register it via `Abstraction.createImplementation({ implementation, dependencies })` (the same pattern used throughout the codebase for `BeforeTransferHook`/`AfterTransferHook`), register the resulting token in your `config.register` callback, then pass the abstraction token itself to the builder: + +```typescript +.beforeExecuteCommands(MyHookToken) +.afterExecuteCommands(MyHookToken) +``` + +**Caveat:** the base `Hook` abstraction (`src/domain/pipeline/abstractions/Hook.ts`) that these methods are typed against is **not currently re-exported from the public `@webiny/data-transfer` package root** — only the higher-level `BeforeTransferHook`/`AfterTransferHook` (whole-transfer lifecycle, registered via `config.register`) and `BeforeLoadPresetHook`/`AfterLoadPresetHook` are public. In practice, reach for those transfer-lifecycle hooks (documented in `configReference.md`) for cross-cutting setup/teardown; treat per-pipeline `.beforeExecuteCommands()`/`.afterExecuteCommands()` as an advanced/internal extension point until `Hook` is added to the public surface. + +## Zero-transformer preset (pure data copy) + +No transformer is required — a pipeline with only a scanner + processor(s) copies records verbatim: + +```typescript +import { createTransferPreset, DdbScanner, DdbProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "copy", + description: "Copy every record verbatim.", + async configure({ runner, pipelineBuilderFactory: factory }) { + const copyAll = await factory + .create({ name: "copy-all", scanner: DdbScanner, processors: [DdbProcessor] }) + .build(); // no .filter → accepts all; no .use → no transformations + + runner.register(copyAll); + } +}); +``` + +`DdbProcessor` (and `OsProcessor`) auto-emit a `PutRecord` for `ctx.record` in their `onEnd` hook — pure-passthrough pipelines still produce writes without you calling `ctx.putRecord()` explicitly. `S3Processor` has no `onEnd`; file copies must be emitted explicitly via a transformer calling `ctx.copyFile(...)` (see `copyFileToTarget` in `writingTransformers.md`). + +## Real example: extending a built-in pipeline's transformer chain + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + createFilter, + isCmsEntry, + addLiveField, + replaceFileUrls, + MigrationConfig +} from "@webiny/data-transfer"; +import { stampMigratedAt } from "./transformers/stampMigratedAt.ts"; + +export default createTransferPreset({ + name: "my-v5-to-v6-ddb", + description: "v5-to-v6-ddb plus a custom stamp on every CMS entry.", + async configure({ runner, pipelineBuilderFactory, container }) { + const config = container.resolve(MigrationConfig); + const cmsEntries = await pipelineBuilderFactory + .create({ name: "CmsEntries", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .use(addLiveField) + .use(replaceFileUrls(config)) // factory — takes the resolved MigrationConfig, needs config.fileUrls set + .use(stampMigratedAt) + .build(); + + runner.register(cmsEntries); + // ...register any additional pipelines this preset needs. + } +}); +``` + +Prefer `PipelineCustomizer` (see `pipeline-customizer.md`) to patch a single pipeline of a built-in preset instead of re-registering all of its pipelines by hand — it's the supported extension point for "built-in preset + a few extra transformers." + +## Conditional blackholing using the resolved config + +Built-in presets use `container.resolve(MigrationConfig)` plus `.blackhole(condition)` to make a pipeline's disposition depend on the parsed config, evaluated once at `configure()` time: + +```typescript +async configure({ runner, pipelineBuilderFactory: factory, container }) { + const config = container.resolve(MigrationConfig); + + const auditLogs = await factory + .create({ name: "AuditLogs", scanner: DdbScanner, processors: [AuditLogProcessor] }) + .filter(createFilter(isAuditLogEntry)) + .use(auditLogTransformers) + .blackhole(() => !config.target.auditLog?.dynamodb?.tableName) + .build(); + + runner.register(auditLogs); +} +``` + +## Built-in presets + +Select by `name` when the wizard asks "Which preset do you want to run?" (or pass `--preset=`): + +| Name | Description | +| --- | --- | +| `v5-to-v6-ddb` | Full Webiny v5 → v6 migration of the primary DynamoDB table (CMS, File Manager, Security, Mailer, Folders, Audit Logs) | +| `v5-to-v6-os` | Migration of the OpenSearch companion DynamoDB table. Run **after** `v5-to-v6-ddb` | +| `copy-ddb` | Verbatim DynamoDB + S3 copy, no transformations | +| `copy-os` | Verbatim OpenSearch companion table copy, no transformations | +| `copy-files` | S3-only file copy | + +Custom presets placed in `pipeline.presetsDir` are listed alongside these five. Source: `src/presets/`. diff --git a/docs/mcp/guides/writingTransformers.md b/docs/mcp/guides/writingTransformers.md new file mode 100644 index 0000000..e2f41c5 --- /dev/null +++ b/docs/mcp/guides/writingTransformers.md @@ -0,0 +1,250 @@ +--- +name: writingTransformers +description: How to write custom transformers — createDdbTransformer/createOsTransformer/createTransformer factories, context types, processor slices, ctx.record, ctx.putRecord(), ctx.blackhole(). +category: Guides +--- + +# Writing transformers + +A transformer is a named function `(ctx) => void | Promise` that mutates `ctx.record`. Register it on a pipeline builder with `.use(...)`; transformers run in registration order, each seeing the mutations of the ones before it. + +Source: `docs/guides/writing-transformers.md`, `src/transformers/createTransformer.ts`, `src/transformers/createDdbTransformer.ts`, `src/transformers/createOsTransformer.ts`, `src/features/TransformContext/abstractions/`. + +## Factories + +```typescript +// src/transformers/createTransformer.ts +function createTransformer( + name: string, + fn: Transformer.Interface +): Transformer.Interface; + +// src/transformers/createDdbTransformer.ts +function createDdbTransformer( + name: string, + fn: Transformer.Interface +): Transformer.Interface; + +// src/transformers/createOsTransformer.ts +function createOsTransformer( + name: string, + fn: Transformer.Interface> +): Transformer.Interface>; +``` + +All three do the same thing at runtime — stamp a non-enumerable `transformerName` property onto `fn` for logging/debugging — and differ only in which context type they bind `fn`'s parameter to: + +- **`createTransformer(name, fn)`** — generic over any context type. Use for `BaseTransformContext` or `DdbCoreTransformContext` transformers, or anything the two convenience factories don't fit. +- **`createDdbTransformer(name, fn)`** — binds `DdbTransformContext.Interface` (Base + `DdbProcessor` slice + `S3Processor` slice). Default choice for v5-to-v6 DDB transformers. +- **`createOsTransformer(name, fn)`** — binds `OsTransformContext.Interface` (Base + `OsProcessor` slice). + +```typescript +import { createDdbTransformer } from "@webiny/data-transfer"; +import type { DdbTransformContext } from "@webiny/data-transfer"; + +export const stampMigratedAt = createDdbTransformer( + "stampMigratedAt", + (ctx: DdbTransformContext.Interface) => { + ctx.record.migratedAt = new Date().toISOString(); + } +); +``` + +**Compile-time contract, not just naming:** whichever context type you bind the transformer to must match the processors actually registered on any pipeline that uses it. A `createDdbTransformer` transformer calling `ctx.copyFile(...)` will only type-check on a pipeline whose `processors` includes `S3Processor`. + +## Context type aliases + +Use the narrowest type that covers what your transformer needs. Source: `src/features/TransformContext/abstractions/contextAliases.ts`. + +| Type | Processors required in pipeline | When to use | +| --- | --- | --- | +| `BaseTransformContext.Interface` | any | Only touches `ctx.record`, `ctx.cache`, `ctx.logger`, etc. — no processor-specific helpers. | +| `DdbCoreTransformContext.Interface` | `DdbProcessor` only | Needs `querySourceRecord` / `queryTargetRecord` / `putRecord` but not S3 helpers. | +| `DdbTransformContext.Interface` | `DdbProcessor` + `S3Processor` | Default for v5-to-v6 DDB transformers that may call `ctx.copyFile` / `ctx.getFile`. | +| `OsTransformContext.Interface` | `OsProcessor` | OS transformers. `ctx.record.data` is the decompressed payload — always present. | + +All four are generic over `TRecord`, defaulting to `BaseRecord`. Import from `@webiny/data-transfer`. + +## Base context API + +Available on every transformer context regardless of pipeline configuration (`src/features/TransformContext/abstractions/BaseTransformContext.ts`): + +```typescript +interface BaseTransformContext { + record: TRecord; + readonly original: Readonly; + readonly modelProvider: ModelProvider.Interface; + readonly cache: Cache.Interface; + readonly logger: Logger.Interface; + readonly compressionHandler: CompressionHandler.Interface; + replace(newRecord: TRecord): void; + addCommand(cmd: Command): void; + blackhole(): void; + readonly isBlackholed: boolean; +} +``` + +| Member | Description | +| --- | --- | +| `ctx.record` | Mutable record. Transformers mutate this directly. | +| `ctx.original` | Frozen, deep-cloned pre-transform snapshot — **always present**, never touched by earlier transformers in the chain. Use for gate-checks or audit comparisons; never modify it. | +| `ctx.replace(newRecord)` | Replace `ctx.record` wholesale (rather than mutating field by field). | +| `ctx.addCommand(cmd)` | Push a raw command onto the command bag. Processor slice helpers (`putRecord`, `copyFile`, `putAuditLog`) are sugar over this — reach for it directly only when emitting a command type no slice helper covers. | +| `ctx.modelProvider` | Loaded CMS models (from DB + `pipeline.modelsDir` JSON files if configured). `ctx.modelProvider.getModel(modelId)`. | +| `ctx.cache` | Shared `Map`-like cache, persists across records **within a shard** (not across shards/workers). Useful for dedup or memoizing lookups. | +| `ctx.logger` | Logger bound to the current worker — use instead of `console.*`; respects `debug.logLevel`. | +| `ctx.compressionHandler` | Gzip compression utility (used internally by OS record handling). Rarely needed directly. | +| `ctx.blackhole()` | Per-record blackholing — suppresses all writes for **this record only**. Remaining transformers and each processor's `onEnd` still run; the runner discards the accumulated commands before forwarding to processors. Irreversible for the record's lifetime. | +| `ctx.isBlackholed` | Read-only flag; `true` after `ctx.blackhole()` has been called for this record. | + +## Processor slices + +Each processor class in a pipeline's `processors` array contributes additional helpers onto the effective context — types intersect via `MergeSlices`, so the context your transformer receives is `BaseTransformContext & `. + +### `DdbProcessor` slice + +Present on `DdbCoreTransformContext` and `DdbTransformContext`. + +```typescript +interface DdbProcessorSlice { + putRecord(record: Record): void; + querySourceRecord>(pk: string, sk?: string): Promise; + queryTargetRecord>(pk: string, sk?: string): Promise; +} +``` + +| Member | Description | +| --- | --- | +| `ctx.putRecord(record)` | Emit an **extra** `PutRecord` to the DDB target, beyond the automatic put at chain end (see Auto-put below). Use for transformers that need to write a second/related record (e.g. a denormalized index entry). | +| `ctx.querySourceRecord(pk, sk?)` | Query the source DDB primary table directly. Returns `null` if not found. | +| `ctx.queryTargetRecord(pk, sk?)` | Query the target DDB primary table directly. Returns `null` if not found. | + +### `S3Processor` slice + +Present on `DdbTransformContext` only (not `DdbCoreTransformContext`). + +```typescript +interface S3ProcessorSlice { + copyFile(sourceKey: string, targetKey: string): void; + getFile(key: string): Promise; +} +``` + +| Member | Description | +| --- | --- | +| `ctx.copyFile(sourceKey, targetKey)` | Emit an S3 copy command (source → target bucket, per config). Keys may differ — e.g. reshaping the storage path during migration. | +| `ctx.getFile(key)` | Read a file from the **source** bucket. Returns `Buffer \| null`. | + +### `OsProcessor` slice + +Present on `OsTransformContext`. Same member names as the DDB slice but reading/writing the OS companion DDB table: + +```typescript +interface OsProcessorSlice { + putRecord(record: Record): void; + querySourceRecord>(pk: string, sk?: string): Promise; + queryTargetRecord>(pk: string, sk?: string): Promise; +} +``` + +Because the key names collide with `DdbProcessorSlice`, `[DdbProcessor, OsProcessor]` cannot be registered together on the same pipeline — the factory's disjoint-keys check rejects it at compile time. + +### `AuditLogProcessor` slice + +Not one of the four context type aliases (no dedicated `AuditLogTransformContext` export), but usable via `createTransformer` with an inline intersection, or accessible implicitly when writing the audit log transformer chain: + +```typescript +interface AuditLogProcessorSlice { + putAuditLog(record: Record): void; +} +``` + +`ctx.putAuditLog(record)` emits a put to the configured audit log table. No-op if `target.auditLog` is not configured. + +## Auto-put behavior + +`DdbProcessor`, `OsProcessor`, and `AuditLogProcessor` each register an `onEnd` hook that fires once per record, after all transformers have run: it calls `putRecord(ctx.record)` (or `putAuditLog(ctx.record)`) automatically. This is why a **zero-transformer pipeline still writes records** — see the "Zero-transformer preset" example in `writingPresets.md`. + +`S3Processor` has **no** `onEnd` — nothing is copied unless a transformer explicitly calls `ctx.copyFile(...)`. + +`onEnd` hooks run **sequentially in processor array order** when a pipeline has multiple processors (e.g. `[DdbProcessor, S3Processor]` → DDB auto-put runs, then S3Processor's `onEnd` runs — which is a no-op since S3Processor doesn't define one). + +## Built-in transformer factories that take config + +Not every built-in transformer is a bare `(ctx) => void` — some are factory functions that must be **called with a config argument** before `.use(...)`: + +```typescript +import { replaceFileUrls, MigrationConfig } from "@webiny/data-transfer"; + +// replaceFileUrls(config: MigrationConfig.Interface) => Transformer> +async configure({ runner, pipelineBuilderFactory: factory, container }) { + const config = container.resolve(MigrationConfig); + + const pipeline = await factory + .create({ name: "cms-entries", scanner: DdbScanner, processors: [DdbProcessor] }) + .use(replaceFileUrls(config)) // NOTE: called, not passed bare + .build(); + + runner.register(pipeline); +} +``` + +`replaceFileUrls` requires a `fileUrls: { source, target }` block at the config root (see `configReference.md`) — it's a no-op transformer if that block is absent. + +Most other built-ins (`wrapInData`, `addGsiTenant`, `removeLocale`, `groupsToRoles`, etc.) are plain transformers — pass them bare to `.use(...)`. Check each transformer's doc under `docs/mcp/transformers/` for its exact call shape before wiring it in. + +## Full example: DDB transformer with a source lookup + +```typescript +import { createDdbTransformer } from "@webiny/data-transfer"; +import type { DdbTransformContext } from "@webiny/data-transfer"; + +export const enrichFromSource = createDdbTransformer( + "enrichFromSource", + async (ctx: DdbTransformContext.Interface) => { + if (ctx.record.TYPE !== "cms.entry") { + return; + } + + const related = await ctx.querySourceRecord<{ title: string }>( + `T#${ctx.record.tenant}#L#en-US#CMS#CMG#category`, + ctx.record.categoryId as string + ); + + if (!related) { + ctx.logger.warn(`enrichFromSource: category not found for ${ctx.record.PK}`); + return; + } + + ctx.record.categoryTitle = related.title; + } +); +``` + +## Full example: OS transformer + +```typescript +import { createOsTransformer } from "@webiny/data-transfer"; +import type { OsTransformContext } from "@webiny/data-transfer"; + +export const dropInternalField = createOsTransformer( + "dropInternalField", + (ctx: OsTransformContext.Interface) => { + const data = ctx.record.data as Record | undefined; + if (data) { + delete data.internalDebugFlag; + } + } +); +``` + +## Built-in processors reference + +| Processor | Slice helpers | Auto-put (`onEnd`) | Notes | +| --- | --- | --- | --- | +| `DdbProcessor` | `putRecord`, `querySourceRecord`, `queryTargetRecord` | Yes | Primary DDB table. | +| `S3Processor` | `copyFile`, `getFile` | No | S3 bucket — emit copies explicitly via `ctx.copyFile`. | +| `OsProcessor` | `putRecord`, `querySourceRecord`, `queryTargetRecord` | Yes | OS companion DDB table. Gzips on write, ensures the target index exists. | +| `AuditLogProcessor` | `putAuditLog` | Yes | Writes to the audit log table. No-op when `target.auditLog` is null/unset. | + +For built-in ready-made transformers (e.g. `copyFileToTarget`), see `docs/mcp/transformers/`. diff --git a/docs/mcp/presets/copy-ddb.md b/docs/mcp/presets/copy-ddb.md new file mode 100644 index 0000000..8ac2b8b --- /dev/null +++ b/docs/mcp/presets/copy-ddb.md @@ -0,0 +1,53 @@ +--- +name: copy-ddb +description: Verbatim copy of a regular DynamoDB table — no transformations. +category: Presets +--- + +# copy-ddb + +**Use when:** you need to copy a Webiny primary DynamoDB table from one environment to another (e.g. prod → dev) with the data left completely untouched — no migration, no reshaping, no filtering. + +**What it does:** + +- Scans every item in the source DynamoDB table. +- Writes every item to the target DynamoDB table unchanged. +- No records are filtered, transformed, or blackholed — everything that is scanned is written. + +**Pipelines registered:** + +| Pipeline | Scanner | Processors | Filter | Transformers | +| ---------------------------------- | ------------ | -------------- | ------ | ------------ | +| `Regular DynamoDB Table Data` | `DdbScanner` | `[DdbProcessor]` | none | none | + +**Transformers applied:** + +None — pure copy. `DdbProcessor.onEnd` emits a `PutRecord` for the scanned record as-is. + +**Example usage in a custom preset:** + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-ddb-copy", + description: "Copy the DynamoDB table, same as copy-ddb.", + configure({ runner, pipelineBuilderFactory }) { + const everything = pipelineBuilderFactory + .create({ + name: "Regular DynamoDB Table Data", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .build(); // no .filter, no .use → verbatim copy + + runner.register(everything); + } +}); +``` + +Select it directly with `--preset=copy-ddb` (or pick it from the wizard) instead of reimplementing it if you just need a plain copy. diff --git a/docs/mcp/presets/copy-files.md b/docs/mcp/presets/copy-files.md new file mode 100644 index 0000000..e34f287 --- /dev/null +++ b/docs/mcp/presets/copy-files.md @@ -0,0 +1,59 @@ +--- +name: copy-files +description: Copy File Manager S3 files (and their DynamoDB records) verbatim — no transformations. +category: Presets +--- + +# copy-files + +**Use when:** you only need to copy File Manager files — the DynamoDB file record plus the underlying S3 object bytes — without touching any other table data and without any migration reshaping. + +**What it does:** + +- Scans the primary DynamoDB table via `DdbScanner`. +- Keeps only records that pass the `isFmFile` filter (File Manager file records). +- Writes the matched DynamoDB record verbatim via `DdbProcessor`. +- Copies the associated S3 object from source bucket to target bucket via `S3Processor`. +- Everything else scanned from the table is skipped by this pipeline (it's the only pipeline in the preset, so non-matching records are simply not written by this preset). + +**Pipelines registered:** + +| Pipeline | Scanner | Processors | Filter | Transformers | +| ------------ | ------------ | ------------------------------ | --------------------- | ------------ | +| `S3 Files` | `DdbScanner` | `[DdbProcessor, S3Processor]` | `createFilter(isFmFile)` | none | + +**Transformers applied:** + +None — pure copy. Both the DynamoDB record and the S3 file bytes are copied as-is; only a filter (`isFmFile`) narrows which records this pipeline claims. + +**Example usage in a custom preset:** + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + S3Processor, + createFilter, + isFmFile +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-files-copy", + description: "Copy File Manager files, same as copy-files.", + configure({ runner, pipelineBuilderFactory }) { + const s3Files = pipelineBuilderFactory + .create({ + name: "S3 Files", + scanner: DdbScanner, + processors: [DdbProcessor, S3Processor] + }) + .filter(createFilter(isFmFile)) + .build(); // no .use → verbatim copy of matched records + + runner.register(s3Files); + } +}); +``` + +Select it directly with `--preset=copy-files`, or combine it with `copy-ddb`/`copy-os` (as separate runs) if you need the rest of the data too — `copy-ddb` already includes File Manager records since it copies the whole table, so `copy-files` is mainly useful when you want *only* the files, or when composing a custom preset that needs the S3 copy step alongside other pipelines. diff --git a/docs/mcp/presets/copy-os.md b/docs/mcp/presets/copy-os.md new file mode 100644 index 0000000..91e980b --- /dev/null +++ b/docs/mcp/presets/copy-os.md @@ -0,0 +1,53 @@ +--- +name: copy-os +description: Verbatim copy of the OpenSearch companion DynamoDB table — no transformations. +category: Presets +--- + +# copy-os + +**Use when:** you need to copy Webiny's OpenSearch companion DynamoDB table (the "OS table" that indexes CMS entries for search) from one environment to another with no transformation — pairs with `copy-ddb` when you also want the primary table copied. + +**What it does:** + +- Scans every item in the source OpenSearch DynamoDB table via `OsScanner` (which decompresses/normalizes the OS record shape). +- Writes every item to the target OS table unchanged via `OsProcessor`. +- No records are filtered, transformed, or blackholed. + +**Pipelines registered:** + +| Pipeline | Scanner | Processors | Filter | Transformers | +| -------------------------------------- | ----------- | --------------- | ------ | ------------ | +| `OpenSearch DynamoDB Table Data` | `OsScanner` | `[OsProcessor]` | none | none | + +**Transformers applied:** + +None — pure copy. `OsProcessor.onEnd` emits a put for the scanned record as-is. + +**Example usage in a custom preset:** + +```typescript +import { + createTransferPreset, + OsScanner, + OsProcessor +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-os-copy", + description: "Copy the OpenSearch DDB table, same as copy-os.", + configure({ runner, pipelineBuilderFactory }) { + const everything = pipelineBuilderFactory + .create({ + name: "OpenSearch DynamoDB Table Data", + scanner: OsScanner, + processors: [OsProcessor] + }) + .build(); // no .filter, no .use → verbatim copy + + runner.register(everything); + } +}); +``` + +Only relevant if your target config has `target.opensearch` configured — the OS scanner/processor pair are registered conditionally on that setting. Select the built-in directly with `--preset=copy-os` instead of hand-rolling it. diff --git a/docs/mcp/presets/v5-to-v6-ddb.md b/docs/mcp/presets/v5-to-v6-ddb.md new file mode 100644 index 0000000..c38989b --- /dev/null +++ b/docs/mcp/presets/v5-to-v6-ddb.md @@ -0,0 +1,91 @@ +--- +name: v5-to-v6-ddb +description: Full Webiny v5 → v6 migration of the primary DynamoDB table (CMS, File Manager, Security, Mailer, Folders, Audit Logs). +category: Presets +--- + +# v5-to-v6-ddb + +**Use when:** migrating a Webiny v5 project's primary DynamoDB table to v6. This is the flagship preset — it applies all the domain-specific reshaping v6 requires (CMS entry/model shape changes, security groups→roles, file manager settings, mailer settings, folder permissions, audit logs) in one pass. Run `v5-to-v6-os` afterward if the project also has an OpenSearch companion table. + +**What it does:** + +- Scans the primary DynamoDB table via `DdbScanner`; runs 15 first-match-wins pipelines over it (registration order below — order is load-bearing because several record shapes overlap, e.g. File Manager files and Form Builder forms are also CMS entries). +- Blackholes (drops) migration-tracking records, ACO search cache records, background tasks, and Form Builder records (no v6 migration path yet for Form Builder). +- Audit logs are blackholed only if `target.auditLog.dynamodb.tableName` is not configured; otherwise they're transformed and written to the configured audit log table. +- Reshapes CMS groups, CMS models, CMS entries, File Manager settings/files, mailer settings, security groups (→ roles) and teams, and folder permissions (FLP records) into their v6 storage format. +- Copies admin user records verbatim (no transformer, still passes through the catch-all `DdbProcessor.onEnd` put). + +**Pipelines registered** (in registration order — first match wins): + +| # | Pipeline | Scanner + Processors | Filter | Notes | +| - | ---------------------- | ------------------------------------------------- | ---------------------------------------------------------------- | ----- | +| 1 | `MigrationRecords` | `DdbScanner` + `[DdbProcessor]` | `isMigrationRecord` | `.blackhole()` — always dropped | +| 2 | `AuditLogs` | `DdbScanner` + `[AuditLogProcessor]` | `isAuditLogEntry` | Must run before `AcoSearchRecordsPage`/`CmsEntries` (shares the `acoSearchRecord` modelId prefix). `.blackhole()` conditionally — only when `target.auditLog?.dynamodb?.tableName` is unset | +| 3 | `AcoSearchRecordsPage` | `DdbScanner` + `[DdbProcessor]` | `isAcoSearchRecord` | `.blackhole()` — always dropped | +| 4 | `ContentModelGroups` | `DdbScanner` + `[DdbProcessor]` | `isCmsGroup` | | +| 5 | `BackgroundTasks` | `DdbScanner` + `[DdbProcessor]` | `isBackgroundTask` | `.blackhole()` — always dropped | +| 6 | `FileManagerSettings` | `DdbScanner` + `[DdbProcessor]` | `byType("fm.settings")` | | +| 7 | `FileManagerFiles` | `DdbScanner` + `[DdbProcessor, S3Processor]` | `isFmFile` | Must run before `CmsEntries` (fm files are also CMS entries) | +| 8 | `MailerSettings` | `DdbScanner` + `[DdbProcessor]` | inline: `record.SK === "L" && record.modelId === "mailerSettings"` | | +| 9 | `SecurityGroups` | `DdbScanner` + `[DdbProcessor]` | inline: `record.TYPE === "security.group" && !isBuiltInSecurityRole(record)` | | +| 10 | `SecurityTeams` | `DdbScanner` + `[DdbProcessor]` | `isSecurityTeam` | | +| 11 | `CmsModels` | `DdbScanner` + `[DdbProcessor]` | `isCmsModel` | | +| 12 | `FolderPermissions` | `DdbScanner` + `[DdbProcessor]` | `isFlpRecord` | | +| 13 | `CmsEntries` | `DdbScanner` + `[DdbProcessor]` | `isCmsEntry` | Catch-all for remaining CMS entries; must run after `FileManagerFiles` | +| 14 | `AdminUsers` | `DdbScanner` + `[DdbProcessor]` | `isAdminUser` | No transformers — verbatim copy | +| 15 | `FormBuilderRecords` | `DdbScanner` + `[DdbProcessor]` | `isFormBuilderRecord` | `.blackhole()` — no v6 migration path yet; must run after `CmsEntries` (FB forms are CMS entries and would otherwise be claimed there) | + +**Transformers applied** (in pipeline order): + +- `AuditLogs`: `coreFieldsTransformer` → `dataFieldsTransformer` → `storageShapeTransformer` (the `auditLogTransformers` bundle) +- `ContentModelGroups`: `wrapInData` → `addGsiTenant` → `removeLocale` → `removeAttributes` +- `FileManagerSettings`: `wrapInData` → `migrateFileManagerSettings` → `removeAttributes` +- `FileManagerFiles`: `cmsEntryTransformers` bundle (`wrapInData` → `addGsiTenant` → `removeLocale` → `fixCmePk` → `fixBrokenStorageKeys` → `transformRichText` → `updateModelIds` → `removeFolderRevision` → `removeAttributes`) → `createMetadata` → `extractImageMetadata` +- `MailerSettings`: `wrapInData` → `migrateMailerSettings` → `removeAttributes` +- `SecurityGroups`: `wrapInData` → `addGsiTenant` → `groupsToRoles` → `transformPermissions` → `removeAttributes` +- `SecurityTeams`: `wrapInData` → `addGsiTenant` → `removeAttributes` +- `CmsModels`: `wrapInData` → `addGsiTenant` → `removeLocale` → `transformModelGroup` → `renameFieldAttributes` → `removeAttributes` +- `FolderPermissions`: `wrapInData` → `addGsiTenant` → `removeLocale` → `removeAttributes` → `updateFlpIds` +- `CmsEntries`: `cmsEntryTransformers` bundle → `addLiveField` → `replaceFileUrls(config)` +- `AdminUsers`: none — pure copy +- Blackholed pipelines (`MigrationRecords`, `AcoSearchRecordsPage`, `BackgroundTasks`, `FormBuilderRecords`, conditionally `AuditLogs`): no transformers run against their output because everything they emit is discarded, but filters still evaluate. + +**Example usage in a custom preset:** + +Extend or override one pipeline from this preset rather than rewriting all 15 — e.g. add a project-specific transformer to the CMS entries catch-all: + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + createFilter, + isCmsEntry, + cmsEntryTransformers, // not currently a public export — copy the chain manually if unavailable + addLiveField, + replaceFileUrls +} from "@webiny/data-transfer"; +import { stampMigratedAt } from "./transformers/stampMigratedAt.ts"; + +export default createTransferPreset({ + name: "my-v5-to-v6-ddb", + description: "v5-to-v6-ddb plus a custom stamp on every CMS entry.", + configure({ runner, pipelineBuilderFactory, container }) { + const cmsEntries = pipelineBuilderFactory + .create({ name: "CmsEntries", scanner: DdbScanner, processors: [DdbProcessor] }) + .filter(createFilter(isCmsEntry)) + .use(cmsEntryTransformers) + .use(addLiveField) + .use(replaceFileUrls(container.resolve(/* MigrationConfig */))) + .use(stampMigratedAt) + .build(); + + runner.register(cmsEntries); + // ...register the remaining 14 pipelines from this preset, in the same order, + // or use PipelineCustomizer to patch the built-in preset instead of copying it. + } +}); +``` + +In practice, prefer `PipelineCustomizer` (see `pipeline-customizer.md`) to patch a single pipeline of this built-in preset instead of re-registering all 15 by hand. diff --git a/docs/mcp/presets/v5-to-v6-os.md b/docs/mcp/presets/v5-to-v6-os.md new file mode 100644 index 0000000..4506ac3 --- /dev/null +++ b/docs/mcp/presets/v5-to-v6-os.md @@ -0,0 +1,98 @@ +--- +name: v5-to-v6-os +description: Webiny v5 → v6 migration of the OpenSearch companion DynamoDB table. +category: Presets +--- + +# v5-to-v6-os + +**Use when:** migrating a Webiny v5 project's OpenSearch companion DynamoDB table (the table that backs CMS search indexing) to v6. Run this **after** `v5-to-v6-ddb` — it only makes sense once the primary table has already been migrated. Only relevant if the project has `target.opensearch` configured. + +**What it does:** + +- Scans the OpenSearch DynamoDB table via `OsScanner` (which decompresses/normalizes the OS record shape); runs 5 first-match-wins pipelines over it via `OsProcessor`. +- Blackholes (drops) ACO search records, background tasks, and mailer settings — none of these have a v6 target in the OS table (mailer settings migrate via the DDB preset into the KV store instead). +- Reshapes File Manager file records and all remaining CMS entries into v6 storage format, mirroring the equivalent DDB-preset transformer chain but adapted for the OS record shape (`data` is already populated by `OsScanner`, so `wrapInData` is not needed). + +**Pipelines registered** (in registration order — first match wins): + +| # | Pipeline | Scanner + Processors | Filter | Notes | +| - | -------------------- | -------------------------------- | ---------------------- | ----- | +| 1 | `AcoSearchRecords` | `OsScanner` + `[OsProcessor]` | `isAcoSearchRecord` | `.blackhole()` — always dropped | +| 2 | `BackgroundTasks` | `OsScanner` + `[OsProcessor]` | `isOsBackgroundTask` | `.blackhole()` — must run before `CmsEntries` (background tasks are CMS entries in the OS table) | +| 3 | `MailerSettings` | `OsScanner` + `[OsProcessor]` | `isOsMailerSettings` | `.blackhole()` — v6 stores mailer settings in the KV store, migrated by the DDB preset; must run before `CmsEntries` | +| 4 | `FileManagerFiles` | `OsScanner` + `[OsProcessor]` | `isFmFile` | Must run before `CmsEntries` (fm files satisfy `isCmsEntry` via TYPE prefix) | +| 5 | `CmsEntries` | `OsScanner` + `[OsProcessor]` | `isCmsEntry` | Catch-all for remaining CMS entries | + +**Transformers applied** (in pipeline order): + +- `AcoSearchRecords`, `BackgroundTasks`, `MailerSettings`: none — everything they emit is discarded via `.blackhole()`, but filters still evaluate. +- `FileManagerFiles`: `osCmsEntryTransformers` bundle (`addGsiTenant` → `removeLocale` → `fixCmePk` → `fixBrokenStorageKeys` → `transformRichText` → `updateModelIds` → `updateOsIndex` → `removeFolderRevision` → `removeAttributes` → `addTransferTimestamp`) +- `CmsEntries`: `osCmsEntryTransformers` bundle → `addLiveField` → `replaceFileUrls(config)` + +**Example usage in a custom preset:** + +Extend the catch-all `CmsEntries` pipeline with a project-specific transformer, keeping the same registration order so the blackholed pipelines still claim their records first: + +```typescript +import { + createTransferPreset, + OsScanner, + OsProcessor, + createFilter, + isAcoSearchRecord, + isOsBackgroundTask, + isOsMailerSettings, + isFmFile, + isCmsEntry, + addLiveField, + replaceFileUrls +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-v5-to-v6-os", + description: "v5-to-v6-os plus a custom transformer on the CMS entries catch-all.", + configure({ runner, pipelineBuilderFactory, container }) { + const acoSearchRecords = pipelineBuilderFactory + .create({ name: "AcoSearchRecords", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isAcoSearchRecord)) + .blackhole() + .build(); + + const backgroundTasks = pipelineBuilderFactory + .create({ name: "BackgroundTasks", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isOsBackgroundTask)) + .blackhole() + .build(); + + const mailerSettings = pipelineBuilderFactory + .create({ name: "MailerSettings", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isOsMailerSettings)) + .blackhole() + .build(); + + const fileManagerFiles = pipelineBuilderFactory + .create({ name: "FileManagerFiles", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isFmFile)) + // osCmsEntryTransformers is not currently a public export — build the equivalent + // chain yourself, or import it from this preset's source as a reference. + .build(); + + const cmsEntries = pipelineBuilderFactory + .create({ name: "CmsEntries", scanner: OsScanner, processors: [OsProcessor] }) + .filter(createFilter(isCmsEntry)) + .use(addLiveField) + .use(replaceFileUrls(container.resolve(/* MigrationConfig */))) + .build(); + + runner + .register(acoSearchRecords) + .register(backgroundTasks) + .register(mailerSettings) + .register(fileManagerFiles) + .register(cmsEntries); + } +}); +``` + +In practice, prefer `PipelineCustomizer` (see `pipeline-customizer.md`) to patch a single pipeline of this built-in preset instead of re-registering all 5 by hand. diff --git a/docs/mcp/processors/auditLogProcessor.md b/docs/mcp/processors/auditLogProcessor.md new file mode 100644 index 0000000..764361c --- /dev/null +++ b/docs/mcp/processors/auditLogProcessor.md @@ -0,0 +1,60 @@ +--- +name: AuditLogProcessor +description: Writes audit-log entries scanned from the source table to a dedicated target audit-log table. +category: Processors +--- + +# AuditLogProcessor + +**Import:** `import { AuditLogProcessor } from "@webiny/data-transfer";` + +**What it does:** a narrow, opt-in persistence processor for Webiny's audit log records. It reads `config.target.auditLog?.dynamodb?.tableName` (nullable — audit log transfer is optional) and, when configured, queues writes to that dedicated table separately from the main `DdbProcessor` target table. `checkAccess()` returns `[]` (no check) when no audit log table is configured; otherwise it `describeTable`s the target audit log table the same way `DdbProcessor` does for its own table. + +**Context slice it adds:** + +- `ctx.putAuditLog(record)` — queues an `AuditLogPutRecord` command (`{ table: auditLogTableName, record }`), but only if **both** conditions hold: an audit log table name is configured, and `record.TYPE === "auditLog.log"`. If either check fails, the call is a silent no-op — this makes it safe to call unconditionally from `onEnd` without every transformer having to check record type or config first. + +**Commands it handles:** `AuditLogPutRecord` (key `AUDIT_LOG_PUT_RECORD`). `execute()` returns immediately if no audit log table is configured; otherwise it maps each `AuditLogPutRecord` to a `PutRecord` (same `table`/`record`, just re-wrapped as the command type `DdbExecutor` understands) and forwards the batch to `DdbExecutor.execute()`. Note this processor does **not** respect `transferContext.dryRun` directly — the guard is purely the missing-table-name check (this differs from `DdbProcessor`/`OsProcessor`/`S3Processor`, all of which check `dryRun` explicitly). + +**`onEnd` hook behavior:** automatically calls `ctx.putAuditLog(ctx.record)` after the pipeline's transformers run. Combined with the slice guard above, this means a pipeline using `AuditLogProcessor` only writes records that are still typed `auditLog.log` by the time `onEnd` fires and only if the target has an audit log table configured — otherwise the pipeline's `.blackhole(() => !config.target.auditLog?.dynamodb?.tableName)` pattern (see usage below) is the idiomatic way to make the intent explicit and avoid the runner's unclaimed-command warning when the table is absent. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { + createTransferPreset, + DdbScanner, + AuditLogProcessor, + MigrationConfig, + createFilter, + isAuditLogEntry, + coreFieldsTransformer, + dataFieldsTransformer, + storageShapeTransformer +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-preset-with-audit-logs", + description: "Audit log transfer, gated on target.auditLog being configured.", + async configure({ runner, pipelineBuilderFactory, container }) { + const config = container.resolve(MigrationConfig); + + const auditLogs = await pipelineBuilderFactory + .create({ + name: "AuditLogs", + scanner: DdbScanner, + processors: [AuditLogProcessor] + }) + .filter(createFilter(isAuditLogEntry)) + .use(coreFieldsTransformer) + .use(dataFieldsTransformer) + .use(storageShapeTransformer) + .blackhole(() => !config.target.auditLog?.dynamodb?.tableName) + .build(); + + runner.register(auditLogs); + } +}); +``` + +This mirrors the pattern used internally in the `v5-to-v6-ddb` preset (which composes the same three transformers as a single internal `auditLogTransformers` array, not part of the public API — the individual transformer functions above are the public equivalent). It must be registered **before** the `AcoSearchRecordsPage` and `CmsEntries` pipelines because audit log records share the same `acoSearchRecord` modelId prefix and would otherwise be claimed by those pipelines first (first-match-wins dispatch). diff --git a/docs/mcp/processors/ddbProcessor.md b/docs/mcp/processors/ddbProcessor.md new file mode 100644 index 0000000..8445413 --- /dev/null +++ b/docs/mcp/processors/ddbProcessor.md @@ -0,0 +1,45 @@ +--- +name: DdbProcessor +description: Writes scanned/transformed records to the target DynamoDB table. +category: Processors +--- + +# DdbProcessor + +**Import:** `import { DdbProcessor } from "@webiny/data-transfer";` + +**What it does:** the default persistence processor for regular DynamoDB table pipelines. It reads `config.source.dynamodb.tableName` / `config.target.dynamodb.tableName` from `MigrationConfig`, exposes helpers on the transform context for queuing writes and looking up records on either side, and drains queued `PutRecord` commands into the target table via `DdbExecutor` at flush time. Its `checkAccess()` probes both the source and target tables with `describeTable` and reports `ok` / `denied` / `missing` / `unknown` so the orchestrator can abort before spawning workers if credentials or table names are wrong. + +**Context slice it adds:** + +- `ctx.putRecord(record)` — queues a `PutRecord` command (`{ table: targetTable, record }`) into the pending buffer via `ctx.addCommand`. Does not write immediately — commands accumulate and are drained in batches (`tuning.flushEvery`, default 500). +- `ctx.querySourceRecord(pk, sk?)` — queries the **source** DynamoDB table directly (bypasses the pipeline/scanner) and returns the first matching item or `null`. Useful when a transformer needs to look up a related record that isn't the one currently being processed. +- `ctx.queryTargetRecord(pk, sk?)` — same as above but against the **target** table. Useful for checking whether something was already migrated. + +**Commands it handles:** `PutRecord` (key `PUT_RECORD`). `execute()` calls `commands.get(PutRecord.key)` — this marks the key as "claimed" so the runner's unclaimed-command warning doesn't fire — and forwards every collected `PutRecord` to `DdbExecutor.execute()`, which performs the actual batched `BatchWriteItem` calls against the target table. If `transferContext.dryRun` is set, `execute()` returns immediately without writing anything (commands are still collected and shown in dry-run reporting, just never sent to AWS). + +**`onEnd` hook behavior:** automatically calls `ctx.putRecord(ctx.record)` — i.e. by default, **the record scanned by `DdbScanner` is written to the target table verbatim**, after any transformers in the pipeline have mutated `ctx.record` in place. This is what gives "zero transformers required" pipelines (like `copy-ddb`) their behavior: a pipeline with `processors: [DdbProcessor]` and no `.use(...)` calls is a pure 1:1 copy. To suppress the write for a pipeline (e.g. to blackhole/drop records), call `.blackhole()` on the pipeline builder — that discards all commands collected during the pipeline, `onEnd` included — rather than trying to prevent `onEnd` from running. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, DdbScanner, DdbProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-ddb-copy", + description: "Copy the DynamoDB table, same as copy-ddb.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "Regular DynamoDB Table Data", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .build(); // no .filter, no .use → verbatim copy of every scanned record + + runner.register(everything); + } +}); +``` + +`DdbProcessor` is also combined with other processors in the same pipeline when a record needs more than one side effect — e.g. `processors: [DdbProcessor, S3Processor]` in the `FileManagerFiles` pipeline of `v5-to-v6-ddb`, where the DDB record is written and an associated S3 file may be copied in the same pass. Multiple processors in one array run sequentially (both `onEnd` and `execute`, in array order) and their context slices are merged, so a single processor's `putRecord`/`copyFile` calls don't collide as long as their slice keys are disjoint. diff --git a/docs/mcp/processors/osProcessor.md b/docs/mcp/processors/osProcessor.md new file mode 100644 index 0000000..560e090 --- /dev/null +++ b/docs/mcp/processors/osProcessor.md @@ -0,0 +1,49 @@ +--- +name: OsProcessor +description: Writes scanned/transformed records to the target OpenSearch-backed DynamoDB table, managing index lifecycle and gzip compression. +category: Processors +--- + +# OsProcessor + +**Import:** `import { OsProcessor } from "@webiny/data-transfer";` + +**What it does:** the persistence processor for OpenSearch-table pipelines (the DynamoDB table that backs a Webiny OpenSearch index, not the search cluster itself). It reads `config.source.opensearch.tableName` / `config.target.opensearch.tableName`, exposes the same put/query helper shape as `DdbProcessor`, and at flush time gzip-compresses each record's `data` field, ensures the target OpenSearch index exists (creating it with resolved mappings/settings if missing, or temporarily disabling `refresh_interval` on an existing one for faster bulk writes), then writes through `DdbExecutor`. `checkAccess()` calls `osClient.listIndexes()` against the target cluster and classifies HTTP 401/403 as `denied`, 404 as `missing`. `afterShard()` persists the list of touched indexes (and their original `refresh_interval`) to a per-segment state file under `.transfer//` so a later orchestrator-side hook can restore refresh settings once all shards finish. + +**Context slice it adds:** + +- `ctx.putRecord(record)` — queues a `PutRecord` (`{ table: targetTable, record }`). Throws at pipeline setup (`extendContext`) if `config.target.opensearch` or `config.source.opensearch` is missing — this fails fast rather than silently writing nowhere. +- `ctx.querySourceRecord(pk, sk?)` — queries the source OpenSearch-table directly by PK/SK. +- `ctx.queryTargetRecord(pk, sk?)` — same, against the target OpenSearch-table. + +**Commands it handles:** `PutRecord` (key `PUT_RECORD`). `execute()`: +1. Returns immediately if `transferContext.dryRun` or there are no queued puts. +2. Gzip-compresses `record.data` on every put, in batches sized by `config.tuning.os.gzipConcurrency` (default 16). +3. Collects the distinct `record.index` values across the batch and calls `ensureIndex()` for each — creating the index or disabling refresh on an existing one, with retry (`config.tuning.os.retryScheduleMs`, default `[5000, 10000, 20000, 30000, 30000]` ms) on retryable AWS errors. +4. Forwards the gzipped puts to `DdbExecutor.execute()`. + +**`onEnd` hook behavior:** automatically calls `ctx.putRecord(ctx.record)` — the scanned/transformed OS record is written to the target table by default, same "zero transformers required" contract as `DdbProcessor`. Use `.blackhole()` on the pipeline builder to suppress the write for a whole pipeline (e.g. `AcoSearchRecords`, `BackgroundTasks`, `MailerSettings` in `v5-to-v6-os` are all blackholed). + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, OsScanner, OsProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-os-copy", + description: "Copy the OpenSearch-backed table, same as copy-os.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "OpenSearch DynamoDB Table Data", + scanner: OsScanner, + processors: [OsProcessor] + }) + .build(); // no .filter, no .use → verbatim copy of every scanned record + + runner.register(everything); + } +}); +``` + +`OsProcessor` requires `OsScanner` as its paired scanner (both operate on the OpenSearch-table record shape, `{ index, data, ...BaseRecord }`) and is only registered by `bootstrap.ts` when `config.target.opensearch != null`. The related `OsIndexPrefixHook` (a `BeforeTransferHook`, not a `Processor`) sets `process.env.OPENSEARCH_INDEX_PREFIX` from `config.target.opensearch.indexPrefix` once before the transfer starts, so index names resolved during `ensureIndex()` carry the right prefix. diff --git a/docs/mcp/processors/s3Processor.md b/docs/mcp/processors/s3Processor.md new file mode 100644 index 0000000..29c15a2 --- /dev/null +++ b/docs/mcp/processors/s3Processor.md @@ -0,0 +1,52 @@ +--- +name: S3Processor +description: Copies S3 objects (e.g. File Manager files) from the source bucket to the target bucket. +category: Processors +--- + +# S3Processor + +**Import:** `import { S3Processor } from "@webiny/data-transfer";` + +**What it does:** handles S3-side side effects for pipelines that also touch DynamoDB records referencing files (typically File Manager entries). It reads `config.source.s3.bucket` / `config.target.s3.bucket`, exposes helpers to read a source object or queue a copy, and at flush time issues batched `CopyObject` calls via `targetS3.batchCopy()`. `checkAccess()` runs `headBucket` against both source and target buckets, and — when source and target accounts differ (cross-account transfer) — adds an extra check that probes the source bucket using the **target** account's credentials (since `CopyObject` executes with target credentials), returning a `hint` explaining the bucket policy needed if that probe is denied. + +**Context slice it adds:** + +- `ctx.copyFile(sourceKey, targetKey)` — queues an `S3Copy` command (`{ sourceBucket, sourceKey, targetBucket, targetKey }`). No file bytes move at call time; the actual copy happens in `execute()`. +- `ctx.getFile(key)` — reads an object from the **source** bucket directly, returning its body as a `Buffer` or `null` if absent. Useful for transformers that need to inspect file content (e.g. `extractImageMetadata`) before deciding what to queue. + +**Commands it handles:** `S3Copy` (key `S3_COPY`). `execute()` returns immediately under `dryRun` or if there are no queued copies; otherwise it maps each `S3Copy` command to a `{ sourceBucket, sourceKey, targetBucket, targetKey }` tuple and calls `targetS3.batchCopy(...)` once for the whole batch. + +**`onEnd` hook behavior:** **none — `S3Processor` has no `onEnd` hook.** Unlike `DdbProcessor`/`OsProcessor`, there is no sensible per-record default for "copy this file" (not every record has an associated file, and the same record's transformers may not want the file copied at all). Transformers must explicitly call `ctx.copyFile(sourceKey, targetKey)` when a copy is wanted; if none do, nothing is queued and `execute()` is a no-op for that pipeline. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { + createTransferPreset, + DdbScanner, + DdbProcessor, + S3Processor, + createFilter, + isFmFile +} from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "copy-files", + description: "Copy all the S3 files loaded via DynamoDB regular table - pure copy.", + async configure({ runner, pipelineBuilderFactory }) { + const s3Files = await pipelineBuilderFactory + .create({ + name: "S3 Files", + scanner: DdbScanner, + processors: [DdbProcessor, S3Processor] + }) + .filter(createFilter(isFmFile)) + .build(); + + runner.register(s3Files); + } +}); +``` + +`S3Processor` is always paired with a DDB-side processor (`DdbProcessor`) in the same pipeline — it never appears alone, since it scans records via `DdbScanner` (the File Manager entry) and only conditionally acts on the file itself. In `copy-files` it runs after a plain `isFmFile` filter with no transformers (pure copy of the FM record plus, if a transformer queued one, the underlying file). In `v5-to-v6-ddb`'s `FileManagerFiles` pipeline it runs alongside `cmsEntryTransformers`, `createMetadata`, and `extractImageMetadata`, though that pipeline currently has file copying commented out (`// .blackhole()` left as a TODO marker) rather than removed. diff --git a/docs/mcp/scanners/ddbScanner.md b/docs/mcp/scanners/ddbScanner.md new file mode 100644 index 0000000..345863f --- /dev/null +++ b/docs/mcp/scanners/ddbScanner.md @@ -0,0 +1,39 @@ +--- +name: DdbScanner +description: Scans every item in the source DynamoDB table, segment by segment. +category: Scanners +--- + +# DdbScanner + +**Import:** `import { DdbScanner } from "@webiny/data-transfer";` + +**Scan behavior:** reads `config.source.dynamodb.tableName` and performs a parallel `Scan` against it via `SourceDynamoDbClient`, using DynamoDB's native segment/totalSegments parallel-scan support. `scan(shard)` is an async generator that yields raw table items one at a time as they page in — there is no buffering of the whole table in memory, and no transformation or filtering happens at the scanner level (that's the pipeline's job downstream). + +**Record shape:** yields `BaseRecord` — `{ PK, SK, _et, _ct, _md, TYPE, [key: string]: unknown }`. This is the raw DynamoDB item exactly as stored, with only the four Webiny bookkeeping fields (`_et`, `_ct`, `_md`, `TYPE`) guaranteed present alongside `PK`/`SK`; everything else is whatever attributes the record happens to have. No decompression or decoding is applied — that distinguishes it from `OsScanner`, which decompresses a `data` payload. + +**Segment support:** `listShards()` returns `total = config.pipeline?.segments ?? 1` shards, each `{ segment: i, total }`. Each shard is handed to a separate worker process (per the runtime model — one worker per shard), and `scan(shard)` passes `{ segment: shard.segment, totalSegments: shard.total }` straight through to the underlying DynamoDB `Scan` call's native `Segment`/`TotalSegments` parameters, so increasing `pipeline.segments` in config directly increases scan parallelism against the source table (and correspondingly the number of worker processes spawned). + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, DdbScanner, DdbProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-ddb-copy", + description: "Copy the DynamoDB table, same as copy-ddb.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "Regular DynamoDB Table Data", + scanner: DdbScanner, + processors: [DdbProcessor] + }) + .build(); + + runner.register(everything); + } +}); +``` + +`DdbScanner` is the scanner for every pipeline in `copy-ddb`, `copy-files`, and `v5-to-v6-ddb` (it's paired with `DdbProcessor` and/or `S3Processor`/`AuditLogProcessor`, never with `OsProcessor`). All pipelines that share a scanner within one preset form a "merge group" — records are scanned once and dispatched first-match-wins across the pipelines registered against that scanner, in registration order. diff --git a/docs/mcp/scanners/osScanner.md b/docs/mcp/scanners/osScanner.md new file mode 100644 index 0000000..e089599 --- /dev/null +++ b/docs/mcp/scanners/osScanner.md @@ -0,0 +1,39 @@ +--- +name: OsScanner +description: Scans the source OpenSearch-backed DynamoDB table, decompressing each record's payload. +category: Scanners +--- + +# OsScanner + +**Import:** `import { OsScanner } from "@webiny/data-transfer";` + +**Scan behavior:** reads `config.source.opensearch.tableName` (throws if `config.source.opensearch` isn't configured) and performs a parallel `Scan` against that table via `SourceDynamoDbClient` — same underlying scan mechanism as `DdbScanner`, just against the OpenSearch-table rather than the regular table. For each raw item, it skips anything with no `index` field (not a valid OS record), then runs the item through `OsRecordDecompressor.decompress()` to inflate the gzip-compressed `data` payload that OS-table records store. If decompression yields nothing, a debug log records the record's PK/SK and `data` is yielded as `{}` rather than throwing — so downstream transformers always get an object, never `null`/`undefined`, for `record.data`. + +**Record shape:** yields `OsRecord`, which extends `BaseRecord` (`PK`, `SK`, `_et`, `_ct`, `_md`, `TYPE`, plus arbitrary attributes) with two additions: `index: string` (the target OpenSearch index name this record belongs to) and `data: Record` (the decompressed document body — this is the field `OsProcessor.execute()` re-compresses before writing to the target). + +**Segment support:** `listShards()` returns `total = config.pipeline?.segments ?? 1` shards as `{ segment, total }`, identical to `DdbScanner`. `scan(shard)` forwards `{ segment: shard.segment, totalSegments: shard.total }` to the underlying parallel `Scan`, so `pipeline.segments` scales OS-table scan parallelism the same way it does for the regular table. + +**Usage in pipelineBuilderFactory.create():** + +```typescript +import { createTransferPreset, OsScanner, OsProcessor } from "@webiny/data-transfer"; + +export default createTransferPreset({ + name: "my-os-copy", + description: "Copy the OpenSearch-backed table, same as copy-os.", + async configure({ runner, pipelineBuilderFactory }) { + const everything = await pipelineBuilderFactory + .create({ + name: "OpenSearch DynamoDB Table Data", + scanner: OsScanner, + processors: [OsProcessor] + }) + .build(); + + runner.register(everything); + } +}); +``` + +`OsScanner` is always paired with `OsProcessor` (the only processor that understands the `{ index, data }` record shape) and is used across every pipeline in `copy-os` and `v5-to-v6-os`. It's only registered — and its preset only selectable — when `config.target.opensearch != null`; `bootstrap.ts` skips OS feature registration entirely otherwise. diff --git a/docs/mcp/transformers/addGsiTenant.md b/docs/mcp/transformers/addGsiTenant.md new file mode 100644 index 0000000..b423674 --- /dev/null +++ b/docs/mcp/transformers/addGsiTenant.md @@ -0,0 +1,17 @@ +--- +name: addGsiTenant +description: Populates the GSI_TENANT attribute from the record's PK or data.tenant. +category: Transformers +--- + +# addGsiTenant + +**Import:** `import { addGsiTenant } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Skips records that already have `GSI_TENANT`. Otherwise extracts the tenant from a `T##...` PK prefix; if that pattern isn't present, falls back to `data.tenant` (requires `wrapInData` to have run first). + +**Record types it targets:** Any record — applied broadly to backfill the `GSI_TENANT` GSI attribute for tenant-scoped queries. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/addLiveField.md b/docs/mcp/transformers/addLiveField.md new file mode 100644 index 0000000..adf394e --- /dev/null +++ b/docs/mcp/transformers/addLiveField.md @@ -0,0 +1,17 @@ +--- +name: addLiveField +description: Computes and attaches the `live` pointer (published revision version) to CMS entry records. +category: Transformers +--- + +# addLiveField + +**Import:** `import { addLiveField } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Resolves the published revision version for a CMS entry and sets `data.live = { version }` (or `null` if none is published). The published (`P`) record and a published latest (`L` with `status: "published"`) record already know their own version; any other revision queries the source `P` record via `ctx.querySourceRecord`. Results are cached per entry PK via `ctx.cache` to avoid repeat queries. Skips internal models (`fmfile`, `wbyfmfile`). + +**Record types it targets:** CMS entry records (`data.modelId` present), keyed like `T##L##CMS#CME#` with `SK` of `P` or a revision number/`L`. + +**Context type required:** `DdbCoreTransformContext` diff --git a/docs/mcp/transformers/addTransferTimestamp.md b/docs/mcp/transformers/addTransferTimestamp.md new file mode 100644 index 0000000..05ff3e8 --- /dev/null +++ b/docs/mcp/transformers/addTransferTimestamp.md @@ -0,0 +1,17 @@ +--- +name: addTransferTimestamp +description: Stamps every record with the transfer time as `_tt`. +category: Transformers +--- + +# addTransferTimestamp + +**Import:** `import { addTransferTimestamp } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Sets `record._tt = Date.now()` on the record, unconditionally. Useful as a generic audit/debug marker for when a record passed through the pipeline. + +**Record types it targets:** Any record. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/copyFileToTarget.md b/docs/mcp/transformers/copyFileToTarget.md new file mode 100644 index 0000000..2f60c85 --- /dev/null +++ b/docs/mcp/transformers/copyFileToTarget.md @@ -0,0 +1,17 @@ +--- +name: copyFileToTarget +description: Emits a verbatim S3 copy for a file-manager record, source key equal to target key. +category: Transformers +--- + +# copyFileToTarget + +**Import:** `import { copyFileToTarget } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** Reads the S3 key from `values["text@key"]`, checking both the raw v5 shape (`record.values`) and the post-`wrapInData` shape (`record.data.values`), and calls `ctx.copyFile(key, key)` to queue an S3 copy at the same key path. Use this when the file's storage key does not need to change; for the v5→v6 key-path migration use `createMetadata` instead. + +**Record types it targets:** File-manager file records with a `text@key` value. + +**Context type required:** `DdbTransformContext` (pipeline must include `S3Processor`) diff --git a/docs/mcp/transformers/coreFieldsTransformer.md b/docs/mcp/transformers/coreFieldsTransformer.md new file mode 100644 index 0000000..8d88ff5 --- /dev/null +++ b/docs/mcp/transformers/coreFieldsTransformer.md @@ -0,0 +1,17 @@ +--- +name: coreFieldsTransformer +description: Resolves an audit-log record's creator identity and creation time, and stamps a fresh id and TTL expiry. +category: Transformers +--- + +# coreFieldsTransformer + +**Import:** `import { coreFieldsTransformer } from "@webiny/data-transfer";` + +**Category:** auditLogs + +**What it does:** Tries root-level fields first (`revisionCreatedBy`/`createdBy`/`savedBy`/`revisionSavedBy` and their `*On` counterparts, in priority order); if unavailable, decompresses the legacy `values["object@data"]["text@data"]` envelope and looks for creator info in the payload or its `before`/`after` sub-objects. Sets `record.id` (new `mdbid()`), `record.createdBy`, `record.createdOn`, and `record.expiresAt` (now + 60 days, ISO string). If no creator can be resolved, logs a warning and leaves the record untouched — downstream pipeline logic is expected to drop such records. + +**Record types it targets:** Audit log source records (v5 `cms.entry`-shaped audit log entries). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/createMetadata.md b/docs/mcp/transformers/createMetadata.md new file mode 100644 index 0000000..fd9ec12 --- /dev/null +++ b/docs/mcp/transformers/createMetadata.md @@ -0,0 +1,17 @@ +--- +name: createMetadata +description: Creates a KeyValueStore file-metadata record and copies the underlying S3 object to its new tenant-scoped path. +category: Transformers +--- + +# createMetadata + +**Import:** `import { createMetadata } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** For `cms.entry.l` file records, computes the new S3 key (`tenants//files/`), emits an S3 copy from the old key to the new key when they differ, and emits a new `KeyValueStore` record (`ctx.putRecord`) at `KV#global:FileManager/File//Metadata` holding `bucketKey`, `contentType`, `id`, `size`, and `tenant`. The file ID has its revision suffix (`#0001`) stripped. + +**Record types it targets:** File-manager file entry records (`TYPE === "cms.entry.l"`) with `text@key`/`text@name` values. + +**Context type required:** `DdbTransformContext` diff --git a/docs/mcp/transformers/dataFieldsTransformer.md b/docs/mcp/transformers/dataFieldsTransformer.md new file mode 100644 index 0000000..3db46a1 --- /dev/null +++ b/docs/mcp/transformers/dataFieldsTransformer.md @@ -0,0 +1,17 @@ +--- +name: dataFieldsTransformer +description: Lifts audit-log content fields (app, action, message, entity, tags, content) out of the legacy values envelope onto the record root. +category: Transformers +--- + +# dataFieldsTransformer + +**Import:** `import { dataFieldsTransformer } from "@webiny/data-transfer";` + +**Category:** auditLogs + +**What it does:** Reads `values["object@data"]` and copies `text@app` → `record.app`, `text@action` → `record.action`, `text@message` → `record.message`, `text@entity` → `record.entity`, and `text@data` → `record.content`; sets `record.entityId` from `record.entryId`; sets `record.tags` from `values["text@tags"]` (defaulting to `[]`). Intended to run before `storageShapeTransformer`, which consumes these root-level fields. + +**Record types it targets:** Audit log source records with `values["object@data"]`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/extractImageMetadata.md b/docs/mcp/transformers/extractImageMetadata.md new file mode 100644 index 0000000..b830bff --- /dev/null +++ b/docs/mcp/transformers/extractImageMetadata.md @@ -0,0 +1,17 @@ +--- +name: extractImageMetadata +description: Extracts image dimensions, EXIF, and IPTC metadata from raster image files and renames the legacy meta field. +category: Transformers +--- + +# extractImageMetadata + +**Import:** `import { extractImageMetadata } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** Deletes the legacy `values["object@meta"]` field. For non-raster or non-image types it sets `values["object@metadata"] = {}` and returns. For raster images, it resolves the file's S3 key (preferring an existing KV metadata record's `bucketKey`, falling back to `text@key`), reads the file via `ctx.getFile`, and uses `sharp` for dimensions/format/orientation plus `exifreader` for EXIF/IPTC tags, writing the result to `values["object@metadata"]`. Results are cached per file ID via `ctx.cache` so each file is only fetched/processed once across records. + +**Record types it targets:** File-manager file records with `data.values["text@type"]` set (image files get full extraction; others get an empty metadata object). + +**Context type required:** `DdbTransformContext` diff --git a/docs/mcp/transformers/fixBrokenStorageKeys.md b/docs/mcp/transformers/fixBrokenStorageKeys.md new file mode 100644 index 0000000..fb84cf9 --- /dev/null +++ b/docs/mcp/transformers/fixBrokenStorageKeys.md @@ -0,0 +1,17 @@ +--- +name: fixBrokenStorageKeys +description: Corrects mismatched field storage keys in CMS entry values against the model's declared storageId. +category: Transformers +--- + +# fixBrokenStorageKeys + +**Import:** `import { fixBrokenStorageKeys } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Walks a CMS entry's `data.values` against its model definition (via `ctx.modelProvider`) and, for each field, moves the value found under a wrong key (the declared `storageId` or `fieldId`) to the correct storage key computed by `getCorrectStorageId`. Logs and skips models it cannot find (warning once per missing model). Skips internal models (`fmfile`, `wbyfmfile`) and fragment-uuid fields. + +**Record types it targets:** CMS entry records with `data.modelId` and `data.values`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/fixCmePk.md b/docs/mcp/transformers/fixCmePk.md new file mode 100644 index 0000000..3f74834 --- /dev/null +++ b/docs/mcp/transformers/fixCmePk.md @@ -0,0 +1,17 @@ +--- +name: fixCmePk +description: Removes a duplicated #CME#CME# segment from a record's PK. +category: Transformers +--- + +# fixCmePk + +**Import:** `import { fixCmePk } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Fixes a known PK corruption where `#CME#` appears twice in a row (e.g. `T#root#L#en-US#CMS#CME#CME#` → `T#root#CMS#CME#`) by replacing the first occurrence of `#CME#CME#` with `#CME#`. No-op if the pattern isn't present. + +**Record types it targets:** Any record whose `PK` contains `#CME#CME#` (CMS entry records). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/groupsToRoles.md b/docs/mcp/transformers/groupsToRoles.md new file mode 100644 index 0000000..fb13fbf --- /dev/null +++ b/docs/mcp/transformers/groupsToRoles.md @@ -0,0 +1,17 @@ +--- +name: groupsToRoles +description: Renames security "group" records and their GROUP/GROUPS key segments to the v6 "role" terminology. +category: Transformers +--- + +# groupsToRoles + +**Import:** `import { groupsToRoles } from "@webiny/data-transfer";` + +**Category:** security + +**What it does:** For records with `TYPE === "security.group"`, sets `TYPE` to `security.role`, updates `_et` from `SecurityGroup` to `SecurityRole` if set, and rewrites `GROUPS`→`ROLES` and `GROUP`→`ROLE` segments (in that order, to avoid partial matches) across `PK`, `SK`, `GSI1_PK/SK`, `GSI2_PK/SK`. + +**Record types it targets:** Security group records (`TYPE === "security.group"`). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/migrateFileManagerSettings.md b/docs/mcp/transformers/migrateFileManagerSettings.md new file mode 100644 index 0000000..0096a74 --- /dev/null +++ b/docs/mcp/transformers/migrateFileManagerSettings.md @@ -0,0 +1,17 @@ +--- +name: migrateFileManagerSettings +description: Converts a legacy File Manager settings record into the v6 KeyValueStore format. +category: Transformers +--- + +# migrateFileManagerSettings + +**Import:** `import { migrateFileManagerSettings } from "@webiny/data-transfer";` + +**Category:** file-manager + +**What it does:** For records with `original.TYPE === "fm.settings"`, replaces the record wholesale (`ctx.replace`) with a `KeyValueStore` shape: `PK: KV#:FileManager/General`, `SK: A`, `data.value` holding all settings fields except `tenant`. Expects `wrapInData` to have run first so settings live under `record.data`. + +**Record types it targets:** File Manager settings records (`TYPE === "fm.settings"`). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/migrateMailerSettings.md b/docs/mcp/transformers/migrateMailerSettings.md new file mode 100644 index 0000000..1cbeb78 --- /dev/null +++ b/docs/mcp/transformers/migrateMailerSettings.md @@ -0,0 +1,17 @@ +--- +name: migrateMailerSettings +description: Converts a legacy Mailer settings record into the v6 KeyValueStore format. +category: Transformers +--- + +# migrateMailerSettings + +**Import:** `import { migrateMailerSettings } from "@webiny/data-transfer";` + +**Category:** mailer + +**What it does:** For records identified by `original.SK === "L"` and `original.modelId === "mailerSettings"`, replaces the record wholesale (`ctx.replace`) with a `KeyValueStore` shape: `PK: KV#:Mailer/Settings/Transport`, `SK: A`, `data.value` holding `data.values`. Expects `wrapInData` to have run first so values live under `record.data.values`. + +**Record types it targets:** Mailer settings records (`SK === "L"`, `modelId === "mailerSettings"`). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeAttributes.md b/docs/mcp/transformers/removeAttributes.md new file mode 100644 index 0000000..05228ee --- /dev/null +++ b/docs/mcp/transformers/removeAttributes.md @@ -0,0 +1,17 @@ +--- +name: removeAttributes +description: Deletes deprecated top-level attributes (currently webinyVersion) from the data envelope. +category: Transformers +--- + +# removeAttributes + +**Import:** `import { removeAttributes } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Deletes `data.webinyVersion` if present — it's no longer needed in v6 (the `tenant` attribute is handled separately, now derived via `GSI_TENANT` from keys). Expects `wrapInData` to have run first so attributes live under `data`. + +**Record types it targets:** Any record with a `data` envelope containing `webinyVersion`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeFolderRevision.md b/docs/mcp/transformers/removeFolderRevision.md new file mode 100644 index 0000000..3162c27 --- /dev/null +++ b/docs/mcp/transformers/removeFolderRevision.md @@ -0,0 +1,17 @@ +--- +name: removeFolderRevision +description: Strips the #0001 revision suffix from folder location IDs and cleans up legacy folder location fields. +category: Transformers +--- + +# removeFolderRevision + +**Import:** `import { removeFolderRevision } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Removes the trailing `#0001` revision marker from `data.location.folderId`, deletes the legacy `data.values["object@location"]` field (location now lives at `data.location`), and for `wbyAcoFolder` records strips a trailing revision number from `data.values["text@parentId"]`. Expects `wrapInData` to have run first so fields are under `data`. + +**Record types it targets:** Folder-related CMS entries, notably `wbyAcoFolder` model records. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeLocale.md b/docs/mcp/transformers/removeLocale.md new file mode 100644 index 0000000..cca1e08 --- /dev/null +++ b/docs/mcp/transformers/removeLocale.md @@ -0,0 +1,17 @@ +--- +name: removeLocale +description: Strips locale segments (e.g. #L#en-US#) from a record's keys and deletes the locale field. +category: Transformers +--- + +# removeLocale + +**Import:** `import { removeLocale } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Regex-strips `#L##` segments from `PK`, `SK`, `GSI1_PK/SK`, `GSI2_PK/SK`, and deletes the top-level `locale` field plus `data.locale` if present. Used broadly during v5→v6 migration since v6 is single-locale-per-tenant at the storage layer. + +**Record types it targets:** Any record whose keys contain a `#L##` segment. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/removeTenant.md b/docs/mcp/transformers/removeTenant.md new file mode 100644 index 0000000..c670a0d --- /dev/null +++ b/docs/mcp/transformers/removeTenant.md @@ -0,0 +1,17 @@ +--- +name: removeTenant +description: Deletes the top-level tenant attribute from security role records. +category: Transformers +--- + +# removeTenant + +**Import:** `import { removeTenant } from "@webiny/data-transfer";` + +**Category:** security + +**What it does:** Deletes `record.tenant` unconditionally. Tenant scoping is derived from keys (`GSI_TENANT`) rather than a plain attribute in v6. + +**Record types it targets:** Security role records carrying a legacy top-level `tenant` field. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/renameFieldAttributes.md b/docs/mcp/transformers/renameFieldAttributes.md new file mode 100644 index 0000000..3d1b866 --- /dev/null +++ b/docs/mcp/transformers/renameFieldAttributes.md @@ -0,0 +1,17 @@ +--- +name: renameFieldAttributes +description: Renames legacy CMS model field attributes (helpText, placeholderText, multipleValues) to their v6 equivalents. +category: Transformers +--- + +# renameFieldAttributes + +**Import:** `import { renameFieldAttributes } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Recursively walks a CMS model's `data.fields` (including nested object fields and dynamic-zone template fields) renaming `helpText` → `note`, `placeholderText` → `placeholder`, and `multipleValues` → `list`. Only renames when the target attribute doesn't already exist, and always deletes the old attribute. + +**Record types it targets:** CMS model definition records (`data.fields` array present). + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/replaceFileUrls.md b/docs/mcp/transformers/replaceFileUrls.md new file mode 100644 index 0000000..94e0fda --- /dev/null +++ b/docs/mcp/transformers/replaceFileUrls.md @@ -0,0 +1,17 @@ +--- +name: replaceFileUrls +description: Rewrites file-manager URLs embedded in CMS "file" and "rich-text" field values from a source domain to a target domain. +category: Transformers +--- + +# replaceFileUrls + +**Import:** `import { replaceFileUrls } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** A factory — call `replaceFileUrls(config)` with the resolved `MigrationConfig` to get the transformer. It looks up the CMS entry's model, walks `data.values` via the shared field visitor, and for `file`-type fields does a plain string replace of `config.fileUrls.source` with `config.fileUrls.target` (including array values); for `rich-text` fields it decompresses the value, replaces the URL substring inside `state`/`html`, and re-compresses. No-ops entirely if `config.fileUrls.source`/`target` aren't both set. + +**Record types it targets:** CMS entry records with `data.modelId` and `data.values` containing `file` or `rich-text` fields. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/storageShapeTransformer.md b/docs/mcp/transformers/storageShapeTransformer.md new file mode 100644 index 0000000..76dbabc --- /dev/null +++ b/docs/mcp/transformers/storageShapeTransformer.md @@ -0,0 +1,17 @@ +--- +name: storageShapeTransformer +description: Builds the final v6 audit-log storage record — nine GSI key sets plus the data envelope and TTL expiry. +category: Transformers +--- + +# storageShapeTransformer + +**Import:** `import { storageShapeTransformer } from "@webiny/data-transfer";` + +**Category:** auditLogs + +**What it does:** Requires `createdBy`/`createdOn` (and the other fields populated by `coreFieldsTransformer` and `dataFieldsTransformer`) already present on the record — logs a warning and skips otherwise. Replaces the record wholesale (`ctx.replace`) with `PK: T##AUDIT_LOG`, `SK: `, `TYPE: auditLog.log`, and nine `GSI_PK/SK` pairs indexing by app, createdBy, entity, entityId, and action in various combinations, plus a `data` envelope mirroring the fields and a root-level `expiresAt` as Unix-seconds TTL (DynamoDB reads this directly). Must run last in the audit-log transformer chain. + +**Record types it targets:** Audit log records that have already passed through `coreFieldsTransformer` and `dataFieldsTransformer`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/transformModelGroup.md b/docs/mcp/transformers/transformModelGroup.md new file mode 100644 index 0000000..8120885 --- /dev/null +++ b/docs/mcp/transformers/transformModelGroup.md @@ -0,0 +1,17 @@ +--- +name: transformModelGroup +description: Resolves a CMS model's group ID reference to its slug string. +category: Transformers +--- + +# transformModelGroup + +**Import:** `import { transformModelGroup } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Replaces `data.group` (an object with `id`/`name`) with a plain slug string by querying the source group record (`T##L##CMS#CMG` / `group.id`). Falls back to a slugified `group.name` (or `"ungrouped"`) if the group record isn't found, logging a warning. Expects `wrapInData` to have run first so `group` lives at `data.group`. + +**Record types it targets:** CMS model definition records with an object-shaped `data.group`. + +**Context type required:** `DdbCoreTransformContext` diff --git a/docs/mcp/transformers/transformPermissions.md b/docs/mcp/transformers/transformPermissions.md new file mode 100644 index 0000000..d8069f9 --- /dev/null +++ b/docs/mcp/transformers/transformPermissions.md @@ -0,0 +1,17 @@ +--- +name: transformPermissions +description: Migrates security role permissions to v6 shape — drops content.i18n, flattens per-locale model lists, and resolves group IDs to slugs. +category: Transformers +--- + +# transformPermissions + +**Import:** `import { transformPermissions } from "@webiny/data-transfer";` + +**Category:** security + +**What it does:** Walks `data.permissions`: drops any `content.i18n` permission entirely; for `cms.contentModel`, flattens a per-locale `models` object (e.g. `{ "en-US": [...] }`) down to the default locale's array; for `cms.contentModelGroup`, resolves each per-locale group ID to its slug by querying `T##GROUP#` and replaces `groups` with the resolved slug array. Default locale is parsed out of the record's own `PK` (`#L##`). Expects `wrapInData` to have run first. + +**Record types it targets:** Security role records with an array `data.permissions`. + +**Context type required:** `DdbCoreTransformContext` diff --git a/docs/mcp/transformers/transformRichText.md b/docs/mcp/transformers/transformRichText.md new file mode 100644 index 0000000..b67e744 --- /dev/null +++ b/docs/mcp/transformers/transformRichText.md @@ -0,0 +1,17 @@ +--- +name: transformRichText +description: Converts legacy Slate-based rich-text field values into the Lexical state + rendered HTML format. +category: Transformers +--- + +# transformRichText + +**Import:** `import { transformRichText } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** For every `rich-text` field in a CMS entry's `data.values`, decompresses the stored value, and if it has a lexical `root` shape, re-renders it: empty `root.children` gets replaced with `generateInitialLexicalValue()`, then the value is re-compressed as `{ state, html }` (state = JSON string, html = rendered via the internal `LexicalRenderer`). Logs and skips a field on transform failure rather than throwing. + +**Record types it targets:** CMS entry records with `data.modelId`/`data.values` containing compressed `rich-text` field values. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/updateFlpIds.md b/docs/mcp/transformers/updateFlpIds.md new file mode 100644 index 0000000..a1f3717 --- /dev/null +++ b/docs/mcp/transformers/updateFlpIds.md @@ -0,0 +1,17 @@ +--- +name: updateFlpIds +description: Strips the #0001 revision suffix from folder-level-page id and parentId fields. +category: Transformers +--- + +# updateFlpIds + +**Import:** `import { updateFlpIds } from "@webiny/data-transfer";` + +**Category:** folders + +**What it does:** Removes a trailing `#0001` revision marker from `data.id` and `data.parentId` on FLP (folder-level-page) records. These records already carry a `data` envelope natively, so `wrapInData` does not re-wrap them. + +**Record types it targets:** FLP records with `data.id`/`data.parentId`. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/updateModelIds.md b/docs/mcp/transformers/updateModelIds.md new file mode 100644 index 0000000..e9bca01 --- /dev/null +++ b/docs/mcp/transformers/updateModelIds.md @@ -0,0 +1,17 @@ +--- +name: updateModelIds +description: Renames legacy system model IDs (fmFile, acoFolder, etc.) to their v6 wby-prefixed equivalents in keys and data.modelId. +category: Transformers +--- + +# updateModelIds + +**Import:** `import { updateModelIds } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Applies a fixed rename map (`fmFile`→`wbyFmFile`, `acoFolder`→`wbyAcoFolder`, `acoFilter`→`wbyAcoFilter`, `webinyTask`→`wbyTask`, `webinyTaskLog`→`wbyTaskLog`, `wby_recordLocking`→`wbyRecordLock`) to every `##`/`#` occurrence in `PK`, `SK`, `GSI1_PK/SK`, `GSI2_PK/SK`, and to `data.modelId` directly. Expects `wrapInData` to have run first. + +**Record types it targets:** Records whose keys or `data.modelId` reference any of the renamed system model IDs. + +**Context type required:** `BaseTransformContext` diff --git a/docs/mcp/transformers/updateOsIndex.md b/docs/mcp/transformers/updateOsIndex.md new file mode 100644 index 0000000..787b400 --- /dev/null +++ b/docs/mcp/transformers/updateOsIndex.md @@ -0,0 +1,17 @@ +--- +name: updateOsIndex +description: Recomputes an OpenSearch record's target index name from its modelId and tenant. +category: Transformers +--- + +# updateOsIndex + +**Import:** `import { updateOsIndex } from "@webiny/data-transfer";` + +**Category:** cms + +**What it does:** Reads `record.data.modelId` and `record.data.tenant`, builds a minimal model shape, and calls `@webiny/api-headless-cms-ddb-es`'s `configurations.es()` to derive the correct index name, then sets `record.index` to it. Logs a warning and skips the record if `modelId` or `tenant` is missing. + +**Record types it targets:** OpenSearch CMS entry records (`data.modelId`, `data.tenant` present). + +**Context type required:** `OsTransformContext` (bound via `createOsTransformer`) diff --git a/docs/mcp/transformers/wrapInData.md b/docs/mcp/transformers/wrapInData.md new file mode 100644 index 0000000..9dea14a --- /dev/null +++ b/docs/mcp/transformers/wrapInData.md @@ -0,0 +1,17 @@ +--- +name: wrapInData +description: Wraps all non-reserved top-level attributes of a record into a `data` envelope. +category: Transformers +--- + +# wrapInData + +**Import:** `import { wrapInData } from "@webiny/data-transfer";` + +**Category:** global + +**What it does:** Moves every attribute not in the reserved set (`PK`, `SK`, `GSI_TENANT`, `GSI1_PK/SK`, `GSI2_PK/SK`, `TYPE`, `data`, `expiresAt`, `_ct`, `_et`, `_md`) into a new `data` object, then replaces the record (`ctx.replace`) with the reserved attributes plus this `data` envelope. No-op if `record.data` already exists. Many other transformers (`transformModelGroup`, `updateModelIds`, `addGsiTenant`, `removeAttributes`, etc.) document that they expect `wrapInData` to run first in the pipeline. + +**Record types it targets:** Any v5-shaped record without an existing `data` envelope. + +**Context type required:** `BaseTransformContext` diff --git a/docs/superpowers/plans/2026-08-03-mcp-server.md b/docs/superpowers/plans/2026-08-03-mcp-server.md new file mode 100644 index 0000000..131f2c9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-mcp-server.md @@ -0,0 +1,557 @@ +# MCP Server Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build an MCP server inside `@webiny/data-transfer` that serves documentation about all public API components (presets, transformers, processors, scanners, filters, config, pipeline runtime) so AI agents can help users write custom presets, transformers, and pipelines. + +**Architecture:** Two MCP tools (`list_topics` / `get_topic`) serve markdown documentation files from `docs/mcp/`. Entry point at `src/mcp/server.ts`, bin entry `webiny-data-transfer-mcp`, stdio transport via `@modelcontextprotocol/sdk`. Documentation is organized by category (presets, transformers, processors, scanners, guides). All docs ship with the published npm package. + +**Tech Stack:** `@modelcontextprotocol/sdk`, `zod` (already a dependency), `front-matter` for YAML front-matter parsing. + +## Global Constraints + +- MCP server lives inside the data-transfer package at `src/mcp/` +- All tools are read-only (annotations: `readOnlyHint: true`) +- Documentation files use YAML front-matter with `name`, `description`, `category` fields +- Transformer docs are small — name, one-line description, what it does, when to use it +- Existing `docs/guides/` are NOT moved — MCP docs are a separate set at `docs/mcp/` +- `docs/mcp/` MUST be included in the published npm package — `ArtifactCopier.copyAssets()` must copy it to `dist/docs/mcp/` +- Bin entry points to `./dist/mcp/bin.js` (compiled from `src/mcp/bin.ts` by tsc) +- `BuildOrchestrator.ensureShebang()` must handle `dist/mcp/bin.js` in addition to `dist/cli.js` +- Follow existing project patterns: ESM, no reflect-metadata imports, camelCase file names +- Transformers: all 27 are public exports from `@webiny/data-transfer`. Docs show import path for each. +- Filters: all 18 exported in public API (including `isAdminUser`). Docs cover all 18 exported filters. + +--- + +### Task 1: MCP server skeleton + two tools + +**Files:** +- Create: `src/mcp/server.ts` +- Create: `src/mcp/discoverDocs.ts` +- Create: `src/mcp/bin.ts` +- Modify: `package.json` (add bin entry, add `@modelcontextprotocol/sdk` + `front-matter` dependencies) +- Modify: `scripts/features/BuildPackages/ArtifactCopier.ts` (copy `docs/mcp/` to dist) +- Modify: `scripts/features/BuildPackages/BuildOrchestrator.ts` (add shebang to `dist/mcp/bin.js`) + +**Interfaces:** +- Produces: `startMcpServer()` function, `discoverDocs(dirs: string[])` returning `Map`, `buildCatalog(docs: Map)` returning formatted markdown + +- [ ] **Step 1: Install dependencies** + +```bash +yarn add @modelcontextprotocol/sdk front-matter +``` + +- [ ] **Step 2: Create `src/mcp/discoverDocs.ts`** + +Recursively finds all `.md` files in given directories, parses YAML front-matter (`name`, `description`, `category`), returns a `Map`. First-match-wins for duplicate names. + +```typescript +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import fm from "front-matter"; + +interface Doc { + name: string; + description: string; + category: string; + filePath: string; + body: string; +} + +function findMarkdownFiles(dir: string): string[] { + const results: string[] = []; + + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const stat = statSync(full); + + if (stat.isDirectory()) { + results.push(...findMarkdownFiles(full)); + } else if (entry.endsWith(".md")) { + results.push(full); + } + } + + return results; +} + +function parseDoc(filePath: string): Doc | null { + const raw = readFileSync(filePath, "utf-8"); + const parsed = fm<{ name?: string; description?: string; category?: string }>(raw); + + if (!parsed.attributes.name || !parsed.attributes.description) { + return null; + } + + return { + name: parsed.attributes.name, + description: parsed.attributes.description, + category: parsed.attributes.category ?? "general", + filePath, + body: parsed.body + }; +} + +export function discoverDocs(dirs: string[]): Map { + const docs = new Map(); + + for (const dir of dirs) { + for (const file of findMarkdownFiles(dir)) { + const doc = parseDoc(file); + if (doc && !docs.has(doc.name)) { + docs.set(doc.name, doc); + } + } + } + + return docs; +} + +export function buildCatalog(docs: Map): string { + const byCategory = new Map(); + + for (const doc of docs.values()) { + const list = byCategory.get(doc.category) ?? []; + list.push(doc); + byCategory.set(doc.category, list); + } + + const lines: string[] = ["# Available Topics\n"]; + + for (const [category, categoryDocs] of [...byCategory.entries()].sort((a, b) => a[0].localeCompare(b[0]))) { + lines.push(`## ${category}\n`); + lines.push("| Topic | Description |"); + lines.push("|-------|-------------|"); + + for (const doc of categoryDocs.sort((a, b) => a.name.localeCompare(b.name))) { + lines.push(`| ${doc.name} | ${doc.description} |`); + } + + lines.push(""); + } + + return lines.join("\n"); +} + +export type { Doc }; +``` + +- [ ] **Step 3: Create `src/mcp/server.ts`** + +Registers `list_topics` and `get_topic` tools, starts stdio transport. The `DEFAULT_DOCS_DIR` resolves relative to compiled output (`dist/docs/mcp` in production, `docs/mcp` in dev). + +```typescript +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { z } from "zod"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { discoverDocs, buildCatalog, type Doc } from "./discoverDocs.ts"; + +const DEFAULT_DOCS_DIR = join(fileURLToPath(new URL(".", import.meta.url)), "../docs/mcp"); + +export async function startMcpServer(docsDirs?: string[]): Promise { + const dirs = docsDirs ?? [DEFAULT_DOCS_DIR]; + let docsCache: Map | null = null; + + function getDocs(): Map { + if (!docsCache) { + docsCache = discoverDocs(dirs); + } + return docsCache; + } + + const server = new McpServer({ name: "webiny-data-transfer", version: "1.0.0" }); + + server.registerTool( + "list_topics", + { + title: "List Data Transfer Topics", + description: + "Returns a catalog of all available @webiny/data-transfer documentation topics. " + + "Call this first to discover what topics are available, then use get_topic to read the full documentation for a specific topic. " + + "Topics cover: presets, transformers, processors, scanners, filters, config, pipeline runtime, and how to write custom components.", + inputSchema: {}, + annotations: { readOnlyHint: true } + }, + async () => ({ + content: [{ type: "text" as const, text: buildCatalog(getDocs()) }] + }) + ); + + server.registerTool( + "get_topic", + { + title: "Get Data Transfer Topic", + description: + "Returns the full documentation for a specific @webiny/data-transfer topic. " + + "Use exact topic names from list_topics.", + inputSchema: { + topic: z.string().describe("Topic name — use exact names from list_topics") + }, + annotations: { readOnlyHint: true } + }, + async ({ topic }) => { + const docs = getDocs(); + const doc = docs.get(topic); + + if (!doc) { + const available = [...docs.keys()].sort().join(", "); + return { + content: [{ type: "text" as const, text: `Topic "${topic}" not found. Available topics: ${available}` }] + }; + } + + return { + content: [{ type: "text" as const, text: doc.body }] + }; + } + ); + + const transport = new StdioServerTransport(); + await server.connect(transport); +} +``` + +- [ ] **Step 4: Create `src/mcp/bin.ts`** + +```typescript +import { startMcpServer } from "./server.ts"; + +await startMcpServer(); +``` + +- [ ] **Step 5: Add bin entry + dependencies to `package.json`** + +Add to existing `bin` field: +```json +"bin": { + "webiny-data-transfer": "./dist/cli.js", + "webiny-data-transfer-mcp": "./dist/mcp/bin.js" +} +``` + +Add to `dependencies`: +```json +"@modelcontextprotocol/sdk": "^1.30.0", +"front-matter": "^4.0.2" +``` + +- [ ] **Step 6: Update `ArtifactCopier.copyAssets()` to copy `docs/mcp/`** + +In `scripts/features/BuildPackages/ArtifactCopier.ts`, add `"docs/mcp"` to the asset directories copied to dist: + +```typescript +public copyAssets(sourceDir: string, distAbsDir: string): void { + for (const dir of ["templates", "projects", "docs/mcp"]) { + const src = join(sourceDir, dir); + if (existsSync(src)) { + cpSync(src, join(distAbsDir, dir), { recursive: true }); + } + } +} +``` + +- [ ] **Step 7: Update `BuildOrchestrator` to add shebang to MCP bin** + +In `scripts/features/BuildPackages/BuildOrchestrator.ts`, extend the shebang logic to handle all bin entries (not just `cli.js`): + +```typescript +for (const binPath of ["dist/cli.js", "dist/mcp/bin.js"]) { + const fullPath = join(rootDir, binPath); + if (!existsSync(fullPath)) { + continue; + } + const content = readFileSync(fullPath, "utf-8"); + if (!content.startsWith("#!")) { + writeFileSync(fullPath, "#!/usr/bin/env node\n" + content); + } +} +``` + +- [ ] **Step 8: Create a test doc file and verify server starts** + +Create `docs/mcp/test.md`: +```markdown +--- +name: test-topic +description: Test topic for verifying MCP server +category: test +--- + +This is a test topic. +``` + +Run: `tsx src/mcp/bin.ts` — should start without errors (ctrl+c to exit). + +- [ ] **Step 9: Commit** + +```bash +git add src/mcp/ package.json yarn.lock docs/mcp/test.md scripts/features/BuildPackages/ArtifactCopier.ts scripts/features/BuildPackages/BuildOrchestrator.ts +git commit -m "feat: add MCP server skeleton with list_topics and get_topic tools" +``` + +--- + +### Task 2: Preset documentation (5 files) + +**Files:** +- Create: `docs/mcp/presets/copy-ddb.md` +- Create: `docs/mcp/presets/copy-os.md` +- Create: `docs/mcp/presets/copy-files.md` +- Create: `docs/mcp/presets/v5-to-v6-ddb.md` +- Create: `docs/mcp/presets/v5-to-v6-os.md` + +**Interfaces:** +- Consumes: reads preset source files in `src/presets/` for accuracy +- Produces: 5 markdown files with front-matter, each documenting one built-in preset + +Each file follows this template: +```markdown +--- +name: +description: +category: Presets +--- + +# + +**Use when:** + +**What it does:** + + +**Pipelines registered:** + + +**Transformers applied:** + + +**Example usage in a custom preset:** + +``` + +- [ ] **Step 1: Read each preset source file** + +Read `src/presets/copy-ddb.ts`, `copy-os.ts`, `copy-files.ts`, `v5-to-v6-ddb.ts`, `v5-to-v6-os.ts` to understand exactly what each registers. + +- [ ] **Step 2: Write all 5 preset doc files** + +Follow the template above. Extract pipeline names, scanner/processor combos, and transformer chains from the source. + +- [ ] **Step 3: Delete test doc file** + +Remove `docs/mcp/test.md`. + +- [ ] **Step 4: Commit** + +```bash +git add docs/mcp/presets/ && git rm docs/mcp/test.md +git commit -m "docs: add MCP documentation for all 5 built-in presets" +``` + +--- + +### Task 3: Processor and scanner documentation (6 files) + +**Files:** +- Create: `docs/mcp/processors/ddbProcessor.md` +- Create: `docs/mcp/processors/osProcessor.md` +- Create: `docs/mcp/processors/s3Processor.md` +- Create: `docs/mcp/processors/auditLogProcessor.md` +- Create: `docs/mcp/scanners/ddbScanner.md` +- Create: `docs/mcp/scanners/osScanner.md` + +**Interfaces:** +- Consumes: reads processor/scanner source files for accuracy +- Produces: 6 markdown files documenting processors and scanners + +Each processor doc follows this template: +```markdown +--- +name: +description: +category: Processors +--- + +# + +**Import:** `import { } from "@webiny/data-transfer";` + +**What it does:** + +**Context slice it adds:** + +**Commands it handles:** + +**`onEnd` hook behavior:** + +**Usage in pipelineBuilderFactory.create():** + +``` + +Scanner docs follow a similar pattern with scan behavior, record shape, segment support. + +- [ ] **Step 1: Read processor/scanner source files** + +- [ ] **Step 2: Write all 6 doc files** + +- [ ] **Step 3: Commit** + +```bash +git add docs/mcp/processors/ docs/mcp/scanners/ +git commit -m "docs: add MCP documentation for processors and scanners" +``` + +--- + +### Task 4: Transformer documentation (27 files) + +**Files:** +- Create: `docs/mcp/transformers/.md` for each of the 27 built-in transformers + +**Interfaces:** +- Consumes: reads transformer source files for accuracy +- Produces: 27 small markdown files + +All 27 transformers are public exports from `@webiny/data-transfer`. + +Each transformer doc is intentionally small: +```markdown +--- +name: +description: +category: Transformers +--- + +# + +**Import:** `import { } from "@webiny/data-transfer";` + +**Category:** + +**What it does:** <2-3 sentences max> + +**Record types it targets:** + +**Context type required:** `` +``` + +The 27 transformers: +`addGsiTenant`, `addLiveField`, `addTransferTimestamp`, `copyFileToTarget`, `coreFieldsTransformer`, `createMetadata`, `dataFieldsTransformer`, `extractImageMetadata`, `fixBrokenStorageKeys`, `fixCmePk`, `groupsToRoles`, `migrateFileManagerSettings`, `migrateMailerSettings`, `removeAttributes`, `removeFolderRevision`, `removeLocale`, `removeTenant`, `renameFieldAttributes`, `replaceFileUrls`, `storageShapeTransformer`, `transformModelGroup`, `transformPermissions`, `transformRichText`, `updateFlpIds`, `updateModelIds`, `updateOsIndex`, `wrapInData` + +- [ ] **Step 1: Read all transformer source files** + +- [ ] **Step 2: Write all 27 doc files** + +Batch by category (cms, security, global, file-manager, folders, mailer, auditLogs). + +- [ ] **Step 3: Commit** + +```bash +git add docs/mcp/transformers/ +git commit -m "docs: add MCP documentation for all 27 built-in transformers" +``` + +--- + +### Task 5: Filter and pipeline guide documentation (4 files) + +**Files:** +- Create: `docs/mcp/guides/filters.md` +- Create: `docs/mcp/guides/writingPresets.md` +- Create: `docs/mcp/guides/writingTransformers.md` +- Create: `docs/mcp/guides/configReference.md` + +**Interfaces:** +- Consumes: existing `docs/guides/` files as source material, `src/domain/transform/filters.ts` for exact filter signatures +- Produces: 4 markdown files with front-matter + +These are MCP-specific versions of the existing guides, optimized for AI agent consumption (more structured, more code examples, less prose). They cover: + +1. **filters** — all 18 exported filter predicates (`byType`, `byTypePrefix`, `isCmsGroup`, `isCmsModel`, `isCmsEntry`, `byIncludesModelId`, `isAcoSearchRecord`, `isAdminUser`, `isBackgroundTask`, `isFmFile`, `isFlpRecord`, `isBuiltInSecurityRole`, `isSecurityTeam`, `isOsBackgroundTask`, `isOsMailerSettings`, `isAuditLogEntry`, `isMigrationRecord`, `isFormBuilderRecord`) with signatures and examples +2. **writingPresets** — how to write a custom preset from scratch, `createTransferPreset` shape, `pipelineBuilderFactory.create()`, `runner.register()`, filter/use/hook composition +3. **writingTransformers** — `createDdbTransformer`/`createOsTransformer`/`createTransformer` factories, context types, processor slices, working with `ctx.record`, `ctx.putRecord()`, `ctx.blackhole()` +4. **configReference** — `createConfig` shape, `fromEnv`/`numberFromEnv`, credentials (`fromAwsProfile`, `fromAwsCredentialChain`, literal), `register` callback, tuning, debug/snapshot + +- [ ] **Step 1: Read existing guides and `filters.ts` for reference** + +- [ ] **Step 2: Write all 4 doc files** + +- [ ] **Step 3: Commit** + +```bash +git add docs/mcp/guides/ +git commit -m "docs: add MCP guide documentation for filters, presets, transformers, and config" +``` + +--- + +### Task 6: Pipeline runtime and public API docs (2 files) + +**Files:** +- Create: `docs/mcp/guides/pipelineRuntime.md` +- Create: `docs/mcp/guides/publicApi.md` + +**Interfaces:** +- Produces: 2 markdown files covering pipeline runtime semantics and full public API surface + +1. **pipelineRuntime** — merge groups, first-match-wins dispatch, unmatched record drops, `onEnd` hooks, `flushEvery`, parallelism (segments/shards), hook ordering +2. **publicApi** — every export from `src/index.ts` with import path, type, one-line description. Organized by category (config, credentials, transformer factories, built-in transformers, filters, scanners, processors, service clients, context types, lifecycle hooks, customization). Mark which are values vs types. + +- [ ] **Step 1: Write both doc files** + +- [ ] **Step 2: Commit** + +```bash +git add docs/mcp/guides/ +git commit -m "docs: add MCP documentation for pipeline runtime and public API surface" +``` + +--- + +### Task 7: Integration — verify, run `yarn full`, final commit + +**Files:** +- Possibly modify: `src/mcp/server.ts` (fix any issues found) +- Possibly modify: various doc files (fix any issues found) + +- [ ] **Step 1: Verify MCP server discovers all docs** + +```bash +tsx -e " +import { discoverDocs } from './src/mcp/discoverDocs.ts'; +const docs = discoverDocs(['./docs/mcp']); +console.log('Total topics:', docs.size); +for (const [name, doc] of docs) { + console.log(' ', doc.category, '/', name, '—', doc.description); +} +" +``` + +Expect: ~44 topics (5 presets + 27 transformers + 6 processors/scanners + 6 guides). + +- [ ] **Step 2: Verify build includes MCP docs and bin** + +```bash +yarn build +ls dist/mcp/bin.js # must exist with shebang +ls dist/docs/mcp/ # must contain all doc files +head -1 dist/mcp/bin.js # must start with #!/usr/bin/env node +``` + +- [ ] **Step 3: Run `yarn full`** + +```bash +yarn full +``` + +Fix any format/lint/typecheck issues. + +- [ ] **Step 4: Commit any fixes** + +```bash +git commit -m "fix: address issues found during MCP integration verification" +``` diff --git a/package.json b/package.json index 9d2c21a..3897395 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "bin": { - "webiny-data-transfer": "./dist/cli.js" + "data-transfer": "./dist/cli.js", + "webiny-data-transfer": "./dist/cli.js", + "webiny-data-transfer-mcp": "./dist/mcp/bin.js" }, "exports": { ".": { @@ -53,6 +55,7 @@ "@aws-sdk/credential-providers": "^3.1100.0", "@inquirer/core": "^11.2.1", "@inquirer/prompts": "^8.5.2", + "@modelcontextprotocol/sdk": "^1.30.0", "@opensearch-project/opensearch": "3.6.0", "@types/node": "^24.13.3", "@webiny/api-headless-cms-ddb-es": "^6.4.5", @@ -65,6 +68,7 @@ "dotenv": "^17.4.2", "execa": "^10.0.1", "exifreader": "^4.41.3", + "front-matter": "^4.0.2", "jsdom": "^30.0.1", "pino": "^10.3.1", "pino-pretty": "^13.1.3", diff --git a/scripts/features/BuildPackages/ArtifactCopier.ts b/scripts/features/BuildPackages/ArtifactCopier.ts index 5d5a286..c2c2c82 100644 --- a/scripts/features/BuildPackages/ArtifactCopier.ts +++ b/scripts/features/BuildPackages/ArtifactCopier.ts @@ -73,7 +73,7 @@ class ArtifactCopierImpl implements ArtifactCopierAbstraction.Interface { public copyAssets(sourceDir: string, distAbsDir: string): void { // presets live in src/presets/ and are compiled by tsc — not copied here - for (const dir of ["templates", "projects"]) { + for (const dir of ["templates", "projects", "docs/mcp"]) { const src = join(sourceDir, dir); if (existsSync(src)) { cpSync(src, join(distAbsDir, dir), { recursive: true }); diff --git a/scripts/features/BuildPackages/BuildOrchestrator.ts b/scripts/features/BuildPackages/BuildOrchestrator.ts index ff27b66..853b1d1 100644 --- a/scripts/features/BuildPackages/BuildOrchestrator.ts +++ b/scripts/features/BuildPackages/BuildOrchestrator.ts @@ -48,13 +48,15 @@ class BuildOrchestratorImpl implements BuildOrchestratorAbstraction.Interface { } private ensureShebang(rootDir: string): void { - const cliPath = join(rootDir, "dist", "cli.js"); - if (!existsSync(cliPath)) { - return; - } - const content = readFileSync(cliPath, "utf-8"); - if (!content.startsWith("#!")) { - writeFileSync(cliPath, "#!/usr/bin/env node\n" + content); + for (const binPath of ["dist/cli.js", "dist/mcp/bin.js"]) { + const fullPath = join(rootDir, binPath); + if (!existsSync(fullPath)) { + continue; + } + const content = readFileSync(fullPath, "utf-8"); + if (!content.startsWith("#!")) { + writeFileSync(fullPath, "#!/usr/bin/env node\n" + content); + } } } } diff --git a/src/index.ts b/src/index.ts index be9572d..4762f34 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,9 +43,41 @@ export { createTransformer } from "./transformers/createTransformer.ts"; export { createDdbTransformer } from "./transformers/createDdbTransformer.ts"; export { createOsTransformer } from "./transformers/createOsTransformer.ts"; -// Built-in transformers — ready-made for common patterns in custom presets. -export { copyFileToTarget } from "./transformers/file-manager/copyFileToTarget.ts"; +// Built-in transformers — ready-made for use in custom presets. +// CMS +export { addLiveField } from "./transformers/cms/addLiveField.ts"; +export { fixBrokenStorageKeys } from "./transformers/cms/fixBrokenStorageKeys.ts"; +export { fixCmePk } from "./transformers/cms/fixCmePk.ts"; +export { removeFolderRevision } from "./transformers/cms/removeFolderRevision.ts"; +export { renameFieldAttributes } from "./transformers/cms/renameFieldAttributes.ts"; export { replaceFileUrls } from "./transformers/cms/replaceFileUrls.ts"; +export { transformModelGroup } from "./transformers/cms/transformModelGroup.ts"; +export { transformRichText } from "./transformers/cms/transformRichText.ts"; +export { updateModelIds } from "./transformers/cms/updateModelIds.ts"; +export { updateOsIndex } from "./transformers/cms/updateOsIndex.ts"; +// File manager +export { copyFileToTarget } from "./transformers/file-manager/copyFileToTarget.ts"; +export { createMetadata } from "./transformers/file-manager/createMetadata.ts"; +export { extractImageMetadata } from "./transformers/file-manager/extractImageMetadata.ts"; +export { migrateFileManagerSettings } from "./transformers/file-manager/migrateFileManagerSettings.ts"; +// Folders +export { updateFlpIds } from "./transformers/folders/updateFlpIds.ts"; +// Global +export { addGsiTenant } from "./transformers/global/addGsiTenant.ts"; +export { addTransferTimestamp } from "./transformers/global/addTransferTimestamp.ts"; +export { removeAttributes } from "./transformers/global/removeAttributes.ts"; +export { removeLocale } from "./transformers/global/removeLocale.ts"; +export { wrapInData } from "./transformers/global/wrapInData.ts"; +// Security +export { groupsToRoles } from "./transformers/security/groupsToRoles.ts"; +export { removeTenant } from "./transformers/security/removeTenant.ts"; +export { transformPermissions } from "./transformers/security/transformPermissions.ts"; +// Mailer +export { migrateMailerSettings } from "./transformers/mailer/migrateMailerSettings.ts"; +// Audit logs +export { coreFieldsTransformer } from "./transformers/auditLogs/coreFieldsTransformer.ts"; +export { dataFieldsTransformer } from "./transformers/auditLogs/dataFieldsTransformer.ts"; +export { storageShapeTransformer } from "./transformers/auditLogs/storageShapeTransformer.ts"; // Pipeline factories export { createFilter, type Filter } from "./domain/pipeline/Filter.ts"; @@ -59,6 +91,7 @@ export { isCmsEntry, byIncludesModelId, isAcoSearchRecord, + isAdminUser, isBackgroundTask, isFmFile, isFlpRecord, diff --git a/src/mcp/bin.ts b/src/mcp/bin.ts new file mode 100644 index 0000000..321bd1e --- /dev/null +++ b/src/mcp/bin.ts @@ -0,0 +1,3 @@ +import { startMcpServer } from "./server.ts"; + +await startMcpServer(); diff --git a/src/mcp/discoverDocs.ts b/src/mcp/discoverDocs.ts new file mode 100644 index 0000000..a123288 --- /dev/null +++ b/src/mcp/discoverDocs.ts @@ -0,0 +1,90 @@ +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; +import fm from "front-matter"; + +interface Doc { + name: string; + description: string; + category: string; + filePath: string; + body: string; +} + +function findMarkdownFiles(dir: string): string[] { + const results: string[] = []; + + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + const stat = statSync(full); + + if (stat.isDirectory()) { + results.push(...findMarkdownFiles(full)); + } else if (entry.endsWith(".md")) { + results.push(full); + } + } + + return results; +} + +function parseDoc(filePath: string): Doc | null { + const raw = readFileSync(filePath, "utf-8"); + const parsed = fm<{ name?: string; description?: string; category?: string }>(raw); + + if (!parsed.attributes.name || !parsed.attributes.description) { + return null; + } + + return { + name: parsed.attributes.name, + description: parsed.attributes.description, + category: parsed.attributes.category ?? "general", + filePath, + body: parsed.body + }; +} + +export function discoverDocs(dirs: string[]): Map { + const docs = new Map(); + + for (const dir of dirs) { + for (const file of findMarkdownFiles(dir)) { + const doc = parseDoc(file); + if (doc && !docs.has(doc.name)) { + docs.set(doc.name, doc); + } + } + } + + return docs; +} + +export function buildCatalog(docs: Map): string { + const byCategory = new Map(); + + for (const doc of docs.values()) { + const list = byCategory.get(doc.category) ?? []; + list.push(doc); + byCategory.set(doc.category, list); + } + + const lines: string[] = ["# Available Topics\n"]; + + for (const [category, categoryDocs] of [...byCategory.entries()].sort((a, b) => + a[0].localeCompare(b[0]) + )) { + lines.push(`## ${category}\n`); + lines.push("| Topic | Description |"); + lines.push("|-------|-------------|"); + + for (const doc of categoryDocs.sort((a, b) => a.name.localeCompare(b.name))) { + lines.push(`| ${doc.name} | ${doc.description} |`); + } + + lines.push(""); + } + + return lines.join("\n"); +} + +export type { Doc }; diff --git a/src/mcp/front-matter.d.ts b/src/mcp/front-matter.d.ts new file mode 100644 index 0000000..8a906ee --- /dev/null +++ b/src/mcp/front-matter.d.ts @@ -0,0 +1,26 @@ +// The upstream `front-matter` package ships an `export default` ambient +// declaration, but its actual runtime export (`module.exports = fm`) is a +// plain CommonJS export. Under `moduleResolution: nodenext` combined with +// this package's own `type: module`, that mismatch makes the shipped types +// unusable (`fm` resolves to the module namespace instead of the callable +// function). This override restates the same shape using `export =`, which +// matches the real CJS export and interops correctly via `esModuleInterop`. +declare module "front-matter" { + interface FrontMatterResult { + readonly attributes: T; + readonly body: string; + readonly bodyBegin: number; + readonly frontmatter?: string; + } + + interface FrontMatterOptions { + allowUnsafe?: boolean; + } + + function fm(file: string, options?: FrontMatterOptions): FrontMatterResult; + namespace fm { + function test(file: string): boolean; + } + + export = fm; +} diff --git a/src/mcp/server.ts b/src/mcp/server.ts new file mode 100644 index 0000000..1ba9f6a --- /dev/null +++ b/src/mcp/server.ts @@ -0,0 +1,81 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; +import { z } from "zod"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { discoverDocs, buildCatalog, type Doc } from "./discoverDocs.ts"; + +const thisDir = fileURLToPath(new URL(".", import.meta.url)); +const DEFAULT_DOCS_DIR = + [ + join(thisDir, "../docs/mcp"), // production (dist/mcp/ → dist/docs/mcp) + join(thisDir, "../../docs/mcp") // dev (src/mcp/ → docs/mcp) + ].find(existsSync) ?? join(thisDir, "../docs/mcp"); + +export async function startMcpServer(docsDirs?: string[]): Promise { + const dirs = docsDirs ?? [DEFAULT_DOCS_DIR]; + let docsCache: Map | null = null; + + function getDocs(): Map { + if (!docsCache) { + docsCache = discoverDocs(dirs); + } + return docsCache; + } + + const server = new McpServer({ name: "webiny-data-transfer", version: "1.0.0" }); + + server.registerTool( + "list_topics", + { + title: "List Data Transfer Topics", + description: + "Returns a catalog of all available @webiny/data-transfer documentation topics. " + + "Call this first to discover what topics are available, then use get_topic to read the full documentation for a specific topic. " + + "Topics cover: presets, transformers, processors, scanners, filters, config, pipeline runtime, and how to write custom components.", + inputSchema: {}, + annotations: { readOnlyHint: true } + }, + async () => ({ + content: [{ type: "text" as const, text: buildCatalog(getDocs()) }] + }) + ); + + server.registerTool( + "get_topic", + { + title: "Get Data Transfer Topic", + description: + "Returns the full documentation for a specific @webiny/data-transfer topic. " + + "Use exact topic names from list_topics.", + inputSchema: { + topic: z.string().describe("Topic name — use exact names from list_topics") + }, + annotations: { readOnlyHint: true } + }, + async ({ topic }) => { + const docs = getDocs(); + const doc = docs.get(topic); + + if (!doc) { + const available = [...docs.keys()].sort().join(", "); + return { + content: [ + { + type: "text" as const, + text: `Topic "${topic}" not found. Available topics: ${available}` + } + ] + }; + } + + return { + content: [{ type: "text" as const, text: doc.body }] + }; + } + ); + + const transport = new StdioServerTransport(); + await server.connect(transport); +} diff --git a/templates/.mcp.json b/templates/.mcp.json new file mode 100644 index 0000000..35887cb --- /dev/null +++ b/templates/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "webiny-data-transfer": { + "command": "npx", + "args": ["webiny-data-transfer-mcp"] + } + } +} diff --git a/templates/README.md b/templates/README.md index f4230db..cdc4b9b 100644 --- a/templates/README.md +++ b/templates/README.md @@ -67,6 +67,25 @@ For v5 to v6 migration, run `v5-to-v6-ddb` first, then `v5-to-v6-os`. Custom presets in `presets/` are listed alongside built-ins. See `presets/example.ts` for a starting point. +## AI agent support (MCP) + +This project includes an MCP server that gives AI agents (Claude, Cursor, Kiro, Copilot, etc.) access to all data-transfer documentation — presets, transformers, processors, config reference, and how-to guides. + +The `.mcp.json` in the project root auto-configures it for agents that support MCP discovery. No setup needed for new scaffolded projects. + +For existing projects, create `.mcp.json` in your project root: + +```json +{ + "mcpServers": { + "webiny-data-transfer": { + "command": "npx", + "args": ["webiny-data-transfer-mcp"] + } + } +} +``` + ## Documentation - [Config reference](https://github.com/webiny/data-transfer/blob/main/docs/guides/config-reference.md) — config.ts setup, env helpers, credentials, IAM, tuning diff --git a/yarn.lock b/yarn.lock index d2e0b7c..32a1e4d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3442,6 +3442,15 @@ __metadata: languageName: node linkType: hard +"@hono/node-server@npm:^1.19.9 || ^2.0.5": + version: 2.0.12 + resolution: "@hono/node-server@npm:2.0.12" + peerDependencies: + hono: ^4 + checksum: 10/7900669baa2c62c97a25c2cd9dc4ba9e56bba572834fc392e8ce9d3d1aac7b78fdad6b72d2cb47bb668295253a964763a6098e546f0484cf9b96edb1d38ccdf3 + languageName: node + linkType: hard + "@iconify/json@npm:2.2.498": version: 2.2.498 resolution: "@iconify/json@npm:2.2.498" @@ -4791,6 +4800,39 @@ __metadata: languageName: node linkType: hard +"@modelcontextprotocol/sdk@npm:^1.30.0": + version: 1.30.0 + resolution: "@modelcontextprotocol/sdk@npm:1.30.0" + dependencies: + "@hono/node-server": "npm:^1.19.9 || ^2.0.5" + ajv: "npm:^8.17.1" + ajv-formats: "npm:^3.0.1" + content-type: "npm:^1.0.5" + cors: "npm:^2.8.5" + cross-spawn: "npm:^7.0.5" + eventsource: "npm:^3.0.2" + eventsource-parser: "npm:^3.0.0" + express: "npm:^5.2.1" + express-rate-limit: "npm:^8.2.1" + hono: "npm:^4.11.4" + jose: "npm:^6.1.3" + json-schema-typed: "npm:^8.0.2" + pkce-challenge: "npm:^5.0.0" + raw-body: "npm:^3.0.0" + zod: "npm:^3.25 || ^4.0" + zod-to-json-schema: "npm:^3.25.1" + peerDependencies: + "@cfworker/json-schema": ^4.1.1 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + "@cfworker/json-schema": + optional: true + zod: + optional: false + checksum: 10/369723a62ce230e55d4fc98549fb080a51916a2004f3907bfca899f5602de8791d9b252038abc6cf120a9d1c75845d2459a471b678c9b9cfc7bae5febf112b63 + languageName: node + linkType: hard + "@monaco-editor/loader@npm:^1.5.0": version: 1.7.0 resolution: "@monaco-editor/loader@npm:1.7.0" @@ -10167,6 +10209,7 @@ __metadata: "@faker-js/faker": "npm:^10.5.0" "@inquirer/core": "npm:^11.2.1" "@inquirer/prompts": "npm:^8.5.2" + "@modelcontextprotocol/sdk": "npm:^1.30.0" "@opensearch-project/opensearch": "npm:3.6.0" "@smithy/util-stream": "npm:^4.7.16" "@types/jsdom": "npm:^28.0.3" @@ -10186,6 +10229,7 @@ __metadata: dynalite: "npm:^4.0.0" execa: "npm:^10.0.1" exifreader: "npm:^4.41.3" + front-matter: "npm:^4.0.2" jsdom: "npm:^30.0.1" oxfmt: "npm:^0.61.0" oxlint: "npm:^1.76.0" @@ -10199,7 +10243,9 @@ __metadata: yargs: "npm:^18.1.0" zod: "npm:^4.4.3" bin: + data-transfer: ./dist/cli.js webiny-data-transfer: ./dist/cli.js + webiny-data-transfer-mcp: ./dist/mcp/bin.js languageName: unknown linkType: soft @@ -10761,6 +10807,16 @@ __metadata: languageName: node linkType: hard +"accepts@npm:^2.0.0": + version: 2.0.0 + resolution: "accepts@npm:2.0.0" + dependencies: + mime-types: "npm:^3.0.0" + negotiator: "npm:^1.0.0" + checksum: 10/ea1343992b40b2bfb3a3113fa9c3c2f918ba0f9197ae565c48d3f84d44b174f6b1d5cd9989decd7655963eb03a272abc36968cc439c2907f999bd5ef8653d5a7 + languageName: node + linkType: hard + "accepts@npm:~1.3.4, accepts@npm:~1.3.8": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -10902,7 +10958,7 @@ __metadata: languageName: node linkType: hard -"ajv@npm:^8.0.0, ajv@npm:^8.12.0": +"ajv@npm:^8.0.0, ajv@npm:^8.12.0, ajv@npm:^8.17.1": version: 8.20.0 resolution: "ajv@npm:8.20.0" dependencies: @@ -11467,6 +11523,23 @@ __metadata: languageName: node linkType: hard +"body-parser@npm:^2.2.1": + version: 2.3.0 + resolution: "body-parser@npm:2.3.0" + dependencies: + bytes: "npm:^3.1.2" + content-type: "npm:^2.0.0" + debug: "npm:^4.4.3" + http-errors: "npm:^2.0.1" + iconv-lite: "npm:^0.7.2" + on-finished: "npm:^2.4.1" + qs: "npm:^6.15.2" + raw-body: "npm:^3.0.2" + type-is: "npm:^2.1.0" + checksum: 10/94f741ff525358dd0d47f0a042936dc61b9d2e348ff1228c5ffd79f1902c673a317bbb9495b3f7d1db4483befe6ffed1c42bc4cdd8da8f41690530810cfe4dad + languageName: node + linkType: hard + "body-parser@npm:~1.20.3, body-parser@npm:~1.20.5": version: 1.20.6 resolution: "body-parser@npm:1.20.6" @@ -11701,7 +11774,7 @@ __metadata: languageName: node linkType: hard -"bytes@npm:3.1.2, bytes@npm:~3.1.2": +"bytes@npm:3.1.2, bytes@npm:^3.1.2, bytes@npm:~3.1.2": version: 3.1.2 resolution: "bytes@npm:3.1.2" checksum: 10/a10abf2ba70c784471d6b4f58778c0beeb2b5d405148e66affa91f23a9f13d07603d0a0354667310ae1d6dc141474ffd44e2a074be0f6e2254edb8fc21445388 @@ -12169,6 +12242,13 @@ __metadata: languageName: node linkType: hard +"content-disposition@npm:^1.0.0": + version: 1.1.0 + resolution: "content-disposition@npm:1.1.0" + checksum: 10/c4f65e3c001a4a8eb87d0d24c0f112abb139836fb13b8ea67276715e7dce09570ef666ba7848ee8b660d467e6588d030c8ed7e8d0128db6ca78a0800dcd8c7a8 + languageName: node + linkType: hard + "content-disposition@npm:~0.5.4": version: 0.5.4 resolution: "content-disposition@npm:0.5.4" @@ -12178,13 +12258,20 @@ __metadata: languageName: node linkType: hard -"content-type@npm:~1.0.4, content-type@npm:~1.0.5": +"content-type@npm:^1.0.5, content-type@npm:~1.0.4, content-type@npm:~1.0.5": version: 1.0.5 resolution: "content-type@npm:1.0.5" checksum: 10/585847d98dc7fb8035c02ae2cb76c7a9bd7b25f84c447e5ed55c45c2175e83617c8813871b4ee22f368126af6b2b167df655829007b21aa10302873ea9c62662 languageName: node linkType: hard +"content-type@npm:^2.0.0": + version: 2.0.0 + resolution: "content-type@npm:2.0.0" + checksum: 10/0bbb276b790ba7e86c479c7d69fae1861b2e908ff3ce2cb01975b516f93eede2216d242902ed6c5b15cd554014611ce9dfdcf51cd35b16569e45a979e50d0cef + languageName: node + linkType: hard + "convert-source-map@npm:^1.5.0": version: 1.9.0 resolution: "convert-source-map@npm:1.9.0" @@ -12199,6 +12286,13 @@ __metadata: languageName: node linkType: hard +"cookie-signature@npm:^1.2.1": + version: 1.2.2 + resolution: "cookie-signature@npm:1.2.2" + checksum: 10/be44a3c9a56f3771aea3a8bd8ad8f0a8e2679bcb967478267f41a510b4eb5ec55085386ba79c706c4ac21605ca76f4251973444b90283e0eb3eeafe8a92c7708 + languageName: node + linkType: hard + "cookie-signature@npm:~1.0.6": version: 1.0.7 resolution: "cookie-signature@npm:1.0.7" @@ -12206,6 +12300,13 @@ __metadata: languageName: node linkType: hard +"cookie@npm:^0.7.1, cookie@npm:~0.7.1, cookie@npm:~0.7.2": + version: 0.7.2 + resolution: "cookie@npm:0.7.2" + checksum: 10/24b286c556420d4ba4e9bc09120c9d3db7d28ace2bd0f8ccee82422ce42322f73c8312441271e5eefafbead725980e5996cc02766dbb89a90ac7f5636ede608f + languageName: node + linkType: hard + "cookie@npm:^1.0.0, cookie@npm:^1.0.1": version: 1.1.1 resolution: "cookie@npm:1.1.1" @@ -12213,13 +12314,6 @@ __metadata: languageName: node linkType: hard -"cookie@npm:~0.7.1, cookie@npm:~0.7.2": - version: 0.7.2 - resolution: "cookie@npm:0.7.2" - checksum: 10/24b286c556420d4ba4e9bc09120c9d3db7d28ace2bd0f8ccee82422ce42322f73c8312441271e5eefafbead725980e5996cc02766dbb89a90ac7f5636ede608f - languageName: node - linkType: hard - "core-js-compat@npm:^3.48.0": version: 3.49.0 resolution: "core-js-compat@npm:3.49.0" @@ -12243,7 +12337,7 @@ __metadata: languageName: node linkType: hard -"cors@npm:2.8.6, cors@npm:~2.8.5": +"cors@npm:2.8.6, cors@npm:^2.8.5, cors@npm:~2.8.5": version: 2.8.6 resolution: "cors@npm:2.8.6" dependencies: @@ -12482,7 +12576,7 @@ __metadata: languageName: node linkType: hard -"debug@npm:4, debug@npm:4.4.3, debug@npm:^4.1.0, debug@npm:^4.3.1, debug@npm:^4.3.4, debug@npm:^4.3.5, debug@npm:^4.4.1, debug@npm:^4.4.3, debug@npm:~4.4.1": +"debug@npm:4, debug@npm:4.4.3, debug@npm:^4.1.0, debug@npm:^4.3.1, debug@npm:^4.3.4, debug@npm:^4.3.5, debug@npm:^4.4.0, debug@npm:^4.4.1, debug@npm:^4.4.3, debug@npm:~4.4.1": version: 4.4.3 resolution: "debug@npm:4.4.3" dependencies: @@ -12663,7 +12757,7 @@ __metadata: languageName: node linkType: hard -"depd@npm:2.0.0, depd@npm:~2.0.0": +"depd@npm:2.0.0, depd@npm:^2.0.0, depd@npm:~2.0.0": version: 2.0.0 resolution: "depd@npm:2.0.0" checksum: 10/c0c8ff36079ce5ada64f46cc9d6fd47ebcf38241105b6e0c98f412e8ad91f084bcf906ff644cc3a4bd876ca27a62accb8b0fff72ea6ed1a414b89d8506f4a5ca @@ -12958,7 +13052,7 @@ __metadata: languageName: node linkType: hard -"encodeurl@npm:~2.0.0": +"encodeurl@npm:^2.0.0, encodeurl@npm:~2.0.0": version: 2.0.0 resolution: "encodeurl@npm:2.0.0" checksum: 10/abf5cd51b78082cf8af7be6785813c33b6df2068ce5191a40ca8b1afe6a86f9230af9a9ce694a5ce4665955e5c1120871826df9c128a642e09c58d592e2807fe @@ -13251,7 +13345,7 @@ __metadata: languageName: node linkType: hard -"escape-html@npm:~1.0.3": +"escape-html@npm:^1.0.3, escape-html@npm:~1.0.3": version: 1.0.3 resolution: "escape-html@npm:1.0.3" checksum: 10/6213ca9ae00d0ab8bccb6d8d4e0a98e76237b2410302cf7df70aaa6591d509a2a37ce8998008cbecae8fc8ffaadf3fb0229535e6a145f3ce0b211d060decbb24 @@ -13298,7 +13392,7 @@ __metadata: languageName: node linkType: hard -"etag@npm:~1.8.1": +"etag@npm:^1.8.1, etag@npm:~1.8.1": version: 1.8.1 resolution: "etag@npm:1.8.1" checksum: 10/571aeb3dbe0f2bbd4e4fadbdb44f325fc75335cd5f6f6b6a091e6a06a9f25ed5392f0863c5442acb0646787446e816f13cbfc6edce5b07658541dff573cab1ff @@ -13335,13 +13429,22 @@ __metadata: languageName: node linkType: hard -"eventsource-parser@npm:^3.0.8": +"eventsource-parser@npm:^3.0.0, eventsource-parser@npm:^3.0.1, eventsource-parser@npm:^3.0.8": version: 3.1.0 resolution: "eventsource-parser@npm:3.1.0" checksum: 10/6aa03b4d6e3450935690fd9cca6e47b9877287c9419dba9705b85a73e741c7dfbc22b4ebeca25adf05c9549e33c4491e3ca71f80ddfac585e469b7da91f76e20 languageName: node linkType: hard +"eventsource@npm:^3.0.2": + version: 3.0.7 + resolution: "eventsource@npm:3.0.7" + dependencies: + eventsource-parser: "npm:^3.0.1" + checksum: 10/e034915bc97068d1d38617951afd798e6776d6a3a78e36a7569c235b177c7afc2625c9fe82656f7341ab72c7eeecb3fd507b7f88e9328f2448872ff9c4742bb6 + languageName: node + linkType: hard + "execa@npm:5.1.1, execa@npm:^5.1.0": version: 5.1.1 resolution: "execa@npm:5.1.1" @@ -13421,6 +13524,18 @@ __metadata: languageName: node linkType: hard +"express-rate-limit@npm:^8.2.1": + version: 8.6.1 + resolution: "express-rate-limit@npm:8.6.1" + dependencies: + debug: "npm:^4.4.3" + ip-address: "npm:^10.2.0" + peerDependencies: + express: ">= 4.11" + checksum: 10/55e40d693fcb9b7dc1adb48e0d83b3552e902982d80ec1812ba3b60deaffda8b0c12392253018b1c70772df4c0fd40c040dad85dc18349e1e025aa3c8fc70aa2 + languageName: node + linkType: hard + "express@npm:4.22.1": version: 4.22.1 resolution: "express@npm:4.22.1" @@ -13499,6 +13614,42 @@ __metadata: languageName: node linkType: hard +"express@npm:^5.2.1": + version: 5.2.1 + resolution: "express@npm:5.2.1" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.1" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10/4aa545d89702ac83f645c77abda1b57bcabe288f0b380fb5580fac4e323ea0eb533005c8e666b4e19152fb16d4abf11ba87b22aa9a10857a0485cd86b94639bd + languageName: node + linkType: hard + "extend@npm:~3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -13781,6 +13932,20 @@ __metadata: languageName: node linkType: hard +"finalhandler@npm:^2.1.0": + version: 2.1.1 + resolution: "finalhandler@npm:2.1.1" + dependencies: + debug: "npm:^4.4.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + on-finished: "npm:^2.4.1" + parseurl: "npm:^1.3.3" + statuses: "npm:^2.0.1" + checksum: 10/f4ba75c23408d8f9d393c3e875b9452e84d68c925411a6e67b7efa678b0bed5075ef33def4bb65ed8e0dd37c92a3ea354bcbde07303cd4dc2550e12b95885067 + languageName: node + linkType: hard + "finalhandler@npm:~1.3.1": version: 1.3.2 resolution: "finalhandler@npm:1.3.2" @@ -13899,6 +14064,13 @@ __metadata: languageName: node linkType: hard +"fresh@npm:^2.0.0": + version: 2.0.0 + resolution: "fresh@npm:2.0.0" + checksum: 10/44e1468488363074641991c1340d2a10c5a6f6d7c353d89fd161c49d120c58ebf9890720f7584f509058385836e3ce50ddb60e9f017315a4ba8c6c3461813bfc + languageName: node + linkType: hard + "fresh@npm:~0.5.2": version: 0.5.2 resolution: "fresh@npm:0.5.2" @@ -13906,6 +14078,15 @@ __metadata: languageName: node linkType: hard +"front-matter@npm:^4.0.2": + version: 4.0.2 + resolution: "front-matter@npm:4.0.2" + dependencies: + js-yaml: "npm:^3.13.1" + checksum: 10/8897a831a82c5d35413b02b806ed421e793068ad8bf75e864163ec07b7f0cfd87e2fcce0893e8ceccc8f6c63a46e953a6c01208e573627626867a8b86cf6abb9 + languageName: node + linkType: hard + "fs-constants@npm:^1.0.0": version: 1.0.0 resolution: "fs-constants@npm:1.0.0" @@ -14416,6 +14597,13 @@ __metadata: languageName: node linkType: hard +"hono@npm:^4.11.4": + version: 4.12.33 + resolution: "hono@npm:4.12.33" + checksum: 10/1a4d8c2f05731d28785b19488d53ed629dd83f10dce79042da75dbbc9581a7205c20a840a6657d79c50c0d638057cb42adc4656ee6e8a73bde6ab0d0c4f68719 + languageName: node + linkType: hard + "hosted-git-info@npm:^7.0.0": version: 7.0.2 resolution: "hosted-git-info@npm:7.0.2" @@ -14488,7 +14676,7 @@ __metadata: languageName: node linkType: hard -"http-errors@npm:2.0.1, http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": +"http-errors@npm:2.0.1, http-errors@npm:^2.0.0, http-errors@npm:^2.0.1, http-errors@npm:~2.0.0, http-errors@npm:~2.0.1": version: 2.0.1 resolution: "http-errors@npm:2.0.1" dependencies: @@ -14605,7 +14793,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2": +"iconv-lite@npm:^0.7.0, iconv-lite@npm:^0.7.2, iconv-lite@npm:~0.7.0": version: 0.7.3 resolution: "iconv-lite@npm:0.7.3" dependencies: @@ -14748,7 +14936,7 @@ __metadata: languageName: node linkType: hard -"ip-address@npm:^10.1.1": +"ip-address@npm:^10.1.1, ip-address@npm:^10.2.0": version: 10.4.0 resolution: "ip-address@npm:10.4.0" checksum: 10/8a286dd112a88c372b10c706caca14dcf5ed029a11c3911062ad8937ca0f951aa513ecacf82e15ffcf6343749ae6c079dd9741282fc94033a06ff1b7bd6ce69a @@ -14974,6 +15162,13 @@ __metadata: languageName: node linkType: hard +"is-promise@npm:^4.0.0": + version: 4.0.0 + resolution: "is-promise@npm:4.0.0" + checksum: 10/0b46517ad47b00b6358fd6553c83ec1f6ba9acd7ffb3d30a0bf519c5c69e7147c132430452351b8a9fc198f8dd6c4f76f8e6f5a7f100f8c77d57d9e0f4261a8a + languageName: node + linkType: hard + "is-regex@npm:^1.1.4, is-regex@npm:^1.2.1": version: 1.2.1 resolution: "is-regex@npm:1.2.1" @@ -15186,6 +15381,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:^6.1.3": + version: 6.2.5 + resolution: "jose@npm:6.2.5" + checksum: 10/d0e06fd0e0c3b3ea949049fc15db6724c60c1fa1fcb51ca830b2d513f6bbcaf70c4f80cc47cc463fed5991905846da6de323215413cdde057dc113cb694fc416 + languageName: node + linkType: hard + "joycon@npm:^3.1.1": version: 3.1.1 resolution: "joycon@npm:3.1.1" @@ -15225,7 +15427,7 @@ __metadata: languageName: node linkType: hard -"js-yaml@npm:^3.14.1, js-yaml@npm:^3.6.1": +"js-yaml@npm:^3.13.1, js-yaml@npm:^3.14.1, js-yaml@npm:^3.6.1": version: 3.15.0 resolution: "js-yaml@npm:3.15.0" dependencies: @@ -15369,6 +15571,13 @@ __metadata: languageName: node linkType: hard +"json-schema-typed@npm:^8.0.2": + version: 8.0.2 + resolution: "json-schema-typed@npm:8.0.2" + checksum: 10/fa866d1fe91e3a94aa4fe007861475cd03dcaf47b719861cab171ef2f8598478007c634d29ae45de94ee34ddff4e13414c63ea5ff06c5b868b613142c699d511 + languageName: node + linkType: hard + "json-schema@npm:0.4.0, json-schema@npm:^0.4.0": version: 0.4.0 resolution: "json-schema@npm:0.4.0" @@ -16222,6 +16431,13 @@ __metadata: languageName: node linkType: hard +"media-typer@npm:^1.1.0": + version: 1.1.1 + resolution: "media-typer@npm:1.1.1" + checksum: 10/ceef972e43912621b846f816e8937c200dcd83d0bdd4fdd19f28be62a9334f4e6605d0c952fe804bfab7f63a07d14c0d1bea1d9357c35ad6d3b04b184f09e397 + languageName: node + linkType: hard + "memfs@npm:^4.64.0": version: 4.64.0 resolution: "memfs@npm:4.64.0" @@ -16264,6 +16480,13 @@ __metadata: languageName: node linkType: hard +"merge-descriptors@npm:^2.0.0": + version: 2.0.0 + resolution: "merge-descriptors@npm:2.0.0" + checksum: 10/e383332e700a94682d0125a36c8be761142a1320fc9feeb18e6e36647c9edf064271645f5669b2c21cf352116e561914fd8aa831b651f34db15ef4038c86696a + languageName: node + linkType: hard + "merge-stream@npm:^2.0.0": version: 2.0.0 resolution: "merge-stream@npm:2.0.0" @@ -16302,7 +16525,7 @@ __metadata: languageName: node linkType: hard -"mime-db@npm:>= 1.43.0 < 2, mime-db@npm:^1.52.0": +"mime-db@npm:>= 1.43.0 < 2, mime-db@npm:^1.52.0, mime-db@npm:^1.54.0": version: 1.54.0 resolution: "mime-db@npm:1.54.0" checksum: 10/9e7834be3d66ae7f10eaa69215732c6d389692b194f876198dca79b2b90cbf96688d9d5d05ef7987b20f749b769b11c01766564264ea5f919c88b32a29011311 @@ -16318,6 +16541,15 @@ __metadata: languageName: node linkType: hard +"mime-types@npm:^3.0.0, mime-types@npm:^3.0.2": + version: 3.0.2 + resolution: "mime-types@npm:3.0.2" + dependencies: + mime-db: "npm:^1.54.0" + checksum: 10/9db0ad31f5eff10ee8f848130779b7f2d056ddfdb6bda696cb69be68d486d33a3457b4f3f9bdeb60d0736edb471bd5a7c0a384375c011c51c889fd0d5c3b893e + languageName: node + linkType: hard + "mime@npm:1.6.0": version: 1.6.0 resolution: "mime@npm:1.6.0" @@ -17064,7 +17296,7 @@ __metadata: languageName: node linkType: hard -"on-finished@npm:~2.4.1": +"on-finished@npm:^2.4.1, on-finished@npm:~2.4.1": version: 2.4.1 resolution: "on-finished@npm:2.4.1" dependencies: @@ -17552,7 +17784,7 @@ __metadata: languageName: node linkType: hard -"parseurl@npm:~1.3.3": +"parseurl@npm:^1.3.3, parseurl@npm:~1.3.3": version: 1.3.3 resolution: "parseurl@npm:1.3.3" checksum: 10/407cee8e0a3a4c5cd472559bca8b6a45b82c124e9a4703302326e9ab60fc1081442ada4e02628efef1eb16197ddc7f8822f5a91fd7d7c86b51f530aedb17dfa2 @@ -17611,7 +17843,7 @@ __metadata: languageName: node linkType: hard -"path-to-regexp@npm:^8.3.0": +"path-to-regexp@npm:^8.0.0, path-to-regexp@npm:^8.3.0": version: 8.4.2 resolution: "path-to-regexp@npm:8.4.2" checksum: 10/70fd2cbce0b962cbcf4d312af07818bfce2bae11c09cf3bd86be99c0e30168238a1a7b02b18b452e73f075897df04597d30d63e56da7be41eecfc37998693389 @@ -17831,6 +18063,13 @@ __metadata: languageName: node linkType: hard +"pkce-challenge@npm:^5.0.0": + version: 5.0.1 + resolution: "pkce-challenge@npm:5.0.1" + checksum: 10/51d11f68d5a78617cfb2e9c2706dadcc2cbe55ffb55b21d42a6ed848ac5159db2657bf6c966a5a414119aa839ceb64240afea35e9e1c06946b57606ed0b43789 + languageName: node + linkType: hard + "pluralize@npm:8.0.0": version: 8.0.0 resolution: "pluralize@npm:8.0.0" @@ -18069,7 +18308,7 @@ __metadata: languageName: node linkType: hard -"proxy-addr@npm:~2.0.7": +"proxy-addr@npm:^2.0.7, proxy-addr@npm:~2.0.7": version: 2.0.7 resolution: "proxy-addr@npm:2.0.7" dependencies: @@ -18124,7 +18363,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.15.2, qs@npm:~6.15.1": +"qs@npm:^6.14.0, qs@npm:^6.15.2, qs@npm:~6.15.1": version: 6.15.3 resolution: "qs@npm:6.15.3" dependencies: @@ -18260,6 +18499,13 @@ __metadata: languageName: node linkType: hard +"range-parser@npm:^1.2.1": + version: 1.3.0 + resolution: "range-parser@npm:1.3.0" + checksum: 10/f0b7a34de67333b80da0a1098cc02579bd5f74da717fdc0807caf87f043af6641d98d94c64a60477a8f88ba7c214aede965fd0e0c3036ea35c9fe89e1149923d + languageName: node + linkType: hard + "range-parser@npm:~1.2.1": version: 1.2.1 resolution: "range-parser@npm:1.2.1" @@ -18267,6 +18513,18 @@ __metadata: languageName: node linkType: hard +"raw-body@npm:^3.0.0, raw-body@npm:^3.0.2": + version: 3.0.2 + resolution: "raw-body@npm:3.0.2" + dependencies: + bytes: "npm:~3.1.2" + http-errors: "npm:~2.0.1" + iconv-lite: "npm:~0.7.0" + unpipe: "npm:~1.0.0" + checksum: 10/4168c82157bd69175d5bd960e59b74e253e237b358213694946a427a6f750a18b8e150f036fed3421b3e83294b071a4e2bb01037a79ccacdac05360c63d3ebba + languageName: node + linkType: hard + "raw-body@npm:~2.5.3": version: 2.5.3 resolution: "raw-body@npm:2.5.3" @@ -18917,6 +19175,19 @@ __metadata: languageName: node linkType: hard +"router@npm:^2.2.0": + version: 2.2.0 + resolution: "router@npm:2.2.0" + dependencies: + debug: "npm:^4.4.0" + depd: "npm:^2.0.0" + is-promise: "npm:^4.0.0" + parseurl: "npm:^1.3.3" + path-to-regexp: "npm:^8.0.0" + checksum: 10/8949bd1d3da5403cc024e2989fee58d7fda0f3ffe9f2dc5b8a192f295f400b3cde307b0b554f7d44851077640f36962ca469a766b3d57410d7d96245a7ba6c91 + languageName: node + linkType: hard + "rsbuild-plugin-dts@npm:0.23.2": version: 0.23.2 resolution: "rsbuild-plugin-dts@npm:0.23.2" @@ -19375,6 +19646,25 @@ __metadata: languageName: node linkType: hard +"send@npm:^1.1.0, send@npm:^1.2.0": + version: 1.2.1 + resolution: "send@npm:1.2.1" + dependencies: + debug: "npm:^4.4.3" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.1" + mime-types: "npm:^3.0.2" + ms: "npm:^2.1.3" + on-finished: "npm:^2.4.1" + range-parser: "npm:^1.2.1" + statuses: "npm:^2.0.2" + checksum: 10/274f842d69ccfa49d4940a85598c6825da58dee6cb8ea33b08d5bd3988e6a82267c4d7c32b23d0e4706aad076ee95b1edfa13f859877db9b589829019397e355 + languageName: node + linkType: hard + "send@npm:~0.19.0, send@npm:~0.19.1": version: 0.19.2 resolution: "send@npm:0.19.2" @@ -19406,6 +19696,18 @@ __metadata: languageName: node linkType: hard +"serve-static@npm:^2.2.0": + version: 2.2.1 + resolution: "serve-static@npm:2.2.1" + dependencies: + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + parseurl: "npm:^1.3.3" + send: "npm:^1.2.0" + checksum: 10/71500fe80cc7163fec04e4297de7591ad1cb682d137fc030e7a53e57040fda5187e8082a9c1b2ef37f1d3f9c27c9a94d4ba61806ebc28938ba4a7c8947c9f71e + languageName: node + linkType: hard + "serve-static@npm:~1.16.2": version: 1.16.3 resolution: "serve-static@npm:1.16.3" @@ -19952,7 +20254,7 @@ __metadata: languageName: node linkType: hard -"statuses@npm:~2.0.1, statuses@npm:~2.0.2": +"statuses@npm:^2.0.1, statuses@npm:^2.0.2, statuses@npm:~2.0.1, statuses@npm:~2.0.2": version: 2.0.2 resolution: "statuses@npm:2.0.2" checksum: 10/6927feb50c2a75b2a4caab2c565491f7a93ad3d8dbad7b1398d52359e9243a20e2ebe35e33726dee945125ef7a515e9097d8a1b910ba2bbd818265a2f6c39879 @@ -20723,6 +21025,17 @@ __metadata: languageName: node linkType: hard +"type-is@npm:^2.0.1, type-is@npm:^2.1.0": + version: 2.1.0 + resolution: "type-is@npm:2.1.0" + dependencies: + content-type: "npm:^2.0.0" + media-typer: "npm:^1.1.0" + mime-types: "npm:^3.0.0" + checksum: 10/f011885fefb6c6882f36fab89cc596596b96eab1d208a3774628537cad7ce1f91232a6d758115e1a6ed03f0f8c21e9654bb6d280a5c6827ff72a35f6df0fa182 + languageName: node + linkType: hard + "type-is@npm:~1.6.18": version: 1.6.18 resolution: "type-is@npm:1.6.18" @@ -21260,7 +21573,7 @@ __metadata: languageName: node linkType: hard -"vary@npm:^1, vary@npm:~1.1.2": +"vary@npm:^1, vary@npm:^1.1.2, vary@npm:~1.1.2": version: 1.1.2 resolution: "vary@npm:1.1.2" checksum: 10/31389debef15a480849b8331b220782230b9815a8e0dbb7b9a8369559aed2e9a7800cd904d4371ea74f4c3527db456dc8e7ac5befce5f0d289014dbdf47b2242 @@ -21976,7 +22289,16 @@ __metadata: languageName: node linkType: hard -"zod@npm:4.4.3, zod@npm:^4.4.3": +"zod-to-json-schema@npm:^3.25.1": + version: 3.25.2 + resolution: "zod-to-json-schema@npm:3.25.2" + peerDependencies: + zod: ^3.25.28 || ^4 + checksum: 10/7035328654113f1a0b8e4c2d34a06f918c93650ef8a50d4fb30ad8f22e47d5762c163af9c82494756b34776bae3c41c26cfc6945105b0eee7dceb528cc07e665 + languageName: node + linkType: hard + +"zod@npm:4.4.3, zod@npm:^3.25 || ^4.0, zod@npm:^4.4.3": version: 4.4.3 resolution: "zod@npm:4.4.3" checksum: 10/804b9a42aa8f35f2b3c5a8dff906291cb749115f83ee2afe3576d70b5b5c53c965365c7f4967690647a9c54af9838ff232a85ff9577a0a36c44b68bc6cdefe36