From 3d9dfcccf3826d45441a2565a2cdfced20cf5217 Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Mon, 25 Nov 2024 23:59:48 -0600 Subject: [PATCH 1/4] feat: recordHookStream --- flatfilers/sandbox/package.json | 4 +- flatfilers/sandbox/src/index.ts | 145 +++----- package-lock.json | 340 ++++++++++++++++-- plugins/record-hook-stream/CHANGELOG.md | 1 + plugins/record-hook-stream/README.md | 332 +++++++++++++++++ plugins/record-hook-stream/package.json | 68 ++++ plugins/record-hook-stream/src/index.ts | 32 ++ .../src/record.hook.stream.ts | 324 +++++++++++++++++ .../record-hook-stream/src/utils/casting.ts | 73 ++++ .../src/utils/is.nullish.ts | 14 + plugins/record-hook-stream/src/utils/item.ts | 337 +++++++++++++++++ plugins/record-hook-stream/tsup.config.mjs | 3 + plugins/record-hook-stream/vitest.config.ts | 4 + plugins/record-hook/src/RecordHook.ts | 22 +- plugins/record-hook/src/record.hook.plugin.ts | 1 + 15 files changed, 1557 insertions(+), 143 deletions(-) create mode 100644 plugins/record-hook-stream/CHANGELOG.md create mode 100644 plugins/record-hook-stream/README.md create mode 100644 plugins/record-hook-stream/package.json create mode 100644 plugins/record-hook-stream/src/index.ts create mode 100644 plugins/record-hook-stream/src/record.hook.stream.ts create mode 100644 plugins/record-hook-stream/src/utils/casting.ts create mode 100644 plugins/record-hook-stream/src/utils/is.nullish.ts create mode 100644 plugins/record-hook-stream/src/utils/item.ts create mode 100644 plugins/record-hook-stream/tsup.config.mjs create mode 100644 plugins/record-hook-stream/vitest.config.ts diff --git a/flatfilers/sandbox/package.json b/flatfilers/sandbox/package.json index f3443c32a..18ad52475 100644 --- a/flatfilers/sandbox/package.json +++ b/flatfilers/sandbox/package.json @@ -7,7 +7,7 @@ "description": "", "license": "ISC", "scripts": { - "dev": "node --enable-source-maps --inspect ../../node_modules/.bin/flatfile develop", + "dev": "node --enable-source-maps --trace-warnings --inspect ../../node_modules/.bin/flatfile develop", "dev:local": "dotenvx run --env-file=.env.local -- npm run dev", "dev:staging": "dotenvx run --env-file=.env.staging -- npm run dev", "dev:prod": "dotenvx run --env-file=.env.prod -- npm run dev", @@ -30,6 +30,6 @@ }, "devDependencies": { "@dotenvx/dotenvx": "^0.39.0", - "flatfile": "^3.6.1" + "flatfile": "^3.8.0" } } diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index 16731d584..1096e1821 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -1,115 +1,52 @@ -import type { FlatfileListener } from '@flatfile/listener' -import { exportDelimitedZip } from '@flatfile/plugin-export-delimited-zip' -import { JSONExtractor } from '@flatfile/plugin-json-extractor' +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { + bulkRecordHook, + type FlatfileRecord, +} from '@flatfile/plugin-record-hook' +import { recordHookStream } from '@flatfile/plugin-record-hook-stream' import { configureSpace } from '@flatfile/plugin-space-configure' +import { contactsSheet, oneHundredSheet } from '../../playground/src/blueprints' export default async function (listener: FlatfileListener) { - listener.use(JSONExtractor()) + // listener.use( + // bulkRecordHook( + // '**', + // (records: FlatfileRecord[], event: FlatfileEvent) => { + // for (const record of records) { + // for (const field of oneHundredSheet.fields) { + // record.addError(field.key, 'Testing streaming') + // } + // // record.addError('one', 'Testing streaming') + // } + // return records + // }, + // { debug: true } + // ) + // ) listener.use( - exportDelimitedZip({ - job: 'export-delimited-zip', - delimiter: '\t', - fileExtension: 'tsv', - debug: true, - }) + recordHookStream( + '**', + (records, event: FlatfileEvent) => { + for (const record of records) { + for (const field of oneHundredSheet.fields) { + record.err(field.key, 'Testing something') + } + } + return records + }, + { includeMessages: true, debug: true } + ) ) listener.use( configureSpace({ workbooks: [ - { - name: 'Sandbox', - sheets: [ - { - name: 'Sales', - slug: 'sales', - fields: [ - { - key: 'date', - type: 'string', - label: 'Date', - }, - { - key: 'product', - type: 'string', - label: 'Product', - }, - { - key: 'category', - type: 'string', - label: 'Category', - }, - { - key: 'region', - type: 'string', - label: 'Region', - }, - { - key: 'salesAmount', - type: 'number', - label: 'Sales Amount', - }, - ], - actions: [ - { - operation: 'generateExampleRecords', - label: 'Generate Example Records', - description: - 'This custom action code generates example records using Anthropic.', - primary: false, - mode: 'foreground', - }, - ], - }, - { - name: 'Sales 2', - slug: 'sales-2', - fields: [ - { - key: 'date', - type: 'string', - label: 'Date', - }, - { - key: 'product', - type: 'string', - label: 'Product', - }, - { - key: 'category', - type: 'string', - label: 'Category', - }, - { - key: 'region', - type: 'string', - label: 'Region', - }, - { - key: 'salesAmount', - type: 'number', - label: 'Sales Amount', - }, - ], - actions: [ - { - operation: 'export-external-api', - label: 'Export to External API', - description: - 'This custom action code exports the records in the Sales sheet to an external API.', - primary: false, - mode: 'foreground', - }, - ], - }, - ], - actions: [ - { - operation: 'export-delimited-zip', - label: 'Export to Delimited ZIP', - mode: 'foreground', - }, - ], - }, + // { + // name: 'Sandbox', + // sheets: [ + // contactsSheet, + // ], + // }, + oneHundredSheet, ], }) ) diff --git a/package-lock.json b/package-lock.json index 376d11792..9a3f1735a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -313,7 +313,36 @@ }, "devDependencies": { "@dotenvx/dotenvx": "^0.39.0", - "flatfile": "^3.6.1" + "flatfile": "^3.8.0" + } + }, + "flatfilers/streaming": { + "name": "@flatfile/flatfilers-streaming", + "version": "0.0.0", + "extraneous": true, + "license": "ISC", + "dependencies": { + "chrono-node": "^2.7.7", + "collect.js": "^4.36.1", + "cross-fetch": "^4.0.0", + "extendable-error": "^0.1.7", + "jsonlines": "^0.1.1", + "modern-async": "^2.0.4" + }, + "devDependencies": { + "@dotenvx/dotenvx": "^0.39.0", + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0", + "@types/jsonlines": "^0.1.5", + "@types/node": "^18.16.0", + "tsx": "^4.7.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@flatfile/api": "^1.9.19", + "@flatfile/listener": "^1.1.0" } }, "import/faker": { @@ -3332,6 +3361,8 @@ }, "node_modules/@dabh/diagnostics": { "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", "dev": true, "license": "MIT", "dependencies": { @@ -3342,6 +3373,8 @@ }, "node_modules/@dotenvx/dotenvx": { "version": "0.39.0", + "resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-0.39.0.tgz", + "integrity": "sha512-OC8m1P10A9hG7yzuQZxQjIGvJlm2nHIZnvHse95DkwhxBgCC8NgZkb/3GnoIIw9Z6VZ2+LF0jLsnMrlcVIFk1w==", "dev": true, "license": "MIT", "dependencies": { @@ -3377,6 +3410,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/brace-expansion": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "license": "MIT", "dependencies": { @@ -3385,6 +3420,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/commander": { "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", "dev": true, "license": "MIT", "engines": { @@ -3392,11 +3429,13 @@ } }, "node_modules/@dotenvx/dotenvx/node_modules/dotenv-expand": { - "version": "11.0.6", + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", + "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "dotenv": "^16.4.4" + "dotenv": "^16.4.5" }, "engines": { "node": ">=12" @@ -3407,6 +3446,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/glob": { "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", "dev": true, "license": "ISC", "dependencies": { @@ -3426,6 +3467,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/isexe": { "version": "3.1.1", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", + "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", "dev": true, "license": "ISC", "engines": { @@ -3434,6 +3477,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/minimatch": { "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { @@ -3448,6 +3493,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/which": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", + "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", "dev": true, "license": "ISC", "dependencies": { @@ -3460,6 +3507,20 @@ "node": "^16.13.0 || >=18.0.0" } }, + "node_modules/@ecies/ciphers": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@ecies/ciphers/-/ciphers-0.2.1.tgz", + "integrity": "sha512-ezMihhjW24VNK/2qQR7lH8xCQY24nk0XHF/kwJ1OuiiY5iEwQXOcKVSy47fSoHPRG8gVGXcK5SgtONDk5xMwtQ==", + "dev": true, + "engines": { + "bun": ">=1", + "deno": ">=2", + "node": ">=16" + }, + "peerDependencies": { + "@noble/ciphers": "^1.0.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -3830,6 +3891,8 @@ }, "node_modules/@fastify/busboy": { "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", "dev": true, "license": "MIT", "engines": { @@ -4096,6 +4159,10 @@ "resolved": "plugins/record-hook", "link": true }, + "node_modules/@flatfile/plugin-record-hook-stream": { + "resolved": "plugins/record-hook-stream", + "link": true + }, "node_modules/@flatfile/plugin-rollout": { "resolved": "plugins/rollout", "link": true @@ -4287,6 +4354,8 @@ }, "node_modules/@inquirer/confirm": { "version": "2.0.17", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-2.0.17.tgz", + "integrity": "sha512-EqzhGryzmGpy2aJf6LxJVhndxYmFs+m8cxXzf8nejb1DE3sabf6mUgBcp4J0jAUEiAcYzqmkqRr7LPFh/WdnXA==", "dev": true, "license": "MIT", "dependencies": { @@ -4300,6 +4369,8 @@ }, "node_modules/@inquirer/core": { "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-6.0.0.tgz", + "integrity": "sha512-fKi63Khkisgda3ohnskNf5uZJj+zXOaBvOllHsOkdsXRA/ubQLJQrZchFFi57NKbZzkTunXiBMdvWOv71alonw==", "dev": true, "license": "MIT", "dependencies": { @@ -4324,6 +4395,8 @@ }, "node_modules/@inquirer/core/node_modules/signal-exit": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "dev": true, "license": "ISC", "engines": { @@ -4335,6 +4408,8 @@ }, "node_modules/@inquirer/core/node_modules/wrap-ansi": { "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", "dev": true, "license": "MIT", "dependencies": { @@ -4347,7 +4422,9 @@ } }, "node_modules/@inquirer/type": { - "version": "1.5.2", + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.5.tgz", + "integrity": "sha512-MzICLu4yS7V8AA61sANROZ9vT1H3ooca5dSmI1FjZkzq7o/koMsRfQSzRtFo+F3Ao4Sf1C0bpLKejpKB/+j6MA==", "dev": true, "license": "MIT", "dependencies": { @@ -4775,30 +4852,39 @@ } }, "node_modules/@noble/ciphers": { - "version": "0.5.3", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.0.0.tgz", + "integrity": "sha512-wH5EHOmLi0rEazphPbecAzmjd12I6/Yv/SiHdkA9LSycsQk7RuuTp7am5/o62qYr0RScE7Pc9icXGBbsr6cesA==", "dev": true, - "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, "funding": { "url": "https://paulmillr.com/funding/" } }, "node_modules/@noble/curves": { - "version": "1.5.0", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.6.0.tgz", + "integrity": "sha512-TlaHRXDehJuRNR9TfZDNQ45mMEd5dwUwmicsafcIX4SsNiqnCHKjE/1alYPd/lDRVhxdhUAlv8uEhMCI5zjIJQ==", "dev": true, - "license": "MIT", "dependencies": { - "@noble/hashes": "1.4.0" + "@noble/hashes": "1.5.0" + }, + "engines": { + "node": "^14.21.3 || >=16" }, "funding": { "url": "https://paulmillr.com/funding/" } }, "node_modules/@noble/hashes": { - "version": "1.4.0", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.5.0.tgz", + "integrity": "sha512-1j6kQFb7QRru7eKN3ZDvRcP13rugwdxZqCjbiAVZfIJwgj2A65UmT4TgARXGlXgnRkORLTDTrO19ZErt7+QXgA==", "dev": true, - "license": "MIT", "engines": { - "node": ">= 16" + "node": "^14.21.3 || >=16" }, "funding": { "url": "https://paulmillr.com/funding/" @@ -6407,6 +6493,8 @@ }, "node_modules/@types/mute-stream": { "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", "dev": true, "license": "MIT", "dependencies": { @@ -6513,6 +6601,8 @@ }, "node_modules/@types/triple-beam": { "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", "dev": true, "license": "MIT" }, @@ -6523,6 +6613,8 @@ }, "node_modules/@types/wrap-ansi": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==", "dev": true, "license": "MIT" }, @@ -6748,6 +6840,8 @@ }, "node_modules/ajv-formats": { "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", "dev": true, "license": "MIT", "dependencies": { @@ -6776,6 +6870,8 @@ }, "node_modules/ansi-escapes": { "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", "dev": true, "license": "MIT", "dependencies": { @@ -6859,6 +6955,8 @@ }, "node_modules/arch": { "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", "dev": true, "funding": [ { @@ -6983,6 +7081,8 @@ }, "node_modules/atomically": { "version": "1.7.0", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-1.7.0.tgz", + "integrity": "sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==", "dev": true, "license": "MIT", "engines": { @@ -7556,6 +7656,8 @@ }, "node_modules/chrono-node": { "version": "2.7.7", + "resolved": "https://registry.npmjs.org/chrono-node/-/chrono-node-2.7.7.tgz", + "integrity": "sha512-p3S7gotuTPu5oqhRL2p1fLwQXGgdQaRTtWR3e8Di9P1Pa9mzkK5DWR5AWBieMUh2ZdOnPgrK+zCrbbtyuA+D/Q==", "license": "MIT", "dependencies": { "dayjs": "^1.10.0" @@ -7668,6 +7770,8 @@ }, "node_modules/cli-width": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", "dev": true, "license": "ISC", "engines": { @@ -7701,6 +7805,8 @@ }, "node_modules/color": { "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", "dev": true, "license": "MIT", "dependencies": { @@ -7724,6 +7830,8 @@ }, "node_modules/color-string": { "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", "dev": true, "license": "MIT", "dependencies": { @@ -7733,6 +7841,8 @@ }, "node_modules/color/node_modules/color-convert": { "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", "dev": true, "license": "MIT", "dependencies": { @@ -7741,6 +7851,8 @@ }, "node_modules/color/node_modules/color-name": { "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true, "license": "MIT" }, @@ -7751,6 +7863,8 @@ }, "node_modules/colorspace": { "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", "dev": true, "license": "MIT", "dependencies": { @@ -7844,6 +7958,8 @@ }, "node_modules/conf": { "version": "10.2.0", + "resolved": "https://registry.npmjs.org/conf/-/conf-10.2.0.tgz", + "integrity": "sha512-8fLl9F04EJqjSqH+QjITQfJF8BrOVaYr1jewVgSRAEWePfxT0sku4w2hrGQ60BC/TNLGQ2pgxNlTbWQmMPFvXg==", "dev": true, "license": "MIT", "dependencies": { @@ -8122,6 +8238,8 @@ }, "node_modules/cuint": { "version": "0.2.2", + "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", + "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==", "dev": true, "license": "MIT" }, @@ -8206,6 +8324,8 @@ }, "node_modules/debounce-fn": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-4.0.0.tgz", + "integrity": "sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==", "dev": true, "license": "MIT", "dependencies": { @@ -8220,6 +8340,8 @@ }, "node_modules/debounce-fn/node_modules/mimic-fn": { "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz", + "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==", "dev": true, "license": "MIT", "engines": { @@ -8367,6 +8489,8 @@ }, "node_modules/diff": { "version": "5.2.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", + "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -8400,6 +8524,8 @@ }, "node_modules/dot-prop": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", "dev": true, "license": "MIT", "dependencies": { @@ -8472,16 +8598,20 @@ } }, "node_modules/eciesjs": { - "version": "0.4.7", + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/eciesjs/-/eciesjs-0.4.11.tgz", + "integrity": "sha512-SmUG449n1w1YGvJD9R30tBGvpxTxA0cnn0rfvpFIBvmezfIhagLjsH2JG8HBHOLS8slXsPh48II7IDUTH/J3Mg==", "dev": true, - "license": "MIT", "dependencies": { - "@noble/ciphers": "^0.5.3", - "@noble/curves": "^1.4.0", - "@noble/hashes": "^1.4.0" + "@ecies/ciphers": "^0.2.1", + "@noble/ciphers": "^1.0.0", + "@noble/curves": "^1.6.0", + "@noble/hashes": "^1.5.0" }, "engines": { - "node": ">=16.0.0" + "bun": ">=1", + "deno": ">=2", + "node": ">=16" } }, "node_modules/ee-first": { @@ -8511,6 +8641,8 @@ }, "node_modules/enabled": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", "dev": true, "license": "MIT" }, @@ -9284,6 +9416,8 @@ }, "node_modules/extendable-error": { "version": "0.1.7", + "resolved": "https://registry.npmjs.org/extendable-error/-/extendable-error-0.1.7.tgz", + "integrity": "sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==", "dev": true, "license": "MIT" }, @@ -9373,6 +9507,8 @@ }, "node_modules/fecha": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "dev": true, "license": "MIT" }, @@ -9394,6 +9530,8 @@ }, "node_modules/figures": { "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", "dev": true, "license": "MIT", "dependencies": { @@ -9408,6 +9546,8 @@ }, "node_modules/figures/node_modules/escape-string-regexp": { "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true, "license": "MIT", "engines": { @@ -10305,6 +10445,8 @@ }, "node_modules/fn.name": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", "dev": true, "license": "MIT" }, @@ -10541,6 +10683,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", + "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/get-uri": { "version": "6.0.3", "license": "MIT", @@ -11459,6 +11615,8 @@ }, "node_modules/is-obj": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", "dev": true, "license": "MIT", "engines": { @@ -11936,6 +12094,8 @@ }, "node_modules/json-schema-typed": { "version": "7.0.3", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-7.0.3.tgz", + "integrity": "sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==", "dev": true, "license": "BSD-2-Clause" }, @@ -12032,6 +12192,8 @@ }, "node_modules/kuler": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "dev": true, "license": "MIT" }, @@ -12217,7 +12379,9 @@ } }, "node_modules/logform": { - "version": "2.6.1", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", "dev": true, "license": "MIT", "dependencies": { @@ -12234,6 +12398,8 @@ }, "node_modules/logform/node_modules/@colors/colors": { "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", "dev": true, "license": "MIT", "engines": { @@ -12242,6 +12408,8 @@ }, "node_modules/logform/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true, "license": "MIT" }, @@ -12926,6 +13094,8 @@ }, "node_modules/modern-async": { "version": "2.0.4", + "resolved": "https://registry.npmjs.org/modern-async/-/modern-async-2.0.4.tgz", + "integrity": "sha512-89Ig+D/NQAFi39/oRUphkwdLb4qo/3Vl21TCMPZY3oCWGQQ+HqNJ1kyWknyAJ2rsiewL9OGBmpKA5bQ0PikkMg==", "license": "MIT", "dependencies": { "core-js-pure": "^3.19.1", @@ -13006,6 +13176,8 @@ }, "node_modules/mute-stream": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", "dev": true, "license": "ISC", "engines": { @@ -13427,6 +13599,8 @@ }, "node_modules/one-time": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", "dev": true, "license": "MIT", "dependencies": { @@ -13928,6 +14102,8 @@ }, "node_modules/pkg-up": { "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", + "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", "dev": true, "license": "MIT", "dependencies": { @@ -13939,6 +14115,8 @@ }, "node_modules/pkg-up/node_modules/find-up": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", "dev": true, "license": "MIT", "dependencies": { @@ -13950,6 +14128,8 @@ }, "node_modules/pkg-up/node_modules/locate-path": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "license": "MIT", "dependencies": { @@ -13962,6 +14142,8 @@ }, "node_modules/pkg-up/node_modules/p-locate": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "dev": true, "license": "MIT", "dependencies": { @@ -13973,6 +14155,8 @@ }, "node_modules/pkg-up/node_modules/path-exists": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", "dev": true, "license": "MIT", "engines": { @@ -14896,6 +15080,17 @@ "node": ">=8" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "license": "MIT", + "optional": true, + "peer": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/restore-cursor": { "version": "3.1.0", "license": "MIT", @@ -15076,6 +15271,8 @@ }, "node_modules/run-async": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", "dev": true, "license": "MIT", "engines": { @@ -15165,6 +15362,8 @@ }, "node_modules/safe-stable-stringify": { "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", "dev": true, "license": "MIT", "engines": { @@ -15375,6 +15574,8 @@ }, "node_modules/simple-swizzle": { "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", "dev": true, "license": "MIT", "dependencies": { @@ -15383,6 +15584,8 @@ }, "node_modules/simple-swizzle/node_modules/is-arrayish": { "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", "dev": true, "license": "MIT" }, @@ -15637,6 +15840,8 @@ }, "node_modules/stack-trace": { "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", "dev": true, "license": "MIT", "engines": { @@ -16194,6 +16399,8 @@ }, "node_modules/text-hex": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", "dev": true, "license": "MIT" }, @@ -16381,6 +16588,8 @@ }, "node_modules/triple-beam": { "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", "dev": true, "license": "MIT", "engines": { @@ -16498,6 +16707,27 @@ "webidl-conversions": "^4.0.2" } }, + "node_modules/tsx": { + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.2.tgz", + "integrity": "sha512-pOUl6Vo2LUq/bSa8S5q7b91cgNSjctn9ugq/+Mvow99qW6x/UZYwzxy/3NmqoT66eHYfCVvFvACC58UBPFf28g==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "esbuild": "~0.23.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/turbo": { "version": "1.13.4", "dev": true, @@ -16535,6 +16765,8 @@ }, "node_modules/type-fest": { "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { @@ -16659,6 +16891,8 @@ }, "node_modules/undici": { "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "dev": true, "license": "MIT", "dependencies": { @@ -16770,6 +17004,8 @@ }, "node_modules/util": { "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", "dev": true, "license": "MIT", "dependencies": { @@ -17207,7 +17443,9 @@ "license": "ISC" }, "node_modules/winston": { - "version": "3.14.2", + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", "dev": true, "license": "MIT", "dependencies": { @@ -17215,24 +17453,26 @@ "@dabh/diagnostics": "^2.0.2", "async": "^3.2.3", "is-stream": "^2.0.0", - "logform": "^2.6.0", + "logform": "^2.7.0", "one-time": "^1.0.0", "readable-stream": "^3.4.0", "safe-stable-stringify": "^2.3.1", "stack-trace": "0.0.x", "triple-beam": "^1.3.0", - "winston-transport": "^4.7.0" + "winston-transport": "^4.9.0" }, "engines": { "node": ">= 12.0.0" } }, "node_modules/winston-transport": { - "version": "4.7.1", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", "dev": true, "license": "MIT", "dependencies": { - "logform": "^2.6.1", + "logform": "^2.7.0", "readable-stream": "^3.6.2", "triple-beam": "^1.3.0" }, @@ -17242,6 +17482,8 @@ }, "node_modules/winston/node_modules/@colors/colors": { "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", "dev": true, "license": "MIT", "engines": { @@ -17352,6 +17594,8 @@ }, "node_modules/xxhashjs": { "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", + "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", "dev": true, "license": "MIT", "dependencies": { @@ -17635,10 +17879,10 @@ }, "plugins/json-extractor": { "name": "@flatfile/plugin-json-extractor", - "version": "0.10.0", + "version": "0.11.0", "license": "ISC", "dependencies": { - "@flatfile/util-extractor": "^2.3.0" + "@flatfile/util-extractor": "^2.4.0" }, "devDependencies": { "@flatfile/bundler-config-tsup": "^0.2.0", @@ -17800,6 +18044,21 @@ "@flatfile/listener": "^1.1.0" } }, + "plugins/record-hook-stream": { + "name": "@flatfile/plugin-record-hook-stream", + "version": "0.0.1", + "license": "ISC", + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@flatfile/listener": "^1.1.0" + } + }, "plugins/rollout": { "name": "@flatfile/plugin-rollout", "version": "1.4.0", @@ -18056,6 +18315,33 @@ "node": ">= 18" } }, + "utils/asdf": { + "name": "@flatfile/util-asdf", + "version": "0.0.0", + "extraneous": true, + "license": "ISC", + "dependencies": { + "chrono-node": "^2.7.7", + "collect.js": "^4.36.1", + "cross-fetch": "^4.0.0", + "extendable-error": "^0.1.7", + "jsonlines": "^0.1.1", + "modern-async": "^2.0.4" + }, + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0", + "@types/jsonlines": "^0.1.5", + "@types/node": "^18.16.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@flatfile/api": "^1.9.19", + "@flatfile/listener": "^1.1.0" + } + }, "utils/common": { "name": "@flatfile/util-common", "version": "1.6.0", @@ -18082,7 +18368,7 @@ }, "utils/extractor": { "name": "@flatfile/util-extractor", - "version": "2.3.0", + "version": "2.4.0", "license": "ISC", "dependencies": { "@flatfile/util-common": "^1.6.0", diff --git a/plugins/record-hook-stream/CHANGELOG.md b/plugins/record-hook-stream/CHANGELOG.md new file mode 100644 index 000000000..645c14f72 --- /dev/null +++ b/plugins/record-hook-stream/CHANGELOG.md @@ -0,0 +1 @@ +# @flatfile/plugin-record-hook-stream \ No newline at end of file diff --git a/plugins/record-hook-stream/README.md b/plugins/record-hook-stream/README.md new file mode 100644 index 000000000..8fe8adec8 --- /dev/null +++ b/plugins/record-hook-stream/README.md @@ -0,0 +1,332 @@ + + +The `@flatfile/plugin-record-hook` plugin offers a convenient way to execute +custom logic on individual data records within Flatfile. By setting up an event +listener for the `commit:created` event, this plugin seamlessly integrates with +the data processing flow. + +**Event Type:** +`listener.on('commit:created')` + + + + +## Parameters + +#### `sheetSlug` - `string` +The `sheetSlug` parameter is the slug of the sheet you want to listen to. + +#### `callback` - `function` +The `callback` parameter takes a function that will be run on the record or records. + +#### `options.chunkSize` - `number` - `default: 10_000` - (optional) +The `chunkSize` parameter allows you to specify the quantity of records to process in each chunk. + +#### `options.parallel` - `number` - `default: 1` - (optional) +The `parallel` parameter allows you to specify the number of chunks to process in parallel. + +#### `options.debug` - `boolean` - `default: false` - (optional) +The `debug` parameter allows you to turn on debug logging. + + + +## Usage + +```bash install +npm i @flatfile/plugin-record-hook @flatfile/hooks +``` + +### Import + + +#### bulkRecordHook + +```js bulkRecordHook +import { FlatfileRecord, bulkRecordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileEvent, FlatfileListener } from "@flatfile/listener"; +``` + +#### recordHook + +```js recordHook +import { FlatfileRecord, recordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileEvent, FlatfileListener } from "@flatfile/listener"; +``` + + + +Pass `bulkRecordHook` or `recordHook` to a Flatfile data listener and provide a function to run when data is added or updated. + + +### Listen for data changes + +Set up a listener to configure Flatfile and respond to data Events. Then use this plugin to set up a hook that responds to data changes. + + +#### JavaScript + +**bulkRecordHook.js** + +```js bulkRecordHook js +import { bulkRecordHook } from "@flatfile/plugin-record-hook"; + +export default async function (listener) { + listener.use( + bulkRecordHook("my-sheet", (records) => { + return records.map((r) => { + //do your work here + return r; + }); + }) + ); +} +``` + +**recordHook.js** + +```js recordHook js +import { recordHook } from "@flatfile/plugin-record-hook"; + +export default async function (listener) { + listener.use( + recordHook("my-sheet", (record) => { + //do your work here + return record; + }) + ); +} +``` + +#### TypeScript + +**bulkRecordHook.ts** + +```js bulkRecordHook ts +import { FlatfileRecord } from "@flatfile/hooks"; +import { bulkRecordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileListener } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + bulkRecordHook("my-sheet", (records: FlatfileRecord[]) => { + return records.map((r) => { + //do your work here + return r; + }); + }) + ); +} +``` + +**recordHook.ts** + +```js recordHook ts +import { FlatfileRecord } from "@flatfile/hooks"; +import { recordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileListener } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + recordHook("my-sheet", (record: FlatfileRecord) => { + //do your work here + return record; + }) + ); +} +``` + + + +### Additional Options + +`bulkRecordHook` can accept additional properties. Props will be passed along to the transformer. + + +#### JavaScript + +**bulkRecordHook.js** + +```js bulkRecordHook js +import { bulkRecordHook } from "@flatfile/plugin-record-hook"; + +export default async function (listener) { + listener.use( + bulkRecordHook("my-sheet", (records) => { + return records.map((r) => { + //do your work here + return r; + }); + }), + { chunkSize: 100, parallel: 2 } + ); +} +``` + + +#### TypeScript + +**bulkRecordHook.ts** + +```js bulkRecordHook ts +import { FlatfileRecord } from "@flatfile/hooks"; +import { bulkRecordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileListener } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + bulkRecordHook( + "my-sheet", + (records: FlatfileRecord[]) => { + return records.map((r) => { + //do your work here + return r; + }); + }, + { chunkSize: 100, parallel: 2 } + ) + ); +} +``` + + + +#### Flexible Options + +#### `chunkSize` *number* *default: 10_000* (optional) +Define how many records you want to process in each batch. This allows you to balance efficiency and resource utilization based on your specific use case. + +#### `parallel` *number* *default: 1* (optional) +Choose whether the records should be processed in parallel. This enables you to optimize the execution time when dealing with large datasets. + + +## Example Usage + +This example sets up a record hook using `listener.use` to modify records in the "my-sheet" sheet. + +When a record is processed by the hook, it checks if an email address is missing, empty, or invalid, and if so, it logs corresponding error messages and adds them to a form validation context (if the r object is related to form validation). This helps ensure that only valid email addresses are accepted in the application. + +In the `bulkRecordHook` example, it passes a `chunkSize` of 100 and asks the hooks to run 2 at a time via the `parallel` property. + + +### JavaScript + +**bulkRecordHook.js** + +```js bulkRecordHook js +import { bulkRecordHook } from "@flatfile/plugin-record-hook"; + +export default async function (listener) { + listener.use( + bulkRecordHook( + "my-sheet", + (records) => { + return records.map((r) => { + const email = r.get("email") as string; + if (!email) { + console.log("Email is required"); + r.addError("email", "Email is required"); + } + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (email !== null && !validEmailAddress.test(email)) { + console.log("Invalid email address"); + r.addError("email", "Invalid email address"); + } + return r; + }); + }, + { chunkSize: 100, parallel: 2 } + ) + ); +} +``` + +**recordHook.js** + +```js recordHook js +import { recordHook } from "@flatfile/plugin-record-hook"; + +export default async function (listener) { + listener.use( + recordHook( + "my-sheet", + (record) => { + const email = record.get("email") as string; + if (!email) { + console.log("Email is required"); + record.addError("email", "Email is required"); + } + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (email !== null && !validEmailAddress.test(email)) { + console.log("Invalid email address"); + record.addError("email", "Invalid email address"); + } + return record; + } + ) + ); +} +``` + + +### TypeScript + +**bulkRecordHook.ts** + +```js bulkRecordHook ts +import { FlatfileRecord } from "@flatfile/hooks"; +import { bulkRecordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileListener } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + bulkRecordHook( + "contacts", + (records: FlatfileRecord[]) => { + return records.map((r) => { + const email = r.get("email") as string; + if (!email) { + console.log("Email is required"); + r.addError("email", "Email is required"); + } + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (email !== null && !validEmailAddress.test(email)) { + console.log("Invalid email address"); + r.addError("email", "Invalid email address"); + } + return r; + }); + }, + { chunkSize: 100, parallel: 2 } + ) + ); +} +``` + +**recordHook.ts** + +```js recordHook ts +import { FlatfileRecord } from "@flatfile/hooks"; +import { recordHook } from "@flatfile/plugin-record-hook"; +import { FlatfileListener } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + recordHook( + "contacts", + (record: FlatfileRecord) => { + const email = record.get("email") as string; + if (!email) { + console.log("Email is required"); + record.addError("email", "Email is required"); + } + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (email !== null && !validEmailAddress.test(email)) { + console.log("Invalid email address"); + record.addError("email", "Invalid email address"); + } + return record; + } + ) + ); +} +``` diff --git a/plugins/record-hook-stream/package.json b/plugins/record-hook-stream/package.json new file mode 100644 index 000000000..558f76a85 --- /dev/null +++ b/plugins/record-hook-stream/package.json @@ -0,0 +1,68 @@ +{ + "name": "@flatfile/plugin-record-hook-stream", + "version": "0.0.1", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/plugins/record-hook-stream", + "description": "A plugin for running custom logic on individual data records in Flatfile.", + "registryMetadata": { + "category": "records" + }, + "type": "module", + "engines": { + "node": ">= 18" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ], + "exports": { + ".": { + "node": { + "types": { + "import": "./dist/index.d.ts", + "require": "./dist/index.d.cts" + }, + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "main": "./dist/index.js", + "module": "./dist/index.js", + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "tsup", + "build:watch": "tsup --watch", + "build:prod": "NODE_ENV=production tsup", + "checks": "tsc --noEmit && attw --pack . && publint .", + "lint": "tsc --noEmit", + "test": "vitest run --mode defaults src/*.spec.ts --passWithNoTests", + "test:unit": "vitest run --mode defaults src/*.spec.ts --passWithNoTests --exclude src/*.e2e.spec.ts", + "test:e2e": "vitest run --mode defaults src/*.e2e.spec.ts --passWithNoTests --no-file-parallelism" + }, + "keywords": [ + "flatfile-plugins", + "category-transform", + "featured" + ], + "author": "Carl Brugger", + "repository": { + "type": "git", + "url": "git+https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/record-hook-stream" + }, + "license": "ISC", + "peerDependencies": { + "@flatfile/listener": "^1.1.0" + }, + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0" + } +} diff --git a/plugins/record-hook-stream/src/index.ts b/plugins/record-hook-stream/src/index.ts new file mode 100644 index 000000000..220309d5c --- /dev/null +++ b/plugins/record-hook-stream/src/index.ts @@ -0,0 +1,32 @@ +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { recordReadWriteStream } from './record.hook.stream' +import { Item } from './utils/item' + +export interface RecordHookStreamOptions { + includeMessages?: boolean + includeMetadata?: boolean + debug?: boolean +} + +export const recordHookStream = ( + sheetSlug: string, + callback: (items: Item[], event: FlatfileEvent) => Item[] | Promise, + options: RecordHookStreamOptions = {} +) => { + return (listener: FlatfileListener) => { + listener.on('commit:created', { sheetSlug }, (event: FlatfileEvent) => + recordReadWriteStream(callback, event, options) + .then( + (results: { totalProcessed: number; totalTimeSeconds: string }) => { + console.log( + `Processed ${results.totalProcessed} records in ${results.totalTimeSeconds} seconds (r/s: ${Math.ceil(results.totalProcessed / parseFloat(results.totalTimeSeconds))})` + ) + } + ) + .catch((error) => { + console.error('Processing failed:', error) + throw error + }) + ) + } +} diff --git a/plugins/record-hook-stream/src/record.hook.stream.ts b/plugins/record-hook-stream/src/record.hook.stream.ts new file mode 100644 index 000000000..d1aa68ad9 --- /dev/null +++ b/plugins/record-hook-stream/src/record.hook.stream.ts @@ -0,0 +1,324 @@ +import type { FlatfileEvent } from '@flatfile/listener' +import { request } from 'https' +import { RecordHookStreamOptions } from '.' +import { Item } from './utils/item' + +const MIN_BATCH_SIZE = 100 +const MAX_BATCH_SIZE = 500 +const TARGET_BATCH_DURATION = 5000 +const MAX_CONCURRENT_BATCHES = 15 +const CHUNK_SIZE = 50 + +export async function recordReadWriteStream( + callback: (items: Item[], event: FlatfileEvent) => Item[] | Promise, + event: FlatfileEvent, + options: RecordHookStreamOptions = {} +) { + const { + includeMessages = false, + includeMetadata = false, + debug = false, + } = options + + const processedRecordIds = new Set() + const startTime = Date.now() + let pendingBatches = 0 + + // Pre-create Item instances in smaller chunks to prevent memory spikes + const processItemsInChunks = async (records: any[]) => { + const results = [] + for (let i = 0; i < records.length; i += CHUNK_SIZE) { + const chunk = records.slice(i, i + CHUNK_SIZE) + const items = chunk.map((record) => new Item(record)) + + // Process all records in chunk at once + chunk.forEach((record) => { + if (!processedRecordIds.has(record.__k)) { + processedRecordIds.add(record.__k) + } + }) + + const transformedItems = await callback(items, event) + const chunkResults = transformedItems.map((item) => item.changeset()) + results.push(...chunkResults) + } + return results + } + + let currentBatchSize = 250 + + const processBatchOfRecords = async (records: any[]) => { + if (records.length === 0) return + + while (pendingBatches >= MAX_CONCURRENT_BATCHES) { + await new Promise((resolve) => setTimeout(resolve, 25)) + } + + pendingBatches++ + const batchStartTime = Date.now() + + try { + const results = await processItemsInChunks(records) + const sendStart = Date.now() + await sendBatch(results, event, options) + + // Adjust batch size based on send duration + const batchDuration = Date.now() - sendStart + if (batchDuration > TARGET_BATCH_DURATION) { + currentBatchSize = Math.max( + MIN_BATCH_SIZE, + Math.floor(currentBatchSize * 0.8) + ) + } else if (batchDuration < TARGET_BATCH_DURATION * 0.8) { + currentBatchSize = Math.min( + MAX_BATCH_SIZE, + Math.floor(currentBatchSize * 1.2) + ) + } + + debug && + console.log( + `Batch metrics - Size: ${records.length}, Process: ${sendStart - batchStartTime}ms, ` + + `Send: ${batchDuration}ms, Next batch size: ${currentBatchSize}` + ) + } catch (error) { + if (error.message.includes('Too Many Requests')) { + currentBatchSize = Math.max( + MIN_BATCH_SIZE, + Math.floor(currentBatchSize * 0.5) + ) + await new Promise((resolve) => setTimeout(resolve, 1000)) + await processBatchOfRecords(records) + return + } + throw error + } finally { + pendingBatches-- + } + } + + return new Promise((resolve, reject) => { + let buffer = '' + let recordBuffer: any[] = [] + let lastProgressLog = Date.now() + let lastRecordCount = 0 + const PROGRESS_INTERVAL = 5000 + + const logProgress = () => { + const now = Date.now() + if (now - lastProgressLog >= PROGRESS_INTERVAL) { + const timeElapsed = (now - lastProgressLog) / 1000 + const recordsSinceLastLog = processedRecordIds.size - lastRecordCount + const currentRecordsPerSecond = recordsSinceLastLog / timeElapsed + const overallRecordsPerSecond = + processedRecordIds.size / ((now - startTime) / 1000) + + // Only log if we've processed new records or have pending batches + if (recordsSinceLastLog > 0 || pendingBatches > 0) { + debug && + console.log( + `Progress: ${processedRecordIds.size} records processed ` + + `(current: ${recordsSinceLastLog > 0 ? Math.round(currentRecordsPerSecond) : 'waiting'} r/s, ` + + `avg: ${Math.round(overallRecordsPerSecond)} r/s)` + + `${pendingBatches > 0 ? ` - ${pendingBatches} batches pending` : ''}` + ) + } + + lastProgressLog = now + lastRecordCount = processedRecordIds.size + } + } + + const options = { + method: 'GET', + headers: { + 'Content-Type': 'application/jsonl', + Authorization: `Bearer ${process.env.FLATFILE_API_KEY || process.env.FLATFILE_BEARER_TOKEN}`, + }, + timeout: 120000, + } + const req = request( + getRecordStreamEndpoint(event, { includeMessages, includeMetadata }), + options + ) + + req.on('response', (res) => { + res.setEncoding('utf8') + + res.on('data', async (chunk: string) => { + try { + buffer += chunk + const lines = buffer.split('\n') + buffer = lines.pop() || '' + + const records = lines + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)) + + if (records.length > 0) { + recordBuffer.push(...records) + + while (recordBuffer.length >= currentBatchSize) { + const batch = recordBuffer.splice(0, currentBatchSize) + await processBatchOfRecords(batch) + logProgress() + } + } + } catch (error) { + reject(error) + } + }) + + res.on('end', async () => { + try { + if (buffer.length > 0) { + const records = buffer + .split('\n') + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)) + recordBuffer.push(...records) + } + + if (recordBuffer.length > 0) { + await processBatchOfRecords(recordBuffer) + recordBuffer = [] + logProgress() + } + + while (pendingBatches > 0) { + logProgress() + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + + const endTime = Date.now() + const totalTimeSeconds = (endTime - startTime) / 1000 + resolve({ + totalProcessed: processedRecordIds.size, + totalTimeSeconds: totalTimeSeconds.toFixed(2), + }) + } catch (error) { + reject(error) + } + }) + }) + + req.on('error', reject) + req.end() + }) +} + +async function sendBatch( + batch: any[], + event: FlatfileEvent, + options: RecordHookStreamOptions +) { + const { debug = false } = options + + const MAX_RETRIES = 5 + const INITIAL_RETRY_DELAY = 250 + + for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { + try { + return await sendBatchWithTimeout(batch, event, options) + } catch (error) { + if (error.message.includes('Too Many Requests')) { + debug && + console.log( + `Rate limited, retry attempt ${attempt + 1} of ${MAX_RETRIES}` + ) + const delay = + INITIAL_RETRY_DELAY * Math.pow(1.5, attempt) * (0.75 + Math.random()) + await new Promise((resolve) => setTimeout(resolve, delay)) + } else { + if (attempt === MAX_RETRIES - 1) throw error + const delay = INITIAL_RETRY_DELAY * Math.pow(1.5, attempt) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + } +} + +async function sendBatchWithTimeout( + batch: any[], + event: FlatfileEvent, + options: RecordHookStreamOptions +) { + const { includeMessages = true, includeMetadata = false } = options + + const RECORD_STREAM_ENDPOINT = getRecordStreamEndpoint(event, { + includeMessages, + includeMetadata, + }) + const TIMEOUT = 60000 + const MAX_RECORDS_PER_REQUEST = 1000 + + if (batch.length > MAX_RECORDS_PER_REQUEST) { + const chunks = [] + for (let i = 0; i < batch.length; i += MAX_RECORDS_PER_REQUEST) { + chunks.push(batch.slice(i, i + MAX_RECORDS_PER_REQUEST)) + } + + for (const chunk of chunks) { + await sendBatchWithTimeout(chunk, event, options) + } + return + } + + return new Promise((resolve, reject) => { + let isRequestEnded = false + let timeout: NodeJS.Timeout + + const cleanup = () => { + clearTimeout(timeout) + if (!isRequestEnded) { + req.destroy() + } + } + + const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/x-ndjson', + Authorization: `Bearer ${process.env.FLATFILE_API_KEY || process.env.FLATFILE_BEARER_TOKEN}`, + }, + } + + const req = request(RECORD_STREAM_ENDPOINT, options, (res) => { + let data = '' + res.on('data', (chunk) => (data += chunk)) + res.on('end', () => { + isRequestEnded = true + cleanup() + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + resolve(undefined) + } else { + reject(new Error(`Failed to write batch: ${data}`)) + } + }) + }) + + timeout = setTimeout(() => { + cleanup() + reject(new Error('Request timeout')) + }, TIMEOUT) + + req.on('error', (error) => { + cleanup() + reject(error) + }) + + for (const record of batch) { + req.write(JSON.stringify(record) + '\n') + } + + req.end() + }) +} + +const getRecordStreamEndpoint = ( + event: FlatfileEvent, + options: Omit +) => { + const { includeMessages = false, includeMetadata = false } = options + return `${process.env.FLATFILE_API_URL}/v2-alpha/records.jsonl?sheetId=${event.context.sheetId}&for=${event.src.id}&stream=true&includeMessages=${includeMessages}&includeMetadata=${includeMetadata}` +} diff --git a/plugins/record-hook-stream/src/utils/casting.ts b/plugins/record-hook-stream/src/utils/casting.ts new file mode 100644 index 000000000..9fe4a2217 --- /dev/null +++ b/plugins/record-hook-stream/src/utils/casting.ts @@ -0,0 +1,73 @@ +export function asDate(input: any): Date | null { + const str_value = asString(input) + if (!str_value) { + return null + } + + const date = new Date(str_value) + // Check if the date is valid + return isNaN(date.getTime()) ? null : date +} + +export function asNumber(input: any): number { + // Check if the input is already a number + if (typeof input === 'number') return input + + // Check if the input is a string that can be converted to a number + if (typeof input === 'string') { + // Trim leading and trailing whitespaces + const trimmedInput = input.trim() + + // Check if the trimmed string is a numeric value + const num = Number(trimmedInput) + + // Check if the result is a valid number and not NaN + if (!isNaN(num)) { + return num + } + } + + // Return 0 for inputs that cannot be converted to a number + return 0 +} + +export function asString(input: any): string { + // Check if the input is null or undefined + if (input === null || input === undefined) { + return '' + } + + // Check if the input is an object or an array (excluding null, which is technically an object in JS) + if (typeof input === 'object') { + // Attempt to convert objects and arrays to a JSON string + try { + return JSON.stringify(input) + } catch (error) { + // In case of circular references or other JSON.stringify errors, return a fallback string + return '[object]' + } + } + + // For numbers, booleans, and other types, convert directly to string + return String(input) +} + +export function asNullableString(input: any): string | undefined { + // Check if the input is null or undefined + if ( + input === null || + input === undefined || + input === 'undefined' || + input === 'null' || + input === '' + ) { + return undefined + } + + return asString(input) +} + +export function asBool(input: any): boolean { + // todo: make this way smarter soon + return !!input +} diff --git a/plugins/record-hook-stream/src/utils/is.nullish.ts b/plugins/record-hook-stream/src/utils/is.nullish.ts new file mode 100644 index 000000000..a24a72877 --- /dev/null +++ b/plugins/record-hook-stream/src/utils/is.nullish.ts @@ -0,0 +1,14 @@ +export function isNullish(value: T): value is null { + return ( + value === null || + value === '' || + value === undefined || + value === 'null' || + value === 'undefined' || + (typeof value === 'number' && isNaN(value)) + ) +} + +export function isPresent(value: T): value is NonNullable { + return !isNullish(value) +} diff --git a/plugins/record-hook-stream/src/utils/item.ts b/plugins/record-hook-stream/src/utils/item.ts new file mode 100644 index 000000000..257030423 --- /dev/null +++ b/plugins/record-hook-stream/src/utils/item.ts @@ -0,0 +1,337 @@ +import { asBool, asDate, asNullableString, asString } from './casting' +import { isPresent } from './is.nullish' + +export const HASH_VALUE_DELIM = '|*|' +export const HASH_PROP_DELIM = '|&|' + +export class Item = Record> { + private _changes: Map = new Map() + private _messages: Map< + string, + Set<{ type: 'error' | 'warn' | 'info'; message: string }> + > = new Map() + private _deleted = false + private _tempId?: string + + constructor( + public data: Readonly>, + dirty = false + ) { + if (dirty) { + this.data = Object.freeze({}) + Object.entries(data).forEach(([key, value]) => { + this.set(key, value) + }) + } else { + Object.freeze(this.data) + } + } + + get id() { + return this.data.__k || this._tempId + } + + get slug() { + return this.data.__n + } + + get sheetId() { + return this.data.__s + } + + get versionId() { + return this.data.__v + } + + set(key: string, value: any) { + if (this.data[key] === value) { + this._changes.delete(key) + return + } + this._changes.set(key, value) + return this + } + + flag(key: string) { + this.set(key, true) + } + + unflag(key: string) { + this.set(key, false) + } + + get(key: string) { + if (this._changes.has(key)) { + return this._changes.get(key) + } + return this.data[key] + } + + has(key: string) { + return isPresent(this.get(key)) + } + + hasAny(...keys: string[]) { + return keys.some((k) => this.has(k)) + } + + hasAll(...keys: string[]) { + return keys.every((k) => this.has(k)) + } + + isEmpty(key: string) { + return !this.has(key) + } + + keys(options?: { omit?: string[]; pick?: string[] }): string[] { + const set = new Set( + Object.keys(this.data).filter((key) => !key.startsWith('__')) + ) + + for (const key of this._changes.keys()) { + if (!key.startsWith('__')) { + set.add(key) + } + } + const res = Array.from(set) + + if (options?.omit) { + return res.filter((key) => !options.omit.includes(key)) + } + if (options?.pick) { + return res.filter((key) => options.pick.includes(key)) + } + return res + } + + keysWithData(props?: { exclude?: Array }): string[] { + const keys = this.keys().filter((k) => this.has(k)) + if (props?.exclude) { + const f = props.exclude.flat() + return keys.filter((k) => !f.includes(k)) + } + return keys + } + + /** + * Intersects exactly with another item on the keys + * + * @param item + * @param keys + */ + intersects(item: Item, keys: string[]) { + return keys.every((key) => { + const value1 = this.str(key) + const value2 = item.str(key) + return value1 === value2 + }) + } + + hash(...keys: string[]) { + return keys + .map((k) => [k, this.get(k)]) + .map(([k, v]) => `${k}${HASH_VALUE_DELIM}${asString(v)}`) + .join(HASH_PROP_DELIM) + } + + isDirty(key?: string): boolean { + if (key) { + return this._changes.has(key) || this._messages.get(key)?.size > 0 + } + return this._changes.size > 0 || this._messages.size > 0 || this._deleted + } + + eachOfKeysPresent( + keys: string[], + callback: (key: string, value: any) => void + ) { + for (const key of keys) { + if (this.has(key)) { + callback(key, this.get(key)) + } + } + } + + isDeleted(): boolean { + return this._deleted + } + + delete() { + this._deleted = true + } + + str(key: string) { + return asNullableString(this.get(key)) + } + + defStr(key: string): string { + return asString(this.get(key)) + } + + bool(key: string) { + return asBool(this.get(key)) + } + + date(key: string) { + return asDate(this.get(key)) + } + + pick(...keys: string[]) { + const obj: Record = {} + for (const key of keys) { + obj[key] = this.get(key) + } + return obj + } + + addMsg(key: string, msg: string, type: 'error' | 'warn' | 'info') { + if ( + this.data.__i && + this.data.__i.some((m) => m.x === key && m.m === msg) + ) { + return this + } + if (!this._messages.has(key)) { + this._messages.set(key, new Set()) + } + this._messages.get(key).add({ type, message: msg }) + return this + } + + info(key: string, msg: string) { + return this.addMsg(key, msg, 'info') + } + + warn(key: string, msg: string) { + return this.addMsg(key, msg, 'warn') + } + + err(key: string, msg: string) { + return this.addMsg(key, msg, 'error') + } + + values() { + return Object.fromEntries(this.entries()) + } + + entries() { + return this.keys().map((key) => [key, this.get(key)]) + } + + merge(item: Item, props: { overwrite?: boolean } = {}) { + for (const key of item.keys()) { + if (props.overwrite) { + this.set(key, item.get(key)) + } else if (!this.has(key)) { + this.set(key, item.get(key)) + } + } + return this + } + + hasConflict(b: Item, keys?: string[]) { + if (keys) { + return keys.some((key) => { + const aValue = this.get(key) + const bValue = b.get(key) + return aValue && bValue && aValue !== bValue + }) + } + return this.entries().some(([key, aValue]) => { + const bValue = b.get(key) + return aValue && bValue && aValue !== bValue + }) + } + + toJSON() { + return { ...this.data, ...this.changeset() } + } + + toString() { + return `${this._deleted ? '❌ ' : ''}${this.slug || this.sheetId}(${this.id ?? 'new'}) ${JSON.stringify(this.values(), null, ' ')}` + } + + [Symbol.for('nodejs.util.inspect.custom') || Symbol()]() { + return this.toString() + } + + copy(props?: { + mixin?: Item + select?: string[] + slug?: string + sheetId?: string + }) { + const newObj = new Item({}) + newObj._tempId = `TEMP_${crypto.randomUUID()}` + if (props.slug) { + newObj.set('__n', props.slug) + } + if (props.sheetId) { + newObj.set('__s', props.sheetId) + } + if (props.select) { + for (const key of props.select) { + newObj.set(key, props.mixin?.get(key) ?? this.get(key)) + } + } else { + for (const key in this.data) { + if (!key.startsWith('__')) { + newObj.set(key, this.get(key)) + } + } + if (props.mixin) { + for (const key in props.mixin.data) { + if (!key.startsWith('__')) { + newObj.set(key, props.mixin.get(key)) + } + } + } + } + return newObj + } + + commit() { + // reset the data object with new changes and unset all pending changes + const newObj: Record = Object.assign({}, this.data) + for (const [key, value] of this._changes) { + newObj[key] = value + } + this._changes.clear() + if (this._messages.size) { + newObj.__i = [] + for (const [key, values] of this._messages) { + for (const value of values) { + newObj.__i.push({ x: key, m: value.message, t: value.type }) + } + } + } + this._messages.clear() + this.data = Object.freeze(newObj) as any + } + + changeset() { + const val = Object.fromEntries(this._changes) + val.__k = this.get('__k') + val.__s = this.get('__s') + val.__n = this.get('__n') + if (this._deleted) { + val.__d = true + } + if (this._messages.size) { + if (!val.__i) { + val.__i = [] + } + for (const [key, values] of this._messages) { + // If there is a message for a key and no changed value, the value is removed. + // This sets the value if not changed to preserve it. + if (!val[key]) { + val[key] = this.get(key) + } + for (const value of values) { + val.__i.push({ x: key, m: value.message, t: value.type }) + } + } + } + return val + } +} diff --git a/plugins/record-hook-stream/tsup.config.mjs b/plugins/record-hook-stream/tsup.config.mjs new file mode 100644 index 000000000..2070a0891 --- /dev/null +++ b/plugins/record-hook-stream/tsup.config.mjs @@ -0,0 +1,3 @@ +import { defineConfig } from '@flatfile/bundler-config-tsup' + +export default defineConfig({ includeBrowser: false }) diff --git a/plugins/record-hook-stream/vitest.config.ts b/plugins/record-hook-stream/vitest.config.ts new file mode 100644 index 000000000..e9e1729f9 --- /dev/null +++ b/plugins/record-hook-stream/vitest.config.ts @@ -0,0 +1,4 @@ +import config from '@flatfile/config-vitest' +import { defineConfig } from 'vitest/config' + +export default defineConfig(config) diff --git a/plugins/record-hook/src/RecordHook.ts b/plugins/record-hook/src/RecordHook.ts index 4f9622c03..b5f9ea2fc 100644 --- a/plugins/record-hook/src/RecordHook.ts +++ b/plugins/record-hook/src/RecordHook.ts @@ -63,9 +63,11 @@ export const BulkRecordHook = async ( options: BulkRecordHookOptions = {} ) => { const { debug = false } = options + debug && console.log('🐢 bulkRecordHook') try { - startTimer('fetch data', debug) + startTimer(`bulkRecordHook duration ${event.src.id}`, debug) + startTimer('fetch records duration', debug) const data = await event.cache.init( 'data', async (): Promise => { @@ -88,15 +90,15 @@ export const BulkRecordHook = async ( 'records', async () => await prepareXRecords(data) ) - endTimer('fetch data', debug) + endTimer('fetch records duration', debug) // Execute client-defined data hooks - startTimer('run handler', debug) + startTimer('handler duration', debug) await asyncBatch(batch.records, handler, options, event) - endTimer('run handler', debug) + endTimer('handler duration', debug) event.afterAll(async () => { - startTimer('filter modified records', debug) + startTimer('filter modified records duration', debug) const { records: batch } = event.cache.get>('records') const records: Flatfile.RecordsWithLinks = @@ -122,19 +124,19 @@ export const BulkRecordHook = async ( await completeCommit(event, debug) return } - endTimer('filter modified records', debug) + endTimer('filter modified records duration', debug) - if (debug) { + debug && logInfo( '@flatfile/plugin-record-hook', `${modifiedRecords.length} modified records` ) - } try { - startTimer('update modified records', debug) + startTimer('write duration', debug) await event.update(modifiedRecords) - endTimer('update modified records', debug) + endTimer('write duration', debug) + endTimer(`bulkRecordHook duration ${event.src.id}`, debug) return } catch (e) { throw new Error('Error updating records') diff --git a/plugins/record-hook/src/record.hook.plugin.ts b/plugins/record-hook/src/record.hook.plugin.ts index 23bd9438f..d99f95997 100644 --- a/plugins/record-hook/src/record.hook.plugin.ts +++ b/plugins/record-hook/src/record.hook.plugin.ts @@ -1,5 +1,6 @@ import type { FlatfileRecord } from '@flatfile/hooks' import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { Item } from '../../record-hook-stream/src/utils/item' import type { BulkRecordHookOptions, RecordHookOptions } from './RecordHook' import { BulkRecordHook, RecordHook } from './RecordHook' From 887e29a4f44917fb166343a393e1de93b80c935b Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Tue, 26 Nov 2024 13:41:13 -0600 Subject: [PATCH 2/4] update messaging --- flatfilers/sandbox/src/index.ts | 17 +++++++++-------- plugins/record-hook-stream/src/index.ts | 2 +- .../src/record.hook.stream.ts | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index 1096e1821..bf272c733 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -13,10 +13,10 @@ export default async function (listener: FlatfileListener) { // '**', // (records: FlatfileRecord[], event: FlatfileEvent) => { // for (const record of records) { - // for (const field of oneHundredSheet.fields) { - // record.addError(field.key, 'Testing streaming') - // } - // // record.addError('one', 'Testing streaming') + // // for (const field of oneHundredSheet.fields) { + // // record.addError(field.key, 'Testing streaming') + // // } + // record.addError('one', 'Testing streaming') // } // return records // }, @@ -28,13 +28,14 @@ export default async function (listener: FlatfileListener) { '**', (records, event: FlatfileEvent) => { for (const record of records) { - for (const field of oneHundredSheet.fields) { - record.err(field.key, 'Testing something') - } + // for (const field of oneHundredSheet.fields) { + // record.err(field.key, 'Testing something') + // } + record.err('one', 'Testing something') } return records }, - { includeMessages: true, debug: true } + { includeMessages: true } ) ) listener.use( diff --git a/plugins/record-hook-stream/src/index.ts b/plugins/record-hook-stream/src/index.ts index 220309d5c..7b22cf542 100644 --- a/plugins/record-hook-stream/src/index.ts +++ b/plugins/record-hook-stream/src/index.ts @@ -19,7 +19,7 @@ export const recordHookStream = ( .then( (results: { totalProcessed: number; totalTimeSeconds: string }) => { console.log( - `Processed ${results.totalProcessed} records in ${results.totalTimeSeconds} seconds (r/s: ${Math.ceil(results.totalProcessed / parseFloat(results.totalTimeSeconds))})` + `[${event.src.id}] Processed ${results.totalProcessed} records in ${results.totalTimeSeconds} seconds (r/s: ${Math.ceil(results.totalProcessed / parseFloat(results.totalTimeSeconds))})` ) } ) diff --git a/plugins/record-hook-stream/src/record.hook.stream.ts b/plugins/record-hook-stream/src/record.hook.stream.ts index d1aa68ad9..6644ccb89 100644 --- a/plugins/record-hook-stream/src/record.hook.stream.ts +++ b/plugins/record-hook-stream/src/record.hook.stream.ts @@ -1,5 +1,5 @@ import type { FlatfileEvent } from '@flatfile/listener' -import { request } from 'https' +import { request } from 'node:https' import { RecordHookStreamOptions } from '.' import { Item } from './utils/item' From 7933d874454774c03176e15c4a01ce4faf6d6b8c Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Wed, 13 Aug 2025 15:17:57 -0500 Subject: [PATCH 3/4] fix: replace unbounded Set with simple counter to prevent memory leak - Replaced processedRecordIds Set with processedRecordCount counter - Prevents unbounded memory growth when processing millions of records - Maintains same functionality for progress tracking without storing IDs - Added comments explaining the memory optimization --- plugins/record-hook-stream/src/index.ts | 14 +++++++----- .../src/record.hook.stream.ts | 22 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/plugins/record-hook-stream/src/index.ts b/plugins/record-hook-stream/src/index.ts index 7b22cf542..2e11e7418 100644 --- a/plugins/record-hook-stream/src/index.ts +++ b/plugins/record-hook-stream/src/index.ts @@ -16,13 +16,15 @@ export const recordHookStream = ( return (listener: FlatfileListener) => { listener.on('commit:created', { sheetSlug }, (event: FlatfileEvent) => recordReadWriteStream(callback, event, options) - .then( - (results: { totalProcessed: number; totalTimeSeconds: string }) => { - console.log( - `[${event.src.id}] Processed ${results.totalProcessed} records in ${results.totalTimeSeconds} seconds (r/s: ${Math.ceil(results.totalProcessed / parseFloat(results.totalTimeSeconds))})` - ) + .then((results) => { + const typedResults = results as { + totalProcessed: number + totalTimeSeconds: string } - ) + console.log( + `[${event.src.id}] Processed ${typedResults.totalProcessed} records in ${typedResults.totalTimeSeconds} seconds (r/s: ${Math.ceil(typedResults.totalProcessed / parseFloat(typedResults.totalTimeSeconds))})` + ) + }) .catch((error) => { console.error('Processing failed:', error) throw error diff --git a/plugins/record-hook-stream/src/record.hook.stream.ts b/plugins/record-hook-stream/src/record.hook.stream.ts index 6644ccb89..85b880ff1 100644 --- a/plugins/record-hook-stream/src/record.hook.stream.ts +++ b/plugins/record-hook-stream/src/record.hook.stream.ts @@ -20,7 +20,9 @@ export async function recordReadWriteStream( debug = false, } = options - const processedRecordIds = new Set() + // Use a simple counter instead of storing all IDs to prevent memory issues + // This avoids unbounded memory growth when processing millions of records + let processedRecordCount = 0 const startTime = Date.now() let pendingBatches = 0 @@ -31,12 +33,8 @@ export async function recordReadWriteStream( const chunk = records.slice(i, i + CHUNK_SIZE) const items = chunk.map((record) => new Item(record)) - // Process all records in chunk at once - chunk.forEach((record) => { - if (!processedRecordIds.has(record.__k)) { - processedRecordIds.add(record.__k) - } - }) + // Simply count the processed records + processedRecordCount += chunk.length const transformedItems = await callback(items, event) const chunkResults = transformedItems.map((item) => item.changeset()) @@ -108,16 +106,16 @@ export async function recordReadWriteStream( const now = Date.now() if (now - lastProgressLog >= PROGRESS_INTERVAL) { const timeElapsed = (now - lastProgressLog) / 1000 - const recordsSinceLastLog = processedRecordIds.size - lastRecordCount + const recordsSinceLastLog = processedRecordCount - lastRecordCount const currentRecordsPerSecond = recordsSinceLastLog / timeElapsed const overallRecordsPerSecond = - processedRecordIds.size / ((now - startTime) / 1000) + processedRecordCount / ((now - startTime) / 1000) // Only log if we've processed new records or have pending batches if (recordsSinceLastLog > 0 || pendingBatches > 0) { debug && console.log( - `Progress: ${processedRecordIds.size} records processed ` + + `Progress: ${processedRecordCount} records processed ` + `(current: ${recordsSinceLastLog > 0 ? Math.round(currentRecordsPerSecond) : 'waiting'} r/s, ` + `avg: ${Math.round(overallRecordsPerSecond)} r/s)` + `${pendingBatches > 0 ? ` - ${pendingBatches} batches pending` : ''}` @@ -125,7 +123,7 @@ export async function recordReadWriteStream( } lastProgressLog = now - lastRecordCount = processedRecordIds.size + lastRecordCount = processedRecordCount } } @@ -193,7 +191,7 @@ export async function recordReadWriteStream( const endTime = Date.now() const totalTimeSeconds = (endTime - startTime) / 1000 resolve({ - totalProcessed: processedRecordIds.size, + totalProcessed: processedRecordCount, totalTimeSeconds: totalTimeSeconds.toFixed(2), }) } catch (error) { From 4d1bdc9261fe64cde57f96774a0c71d80e65a7ae Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Wed, 13 Aug 2025 16:27:53 -0500 Subject: [PATCH 4/4] docs: update README with accurate streaming documentation - Replaced copied record-hook documentation with stream-specific content - Added comprehensive Item API reference - Documented streaming benefits and performance optimizations - Added examples showing proper usage of Item methods (err, warn, info) - Included guidance on when to use streaming vs standard hooks - Documented includeMessages and includeMetadata options --- plugins/record-hook-stream/README.md | 397 ++++++++++++++------------- 1 file changed, 200 insertions(+), 197 deletions(-) diff --git a/plugins/record-hook-stream/README.md b/plugins/record-hook-stream/README.md index 8fe8adec8..be8a50be6 100644 --- a/plugins/record-hook-stream/README.md +++ b/plugins/record-hook-stream/README.md @@ -1,9 +1,6 @@ -The `@flatfile/plugin-record-hook` plugin offers a convenient way to execute -custom logic on individual data records within Flatfile. By setting up an event -listener for the `commit:created` event, this plugin seamlessly integrates with -the data processing flow. +The `@flatfile/plugin-record-hook-stream` plugin provides a high-performance streaming interface for processing large datasets in Flatfile. It uses adaptive batch processing and concurrent execution to efficiently handle millions of records without memory issues. This plugin is ideal for processing large files where standard record hooks might encounter performance or memory limitations. **Event Type:** `listener.on('commit:created')` @@ -17,147 +14,127 @@ the data processing flow. The `sheetSlug` parameter is the slug of the sheet you want to listen to. #### `callback` - `function` -The `callback` parameter takes a function that will be run on the record or records. +The `callback` parameter takes a function that will be run on batches of records. The function receives an array of `Item` objects and must return an array of `Item` objects. -#### `options.chunkSize` - `number` - `default: 10_000` - (optional) -The `chunkSize` parameter allows you to specify the quantity of records to process in each chunk. +#### `options.includeMessages` - `boolean` - `default: false` - (optional) +When true, includes existing messages on records when streaming data. -#### `options.parallel` - `number` - `default: 1` - (optional) -The `parallel` parameter allows you to specify the number of chunks to process in parallel. +#### `options.includeMetadata` - `boolean` - `default: false` - (optional) +When true, includes record metadata in the stream. #### `options.debug` - `boolean` - `default: false` - (optional) -The `debug` parameter allows you to turn on debug logging. +The `debug` parameter enables detailed logging of processing progress, batch metrics, and performance statistics. ## Usage ```bash install -npm i @flatfile/plugin-record-hook @flatfile/hooks +npm i @flatfile/plugin-record-hook-stream ``` ### Import -#### bulkRecordHook - -```js bulkRecordHook -import { FlatfileRecord, bulkRecordHook } from "@flatfile/plugin-record-hook"; -import { FlatfileEvent, FlatfileListener } from "@flatfile/listener"; -``` - -#### recordHook - -```js recordHook -import { FlatfileRecord, recordHook } from "@flatfile/plugin-record-hook"; +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; import { FlatfileEvent, FlatfileListener } from "@flatfile/listener"; ``` -Pass `bulkRecordHook` or `recordHook` to a Flatfile data listener and provide a function to run when data is added or updated. +Pass `recordHookStream` to a Flatfile data listener and provide a function to run when data is added or updated. ### Listen for data changes -Set up a listener to configure Flatfile and respond to data Events. Then use this plugin to set up a hook that responds to data changes. +Set up a listener to configure Flatfile and respond to data Events. Then use this plugin to set up a streaming hook that responds to data changes. #### JavaScript -**bulkRecordHook.js** - -```js bulkRecordHook js -import { bulkRecordHook } from "@flatfile/plugin-record-hook"; +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; export default async function (listener) { listener.use( - bulkRecordHook("my-sheet", (records) => { - return records.map((r) => { - //do your work here - return r; + recordHookStream("my-sheet", (items) => { + return items.map((item) => { + // Access record data + const email = item.get("email"); + + // Validate and add errors + if (!email) { + item.err("email", "Email is required"); + } + + // Modify values + item.set("processed", true); + + return item; }); }) ); } ``` -**recordHook.js** - -```js recordHook js -import { recordHook } from "@flatfile/plugin-record-hook"; - -export default async function (listener) { - listener.use( - recordHook("my-sheet", (record) => { - //do your work here - return record; - }) - ); -} -``` #### TypeScript -**bulkRecordHook.ts** - -```js bulkRecordHook ts -import { FlatfileRecord } from "@flatfile/hooks"; -import { bulkRecordHook } from "@flatfile/plugin-record-hook"; -import { FlatfileListener } from "@flatfile/listener"; +```typescript +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileListener, FlatfileEvent } from "@flatfile/listener"; export default async function (listener: FlatfileListener) { listener.use( - bulkRecordHook("my-sheet", (records: FlatfileRecord[]) => { - return records.map((r) => { - //do your work here - return r; + recordHookStream("my-sheet", (items, event: FlatfileEvent) => { + return items.map((item) => { + // Access record data with type safety + const email = item.get("email") as string; + + // Validate and add errors + if (!email) { + item.err("email", "Email is required"); + } + + // Modify values + item.set("processed", true); + + return item; }); }) ); } ``` -**recordHook.ts** - -```js recordHook ts -import { FlatfileRecord } from "@flatfile/hooks"; -import { recordHook } from "@flatfile/plugin-record-hook"; -import { FlatfileListener } from "@flatfile/listener"; - -export default async function (listener: FlatfileListener) { - listener.use( - recordHook("my-sheet", (record: FlatfileRecord) => { - //do your work here - return record; - }) - ); -} -``` - ### Additional Options -`bulkRecordHook` can accept additional properties. Props will be passed along to the transformer. +`recordHookStream` can accept additional options to control streaming behavior and performance. #### JavaScript -**bulkRecordHook.js** - -```js bulkRecordHook js -import { bulkRecordHook } from "@flatfile/plugin-record-hook"; +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; export default async function (listener) { listener.use( - bulkRecordHook("my-sheet", (records) => { - return records.map((r) => { - //do your work here - return r; - }); - }), - { chunkSize: 100, parallel: 2 } + recordHookStream( + "my-sheet", + (items) => { + return items.map((item) => { + // Process each item + return item; + }); + }, + { + includeMessages: true, + includeMetadata: false, + debug: true + } + ) ); } ``` @@ -165,24 +142,25 @@ export default async function (listener) { #### TypeScript -**bulkRecordHook.ts** - -```js bulkRecordHook ts -import { FlatfileRecord } from "@flatfile/hooks"; -import { bulkRecordHook } from "@flatfile/plugin-record-hook"; -import { FlatfileListener } from "@flatfile/listener"; +```typescript +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileListener, FlatfileEvent } from "@flatfile/listener"; export default async function (listener: FlatfileListener) { listener.use( - bulkRecordHook( + recordHookStream( "my-sheet", - (records: FlatfileRecord[]) => { - return records.map((r) => { - //do your work here - return r; + (items, event: FlatfileEvent) => { + return items.map((item) => { + // Process each item + return item; }); }, - { chunkSize: 100, parallel: 2 } + { + includeMessages: true, + includeMetadata: false, + debug: true + } ) ); } @@ -190,143 +168,168 @@ export default async function (listener: FlatfileListener) { -#### Flexible Options +## Streaming Benefits -#### `chunkSize` *number* *default: 10_000* (optional) -Define how many records you want to process in each batch. This allows you to balance efficiency and resource utilization based on your specific use case. +The streaming architecture provides several advantages over traditional record hooks: -#### `parallel` *number* *default: 1* (optional) -Choose whether the records should be processed in parallel. This enables you to optimize the execution time when dealing with large datasets. +### Performance Optimization +- **Adaptive batch sizing**: Automatically adjusts batch size (100-500 records) based on processing speed +- **Concurrent processing**: Processes up to 15 batches simultaneously for maximum throughput +- **Memory efficiency**: Uses a simple counter instead of storing record IDs, preventing memory issues with large datasets +- **Chunk processing**: Processes records in chunks of 50 to prevent memory spikes +### Reliability Features +- **Automatic retry logic**: Implements exponential backoff for rate limiting and network errors +- **Progress tracking**: Real-time metrics showing records/second processing speed +- **Error handling**: Robust error handling with configurable retry behavior -## Example Usage +### When to Use Stream Processing -This example sets up a record hook using `listener.use` to modify records in the "my-sheet" sheet. +Use `recordHookStream` when: +- Processing files with more than 10,000 records +- Dealing with memory constraints on large datasets +- Needing real-time progress updates during processing +- Requiring high throughput for data transformations -When a record is processed by the hook, it checks if an email address is missing, empty, or invalid, and if so, it logs corresponding error messages and adds them to a form validation context (if the r object is related to form validation). This helps ensure that only valid email addresses are accepted in the application. +Use standard `recordHook` when: +- Processing small datasets (< 10,000 records) +- Needing simple, synchronous processing +- Working with records that require complex inter-record logic -In the `bulkRecordHook` example, it passes a `chunkSize` of 100 and asks the hooks to run 2 at a time via the `parallel` property. +## Item API Reference -### JavaScript +The `Item` class provides methods for working with record data: -**bulkRecordHook.js** +### Data Access Methods +- `get(key: string)`: Get a field value +- `set(key: string, value: any)`: Set a field value +- `has(key: string)`: Check if a field has a value +- `isEmpty(key: string)`: Check if a field is empty -```js bulkRecordHook js -import { bulkRecordHook } from "@flatfile/plugin-record-hook"; +### Validation Methods +- `err(key: string, message: string)`: Add an error message +- `warn(key: string, message: string)`: Add a warning message +- `info(key: string, message: string)`: Add an info message -export default async function (listener) { - listener.use( - bulkRecordHook( - "my-sheet", - (records) => { - return records.map((r) => { - const email = r.get("email") as string; - if (!email) { - console.log("Email is required"); - r.addError("email", "Email is required"); - } - const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - if (email !== null && !validEmailAddress.test(email)) { - console.log("Invalid email address"); - r.addError("email", "Invalid email address"); - } - return r; - }); - }, - { chunkSize: 100, parallel: 2 } - ) - ); -} -``` +### Type Conversion Methods +- `str(key: string)`: Get field as nullable string +- `defStr(key: string)`: Get field as string (never null) +- `bool(key: string)`: Get field as boolean +- `date(key: string)`: Get field as Date -**recordHook.js** +### Utility Methods +- `delete()`: Mark record for deletion +- `isDeleted()`: Check if record is marked for deletion +- `isDirty(key?: string)`: Check if record or field has changes +- `keys()`: Get all field keys -```js recordHook js -import { recordHook } from "@flatfile/plugin-record-hook"; -export default async function (listener) { - listener.use( - recordHook( - "my-sheet", - (record) => { - const email = record.get("email") as string; - if (!email) { - console.log("Email is required"); - record.addError("email", "Email is required"); - } - const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - if (email !== null && !validEmailAddress.test(email)) { - console.log("Invalid email address"); - record.addError("email", "Invalid email address"); - } - return record; - } - ) - ); -} -``` +## Example Usage +This example demonstrates a complete implementation with email validation, data transformation, and error handling: -### TypeScript -**bulkRecordHook.ts** +### JavaScript -```js bulkRecordHook ts -import { FlatfileRecord } from "@flatfile/hooks"; -import { bulkRecordHook } from "@flatfile/plugin-record-hook"; -import { FlatfileListener } from "@flatfile/listener"; +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; -export default async function (listener: FlatfileListener) { +export default async function (listener) { listener.use( - bulkRecordHook( + recordHookStream( "contacts", - (records: FlatfileRecord[]) => { - return records.map((r) => { - const email = r.get("email") as string; + (items) => { + return items.map((item) => { + // Email validation + const email = item.get("email"); if (!email) { - console.log("Email is required"); - r.addError("email", "Email is required"); + item.err("email", "Email is required"); + } else { + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!validEmailAddress.test(email)) { + item.err("email", "Invalid email address"); + } } - const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - if (email !== null && !validEmailAddress.test(email)) { - console.log("Invalid email address"); - r.addError("email", "Invalid email address"); + + // Data transformation + const firstName = item.str("firstName"); + const lastName = item.str("lastName"); + if (firstName && lastName) { + item.set("fullName", `${firstName} ${lastName}`); } - return r; + + // Conditional logic + if (item.bool("isVIP")) { + item.set("priority", "high"); + } + + // Add processing timestamp + item.set("processedAt", new Date().toISOString()); + + return item; }); }, - { chunkSize: 100, parallel: 2 } + { debug: true } ) ); } ``` -**recordHook.ts** -```js recordHook ts -import { FlatfileRecord } from "@flatfile/hooks"; -import { recordHook } from "@flatfile/plugin-record-hook"; -import { FlatfileListener } from "@flatfile/listener"; +### TypeScript + +```typescript +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileListener, FlatfileEvent } from "@flatfile/listener"; + +interface ContactRecord { + email?: string; + firstName?: string; + lastName?: string; + fullName?: string; + isVIP?: boolean; + priority?: string; + processedAt?: string; +} export default async function (listener: FlatfileListener) { listener.use( - recordHook( + recordHookStream( "contacts", - (record: FlatfileRecord) => { - const email = record.get("email") as string; - if (!email) { - console.log("Email is required"); - record.addError("email", "Email is required"); - } - const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; - if (email !== null && !validEmailAddress.test(email)) { - console.log("Invalid email address"); - record.addError("email", "Invalid email address"); - } - return record; - } + (items, event: FlatfileEvent) => { + return items.map((item) => { + // Email validation with type safety + const email = item.get("email") as string | undefined; + if (!email) { + item.err("email", "Email is required"); + } else { + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!validEmailAddress.test(email)) { + item.err("email", "Invalid email address"); + } + } + + // Data transformation + const firstName = item.str("firstName"); + const lastName = item.str("lastName"); + if (firstName && lastName) { + item.set("fullName", `${firstName} ${lastName}`); + } + + // Conditional logic + if (item.bool("isVIP")) { + item.set("priority", "high"); + } + + // Add processing timestamp + item.set("processedAt", new Date().toISOString()); + + return item; + }); + }, + { debug: true } ) ); } -``` +``` \ No newline at end of file