diff --git a/flatfilers/sandbox/package.json b/flatfilers/sandbox/package.json index f3443c32a..18ad52475 100644 --- a/flatfilers/sandbox/package.json +++ b/flatfilers/sandbox/package.json @@ -7,7 +7,7 @@ "description": "", "license": "ISC", "scripts": { - "dev": "node --enable-source-maps --inspect ../../node_modules/.bin/flatfile develop", + "dev": "node --enable-source-maps --trace-warnings --inspect ../../node_modules/.bin/flatfile develop", "dev:local": "dotenvx run --env-file=.env.local -- npm run dev", "dev:staging": "dotenvx run --env-file=.env.staging -- npm run dev", "dev:prod": "dotenvx run --env-file=.env.prod -- npm run dev", @@ -30,6 +30,6 @@ }, "devDependencies": { "@dotenvx/dotenvx": "^0.39.0", - "flatfile": "^3.6.1" + "flatfile": "^3.8.0" } } diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index 16731d584..bf272c733 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -1,115 +1,53 @@ -import type { FlatfileListener } from '@flatfile/listener' -import { exportDelimitedZip } from '@flatfile/plugin-export-delimited-zip' -import { JSONExtractor } from '@flatfile/plugin-json-extractor' +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { + bulkRecordHook, + type FlatfileRecord, +} from '@flatfile/plugin-record-hook' +import { recordHookStream } from '@flatfile/plugin-record-hook-stream' import { configureSpace } from '@flatfile/plugin-space-configure' +import { contactsSheet, oneHundredSheet } from '../../playground/src/blueprints' export default async function (listener: FlatfileListener) { - listener.use(JSONExtractor()) + // listener.use( + // bulkRecordHook( + // '**', + // (records: FlatfileRecord[], event: FlatfileEvent) => { + // for (const record of records) { + // // for (const field of oneHundredSheet.fields) { + // // record.addError(field.key, 'Testing streaming') + // // } + // record.addError('one', 'Testing streaming') + // } + // return records + // }, + // { debug: true } + // ) + // ) listener.use( - exportDelimitedZip({ - job: 'export-delimited-zip', - delimiter: '\t', - fileExtension: 'tsv', - debug: true, - }) + recordHookStream( + '**', + (records, event: FlatfileEvent) => { + for (const record of records) { + // for (const field of oneHundredSheet.fields) { + // record.err(field.key, 'Testing something') + // } + record.err('one', 'Testing something') + } + return records + }, + { includeMessages: true } + ) ) listener.use( configureSpace({ workbooks: [ - { - name: 'Sandbox', - sheets: [ - { - name: 'Sales', - slug: 'sales', - fields: [ - { - key: 'date', - type: 'string', - label: 'Date', - }, - { - key: 'product', - type: 'string', - label: 'Product', - }, - { - key: 'category', - type: 'string', - label: 'Category', - }, - { - key: 'region', - type: 'string', - label: 'Region', - }, - { - key: 'salesAmount', - type: 'number', - label: 'Sales Amount', - }, - ], - actions: [ - { - operation: 'generateExampleRecords', - label: 'Generate Example Records', - description: - 'This custom action code generates example records using Anthropic.', - primary: false, - mode: 'foreground', - }, - ], - }, - { - name: 'Sales 2', - slug: 'sales-2', - fields: [ - { - key: 'date', - type: 'string', - label: 'Date', - }, - { - key: 'product', - type: 'string', - label: 'Product', - }, - { - key: 'category', - type: 'string', - label: 'Category', - }, - { - key: 'region', - type: 'string', - label: 'Region', - }, - { - key: 'salesAmount', - type: 'number', - label: 'Sales Amount', - }, - ], - actions: [ - { - operation: 'export-external-api', - label: 'Export to External API', - description: - 'This custom action code exports the records in the Sales sheet to an external API.', - primary: false, - mode: 'foreground', - }, - ], - }, - ], - actions: [ - { - operation: 'export-delimited-zip', - label: 'Export to Delimited ZIP', - mode: 'foreground', - }, - ], - }, + // { + // name: 'Sandbox', + // sheets: [ + // contactsSheet, + // ], + // }, + oneHundredSheet, ], }) ) diff --git a/package-lock.json b/package-lock.json index 376d11792..9a3f1735a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -313,7 +313,36 @@ }, "devDependencies": { "@dotenvx/dotenvx": "^0.39.0", - "flatfile": "^3.6.1" + "flatfile": "^3.8.0" + } + }, + "flatfilers/streaming": { + "name": "@flatfile/flatfilers-streaming", + "version": "0.0.0", + "extraneous": true, + "license": "ISC", + "dependencies": { + "chrono-node": "^2.7.7", + "collect.js": "^4.36.1", + "cross-fetch": "^4.0.0", + "extendable-error": "^0.1.7", + "jsonlines": "^0.1.1", + "modern-async": "^2.0.4" + }, + "devDependencies": { + "@dotenvx/dotenvx": "^0.39.0", + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0", + "@types/jsonlines": "^0.1.5", + "@types/node": "^18.16.0", + "tsx": "^4.7.1" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@flatfile/api": "^1.9.19", + "@flatfile/listener": "^1.1.0" } }, "import/faker": { @@ -3332,6 +3361,8 @@ }, "node_modules/@dabh/diagnostics": { "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", "dev": true, "license": "MIT", "dependencies": { @@ -3342,6 +3373,8 @@ }, "node_modules/@dotenvx/dotenvx": { "version": "0.39.0", + "resolved": "https://registry.npmjs.org/@dotenvx/dotenvx/-/dotenvx-0.39.0.tgz", + "integrity": "sha512-OC8m1P10A9hG7yzuQZxQjIGvJlm2nHIZnvHse95DkwhxBgCC8NgZkb/3GnoIIw9Z6VZ2+LF0jLsnMrlcVIFk1w==", "dev": true, "license": "MIT", "dependencies": { @@ -3377,6 +3410,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/brace-expansion": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "license": "MIT", "dependencies": { @@ -3385,6 +3420,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/commander": { "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", "dev": true, "license": "MIT", "engines": { @@ -3392,11 +3429,13 @@ } }, "node_modules/@dotenvx/dotenvx/node_modules/dotenv-expand": { - "version": "11.0.6", + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", + "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "dotenv": "^16.4.4" + "dotenv": "^16.4.5" }, "engines": { "node": ">=12" @@ -3407,6 +3446,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/glob": { "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", "dev": true, "license": "ISC", "dependencies": { @@ -3426,6 +3467,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/isexe": { "version": "3.1.1", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.1.tgz", + "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", "dev": true, "license": "ISC", "engines": { @@ -3434,6 +3477,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/minimatch": { "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "license": "ISC", "dependencies": { @@ -3448,6 +3493,8 @@ }, "node_modules/@dotenvx/dotenvx/node_modules/which": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz", + "integrity": "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg==", "dev": true, "license": "ISC", "dependencies": { @@ -3460,6 +3507,20 @@ "node": "^16.13.0 || >=18.0.0" } }, + "node_modules/@ecies/ciphers": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@ecies/ciphers/-/ciphers-0.2.1.tgz", + "integrity": "sha512-ezMihhjW24VNK/2qQR7lH8xCQY24nk0XHF/kwJ1OuiiY5iEwQXOcKVSy47fSoHPRG8gVGXcK5SgtONDk5xMwtQ==", + "dev": true, + "engines": { + "bun": ">=1", + "deno": ">=2", + "node": ">=16" + }, + "peerDependencies": { + "@noble/ciphers": "^1.0.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -3830,6 +3891,8 @@ }, "node_modules/@fastify/busboy": { "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", "dev": true, "license": "MIT", "engines": { @@ -4096,6 +4159,10 @@ "resolved": "plugins/record-hook", "link": true }, + "node_modules/@flatfile/plugin-record-hook-stream": { + "resolved": "plugins/record-hook-stream", + "link": true + }, "node_modules/@flatfile/plugin-rollout": { "resolved": "plugins/rollout", "link": true @@ -4287,6 +4354,8 @@ }, "node_modules/@inquirer/confirm": { "version": "2.0.17", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-2.0.17.tgz", + "integrity": "sha512-EqzhGryzmGpy2aJf6LxJVhndxYmFs+m8cxXzf8nejb1DE3sabf6mUgBcp4J0jAUEiAcYzqmkqRr7LPFh/WdnXA==", "dev": true, "license": "MIT", "dependencies": { @@ -4300,6 +4369,8 @@ }, "node_modules/@inquirer/core": { "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-6.0.0.tgz", + "integrity": "sha512-fKi63Khkisgda3ohnskNf5uZJj+zXOaBvOllHsOkdsXRA/ubQLJQrZchFFi57NKbZzkTunXiBMdvWOv71alonw==", "dev": true, "license": "MIT", "dependencies": { @@ -4324,6 +4395,8 @@ }, "node_modules/@inquirer/core/node_modules/signal-exit": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "dev": true, "license": "ISC", "engines": { @@ -4335,6 +4408,8 @@ }, "node_modules/@inquirer/core/node_modules/wrap-ansi": { "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", "dev": true, "license": "MIT", "dependencies": { @@ -4347,7 +4422,9 @@ } }, "node_modules/@inquirer/type": { - "version": "1.5.2", + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.5.tgz", + "integrity": "sha512-MzICLu4yS7V8AA61sANROZ9vT1H3ooca5dSmI1FjZkzq7o/koMsRfQSzRtFo+F3Ao4Sf1C0bpLKejpKB/+j6MA==", "dev": true, "license": "MIT", "dependencies": { @@ -4775,30 +4852,39 @@ } }, "node_modules/@noble/ciphers": { - "version": "0.5.3", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.0.0.tgz", + "integrity": "sha512-wH5EHOmLi0rEazphPbecAzmjd12I6/Yv/SiHdkA9LSycsQk7RuuTp7am5/o62qYr0RScE7Pc9icXGBbsr6cesA==", "dev": true, - "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, "funding": { "url": "https://paulmillr.com/funding/" } }, "node_modules/@noble/curves": { - "version": "1.5.0", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.6.0.tgz", + "integrity": "sha512-TlaHRXDehJuRNR9TfZDNQ45mMEd5dwUwmicsafcIX4SsNiqnCHKjE/1alYPd/lDRVhxdhUAlv8uEhMCI5zjIJQ==", "dev": true, - "license": "MIT", "dependencies": { - "@noble/hashes": "1.4.0" + "@noble/hashes": "1.5.0" + }, + "engines": { + "node": "^14.21.3 || >=16" }, "funding": { "url": "https://paulmillr.com/funding/" } }, "node_modules/@noble/hashes": { - "version": "1.4.0", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.5.0.tgz", + "integrity": "sha512-1j6kQFb7QRru7eKN3ZDvRcP13rugwdxZqCjbiAVZfIJwgj2A65UmT4TgARXGlXgnRkORLTDTrO19ZErt7+QXgA==", "dev": true, - "license": "MIT", "engines": { - "node": ">= 16" + "node": "^14.21.3 || >=16" }, "funding": { "url": "https://paulmillr.com/funding/" @@ -6407,6 +6493,8 @@ }, "node_modules/@types/mute-stream": { "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", "dev": true, "license": "MIT", "dependencies": { @@ -6513,6 +6601,8 @@ }, "node_modules/@types/triple-beam": { "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", "dev": true, "license": "MIT" }, @@ -6523,6 +6613,8 @@ }, "node_modules/@types/wrap-ansi": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==", "dev": true, "license": "MIT" }, @@ -6748,6 +6840,8 @@ }, "node_modules/ajv-formats": { "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", "dev": true, "license": "MIT", "dependencies": { @@ -6776,6 +6870,8 @@ }, "node_modules/ansi-escapes": { "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", "dev": true, "license": "MIT", "dependencies": { @@ -6859,6 +6955,8 @@ }, "node_modules/arch": { "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", "dev": true, "funding": [ { @@ -6983,6 +7081,8 @@ }, "node_modules/atomically": { "version": "1.7.0", + "resolved": "https://registry.npmjs.org/atomically/-/atomically-1.7.0.tgz", + "integrity": "sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==", "dev": true, "license": "MIT", "engines": { @@ -7556,6 +7656,8 @@ }, "node_modules/chrono-node": { "version": "2.7.7", + "resolved": "https://registry.npmjs.org/chrono-node/-/chrono-node-2.7.7.tgz", + "integrity": "sha512-p3S7gotuTPu5oqhRL2p1fLwQXGgdQaRTtWR3e8Di9P1Pa9mzkK5DWR5AWBieMUh2ZdOnPgrK+zCrbbtyuA+D/Q==", "license": "MIT", "dependencies": { "dayjs": "^1.10.0" @@ -7668,6 +7770,8 @@ }, "node_modules/cli-width": { "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", "dev": true, "license": "ISC", "engines": { @@ -7701,6 +7805,8 @@ }, "node_modules/color": { "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", "dev": true, "license": "MIT", "dependencies": { @@ -7724,6 +7830,8 @@ }, "node_modules/color-string": { "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", "dev": true, "license": "MIT", "dependencies": { @@ -7733,6 +7841,8 @@ }, "node_modules/color/node_modules/color-convert": { "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", "dev": true, "license": "MIT", "dependencies": { @@ -7741,6 +7851,8 @@ }, "node_modules/color/node_modules/color-name": { "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true, "license": "MIT" }, @@ -7751,6 +7863,8 @@ }, "node_modules/colorspace": { "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", "dev": true, "license": "MIT", "dependencies": { @@ -7844,6 +7958,8 @@ }, "node_modules/conf": { "version": "10.2.0", + "resolved": "https://registry.npmjs.org/conf/-/conf-10.2.0.tgz", + "integrity": "sha512-8fLl9F04EJqjSqH+QjITQfJF8BrOVaYr1jewVgSRAEWePfxT0sku4w2hrGQ60BC/TNLGQ2pgxNlTbWQmMPFvXg==", "dev": true, "license": "MIT", "dependencies": { @@ -8122,6 +8238,8 @@ }, "node_modules/cuint": { "version": "0.2.2", + "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", + "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==", "dev": true, "license": "MIT" }, @@ -8206,6 +8324,8 @@ }, "node_modules/debounce-fn": { "version": "4.0.0", + "resolved": "https://registry.npmjs.org/debounce-fn/-/debounce-fn-4.0.0.tgz", + "integrity": "sha512-8pYCQiL9Xdcg0UPSD3d+0KMlOjp+KGU5EPwYddgzQ7DATsg4fuUDjQtsYLmWjnk2obnNHgV3vE2Y4jejSOJVBQ==", "dev": true, "license": "MIT", "dependencies": { @@ -8220,6 +8340,8 @@ }, "node_modules/debounce-fn/node_modules/mimic-fn": { "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz", + "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==", "dev": true, "license": "MIT", "engines": { @@ -8367,6 +8489,8 @@ }, "node_modules/diff": { "version": "5.2.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", + "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", "dev": true, "license": "BSD-3-Clause", "engines": { @@ -8400,6 +8524,8 @@ }, "node_modules/dot-prop": { "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", "dev": true, "license": "MIT", "dependencies": { @@ -8472,16 +8598,20 @@ } }, "node_modules/eciesjs": { - "version": "0.4.7", + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/eciesjs/-/eciesjs-0.4.11.tgz", + "integrity": "sha512-SmUG449n1w1YGvJD9R30tBGvpxTxA0cnn0rfvpFIBvmezfIhagLjsH2JG8HBHOLS8slXsPh48II7IDUTH/J3Mg==", "dev": true, - "license": "MIT", "dependencies": { - "@noble/ciphers": "^0.5.3", - "@noble/curves": "^1.4.0", - "@noble/hashes": "^1.4.0" + "@ecies/ciphers": "^0.2.1", + "@noble/ciphers": "^1.0.0", + "@noble/curves": "^1.6.0", + "@noble/hashes": "^1.5.0" }, "engines": { - "node": ">=16.0.0" + "bun": ">=1", + "deno": ">=2", + "node": ">=16" } }, "node_modules/ee-first": { @@ -8511,6 +8641,8 @@ }, "node_modules/enabled": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", "dev": true, "license": "MIT" }, @@ -9284,6 +9416,8 @@ }, "node_modules/extendable-error": { "version": "0.1.7", + "resolved": "https://registry.npmjs.org/extendable-error/-/extendable-error-0.1.7.tgz", + "integrity": "sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==", "dev": true, "license": "MIT" }, @@ -9373,6 +9507,8 @@ }, "node_modules/fecha": { "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "dev": true, "license": "MIT" }, @@ -9394,6 +9530,8 @@ }, "node_modules/figures": { "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", "dev": true, "license": "MIT", "dependencies": { @@ -9408,6 +9546,8 @@ }, "node_modules/figures/node_modules/escape-string-regexp": { "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true, "license": "MIT", "engines": { @@ -10305,6 +10445,8 @@ }, "node_modules/fn.name": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", "dev": true, "license": "MIT" }, @@ -10541,6 +10683,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", + "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/get-uri": { "version": "6.0.3", "license": "MIT", @@ -11459,6 +11615,8 @@ }, "node_modules/is-obj": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", "dev": true, "license": "MIT", "engines": { @@ -11936,6 +12094,8 @@ }, "node_modules/json-schema-typed": { "version": "7.0.3", + "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-7.0.3.tgz", + "integrity": "sha512-7DE8mpG+/fVw+dTpjbxnx47TaMnDfOI1jwft9g1VybltZCduyRQPJPvc+zzKY9WPHxhPWczyFuYa6I8Mw4iU5A==", "dev": true, "license": "BSD-2-Clause" }, @@ -12032,6 +12192,8 @@ }, "node_modules/kuler": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", "dev": true, "license": "MIT" }, @@ -12217,7 +12379,9 @@ } }, "node_modules/logform": { - "version": "2.6.1", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", "dev": true, "license": "MIT", "dependencies": { @@ -12234,6 +12398,8 @@ }, "node_modules/logform/node_modules/@colors/colors": { "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", "dev": true, "license": "MIT", "engines": { @@ -12242,6 +12408,8 @@ }, "node_modules/logform/node_modules/ms": { "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true, "license": "MIT" }, @@ -12926,6 +13094,8 @@ }, "node_modules/modern-async": { "version": "2.0.4", + "resolved": "https://registry.npmjs.org/modern-async/-/modern-async-2.0.4.tgz", + "integrity": "sha512-89Ig+D/NQAFi39/oRUphkwdLb4qo/3Vl21TCMPZY3oCWGQQ+HqNJ1kyWknyAJ2rsiewL9OGBmpKA5bQ0PikkMg==", "license": "MIT", "dependencies": { "core-js-pure": "^3.19.1", @@ -13006,6 +13176,8 @@ }, "node_modules/mute-stream": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", "dev": true, "license": "ISC", "engines": { @@ -13427,6 +13599,8 @@ }, "node_modules/one-time": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", "dev": true, "license": "MIT", "dependencies": { @@ -13928,6 +14102,8 @@ }, "node_modules/pkg-up": { "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-up/-/pkg-up-3.1.0.tgz", + "integrity": "sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==", "dev": true, "license": "MIT", "dependencies": { @@ -13939,6 +14115,8 @@ }, "node_modules/pkg-up/node_modules/find-up": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", "dev": true, "license": "MIT", "dependencies": { @@ -13950,6 +14128,8 @@ }, "node_modules/pkg-up/node_modules/locate-path": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "license": "MIT", "dependencies": { @@ -13962,6 +14142,8 @@ }, "node_modules/pkg-up/node_modules/p-locate": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "dev": true, "license": "MIT", "dependencies": { @@ -13973,6 +14155,8 @@ }, "node_modules/pkg-up/node_modules/path-exists": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", "dev": true, "license": "MIT", "engines": { @@ -14896,6 +15080,17 @@ "node": ">=8" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "license": "MIT", + "optional": true, + "peer": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/restore-cursor": { "version": "3.1.0", "license": "MIT", @@ -15076,6 +15271,8 @@ }, "node_modules/run-async": { "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", "dev": true, "license": "MIT", "engines": { @@ -15165,6 +15362,8 @@ }, "node_modules/safe-stable-stringify": { "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", "dev": true, "license": "MIT", "engines": { @@ -15375,6 +15574,8 @@ }, "node_modules/simple-swizzle": { "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", "dev": true, "license": "MIT", "dependencies": { @@ -15383,6 +15584,8 @@ }, "node_modules/simple-swizzle/node_modules/is-arrayish": { "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", "dev": true, "license": "MIT" }, @@ -15637,6 +15840,8 @@ }, "node_modules/stack-trace": { "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", "dev": true, "license": "MIT", "engines": { @@ -16194,6 +16399,8 @@ }, "node_modules/text-hex": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", "dev": true, "license": "MIT" }, @@ -16381,6 +16588,8 @@ }, "node_modules/triple-beam": { "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", "dev": true, "license": "MIT", "engines": { @@ -16498,6 +16707,27 @@ "webidl-conversions": "^4.0.2" } }, + "node_modules/tsx": { + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.2.tgz", + "integrity": "sha512-pOUl6Vo2LUq/bSa8S5q7b91cgNSjctn9ugq/+Mvow99qW6x/UZYwzxy/3NmqoT66eHYfCVvFvACC58UBPFf28g==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "esbuild": "~0.23.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/turbo": { "version": "1.13.4", "dev": true, @@ -16535,6 +16765,8 @@ }, "node_modules/type-fest": { "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "dev": true, "license": "(MIT OR CC0-1.0)", "engines": { @@ -16659,6 +16891,8 @@ }, "node_modules/undici": { "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "dev": true, "license": "MIT", "dependencies": { @@ -16770,6 +17004,8 @@ }, "node_modules/util": { "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", "dev": true, "license": "MIT", "dependencies": { @@ -17207,7 +17443,9 @@ "license": "ISC" }, "node_modules/winston": { - "version": "3.14.2", + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", "dev": true, "license": "MIT", "dependencies": { @@ -17215,24 +17453,26 @@ "@dabh/diagnostics": "^2.0.2", "async": "^3.2.3", "is-stream": "^2.0.0", - "logform": "^2.6.0", + "logform": "^2.7.0", "one-time": "^1.0.0", "readable-stream": "^3.4.0", "safe-stable-stringify": "^2.3.1", "stack-trace": "0.0.x", "triple-beam": "^1.3.0", - "winston-transport": "^4.7.0" + "winston-transport": "^4.9.0" }, "engines": { "node": ">= 12.0.0" } }, "node_modules/winston-transport": { - "version": "4.7.1", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", "dev": true, "license": "MIT", "dependencies": { - "logform": "^2.6.1", + "logform": "^2.7.0", "readable-stream": "^3.6.2", "triple-beam": "^1.3.0" }, @@ -17242,6 +17482,8 @@ }, "node_modules/winston/node_modules/@colors/colors": { "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", "dev": true, "license": "MIT", "engines": { @@ -17352,6 +17594,8 @@ }, "node_modules/xxhashjs": { "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", + "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", "dev": true, "license": "MIT", "dependencies": { @@ -17635,10 +17879,10 @@ }, "plugins/json-extractor": { "name": "@flatfile/plugin-json-extractor", - "version": "0.10.0", + "version": "0.11.0", "license": "ISC", "dependencies": { - "@flatfile/util-extractor": "^2.3.0" + "@flatfile/util-extractor": "^2.4.0" }, "devDependencies": { "@flatfile/bundler-config-tsup": "^0.2.0", @@ -17800,6 +18044,21 @@ "@flatfile/listener": "^1.1.0" } }, + "plugins/record-hook-stream": { + "name": "@flatfile/plugin-record-hook-stream", + "version": "0.0.1", + "license": "ISC", + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@flatfile/listener": "^1.1.0" + } + }, "plugins/rollout": { "name": "@flatfile/plugin-rollout", "version": "1.4.0", @@ -18056,6 +18315,33 @@ "node": ">= 18" } }, + "utils/asdf": { + "name": "@flatfile/util-asdf", + "version": "0.0.0", + "extraneous": true, + "license": "ISC", + "dependencies": { + "chrono-node": "^2.7.7", + "collect.js": "^4.36.1", + "cross-fetch": "^4.0.0", + "extendable-error": "^0.1.7", + "jsonlines": "^0.1.1", + "modern-async": "^2.0.4" + }, + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0", + "@types/jsonlines": "^0.1.5", + "@types/node": "^18.16.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@flatfile/api": "^1.9.19", + "@flatfile/listener": "^1.1.0" + } + }, "utils/common": { "name": "@flatfile/util-common", "version": "1.6.0", @@ -18082,7 +18368,7 @@ }, "utils/extractor": { "name": "@flatfile/util-extractor", - "version": "2.3.0", + "version": "2.4.0", "license": "ISC", "dependencies": { "@flatfile/util-common": "^1.6.0", diff --git a/plugins/record-hook-stream/CHANGELOG.md b/plugins/record-hook-stream/CHANGELOG.md new file mode 100644 index 000000000..645c14f72 --- /dev/null +++ b/plugins/record-hook-stream/CHANGELOG.md @@ -0,0 +1 @@ +# @flatfile/plugin-record-hook-stream \ No newline at end of file diff --git a/plugins/record-hook-stream/README.md b/plugins/record-hook-stream/README.md new file mode 100644 index 000000000..be8a50be6 --- /dev/null +++ b/plugins/record-hook-stream/README.md @@ -0,0 +1,335 @@ + + +The `@flatfile/plugin-record-hook-stream` plugin provides a high-performance streaming interface for processing large datasets in Flatfile. It uses adaptive batch processing and concurrent execution to efficiently handle millions of records without memory issues. This plugin is ideal for processing large files where standard record hooks might encounter performance or memory limitations. + +**Event Type:** +`listener.on('commit:created')` + + + + +## Parameters + +#### `sheetSlug` - `string` +The `sheetSlug` parameter is the slug of the sheet you want to listen to. + +#### `callback` - `function` +The `callback` parameter takes a function that will be run on batches of records. The function receives an array of `Item` objects and must return an array of `Item` objects. + +#### `options.includeMessages` - `boolean` - `default: false` - (optional) +When true, includes existing messages on records when streaming data. + +#### `options.includeMetadata` - `boolean` - `default: false` - (optional) +When true, includes record metadata in the stream. + +#### `options.debug` - `boolean` - `default: false` - (optional) +The `debug` parameter enables detailed logging of processing progress, batch metrics, and performance statistics. + + + +## Usage + +```bash install +npm i @flatfile/plugin-record-hook-stream +``` + +### Import + + +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileEvent, FlatfileListener } from "@flatfile/listener"; +``` + + + +Pass `recordHookStream` to a Flatfile data listener and provide a function to run when data is added or updated. + + +### Listen for data changes + +Set up a listener to configure Flatfile and respond to data Events. Then use this plugin to set up a streaming hook that responds to data changes. + + +#### JavaScript + +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; + +export default async function (listener) { + listener.use( + recordHookStream("my-sheet", (items) => { + return items.map((item) => { + // Access record data + const email = item.get("email"); + + // Validate and add errors + if (!email) { + item.err("email", "Email is required"); + } + + // Modify values + item.set("processed", true); + + return item; + }); + }) + ); +} +``` + + +#### TypeScript + +```typescript +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileListener, FlatfileEvent } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + recordHookStream("my-sheet", (items, event: FlatfileEvent) => { + return items.map((item) => { + // Access record data with type safety + const email = item.get("email") as string; + + // Validate and add errors + if (!email) { + item.err("email", "Email is required"); + } + + // Modify values + item.set("processed", true); + + return item; + }); + }) + ); +} +``` + + + +### Additional Options + +`recordHookStream` can accept additional options to control streaming behavior and performance. + + +#### JavaScript + +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; + +export default async function (listener) { + listener.use( + recordHookStream( + "my-sheet", + (items) => { + return items.map((item) => { + // Process each item + return item; + }); + }, + { + includeMessages: true, + includeMetadata: false, + debug: true + } + ) + ); +} +``` + + +#### TypeScript + +```typescript +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileListener, FlatfileEvent } from "@flatfile/listener"; + +export default async function (listener: FlatfileListener) { + listener.use( + recordHookStream( + "my-sheet", + (items, event: FlatfileEvent) => { + return items.map((item) => { + // Process each item + return item; + }); + }, + { + includeMessages: true, + includeMetadata: false, + debug: true + } + ) + ); +} +``` + + + +## Streaming Benefits + +The streaming architecture provides several advantages over traditional record hooks: + +### Performance Optimization +- **Adaptive batch sizing**: Automatically adjusts batch size (100-500 records) based on processing speed +- **Concurrent processing**: Processes up to 15 batches simultaneously for maximum throughput +- **Memory efficiency**: Uses a simple counter instead of storing record IDs, preventing memory issues with large datasets +- **Chunk processing**: Processes records in chunks of 50 to prevent memory spikes + +### Reliability Features +- **Automatic retry logic**: Implements exponential backoff for rate limiting and network errors +- **Progress tracking**: Real-time metrics showing records/second processing speed +- **Error handling**: Robust error handling with configurable retry behavior + +### When to Use Stream Processing + +Use `recordHookStream` when: +- Processing files with more than 10,000 records +- Dealing with memory constraints on large datasets +- Needing real-time progress updates during processing +- Requiring high throughput for data transformations + +Use standard `recordHook` when: +- Processing small datasets (< 10,000 records) +- Needing simple, synchronous processing +- Working with records that require complex inter-record logic + + +## Item API Reference + +The `Item` class provides methods for working with record data: + +### Data Access Methods +- `get(key: string)`: Get a field value +- `set(key: string, value: any)`: Set a field value +- `has(key: string)`: Check if a field has a value +- `isEmpty(key: string)`: Check if a field is empty + +### Validation Methods +- `err(key: string, message: string)`: Add an error message +- `warn(key: string, message: string)`: Add a warning message +- `info(key: string, message: string)`: Add an info message + +### Type Conversion Methods +- `str(key: string)`: Get field as nullable string +- `defStr(key: string)`: Get field as string (never null) +- `bool(key: string)`: Get field as boolean +- `date(key: string)`: Get field as Date + +### Utility Methods +- `delete()`: Mark record for deletion +- `isDeleted()`: Check if record is marked for deletion +- `isDirty(key?: string)`: Check if record or field has changes +- `keys()`: Get all field keys + + +## Example Usage + +This example demonstrates a complete implementation with email validation, data transformation, and error handling: + + +### JavaScript + +```js +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; + +export default async function (listener) { + listener.use( + recordHookStream( + "contacts", + (items) => { + return items.map((item) => { + // Email validation + const email = item.get("email"); + if (!email) { + item.err("email", "Email is required"); + } else { + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!validEmailAddress.test(email)) { + item.err("email", "Invalid email address"); + } + } + + // Data transformation + const firstName = item.str("firstName"); + const lastName = item.str("lastName"); + if (firstName && lastName) { + item.set("fullName", `${firstName} ${lastName}`); + } + + // Conditional logic + if (item.bool("isVIP")) { + item.set("priority", "high"); + } + + // Add processing timestamp + item.set("processedAt", new Date().toISOString()); + + return item; + }); + }, + { debug: true } + ) + ); +} +``` + + +### TypeScript + +```typescript +import { recordHookStream } from "@flatfile/plugin-record-hook-stream"; +import { FlatfileListener, FlatfileEvent } from "@flatfile/listener"; + +interface ContactRecord { + email?: string; + firstName?: string; + lastName?: string; + fullName?: string; + isVIP?: boolean; + priority?: string; + processedAt?: string; +} + +export default async function (listener: FlatfileListener) { + listener.use( + recordHookStream( + "contacts", + (items, event: FlatfileEvent) => { + return items.map((item) => { + // Email validation with type safety + const email = item.get("email") as string | undefined; + if (!email) { + item.err("email", "Email is required"); + } else { + const validEmailAddress = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + if (!validEmailAddress.test(email)) { + item.err("email", "Invalid email address"); + } + } + + // Data transformation + const firstName = item.str("firstName"); + const lastName = item.str("lastName"); + if (firstName && lastName) { + item.set("fullName", `${firstName} ${lastName}`); + } + + // Conditional logic + if (item.bool("isVIP")) { + item.set("priority", "high"); + } + + // Add processing timestamp + item.set("processedAt", new Date().toISOString()); + + return item; + }); + }, + { debug: true } + ) + ); +} +``` \ No newline at end of file diff --git a/plugins/record-hook-stream/package.json b/plugins/record-hook-stream/package.json new file mode 100644 index 000000000..558f76a85 --- /dev/null +++ b/plugins/record-hook-stream/package.json @@ -0,0 +1,68 @@ +{ + "name": "@flatfile/plugin-record-hook-stream", + "version": "0.0.1", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/plugins/record-hook-stream", + "description": "A plugin for running custom logic on individual data records in Flatfile.", + "registryMetadata": { + "category": "records" + }, + "type": "module", + "engines": { + "node": ">= 18" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ], + "exports": { + ".": { + "node": { + "types": { + "import": "./dist/index.d.ts", + "require": "./dist/index.d.cts" + }, + "import": "./dist/index.js", + "require": "./dist/index.cjs" + }, + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "main": "./dist/index.js", + "module": "./dist/index.js", + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "tsup", + "build:watch": "tsup --watch", + "build:prod": "NODE_ENV=production tsup", + "checks": "tsc --noEmit && attw --pack . && publint .", + "lint": "tsc --noEmit", + "test": "vitest run --mode defaults src/*.spec.ts --passWithNoTests", + "test:unit": "vitest run --mode defaults src/*.spec.ts --passWithNoTests --exclude src/*.e2e.spec.ts", + "test:e2e": "vitest run --mode defaults src/*.e2e.spec.ts --passWithNoTests --no-file-parallelism" + }, + "keywords": [ + "flatfile-plugins", + "category-transform", + "featured" + ], + "author": "Carl Brugger", + "repository": { + "type": "git", + "url": "git+https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/record-hook-stream" + }, + "license": "ISC", + "peerDependencies": { + "@flatfile/listener": "^1.1.0" + }, + "devDependencies": { + "@flatfile/bundler-config-tsup": "^0.2.0", + "@flatfile/config-vitest": "^0.0.0" + } +} diff --git a/plugins/record-hook-stream/src/index.ts b/plugins/record-hook-stream/src/index.ts new file mode 100644 index 000000000..2e11e7418 --- /dev/null +++ b/plugins/record-hook-stream/src/index.ts @@ -0,0 +1,34 @@ +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { recordReadWriteStream } from './record.hook.stream' +import { Item } from './utils/item' + +export interface RecordHookStreamOptions { + includeMessages?: boolean + includeMetadata?: boolean + debug?: boolean +} + +export const recordHookStream = ( + sheetSlug: string, + callback: (items: Item[], event: FlatfileEvent) => Item[] | Promise, + options: RecordHookStreamOptions = {} +) => { + return (listener: FlatfileListener) => { + listener.on('commit:created', { sheetSlug }, (event: FlatfileEvent) => + recordReadWriteStream(callback, event, options) + .then((results) => { + const typedResults = results as { + totalProcessed: number + totalTimeSeconds: string + } + console.log( + `[${event.src.id}] Processed ${typedResults.totalProcessed} records in ${typedResults.totalTimeSeconds} seconds (r/s: ${Math.ceil(typedResults.totalProcessed / parseFloat(typedResults.totalTimeSeconds))})` + ) + }) + .catch((error) => { + console.error('Processing failed:', error) + throw error + }) + ) + } +} diff --git a/plugins/record-hook-stream/src/record.hook.stream.ts b/plugins/record-hook-stream/src/record.hook.stream.ts new file mode 100644 index 000000000..85b880ff1 --- /dev/null +++ b/plugins/record-hook-stream/src/record.hook.stream.ts @@ -0,0 +1,322 @@ +import type { FlatfileEvent } from '@flatfile/listener' +import { request } from 'node:https' +import { RecordHookStreamOptions } from '.' +import { Item } from './utils/item' + +const MIN_BATCH_SIZE = 100 +const MAX_BATCH_SIZE = 500 +const TARGET_BATCH_DURATION = 5000 +const MAX_CONCURRENT_BATCHES = 15 +const CHUNK_SIZE = 50 + +export async function recordReadWriteStream( + callback: (items: Item[], event: FlatfileEvent) => Item[] | Promise, + event: FlatfileEvent, + options: RecordHookStreamOptions = {} +) { + const { + includeMessages = false, + includeMetadata = false, + debug = false, + } = options + + // Use a simple counter instead of storing all IDs to prevent memory issues + // This avoids unbounded memory growth when processing millions of records + let processedRecordCount = 0 + const startTime = Date.now() + let pendingBatches = 0 + + // Pre-create Item instances in smaller chunks to prevent memory spikes + const processItemsInChunks = async (records: any[]) => { + const results = [] + for (let i = 0; i < records.length; i += CHUNK_SIZE) { + const chunk = records.slice(i, i + CHUNK_SIZE) + const items = chunk.map((record) => new Item(record)) + + // Simply count the processed records + processedRecordCount += chunk.length + + const transformedItems = await callback(items, event) + const chunkResults = transformedItems.map((item) => item.changeset()) + results.push(...chunkResults) + } + return results + } + + let currentBatchSize = 250 + + const processBatchOfRecords = async (records: any[]) => { + if (records.length === 0) return + + while (pendingBatches >= MAX_CONCURRENT_BATCHES) { + await new Promise((resolve) => setTimeout(resolve, 25)) + } + + pendingBatches++ + const batchStartTime = Date.now() + + try { + const results = await processItemsInChunks(records) + const sendStart = Date.now() + await sendBatch(results, event, options) + + // Adjust batch size based on send duration + const batchDuration = Date.now() - sendStart + if (batchDuration > TARGET_BATCH_DURATION) { + currentBatchSize = Math.max( + MIN_BATCH_SIZE, + Math.floor(currentBatchSize * 0.8) + ) + } else if (batchDuration < TARGET_BATCH_DURATION * 0.8) { + currentBatchSize = Math.min( + MAX_BATCH_SIZE, + Math.floor(currentBatchSize * 1.2) + ) + } + + debug && + console.log( + `Batch metrics - Size: ${records.length}, Process: ${sendStart - batchStartTime}ms, ` + + `Send: ${batchDuration}ms, Next batch size: ${currentBatchSize}` + ) + } catch (error) { + if (error.message.includes('Too Many Requests')) { + currentBatchSize = Math.max( + MIN_BATCH_SIZE, + Math.floor(currentBatchSize * 0.5) + ) + await new Promise((resolve) => setTimeout(resolve, 1000)) + await processBatchOfRecords(records) + return + } + throw error + } finally { + pendingBatches-- + } + } + + return new Promise((resolve, reject) => { + let buffer = '' + let recordBuffer: any[] = [] + let lastProgressLog = Date.now() + let lastRecordCount = 0 + const PROGRESS_INTERVAL = 5000 + + const logProgress = () => { + const now = Date.now() + if (now - lastProgressLog >= PROGRESS_INTERVAL) { + const timeElapsed = (now - lastProgressLog) / 1000 + const recordsSinceLastLog = processedRecordCount - lastRecordCount + const currentRecordsPerSecond = recordsSinceLastLog / timeElapsed + const overallRecordsPerSecond = + processedRecordCount / ((now - startTime) / 1000) + + // Only log if we've processed new records or have pending batches + if (recordsSinceLastLog > 0 || pendingBatches > 0) { + debug && + console.log( + `Progress: ${processedRecordCount} records processed ` + + `(current: ${recordsSinceLastLog > 0 ? Math.round(currentRecordsPerSecond) : 'waiting'} r/s, ` + + `avg: ${Math.round(overallRecordsPerSecond)} r/s)` + + `${pendingBatches > 0 ? ` - ${pendingBatches} batches pending` : ''}` + ) + } + + lastProgressLog = now + lastRecordCount = processedRecordCount + } + } + + const options = { + method: 'GET', + headers: { + 'Content-Type': 'application/jsonl', + Authorization: `Bearer ${process.env.FLATFILE_API_KEY || process.env.FLATFILE_BEARER_TOKEN}`, + }, + timeout: 120000, + } + const req = request( + getRecordStreamEndpoint(event, { includeMessages, includeMetadata }), + options + ) + + req.on('response', (res) => { + res.setEncoding('utf8') + + res.on('data', async (chunk: string) => { + try { + buffer += chunk + const lines = buffer.split('\n') + buffer = lines.pop() || '' + + const records = lines + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)) + + if (records.length > 0) { + recordBuffer.push(...records) + + while (recordBuffer.length >= currentBatchSize) { + const batch = recordBuffer.splice(0, currentBatchSize) + await processBatchOfRecords(batch) + logProgress() + } + } + } catch (error) { + reject(error) + } + }) + + res.on('end', async () => { + try { + if (buffer.length > 0) { + const records = buffer + .split('\n') + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)) + recordBuffer.push(...records) + } + + if (recordBuffer.length > 0) { + await processBatchOfRecords(recordBuffer) + recordBuffer = [] + logProgress() + } + + while (pendingBatches > 0) { + logProgress() + await new Promise((resolve) => setTimeout(resolve, 1000)) + } + + const endTime = Date.now() + const totalTimeSeconds = (endTime - startTime) / 1000 + resolve({ + totalProcessed: processedRecordCount, + totalTimeSeconds: totalTimeSeconds.toFixed(2), + }) + } catch (error) { + reject(error) + } + }) + }) + + req.on('error', reject) + req.end() + }) +} + +async function sendBatch( + batch: any[], + event: FlatfileEvent, + options: RecordHookStreamOptions +) { + const { debug = false } = options + + const MAX_RETRIES = 5 + const INITIAL_RETRY_DELAY = 250 + + for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { + try { + return await sendBatchWithTimeout(batch, event, options) + } catch (error) { + if (error.message.includes('Too Many Requests')) { + debug && + console.log( + `Rate limited, retry attempt ${attempt + 1} of ${MAX_RETRIES}` + ) + const delay = + INITIAL_RETRY_DELAY * Math.pow(1.5, attempt) * (0.75 + Math.random()) + await new Promise((resolve) => setTimeout(resolve, delay)) + } else { + if (attempt === MAX_RETRIES - 1) throw error + const delay = INITIAL_RETRY_DELAY * Math.pow(1.5, attempt) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + } +} + +async function sendBatchWithTimeout( + batch: any[], + event: FlatfileEvent, + options: RecordHookStreamOptions +) { + const { includeMessages = true, includeMetadata = false } = options + + const RECORD_STREAM_ENDPOINT = getRecordStreamEndpoint(event, { + includeMessages, + includeMetadata, + }) + const TIMEOUT = 60000 + const MAX_RECORDS_PER_REQUEST = 1000 + + if (batch.length > MAX_RECORDS_PER_REQUEST) { + const chunks = [] + for (let i = 0; i < batch.length; i += MAX_RECORDS_PER_REQUEST) { + chunks.push(batch.slice(i, i + MAX_RECORDS_PER_REQUEST)) + } + + for (const chunk of chunks) { + await sendBatchWithTimeout(chunk, event, options) + } + return + } + + return new Promise((resolve, reject) => { + let isRequestEnded = false + let timeout: NodeJS.Timeout + + const cleanup = () => { + clearTimeout(timeout) + if (!isRequestEnded) { + req.destroy() + } + } + + const options = { + method: 'POST', + headers: { + 'Content-Type': 'application/x-ndjson', + Authorization: `Bearer ${process.env.FLATFILE_API_KEY || process.env.FLATFILE_BEARER_TOKEN}`, + }, + } + + const req = request(RECORD_STREAM_ENDPOINT, options, (res) => { + let data = '' + res.on('data', (chunk) => (data += chunk)) + res.on('end', () => { + isRequestEnded = true + cleanup() + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + resolve(undefined) + } else { + reject(new Error(`Failed to write batch: ${data}`)) + } + }) + }) + + timeout = setTimeout(() => { + cleanup() + reject(new Error('Request timeout')) + }, TIMEOUT) + + req.on('error', (error) => { + cleanup() + reject(error) + }) + + for (const record of batch) { + req.write(JSON.stringify(record) + '\n') + } + + req.end() + }) +} + +const getRecordStreamEndpoint = ( + event: FlatfileEvent, + options: Omit +) => { + const { includeMessages = false, includeMetadata = false } = options + return `${process.env.FLATFILE_API_URL}/v2-alpha/records.jsonl?sheetId=${event.context.sheetId}&for=${event.src.id}&stream=true&includeMessages=${includeMessages}&includeMetadata=${includeMetadata}` +} diff --git a/plugins/record-hook-stream/src/utils/casting.ts b/plugins/record-hook-stream/src/utils/casting.ts new file mode 100644 index 000000000..9fe4a2217 --- /dev/null +++ b/plugins/record-hook-stream/src/utils/casting.ts @@ -0,0 +1,73 @@ +export function asDate(input: any): Date | null { + const str_value = asString(input) + if (!str_value) { + return null + } + + const date = new Date(str_value) + // Check if the date is valid + return isNaN(date.getTime()) ? null : date +} + +export function asNumber(input: any): number { + // Check if the input is already a number + if (typeof input === 'number') return input + + // Check if the input is a string that can be converted to a number + if (typeof input === 'string') { + // Trim leading and trailing whitespaces + const trimmedInput = input.trim() + + // Check if the trimmed string is a numeric value + const num = Number(trimmedInput) + + // Check if the result is a valid number and not NaN + if (!isNaN(num)) { + return num + } + } + + // Return 0 for inputs that cannot be converted to a number + return 0 +} + +export function asString(input: any): string { + // Check if the input is null or undefined + if (input === null || input === undefined) { + return '' + } + + // Check if the input is an object or an array (excluding null, which is technically an object in JS) + if (typeof input === 'object') { + // Attempt to convert objects and arrays to a JSON string + try { + return JSON.stringify(input) + } catch (error) { + // In case of circular references or other JSON.stringify errors, return a fallback string + return '[object]' + } + } + + // For numbers, booleans, and other types, convert directly to string + return String(input) +} + +export function asNullableString(input: any): string | undefined { + // Check if the input is null or undefined + if ( + input === null || + input === undefined || + input === 'undefined' || + input === 'null' || + input === '' + ) { + return undefined + } + + return asString(input) +} + +export function asBool(input: any): boolean { + // todo: make this way smarter soon + return !!input +} diff --git a/plugins/record-hook-stream/src/utils/is.nullish.ts b/plugins/record-hook-stream/src/utils/is.nullish.ts new file mode 100644 index 000000000..a24a72877 --- /dev/null +++ b/plugins/record-hook-stream/src/utils/is.nullish.ts @@ -0,0 +1,14 @@ +export function isNullish(value: T): value is null { + return ( + value === null || + value === '' || + value === undefined || + value === 'null' || + value === 'undefined' || + (typeof value === 'number' && isNaN(value)) + ) +} + +export function isPresent(value: T): value is NonNullable { + return !isNullish(value) +} diff --git a/plugins/record-hook-stream/src/utils/item.ts b/plugins/record-hook-stream/src/utils/item.ts new file mode 100644 index 000000000..257030423 --- /dev/null +++ b/plugins/record-hook-stream/src/utils/item.ts @@ -0,0 +1,337 @@ +import { asBool, asDate, asNullableString, asString } from './casting' +import { isPresent } from './is.nullish' + +export const HASH_VALUE_DELIM = '|*|' +export const HASH_PROP_DELIM = '|&|' + +export class Item = Record> { + private _changes: Map = new Map() + private _messages: Map< + string, + Set<{ type: 'error' | 'warn' | 'info'; message: string }> + > = new Map() + private _deleted = false + private _tempId?: string + + constructor( + public data: Readonly>, + dirty = false + ) { + if (dirty) { + this.data = Object.freeze({}) + Object.entries(data).forEach(([key, value]) => { + this.set(key, value) + }) + } else { + Object.freeze(this.data) + } + } + + get id() { + return this.data.__k || this._tempId + } + + get slug() { + return this.data.__n + } + + get sheetId() { + return this.data.__s + } + + get versionId() { + return this.data.__v + } + + set(key: string, value: any) { + if (this.data[key] === value) { + this._changes.delete(key) + return + } + this._changes.set(key, value) + return this + } + + flag(key: string) { + this.set(key, true) + } + + unflag(key: string) { + this.set(key, false) + } + + get(key: string) { + if (this._changes.has(key)) { + return this._changes.get(key) + } + return this.data[key] + } + + has(key: string) { + return isPresent(this.get(key)) + } + + hasAny(...keys: string[]) { + return keys.some((k) => this.has(k)) + } + + hasAll(...keys: string[]) { + return keys.every((k) => this.has(k)) + } + + isEmpty(key: string) { + return !this.has(key) + } + + keys(options?: { omit?: string[]; pick?: string[] }): string[] { + const set = new Set( + Object.keys(this.data).filter((key) => !key.startsWith('__')) + ) + + for (const key of this._changes.keys()) { + if (!key.startsWith('__')) { + set.add(key) + } + } + const res = Array.from(set) + + if (options?.omit) { + return res.filter((key) => !options.omit.includes(key)) + } + if (options?.pick) { + return res.filter((key) => options.pick.includes(key)) + } + return res + } + + keysWithData(props?: { exclude?: Array }): string[] { + const keys = this.keys().filter((k) => this.has(k)) + if (props?.exclude) { + const f = props.exclude.flat() + return keys.filter((k) => !f.includes(k)) + } + return keys + } + + /** + * Intersects exactly with another item on the keys + * + * @param item + * @param keys + */ + intersects(item: Item, keys: string[]) { + return keys.every((key) => { + const value1 = this.str(key) + const value2 = item.str(key) + return value1 === value2 + }) + } + + hash(...keys: string[]) { + return keys + .map((k) => [k, this.get(k)]) + .map(([k, v]) => `${k}${HASH_VALUE_DELIM}${asString(v)}`) + .join(HASH_PROP_DELIM) + } + + isDirty(key?: string): boolean { + if (key) { + return this._changes.has(key) || this._messages.get(key)?.size > 0 + } + return this._changes.size > 0 || this._messages.size > 0 || this._deleted + } + + eachOfKeysPresent( + keys: string[], + callback: (key: string, value: any) => void + ) { + for (const key of keys) { + if (this.has(key)) { + callback(key, this.get(key)) + } + } + } + + isDeleted(): boolean { + return this._deleted + } + + delete() { + this._deleted = true + } + + str(key: string) { + return asNullableString(this.get(key)) + } + + defStr(key: string): string { + return asString(this.get(key)) + } + + bool(key: string) { + return asBool(this.get(key)) + } + + date(key: string) { + return asDate(this.get(key)) + } + + pick(...keys: string[]) { + const obj: Record = {} + for (const key of keys) { + obj[key] = this.get(key) + } + return obj + } + + addMsg(key: string, msg: string, type: 'error' | 'warn' | 'info') { + if ( + this.data.__i && + this.data.__i.some((m) => m.x === key && m.m === msg) + ) { + return this + } + if (!this._messages.has(key)) { + this._messages.set(key, new Set()) + } + this._messages.get(key).add({ type, message: msg }) + return this + } + + info(key: string, msg: string) { + return this.addMsg(key, msg, 'info') + } + + warn(key: string, msg: string) { + return this.addMsg(key, msg, 'warn') + } + + err(key: string, msg: string) { + return this.addMsg(key, msg, 'error') + } + + values() { + return Object.fromEntries(this.entries()) + } + + entries() { + return this.keys().map((key) => [key, this.get(key)]) + } + + merge(item: Item, props: { overwrite?: boolean } = {}) { + for (const key of item.keys()) { + if (props.overwrite) { + this.set(key, item.get(key)) + } else if (!this.has(key)) { + this.set(key, item.get(key)) + } + } + return this + } + + hasConflict(b: Item, keys?: string[]) { + if (keys) { + return keys.some((key) => { + const aValue = this.get(key) + const bValue = b.get(key) + return aValue && bValue && aValue !== bValue + }) + } + return this.entries().some(([key, aValue]) => { + const bValue = b.get(key) + return aValue && bValue && aValue !== bValue + }) + } + + toJSON() { + return { ...this.data, ...this.changeset() } + } + + toString() { + return `${this._deleted ? '❌ ' : ''}${this.slug || this.sheetId}(${this.id ?? 'new'}) ${JSON.stringify(this.values(), null, ' ')}` + } + + [Symbol.for('nodejs.util.inspect.custom') || Symbol()]() { + return this.toString() + } + + copy(props?: { + mixin?: Item + select?: string[] + slug?: string + sheetId?: string + }) { + const newObj = new Item({}) + newObj._tempId = `TEMP_${crypto.randomUUID()}` + if (props.slug) { + newObj.set('__n', props.slug) + } + if (props.sheetId) { + newObj.set('__s', props.sheetId) + } + if (props.select) { + for (const key of props.select) { + newObj.set(key, props.mixin?.get(key) ?? this.get(key)) + } + } else { + for (const key in this.data) { + if (!key.startsWith('__')) { + newObj.set(key, this.get(key)) + } + } + if (props.mixin) { + for (const key in props.mixin.data) { + if (!key.startsWith('__')) { + newObj.set(key, props.mixin.get(key)) + } + } + } + } + return newObj + } + + commit() { + // reset the data object with new changes and unset all pending changes + const newObj: Record = Object.assign({}, this.data) + for (const [key, value] of this._changes) { + newObj[key] = value + } + this._changes.clear() + if (this._messages.size) { + newObj.__i = [] + for (const [key, values] of this._messages) { + for (const value of values) { + newObj.__i.push({ x: key, m: value.message, t: value.type }) + } + } + } + this._messages.clear() + this.data = Object.freeze(newObj) as any + } + + changeset() { + const val = Object.fromEntries(this._changes) + val.__k = this.get('__k') + val.__s = this.get('__s') + val.__n = this.get('__n') + if (this._deleted) { + val.__d = true + } + if (this._messages.size) { + if (!val.__i) { + val.__i = [] + } + for (const [key, values] of this._messages) { + // If there is a message for a key and no changed value, the value is removed. + // This sets the value if not changed to preserve it. + if (!val[key]) { + val[key] = this.get(key) + } + for (const value of values) { + val.__i.push({ x: key, m: value.message, t: value.type }) + } + } + } + return val + } +} diff --git a/plugins/record-hook-stream/tsup.config.mjs b/plugins/record-hook-stream/tsup.config.mjs new file mode 100644 index 000000000..2070a0891 --- /dev/null +++ b/plugins/record-hook-stream/tsup.config.mjs @@ -0,0 +1,3 @@ +import { defineConfig } from '@flatfile/bundler-config-tsup' + +export default defineConfig({ includeBrowser: false }) diff --git a/plugins/record-hook-stream/vitest.config.ts b/plugins/record-hook-stream/vitest.config.ts new file mode 100644 index 000000000..e9e1729f9 --- /dev/null +++ b/plugins/record-hook-stream/vitest.config.ts @@ -0,0 +1,4 @@ +import config from '@flatfile/config-vitest' +import { defineConfig } from 'vitest/config' + +export default defineConfig(config) diff --git a/plugins/record-hook/src/RecordHook.ts b/plugins/record-hook/src/RecordHook.ts index 4f9622c03..b5f9ea2fc 100644 --- a/plugins/record-hook/src/RecordHook.ts +++ b/plugins/record-hook/src/RecordHook.ts @@ -63,9 +63,11 @@ export const BulkRecordHook = async ( options: BulkRecordHookOptions = {} ) => { const { debug = false } = options + debug && console.log('🐢 bulkRecordHook') try { - startTimer('fetch data', debug) + startTimer(`bulkRecordHook duration ${event.src.id}`, debug) + startTimer('fetch records duration', debug) const data = await event.cache.init( 'data', async (): Promise => { @@ -88,15 +90,15 @@ export const BulkRecordHook = async ( 'records', async () => await prepareXRecords(data) ) - endTimer('fetch data', debug) + endTimer('fetch records duration', debug) // Execute client-defined data hooks - startTimer('run handler', debug) + startTimer('handler duration', debug) await asyncBatch(batch.records, handler, options, event) - endTimer('run handler', debug) + endTimer('handler duration', debug) event.afterAll(async () => { - startTimer('filter modified records', debug) + startTimer('filter modified records duration', debug) const { records: batch } = event.cache.get>('records') const records: Flatfile.RecordsWithLinks = @@ -122,19 +124,19 @@ export const BulkRecordHook = async ( await completeCommit(event, debug) return } - endTimer('filter modified records', debug) + endTimer('filter modified records duration', debug) - if (debug) { + debug && logInfo( '@flatfile/plugin-record-hook', `${modifiedRecords.length} modified records` ) - } try { - startTimer('update modified records', debug) + startTimer('write duration', debug) await event.update(modifiedRecords) - endTimer('update modified records', debug) + endTimer('write duration', debug) + endTimer(`bulkRecordHook duration ${event.src.id}`, debug) return } catch (e) { throw new Error('Error updating records') diff --git a/plugins/record-hook/src/record.hook.plugin.ts b/plugins/record-hook/src/record.hook.plugin.ts index 23bd9438f..d99f95997 100644 --- a/plugins/record-hook/src/record.hook.plugin.ts +++ b/plugins/record-hook/src/record.hook.plugin.ts @@ -1,5 +1,6 @@ import type { FlatfileRecord } from '@flatfile/hooks' import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { Item } from '../../record-hook-stream/src/utils/item' import type { BulkRecordHookOptions, RecordHookOptions } from './RecordHook' import { BulkRecordHook, RecordHook } from './RecordHook'