diff --git a/.changeset/short-papayas-share.md b/.changeset/short-papayas-share.md new file mode 100644 index 000000000..c40ebab36 --- /dev/null +++ b/.changeset/short-papayas-share.md @@ -0,0 +1,5 @@ +--- +'@flatfile/plugin-export-external-api': minor +--- + +Initial Release diff --git a/export/external-api/README.MD b/export/external-api/README.MD new file mode 100644 index 000000000..4e5c6164d --- /dev/null +++ b/export/external-api/README.MD @@ -0,0 +1,66 @@ + + +# @flatfile/plugin-export-external-api + +This plugin for Flatfile enables exporting data to an external API with advanced features such as batching, error handling, retry logic, and user notifications. It processes records from Flatfile, maps them to the desired format, and exports them in batches to a specified external API endpoint. + +**Event Type:** `job:ready` + + + +## Features + +- Batch processing for efficient exports +- Configurable batch size +- Retry logic with configurable max retries and delay +- Detailed progress tracking and job status updates +- Error handling and logging +- User notifications for export completion or failure +- Support for custom record validation + +## Parameters + +#### `job` - `string` - (required) +The type of job to create. + +#### `apiEndpoint` - `string` - (required) +The URL of the external API endpoint to send data to. + +#### `secretName` - `string` - (required) +The name of the Flatfile Secret that contains the authentication token for the external API. + +#### `batchSize` - `number` - (required) +The number of records to process in each batch. + +#### `maxRetries` - `number` - (required) +The maximum number of retry attempts for failed API calls. + +#### `retryDelay` - `number` - (required) +The delay (in milliseconds) between retry attempts. + + +## Installation + +To install the plugin, use npm: + +```bash +npm install @flatfile/plugin-export-external-api +``` + +## Example Usage + +```javascript +import { exportToExternalAPIPlugin } from '@flatfile/plugin-export-external-api'; + +export default function (listener) { + listener.use( + exportToExternalAPIPlugin({ + apiEndpoint: 'https://api.example.com/import', + secretName: 'YOUR_SECRET_NAME', + batchSize: 100, + maxRetries: 3, + retryDelay: 1000 + }) + ); +} +``` diff --git a/export/external-api/jest.config.js b/export/external-api/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/export/external-api/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/export/external-api/package.json b/export/external-api/package.json new file mode 100644 index 000000000..5d235d307 --- /dev/null +++ b/export/external-api/package.json @@ -0,0 +1,72 @@ +{ + "name": "@flatfile/plugin-export-external-api", + "version": "0.0.0", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/export/external-api", + "description": "A Flatfile plugin for exporting data to an external API with batching, error handling, and retry logic", + "registryMetadata": { + "category": "export" + }, + "engines": { + "node": ">= 16" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ], + "browser": { + "./dist/index.js": "./dist/index.browser.js", + "./dist/index.mjs": "./dist/index.browser.mjs" + }, + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "browser": { + "require": "./dist/index.browser.js", + "import": "./dist/index.browser.mjs" + }, + "default": "./dist/index.mjs" + }, + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "source": "./src/index.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" + }, + "keywords": [ + "flatfile-plugins", + "category-export" + ], + "author": "Flatfile, Inc.", + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "export/external-api" + }, + "license": "ISC", + "dependencies": { + "@flatfile/plugin-job-handler": "^0.6.1", + "cross-fetch": "^4.0.0" + }, + "peerDependencies": { + "@flatfile/util-common": "^1.4.1" + }, + "devDependencies": { + "@flatfile/api": "^1.9.15", + "@flatfile/listener": "^1.0.5", + "@flatfile/rollup-config": "^0.1.1" + } +} \ No newline at end of file diff --git a/export/external-api/rollup.config.mjs b/export/external-api/rollup.config.mjs new file mode 100644 index 000000000..fafa813c6 --- /dev/null +++ b/export/external-api/rollup.config.mjs @@ -0,0 +1,5 @@ +import { buildConfig } from '@flatfile/rollup-config' + +const config = buildConfig({}) + +export default config diff --git a/export/external-api/src/external.api.plugin.ts b/export/external-api/src/external.api.plugin.ts new file mode 100644 index 000000000..69065c28d --- /dev/null +++ b/export/external-api/src/external.api.plugin.ts @@ -0,0 +1,81 @@ +import type { Flatfile } from '@flatfile/api' +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' +import { jobHandler } from '@flatfile/plugin-job-handler' +import { getSheetLength, Simplified } from '@flatfile/util-common' +import { exportToExternalAPI, retryOperation } from './external.api.utils' + +export interface PluginConfig { + job: string + apiEndpoint: string + secretName: string + batchSize: number + maxRetries: number + retryDelay: number +} + +export const exportToExternalAPIPlugin = (config: PluginConfig) => { + return (listener: FlatfileListener) => { + listener.use( + jobHandler( + `sheet:${config.job}`, + async ( + event: FlatfileEvent, + tick: ( + progress: number, + message?: string + ) => Promise + ) => { + const { sheetId } = event.context + + const authToken = await event.secrets(config.secretName) + + try { + let totalExported = 0 + let failedRecords = 0 + let successfulBatches = 0 + + const sheetLength = await getSheetLength(sheetId) + const batchCount = Math.ceil(sheetLength / config.batchSize) + + let pageNumber = 1 + while (pageNumber <= batchCount) { + const records = await Simplified.getAllRecords(sheetId, { + pageSize: config.batchSize, + pageNumber, + }) + + try { + await retryOperation( + () => + exportToExternalAPI(records, config.apiEndpoint, authToken), + config.maxRetries, + config.retryDelay + ) + totalExported += records.length + successfulBatches++ + } catch (error) { + console.error('Failed to export batch after retries:', error) + failedRecords += records.length + } + + const progress = ((pageNumber - 1) / batchCount) * 100 + await tick( + progress, + `Exported ${totalExported} records. Failed to export ${failedRecords} records.` + ) + pageNumber++ + } + + return { + info: `Exported ${totalExported} records. Failed to export ${failedRecords} records.`, + } + } catch (error) { + console.error('Error during export:', (error as Error).message) + + throw new Error('An error occurred during export.') + } + } + ) + ) + } +} diff --git a/export/external-api/src/external.api.utils.ts b/export/external-api/src/external.api.utils.ts new file mode 100644 index 000000000..3f45cd2b4 --- /dev/null +++ b/export/external-api/src/external.api.utils.ts @@ -0,0 +1,37 @@ +import fetch from 'cross-fetch' + +export async function exportToExternalAPI( + data: Record[], + apiEndpoint: string, + authToken: string +): Promise { + const response = await fetch(apiEndpoint, { + method: 'POST', + headers: { + Authorization: `Bearer ${authToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }) + + if (!response.ok) { + throw new Error(`API request failed: ${response.statusText}`) + } +} + +export async function retryOperation( + operation: () => Promise, + maxRetries: number, + delay: number +): Promise { + let lastError: Error | null = null + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await operation() + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + throw lastError || new Error('Operation failed after max retries') +} diff --git a/export/external-api/src/index.ts b/export/external-api/src/index.ts new file mode 100644 index 000000000..376eee838 --- /dev/null +++ b/export/external-api/src/index.ts @@ -0,0 +1 @@ +export * from './external.api.plugin' diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index d8f6d0935..dcf57a69c 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -1,14 +1,15 @@ import type { FlatfileListener } from '@flatfile/listener' -import { pivotTablePlugin } from '@flatfile/plugin-export-pivot-table' +import { exportToExternalAPIPlugin } from '@flatfile/plugin-export-external-api' import { configureSpace } from '@flatfile/plugin-space-configure' - export default async function (listener: FlatfileListener) { listener.use( - pivotTablePlugin({ - pivotColumn: 'product', - aggregateColumn: 'salesAmount', - aggregationMethod: 'sum', - groupByColumn: 'region', + exportToExternalAPIPlugin({ + job: 'export-external-api', + apiEndpoint: 'http://localhost:5678/api/import', + secretName: 'EXTERNAL_API_AUTH_SECRET', + batchSize: 100, + maxRetries: 3, + retryDelay: 1000, }) ) listener.use( @@ -47,17 +48,17 @@ export default async function (listener: FlatfileListener) { label: 'Sales Amount', }, ], - }, - ], - actions: [ - { - operation: 'generatePivotTable', - label: 'Generate Pivot Table', - description: - 'This custom action code generates a pivot table from the records in the People sheet.', - primary: false, - mode: 'foreground', - type: 'string', + actions: [ + { + operation: 'export-external-api', + label: 'Export to External API', + description: + 'This custom action code exports the records in the Sales sheet to an external API.', + primary: false, + mode: 'foreground', + type: 'string', + }, + ], }, ], }, diff --git a/package-lock.json b/package-lock.json index 921b1af24..a65d20c29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -140,8 +140,9 @@ "license": "MIT" }, "enrich/gpx": { - "version": "0.0.0", - "license": "MIT", + "name": "@flatfile/plugin-enrich-gpx", + "version": "0.1.0", + "license": "ISC", "dependencies": { "@flatfile/plugin-record-hook": "^1.6.1", "xml2js": "^0.6.2" @@ -200,9 +201,29 @@ "@flatfile/listener": "^1.1.0" } }, + "export/external-api": { + "name": "@flatfile/plugin-export-external-api", + "version": "0.0.0", + "license": "ISC", + "dependencies": { + "@flatfile/plugin-job-handler": "^0.6.1", + "cross-fetch": "^4.0.0" + }, + "devDependencies": { + "@flatfile/api": "^1.9.15", + "@flatfile/listener": "^1.0.5", + "@flatfile/rollup-config": "^0.1.1" + }, + "engines": { + "node": ">= 16" + }, + "peerDependencies": { + "@flatfile/util-common": "^1.4.1" + } + }, "export/pivot-table": { "name": "@flatfile/plugin-export-pivot-table", - "version": "0.0.0", + "version": "0.1.0", "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.15" @@ -246,7 +267,6 @@ "flatfilers/playground": { "name": "@private/playground", "version": "0.0.0", - "extraneous": true, "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.19", @@ -3378,6 +3398,10 @@ "resolved": "enrich/summarize", "link": true }, + "node_modules/@flatfile/plugin-export-external-api": { + "resolved": "export/external-api", + "link": true + }, "node_modules/@flatfile/plugin-export-pivot-table": { "resolved": "export/pivot-table", "link": true @@ -6412,6 +6436,10 @@ "node": ">=14" } }, + "node_modules/@private/playground": { + "resolved": "flatfilers/playground", + "link": true + }, "node_modules/@private/sandbox": { "resolved": "flatfilers/sandbox", "link": true