From 49f395ee1b465c53920f368c5bf635c3aa34d46d Mon Sep 17 00:00:00 2001 From: "Alex Rock (Koala)" Date: Wed, 25 Sep 2024 09:58:27 -0600 Subject: [PATCH 1/6] koala: initial commit --- validators/ExternalApiPublisher/README.MD | 75 ++++++ validators/ExternalApiPublisher/metadata.json | 77 ++++++ validators/ExternalApiPublisher/package.json | 62 +++++ .../ExternalApiPublisher/rollup.config.mjs | 42 +++ validators/ExternalApiPublisher/src/index.ts | 247 ++++++++++++++++++ 5 files changed, 503 insertions(+) create mode 100644 validators/ExternalApiPublisher/README.MD create mode 100644 validators/ExternalApiPublisher/metadata.json create mode 100644 validators/ExternalApiPublisher/package.json create mode 100644 validators/ExternalApiPublisher/rollup.config.mjs create mode 100644 validators/ExternalApiPublisher/src/index.ts diff --git a/validators/ExternalApiPublisher/README.MD b/validators/ExternalApiPublisher/README.MD new file mode 100644 index 000000000..b93b76804 --- /dev/null +++ b/validators/ExternalApiPublisher/README.MD @@ -0,0 +1,75 @@ +# Flatfile External API Export Plugin + +This plugin for Flatfile enables exporting data to an external API with advanced features such as batching, error handling, retry logic, and user notifications. It processes records from Flatfile, maps them to the desired format, and exports them in batches to a specified external API endpoint. + +## Features + +- Data mapping from Flatfile fields to external API fields +- Batch processing for efficient exports +- Configurable batch size +- Retry logic with configurable max retries and delay +- Detailed progress tracking and job status updates +- Error handling and logging +- User notifications for export completion or failure +- Support for custom record validation + +## Installation + +To install the plugin, use npm: + +```bash +npm install @flatfile/plugin-external-api-export +``` + +## Example Usage + +```javascript +import { FlatfileListener } from '@flatfile/listener'; +import externalApiExportPlugin from '@flatfile/plugin-external-api-export'; + +const listener = new FlatfileListener(); + +listener.use( + externalApiExportPlugin({ + apiEndpoint: 'https://api.example.com/import', + authToken: 'your-api-token', + dataMapping: { + 'First Name': 'firstName', + 'Last Name': 'lastName', + 'Email': 'emailAddress' + }, + batchSize: 100, + maxRetries: 3, + retryDelay: 1000 + }) +); + +listener.on('job:ready', async ({ context, payload }) => { + // Additional job:ready handling if needed +}); +``` + +## Configuration + +The plugin accepts the following configuration options: + +- `apiEndpoint` (string): The URL of the external API endpoint to send data to. +- `authToken` (string): The authentication token for the external API. +- `dataMapping` (object): A mapping of Flatfile field names to external API field names. +- `batchSize` (number): The number of records to process in each batch. +- `maxRetries` (number): The maximum number of retry attempts for failed API calls. +- `retryDelay` (number): The delay (in milliseconds) between retry attempts. + +## Behavior + +1. When a job is ready, the plugin retrieves all records for the specified file. +2. Records are processed in batches according to the configured `batchSize`. +3. Each record is mapped to the format expected by the external API using the `dataMapping` configuration. +4. The plugin attempts to export each batch to the external API. +5. If an export fails, the plugin will retry the operation up to `maxRetries` times, with a delay of `retryDelay` milliseconds between attempts. +6. Progress is tracked and reported throughout the export process. +7. Successful exports are logged, and failed exports are recorded for error reporting. +8. Upon completion, a summary of the export process is provided, including the number of successful and failed exports. +9. User notifications are sent to inform about the export status. + +The plugin also includes a record hook for the 'contacts' sheet, allowing for custom validation logic to be implemented before export. \ No newline at end of file diff --git a/validators/ExternalApiPublisher/metadata.json b/validators/ExternalApiPublisher/metadata.json new file mode 100644 index 000000000..3a57ddeec --- /dev/null +++ b/validators/ExternalApiPublisher/metadata.json @@ -0,0 +1,77 @@ +{ + "timestamp": "2024-09-25T06-07-30-460Z", + "task": "Create a Flatfile Listener plugin to export Flatfile Workbook data to an External API:\n - Listen for a custom action to publish processed data to an external API, that is configurable in the Plugin settings\n - Allow configuration of API endpoint, authentication, and data mapping\n - Handle batching of records for efficient API calls\n - Implement error handling and retries for failed API requests\n - Provide feedback on successful and failed data publishing attempts", + "summary": "This code implements a Flatfile Listener plugin for exporting data to an external API. It includes features such as batching, error handling, retry logic, and user notifications. The plugin processes records, exports them in batches, and provides detailed feedback on the export process.", + "steps": [ + [ + "Retrieve information about Flatfile Listeners and the Record Hook plugin to understand the structure and best practices.\n", + "#E1", + "PineconeAssistant", + "Provide information about Flatfile Listeners and the Record Hook plugin, including their structure and best practices", + "Plan: Retrieve information about Flatfile Listeners and the Record Hook plugin to understand the structure and best practices.\n#E1 = PineconeAssistant[Provide information about Flatfile Listeners and the Record Hook plugin, including their structure and best practices]" + ], + [ + "Create the basic structure of the Flatfile Listener plugin with the necessary imports and configurations.\n", + "#E2", + "LLM", + "Create the basic structure of a Flatfile Listener plugin for exporting data to an external API, using the information from #E1. Include necessary imports and plugin configuration", + "Plan: Create the basic structure of the Flatfile Listener plugin with the necessary imports and configurations.\n#E2 = LLM[Create the basic structure of a Flatfile Listener plugin for exporting data to an external API, using the information from #E1. Include necessary imports and plugin configuration]" + ], + [ + "Implement the custom action listener for publishing processed data to the external API.\n", + "#E3", + "LLM", + "Implement a custom action listener for the Flatfile plugin created in #E2, which will trigger the data export process", + "Plan: Implement the custom action listener for publishing processed data to the external API.\n#E3 = LLM[Implement a custom action listener for the Flatfile plugin created in #E2, which will trigger the data export process]" + ], + [ + "Add configuration options for the API endpoint, authentication, and data mapping.\n", + "#E4", + "LLM", + "Add configuration options to the Flatfile plugin from #E3 for API endpoint, authentication, and data mapping", + "Plan: Add configuration options for the API endpoint, authentication, and data mapping.\n#E4 = LLM[Add configuration options to the Flatfile plugin from #E3 for API endpoint, authentication, and data mapping]" + ], + [ + "Implement batching functionality for efficient API calls.\n", + "#E5", + "LLM", + "Add batching functionality to the Flatfile plugin from #E4 for efficient API calls", + "Plan: Implement batching functionality for efficient API calls.\n#E5 = LLM[Add batching functionality to the Flatfile plugin from #E4 for efficient API calls]" + ], + [ + "Implement error handling and retry logic for failed API requests.\n", + "#E6", + "LLM", + "Add error handling and retry logic to the Flatfile plugin from #E5 for failed API requests", + "Plan: Implement error handling and retry logic for failed API requests.\n#E6 = LLM[Add error handling and retry logic to the Flatfile plugin from #E5 for failed API requests]" + ], + [ + "Implement feedback mechanisms for successful and failed data publishing attempts.\n", + "#E7", + "LLM", + "Add feedback mechanisms to the Flatfile plugin from #E6 for successful and failed data publishing attempts", + "Plan: Implement feedback mechanisms for successful and failed data publishing attempts.\n#E7 = LLM[Add feedback mechanisms to the Flatfile plugin from #E6 for successful and failed data publishing attempts]" + ], + [ + "Review and optimize the complete Flatfile Listener plugin code.\n", + "#E8", + "LLM", + "Review and optimize the complete Flatfile Listener plugin code from #E7, ensuring all requirements are met and the code follows best practices", + "Plan: Review and optimize the complete Flatfile Listener plugin code.\n#E8 = LLM[Review and optimize the complete Flatfile Listener plugin code from #E7, ensuring all requirements are met and the code follows best practices]" + ], + [ + "Validate the final code, check for unused imports, and ensure correct Event Topic usage.\n", + "#E9", + "PineconeAssistant", + "Validate the Flatfile Listener plugin code from #E8, check for unused imports, and ensure correct Event Topic usage. Provide any necessary corrections or optimizations", + "Plan: Validate the final code, check for unused imports, and ensure correct Event Topic usage.\n#E9 = PineconeAssistant[Validate the Flatfile Listener plugin code from #E8, check for unused imports, and ensure correct Event Topic usage. Provide any necessary corrections or optimizations]" + ] + ], + "metrics": { + "tokens": { + "plan": 4694, + "state": 6200, + "total": 10894 + } + } +} \ No newline at end of file diff --git a/validators/ExternalApiPublisher/package.json b/validators/ExternalApiPublisher/package.json new file mode 100644 index 000000000..9c5682b3d --- /dev/null +++ b/validators/ExternalApiPublisher/package.json @@ -0,0 +1,62 @@ +{ + "name": "@flatfile/plugin-external-api-exporter", + "version": "1.0.0", + "description": "A Flatfile plugin for exporting data to an external API with batching, error handling, and retry logic", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "default": "./dist/index.mjs" + }, + "source": "./src/index.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + }, + "keywords": [ + "flatfile", + "plugin", + "api", + "exporter", + "flatfile-plugins", + "category-transform" + ], + "author": "Your Name", + "license": "MIT", + "dependencies": { + "@flatfile/api": "^1.9.15", + "@flatfile/plugin-record-hook": "^1.7.0", + "axios": "^1.7.7" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + }, + "devDependencies": { + "@flatfile/hooks": "^1.5.0", + "@flatfile/rollup-config": "^0.1.1", + "@types/node": "^22.7.0", + "typescript": "^5.6.2", + "rollup": "^4.22.4", + "@rollup/plugin-typescript": "^12.1.0", + "@rollup/plugin-node-resolve": "^15.3.0", + "@rollup/plugin-commonjs": "^28.0.0", + "jest": "^29.7.0", + "@types/jest": "^29.5.13" + }, + "repository": { + "type": "git", + "url": "https://github.com/YourGitHubUsername/flatfile-plugins.git", + "directory": "plugins/external-api-exporter" + } +} \ No newline at end of file diff --git a/validators/ExternalApiPublisher/rollup.config.mjs b/validators/ExternalApiPublisher/rollup.config.mjs new file mode 100644 index 000000000..ce1befc71 --- /dev/null +++ b/validators/ExternalApiPublisher/rollup.config.mjs @@ -0,0 +1,42 @@ +import { buildConfig } from '@flatfile/rollup-config'; +import typescript from '@rollup/plugin-typescript'; +import resolve from '@rollup/plugin-node-resolve'; +import commonjs from '@rollup/plugin-commonjs'; +import json from '@rollup/plugin-json'; + +const umdExternals = [ + '@flatfile/api', + '@flatfile/hooks', + '@flatfile/listener', + '@flatfile/util-common', + '@flatfile/plugin-record-hook', + 'axios' +]; + +const config = buildConfig({ + input: 'src/index.ts', // Adjust this to your main entry file + includeUmd: true, + umdConfig: { + name: 'FlatfileExportPlugin', // Replace with your plugin name + external: umdExternals + }, + plugins: [ + typescript({ + tsconfig: './tsconfig.json', + declaration: true, + declarationDir: './dist/types', + }), + resolve({ + browser: true, + preferBuiltins: true, + }), + commonjs(), + json(), + ], + external: [ + ...umdExternals, + 'axios', + ] +}); + +export default config; \ No newline at end of file diff --git a/validators/ExternalApiPublisher/src/index.ts b/validators/ExternalApiPublisher/src/index.ts new file mode 100644 index 000000000..0dc2f9c25 --- /dev/null +++ b/validators/ExternalApiPublisher/src/index.ts @@ -0,0 +1,247 @@ +import { + FlatfileListener, + FlatfileEvent, + FlatfileRecord, +} from '@flatfile/listener' +import { recordHook } from '@flatfile/plugin-record-hook' +import { ActionExecutionContext } from '@flatfile/hooks' +import api from '@flatfile/api' +import axios from 'axios' + +interface PluginConfig { + apiEndpoint: string + authToken: string + dataMapping: { + readonly [key: string]: string + } + batchSize: number + maxRetries: number + retryDelay: number +} + +export default function ( + listener: FlatfileListener, + config: PluginConfig +): void { + const { + apiEndpoint, + authToken, + dataMapping, + batchSize, + maxRetries, + retryDelay, + } = config + + listener.use( + recordHook('contacts', (record: FlatfileRecord) => { + // Validation logic here + return record + }) + ) + + listener.on('job:ready', async ({ context, payload }) => { + const { jobId } = payload + + try { + const records = await api.records.get(payload.spaceId, payload.fileId) + const batches = chunkArray(records.data, batchSize) + let totalExported = 0 + const failedRecords: FlatfileRecord[] = [] + let successfulBatches = 0 + let failedBatches = 0 + + for (const batch of batches) { + const processedBatch = batch.map((record) => + processRecord(record, dataMapping) + ) + try { + await retryOperation( + () => exportToExternalAPI(processedBatch, apiEndpoint, authToken), + maxRetries, + retryDelay + ) + totalExported += batch.length + successfulBatches++ + await logSuccess(context, jobId, batch.length) + } catch (error) { + console.error('Failed to export batch after retries:', error) + failedRecords.push(...batch) + failedBatches++ + await logFailure(context, jobId, batch.length, error) + } + + await updateJobProgress( + context, + jobId, + totalExported, + records.data.length + ) + } + + await completeJob( + context, + jobId, + totalExported, + failedRecords, + successfulBatches, + failedBatches + ) + await sendUserNotification( + context, + jobId, + totalExported, + failedRecords.length + ) + } catch (error) { + await handleExportError(context, jobId, error) + } + }) +} + +function chunkArray(array: T[], size: number): T[][] { + return Array.from({ length: Math.ceil(array.length / size) }, (_, index) => + array.slice(index * size, (index + 1) * size) + ) +} + +function processRecord( + record: FlatfileRecord, + mapping: PluginConfig['dataMapping'] +): Record { + return Object.entries(mapping).reduce((acc, [from, to]) => { + acc[to] = record.get(from) + return acc + }, {} as Record) +} + +async function exportToExternalAPI( + data: Record[], + apiEndpoint: string, + authToken: string +): Promise { + try { + await axios.post(apiEndpoint, data, { + headers: { Authorization: `Bearer ${authToken}` }, + }) + } catch (error) { + if (axios.isAxiosError(error)) { + throw new Error(`API request failed: ${error.message}`) + } + throw error + } +} + +async function retryOperation( + operation: () => Promise, + maxRetries: number, + delay: number +): Promise { + let lastError: Error | null = null + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await operation() + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + throw lastError || new Error('Operation failed after max retries') +} + +async function logSuccess( + context: ActionExecutionContext, + jobId: string, + count: number +): Promise { + await api.jobs.log(jobId, { + message: `Successfully exported ${count} records.`, + level: 'info', + }) +} + +async function logFailure( + context: ActionExecutionContext, + jobId: string, + count: number, + error: unknown +): Promise { + const errorMessage = error instanceof Error ? error.message : String(error) + await api.jobs.log(jobId, { + message: `Failed to export ${count} records. Error: ${errorMessage}`, + level: 'error', + }) +} + +async function updateJobProgress( + context: ActionExecutionContext, + jobId: string, + exported: number, + total: number +): Promise { + await api.jobs.update(jobId, { + status: 'active', + progress: { + completed: exported, + total: total, + }, + }) +} + +async function completeJob( + context: ActionExecutionContext, + jobId: string, + totalExported: number, + failedRecords: FlatfileRecord[], + successfulBatches: number, + failedBatches: number +): Promise { + const outcome = + failedRecords.length > 0 + ? { + message: `Exported ${totalExported} records. Failed to export ${failedRecords.length} records.`, + failedRecords: failedRecords.map((record) => record.id), + successfulBatches, + failedBatches, + } + : { + message: `Successfully exported ${totalExported} records to the external API.`, + successfulBatches, + } + + await api.jobs.complete(jobId, { outcome }) +} + +async function sendUserNotification( + context: ActionExecutionContext, + jobId: string, + successCount: number, + failureCount: number, + errorMessage?: string +): Promise { + const message = errorMessage + ? `Export job ${jobId} failed. Error: ${errorMessage}` + : `Export job ${jobId} completed. Successfully exported ${successCount} records. Failed to export ${failureCount} records.` + + await api.jobs.log(jobId, { + message: `User notification: ${message}`, + level: 'info', + }) + + // Implement actual user notification logic here (e.g., email, webhook) +} + +async function handleExportError( + context: ActionExecutionContext, + jobId: string, + error: unknown +): Promise { + const errorMessage = error instanceof Error ? error.message : String(error) + console.error('Error during export:', errorMessage) + await api.jobs.fail(jobId, { + outcome: { + message: 'Failed to export records to the external API.', + error: errorMessage, + }, + }) + await sendUserNotification(context, jobId, 0, 0, errorMessage) +} From 438dfb7e0c4a3b0b7e684ec14dcbd41e8dcbd69e Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Thu, 17 Oct 2024 19:59:45 -0500 Subject: [PATCH 2/6] partial cleanup --- export/external-api/README.MD | 75 ++++++ export/external-api/jest.config.js | 16 ++ export/external-api/package.json | 72 +++++ export/external-api/rollup.config.mjs | 5 + .../external-api/src/external.api.plugin.ts | 93 +++++++ export/external-api/src/external.api.utils.ts | 52 ++++ export/external-api/src/index.ts | 1 + package-lock.json | 37 ++- validators/ExternalApiPublisher/README.MD | 75 ------ validators/ExternalApiPublisher/metadata.json | 77 ------ validators/ExternalApiPublisher/package.json | 62 ----- .../ExternalApiPublisher/rollup.config.mjs | 42 --- validators/ExternalApiPublisher/src/index.ts | 247 ------------------ 13 files changed, 347 insertions(+), 507 deletions(-) create mode 100644 export/external-api/README.MD create mode 100644 export/external-api/jest.config.js create mode 100644 export/external-api/package.json create mode 100644 export/external-api/rollup.config.mjs create mode 100644 export/external-api/src/external.api.plugin.ts create mode 100644 export/external-api/src/external.api.utils.ts create mode 100644 export/external-api/src/index.ts delete mode 100644 validators/ExternalApiPublisher/README.MD delete mode 100644 validators/ExternalApiPublisher/metadata.json delete mode 100644 validators/ExternalApiPublisher/package.json delete mode 100644 validators/ExternalApiPublisher/rollup.config.mjs delete mode 100644 validators/ExternalApiPublisher/src/index.ts diff --git a/export/external-api/README.MD b/export/external-api/README.MD new file mode 100644 index 000000000..769282533 --- /dev/null +++ b/export/external-api/README.MD @@ -0,0 +1,75 @@ + + +# @flatfile/plugin-export-external-api + +This plugin for Flatfile enables exporting data to an external API with advanced features such as batching, error handling, retry logic, and user notifications. It processes records from Flatfile, maps them to the desired format, and exports them in batches to a specified external API endpoint. + +**Event Type:** `job:ready` + + + +## Features + +- Data mapping from Flatfile fields to external API fields +- Batch processing for efficient exports +- Configurable batch size +- Retry logic with configurable max retries and delay +- Detailed progress tracking and job status updates +- Error handling and logging +- User notifications for export completion or failure +- Support for custom record validation + +## Parameters + +#### `job` - `string` - (required) +The type of job to create. + +#### `apiEndpoint` - `string` - (required) +The URL of the external API endpoint to send data to. + +#### `authToken` - `string` - (required) +The authentication token for the external API. + +#### `dataMapping` - `object` - (required) +A mapping of Flatfile field names to external API field names. + +#### `batchSize` - `number` - (required) +The number of records to process in each batch. + +#### `maxRetries` - `number` - (required) +The maximum number of retry attempts for failed API calls. + +#### `retryDelay` - `number` - (required) +The delay (in milliseconds) between retry attempts. + + +## Installation + +To install the plugin, use npm: + +```bash +npm install @flatfile/plugin-external-api-export +``` + +## Example Usage + +```javascript +import externalApiExportPlugin from '@flatfile/plugin-external-api-export'; + +export default function (listener) { + listener.use( + externalApiExportPlugin({ + apiEndpoint: 'https://api.example.com/import', + authToken: 'your-api-token', + dataMapping: { + 'First Name': 'firstName', + 'Last Name': 'lastName', + 'Email': 'emailAddress' + }, + batchSize: 100, + maxRetries: 3, + retryDelay: 1000 + }) + ); +} +``` diff --git a/export/external-api/jest.config.js b/export/external-api/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/export/external-api/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/export/external-api/package.json b/export/external-api/package.json new file mode 100644 index 000000000..5d235d307 --- /dev/null +++ b/export/external-api/package.json @@ -0,0 +1,72 @@ +{ + "name": "@flatfile/plugin-export-external-api", + "version": "0.0.0", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/export/external-api", + "description": "A Flatfile plugin for exporting data to an external API with batching, error handling, and retry logic", + "registryMetadata": { + "category": "export" + }, + "engines": { + "node": ">= 16" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ], + "browser": { + "./dist/index.js": "./dist/index.browser.js", + "./dist/index.mjs": "./dist/index.browser.mjs" + }, + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "browser": { + "require": "./dist/index.browser.js", + "import": "./dist/index.browser.mjs" + }, + "default": "./dist/index.mjs" + }, + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "source": "./src/index.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" + }, + "keywords": [ + "flatfile-plugins", + "category-export" + ], + "author": "Flatfile, Inc.", + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "export/external-api" + }, + "license": "ISC", + "dependencies": { + "@flatfile/plugin-job-handler": "^0.6.1", + "cross-fetch": "^4.0.0" + }, + "peerDependencies": { + "@flatfile/util-common": "^1.4.1" + }, + "devDependencies": { + "@flatfile/api": "^1.9.15", + "@flatfile/listener": "^1.0.5", + "@flatfile/rollup-config": "^0.1.1" + } +} \ No newline at end of file diff --git a/export/external-api/rollup.config.mjs b/export/external-api/rollup.config.mjs new file mode 100644 index 000000000..fafa813c6 --- /dev/null +++ b/export/external-api/rollup.config.mjs @@ -0,0 +1,5 @@ +import { buildConfig } from '@flatfile/rollup-config' + +const config = buildConfig({}) + +export default config diff --git a/export/external-api/src/external.api.plugin.ts b/export/external-api/src/external.api.plugin.ts new file mode 100644 index 000000000..bb9611125 --- /dev/null +++ b/export/external-api/src/external.api.plugin.ts @@ -0,0 +1,93 @@ +import { type Flatfile } from '@flatfile/api' +import { type FlatfileEvent, type FlatfileListener } from '@flatfile/listener' +import { jobHandler } from '@flatfile/plugin-job-handler' +import { getSheetLength, Simplified } from '@flatfile/util-common' +import { + exportToExternalAPI, + processRecord, + retryOperation, +} from './external.api.utils' + +export interface PluginConfig { + job: string + apiEndpoint: string + authToken: string + dataMapping: { + readonly [key: string]: string + } + batchSize: number + maxRetries: number + retryDelay: number +} + +export const externalApiExportPlugin = (config: PluginConfig) => { + return (listener: FlatfileListener) => { + listener.use( + jobHandler( + { operation: config.job }, + async ( + event: FlatfileEvent, + tick: ( + progress: number, + message?: string + ) => Promise + ) => { + const { sheetId } = event.context + + try { + let totalExported = 0 + let failedRecords = 0 + let successfulBatches = 0 + + const sheetLength = await getSheetLength(sheetId) + const batchCount = Math.ceil(sheetLength / config.batchSize) + + let pageNumber = 1 + while (pageNumber <= batchCount) { + const records = await Simplified.getAllRecords(sheetId, { + pageSize: config.batchSize, + pageNumber, + }) + + const processedBatch = records.map((record) => + processRecord(record, config.dataMapping) + ) + + try { + await retryOperation( + () => + exportToExternalAPI( + processedBatch, + config.apiEndpoint, + config.authToken + ), + config.maxRetries, + config.retryDelay + ) + totalExported += processedBatch.length + successfulBatches++ + } catch (error) { + console.error('Failed to export batch after retries:', error) + failedRecords += processedBatch.length + } + + const progress = (successfulBatches / batchCount) * 100 + await tick( + progress, + `Exported ${totalExported} records. Failed to export ${failedRecords} records.` + ) + } + + return { + info: `Exported ${totalExported} records. Failed to export ${failedRecords} records.`, + } + } catch (error) { + console.error('Error during export:', (error as Error).message) + + throw new Error('An error occurred during export.') + } + } + ) + ) + } +} diff --git a/export/external-api/src/external.api.utils.ts b/export/external-api/src/external.api.utils.ts new file mode 100644 index 000000000..fafdf9d5c --- /dev/null +++ b/export/external-api/src/external.api.utils.ts @@ -0,0 +1,52 @@ +import { type SimpleRecord } from '@flatfile/util-common' +import fetch from 'cross-fetch' +import { type PluginConfig } from './external.api.plugin' + +export function processRecord( + record: SimpleRecord, + mapping: PluginConfig['dataMapping'] +): Record { + return Object.entries(mapping).reduce( + (acc, [from, to]) => { + acc[to] = record[from] + return acc + }, + {} as Record + ) +} + +export async function exportToExternalAPI( + data: Record[], + apiEndpoint: string, + authToken: string +): Promise { + const response = await fetch(apiEndpoint, { + method: 'POST', + headers: { + Authorization: `Bearer ${authToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }) + + if (!response.ok) { + throw new Error(`API request failed: ${response.statusText}`) + } +} + +export async function retryOperation( + operation: () => Promise, + maxRetries: number, + delay: number +): Promise { + let lastError: Error | null = null + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await operation() + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)) + await new Promise((resolve) => setTimeout(resolve, delay)) + } + } + throw lastError || new Error('Operation failed after max retries') +} diff --git a/export/external-api/src/index.ts b/export/external-api/src/index.ts new file mode 100644 index 000000000..8e0c2cf49 --- /dev/null +++ b/export/external-api/src/index.ts @@ -0,0 +1 @@ +export { externalApiExportPlugin } from './external.api.plugin' diff --git a/package-lock.json b/package-lock.json index 921b1af24..8a466082a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -140,8 +140,9 @@ "license": "MIT" }, "enrich/gpx": { - "version": "0.0.0", - "license": "MIT", + "name": "@flatfile/plugin-enrich-gpx", + "version": "0.1.0", + "license": "ISC", "dependencies": { "@flatfile/plugin-record-hook": "^1.6.1", "xml2js": "^0.6.2" @@ -200,9 +201,30 @@ "@flatfile/listener": "^1.1.0" } }, + "export/external-api": { + "name": "@flatfile/plugin-export-external-api", + "version": "0.0.0", + "license": "ISC", + "dependencies": { + "@flatfile/plugin-job-handler": "^0.6.1", + "cross-fetch": "^4.0.0" + }, + "devDependencies": { + "@flatfile/rollup-config": "^0.1.1" + }, + "engines": { + "node": ">= 16" + }, + "peerDependencies": { + "@flatfile/api": "^1.9.15", + "@flatfile/hooks": "^1.5.0", + "@flatfile/listener": "^1.0.5", + "@flatfile/util-common": "^1.4.1" + } + }, "export/pivot-table": { "name": "@flatfile/plugin-export-pivot-table", - "version": "0.0.0", + "version": "0.1.0", "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.15" @@ -246,7 +268,6 @@ "flatfilers/playground": { "name": "@private/playground", "version": "0.0.0", - "extraneous": true, "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.19", @@ -3378,6 +3399,10 @@ "resolved": "enrich/summarize", "link": true }, + "node_modules/@flatfile/plugin-export-external-api": { + "resolved": "export/external-api", + "link": true + }, "node_modules/@flatfile/plugin-export-pivot-table": { "resolved": "export/pivot-table", "link": true @@ -6412,6 +6437,10 @@ "node": ">=14" } }, + "node_modules/@private/playground": { + "resolved": "flatfilers/playground", + "link": true + }, "node_modules/@private/sandbox": { "resolved": "flatfilers/sandbox", "link": true diff --git a/validators/ExternalApiPublisher/README.MD b/validators/ExternalApiPublisher/README.MD deleted file mode 100644 index b93b76804..000000000 --- a/validators/ExternalApiPublisher/README.MD +++ /dev/null @@ -1,75 +0,0 @@ -# Flatfile External API Export Plugin - -This plugin for Flatfile enables exporting data to an external API with advanced features such as batching, error handling, retry logic, and user notifications. It processes records from Flatfile, maps them to the desired format, and exports them in batches to a specified external API endpoint. - -## Features - -- Data mapping from Flatfile fields to external API fields -- Batch processing for efficient exports -- Configurable batch size -- Retry logic with configurable max retries and delay -- Detailed progress tracking and job status updates -- Error handling and logging -- User notifications for export completion or failure -- Support for custom record validation - -## Installation - -To install the plugin, use npm: - -```bash -npm install @flatfile/plugin-external-api-export -``` - -## Example Usage - -```javascript -import { FlatfileListener } from '@flatfile/listener'; -import externalApiExportPlugin from '@flatfile/plugin-external-api-export'; - -const listener = new FlatfileListener(); - -listener.use( - externalApiExportPlugin({ - apiEndpoint: 'https://api.example.com/import', - authToken: 'your-api-token', - dataMapping: { - 'First Name': 'firstName', - 'Last Name': 'lastName', - 'Email': 'emailAddress' - }, - batchSize: 100, - maxRetries: 3, - retryDelay: 1000 - }) -); - -listener.on('job:ready', async ({ context, payload }) => { - // Additional job:ready handling if needed -}); -``` - -## Configuration - -The plugin accepts the following configuration options: - -- `apiEndpoint` (string): The URL of the external API endpoint to send data to. -- `authToken` (string): The authentication token for the external API. -- `dataMapping` (object): A mapping of Flatfile field names to external API field names. -- `batchSize` (number): The number of records to process in each batch. -- `maxRetries` (number): The maximum number of retry attempts for failed API calls. -- `retryDelay` (number): The delay (in milliseconds) between retry attempts. - -## Behavior - -1. When a job is ready, the plugin retrieves all records for the specified file. -2. Records are processed in batches according to the configured `batchSize`. -3. Each record is mapped to the format expected by the external API using the `dataMapping` configuration. -4. The plugin attempts to export each batch to the external API. -5. If an export fails, the plugin will retry the operation up to `maxRetries` times, with a delay of `retryDelay` milliseconds between attempts. -6. Progress is tracked and reported throughout the export process. -7. Successful exports are logged, and failed exports are recorded for error reporting. -8. Upon completion, a summary of the export process is provided, including the number of successful and failed exports. -9. User notifications are sent to inform about the export status. - -The plugin also includes a record hook for the 'contacts' sheet, allowing for custom validation logic to be implemented before export. \ No newline at end of file diff --git a/validators/ExternalApiPublisher/metadata.json b/validators/ExternalApiPublisher/metadata.json deleted file mode 100644 index 3a57ddeec..000000000 --- a/validators/ExternalApiPublisher/metadata.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "timestamp": "2024-09-25T06-07-30-460Z", - "task": "Create a Flatfile Listener plugin to export Flatfile Workbook data to an External API:\n - Listen for a custom action to publish processed data to an external API, that is configurable in the Plugin settings\n - Allow configuration of API endpoint, authentication, and data mapping\n - Handle batching of records for efficient API calls\n - Implement error handling and retries for failed API requests\n - Provide feedback on successful and failed data publishing attempts", - "summary": "This code implements a Flatfile Listener plugin for exporting data to an external API. It includes features such as batching, error handling, retry logic, and user notifications. The plugin processes records, exports them in batches, and provides detailed feedback on the export process.", - "steps": [ - [ - "Retrieve information about Flatfile Listeners and the Record Hook plugin to understand the structure and best practices.\n", - "#E1", - "PineconeAssistant", - "Provide information about Flatfile Listeners and the Record Hook plugin, including their structure and best practices", - "Plan: Retrieve information about Flatfile Listeners and the Record Hook plugin to understand the structure and best practices.\n#E1 = PineconeAssistant[Provide information about Flatfile Listeners and the Record Hook plugin, including their structure and best practices]" - ], - [ - "Create the basic structure of the Flatfile Listener plugin with the necessary imports and configurations.\n", - "#E2", - "LLM", - "Create the basic structure of a Flatfile Listener plugin for exporting data to an external API, using the information from #E1. Include necessary imports and plugin configuration", - "Plan: Create the basic structure of the Flatfile Listener plugin with the necessary imports and configurations.\n#E2 = LLM[Create the basic structure of a Flatfile Listener plugin for exporting data to an external API, using the information from #E1. Include necessary imports and plugin configuration]" - ], - [ - "Implement the custom action listener for publishing processed data to the external API.\n", - "#E3", - "LLM", - "Implement a custom action listener for the Flatfile plugin created in #E2, which will trigger the data export process", - "Plan: Implement the custom action listener for publishing processed data to the external API.\n#E3 = LLM[Implement a custom action listener for the Flatfile plugin created in #E2, which will trigger the data export process]" - ], - [ - "Add configuration options for the API endpoint, authentication, and data mapping.\n", - "#E4", - "LLM", - "Add configuration options to the Flatfile plugin from #E3 for API endpoint, authentication, and data mapping", - "Plan: Add configuration options for the API endpoint, authentication, and data mapping.\n#E4 = LLM[Add configuration options to the Flatfile plugin from #E3 for API endpoint, authentication, and data mapping]" - ], - [ - "Implement batching functionality for efficient API calls.\n", - "#E5", - "LLM", - "Add batching functionality to the Flatfile plugin from #E4 for efficient API calls", - "Plan: Implement batching functionality for efficient API calls.\n#E5 = LLM[Add batching functionality to the Flatfile plugin from #E4 for efficient API calls]" - ], - [ - "Implement error handling and retry logic for failed API requests.\n", - "#E6", - "LLM", - "Add error handling and retry logic to the Flatfile plugin from #E5 for failed API requests", - "Plan: Implement error handling and retry logic for failed API requests.\n#E6 = LLM[Add error handling and retry logic to the Flatfile plugin from #E5 for failed API requests]" - ], - [ - "Implement feedback mechanisms for successful and failed data publishing attempts.\n", - "#E7", - "LLM", - "Add feedback mechanisms to the Flatfile plugin from #E6 for successful and failed data publishing attempts", - "Plan: Implement feedback mechanisms for successful and failed data publishing attempts.\n#E7 = LLM[Add feedback mechanisms to the Flatfile plugin from #E6 for successful and failed data publishing attempts]" - ], - [ - "Review and optimize the complete Flatfile Listener plugin code.\n", - "#E8", - "LLM", - "Review and optimize the complete Flatfile Listener plugin code from #E7, ensuring all requirements are met and the code follows best practices", - "Plan: Review and optimize the complete Flatfile Listener plugin code.\n#E8 = LLM[Review and optimize the complete Flatfile Listener plugin code from #E7, ensuring all requirements are met and the code follows best practices]" - ], - [ - "Validate the final code, check for unused imports, and ensure correct Event Topic usage.\n", - "#E9", - "PineconeAssistant", - "Validate the Flatfile Listener plugin code from #E8, check for unused imports, and ensure correct Event Topic usage. Provide any necessary corrections or optimizations", - "Plan: Validate the final code, check for unused imports, and ensure correct Event Topic usage.\n#E9 = PineconeAssistant[Validate the Flatfile Listener plugin code from #E8, check for unused imports, and ensure correct Event Topic usage. Provide any necessary corrections or optimizations]" - ] - ], - "metrics": { - "tokens": { - "plan": 4694, - "state": 6200, - "total": 10894 - } - } -} \ No newline at end of file diff --git a/validators/ExternalApiPublisher/package.json b/validators/ExternalApiPublisher/package.json deleted file mode 100644 index 9c5682b3d..000000000 --- a/validators/ExternalApiPublisher/package.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "name": "@flatfile/plugin-external-api-exporter", - "version": "1.0.0", - "description": "A Flatfile plugin for exporting data to an external API with batching, error handling, and retry logic", - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", - "exports": { - "types": "./dist/index.d.ts", - "node": { - "import": "./dist/index.mjs", - "require": "./dist/index.js" - }, - "default": "./dist/index.mjs" - }, - "source": "./src/index.ts", - "files": [ - "dist/**" - ], - "scripts": { - "build": "rollup -c", - "build:watch": "rollup -c --watch", - "build:prod": "NODE_ENV=production rollup -c", - "check": "tsc ./**/*.ts --noEmit --esModuleInterop", - "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" - }, - "keywords": [ - "flatfile", - "plugin", - "api", - "exporter", - "flatfile-plugins", - "category-transform" - ], - "author": "Your Name", - "license": "MIT", - "dependencies": { - "@flatfile/api": "^1.9.15", - "@flatfile/plugin-record-hook": "^1.7.0", - "axios": "^1.7.7" - }, - "peerDependencies": { - "@flatfile/listener": "^1.0.5" - }, - "devDependencies": { - "@flatfile/hooks": "^1.5.0", - "@flatfile/rollup-config": "^0.1.1", - "@types/node": "^22.7.0", - "typescript": "^5.6.2", - "rollup": "^4.22.4", - "@rollup/plugin-typescript": "^12.1.0", - "@rollup/plugin-node-resolve": "^15.3.0", - "@rollup/plugin-commonjs": "^28.0.0", - "jest": "^29.7.0", - "@types/jest": "^29.5.13" - }, - "repository": { - "type": "git", - "url": "https://github.com/YourGitHubUsername/flatfile-plugins.git", - "directory": "plugins/external-api-exporter" - } -} \ No newline at end of file diff --git a/validators/ExternalApiPublisher/rollup.config.mjs b/validators/ExternalApiPublisher/rollup.config.mjs deleted file mode 100644 index ce1befc71..000000000 --- a/validators/ExternalApiPublisher/rollup.config.mjs +++ /dev/null @@ -1,42 +0,0 @@ -import { buildConfig } from '@flatfile/rollup-config'; -import typescript from '@rollup/plugin-typescript'; -import resolve from '@rollup/plugin-node-resolve'; -import commonjs from '@rollup/plugin-commonjs'; -import json from '@rollup/plugin-json'; - -const umdExternals = [ - '@flatfile/api', - '@flatfile/hooks', - '@flatfile/listener', - '@flatfile/util-common', - '@flatfile/plugin-record-hook', - 'axios' -]; - -const config = buildConfig({ - input: 'src/index.ts', // Adjust this to your main entry file - includeUmd: true, - umdConfig: { - name: 'FlatfileExportPlugin', // Replace with your plugin name - external: umdExternals - }, - plugins: [ - typescript({ - tsconfig: './tsconfig.json', - declaration: true, - declarationDir: './dist/types', - }), - resolve({ - browser: true, - preferBuiltins: true, - }), - commonjs(), - json(), - ], - external: [ - ...umdExternals, - 'axios', - ] -}); - -export default config; \ No newline at end of file diff --git a/validators/ExternalApiPublisher/src/index.ts b/validators/ExternalApiPublisher/src/index.ts deleted file mode 100644 index 0dc2f9c25..000000000 --- a/validators/ExternalApiPublisher/src/index.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { - FlatfileListener, - FlatfileEvent, - FlatfileRecord, -} from '@flatfile/listener' -import { recordHook } from '@flatfile/plugin-record-hook' -import { ActionExecutionContext } from '@flatfile/hooks' -import api from '@flatfile/api' -import axios from 'axios' - -interface PluginConfig { - apiEndpoint: string - authToken: string - dataMapping: { - readonly [key: string]: string - } - batchSize: number - maxRetries: number - retryDelay: number -} - -export default function ( - listener: FlatfileListener, - config: PluginConfig -): void { - const { - apiEndpoint, - authToken, - dataMapping, - batchSize, - maxRetries, - retryDelay, - } = config - - listener.use( - recordHook('contacts', (record: FlatfileRecord) => { - // Validation logic here - return record - }) - ) - - listener.on('job:ready', async ({ context, payload }) => { - const { jobId } = payload - - try { - const records = await api.records.get(payload.spaceId, payload.fileId) - const batches = chunkArray(records.data, batchSize) - let totalExported = 0 - const failedRecords: FlatfileRecord[] = [] - let successfulBatches = 0 - let failedBatches = 0 - - for (const batch of batches) { - const processedBatch = batch.map((record) => - processRecord(record, dataMapping) - ) - try { - await retryOperation( - () => exportToExternalAPI(processedBatch, apiEndpoint, authToken), - maxRetries, - retryDelay - ) - totalExported += batch.length - successfulBatches++ - await logSuccess(context, jobId, batch.length) - } catch (error) { - console.error('Failed to export batch after retries:', error) - failedRecords.push(...batch) - failedBatches++ - await logFailure(context, jobId, batch.length, error) - } - - await updateJobProgress( - context, - jobId, - totalExported, - records.data.length - ) - } - - await completeJob( - context, - jobId, - totalExported, - failedRecords, - successfulBatches, - failedBatches - ) - await sendUserNotification( - context, - jobId, - totalExported, - failedRecords.length - ) - } catch (error) { - await handleExportError(context, jobId, error) - } - }) -} - -function chunkArray(array: T[], size: number): T[][] { - return Array.from({ length: Math.ceil(array.length / size) }, (_, index) => - array.slice(index * size, (index + 1) * size) - ) -} - -function processRecord( - record: FlatfileRecord, - mapping: PluginConfig['dataMapping'] -): Record { - return Object.entries(mapping).reduce((acc, [from, to]) => { - acc[to] = record.get(from) - return acc - }, {} as Record) -} - -async function exportToExternalAPI( - data: Record[], - apiEndpoint: string, - authToken: string -): Promise { - try { - await axios.post(apiEndpoint, data, { - headers: { Authorization: `Bearer ${authToken}` }, - }) - } catch (error) { - if (axios.isAxiosError(error)) { - throw new Error(`API request failed: ${error.message}`) - } - throw error - } -} - -async function retryOperation( - operation: () => Promise, - maxRetries: number, - delay: number -): Promise { - let lastError: Error | null = null - for (let attempt = 0; attempt < maxRetries; attempt++) { - try { - return await operation() - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)) - await new Promise((resolve) => setTimeout(resolve, delay)) - } - } - throw lastError || new Error('Operation failed after max retries') -} - -async function logSuccess( - context: ActionExecutionContext, - jobId: string, - count: number -): Promise { - await api.jobs.log(jobId, { - message: `Successfully exported ${count} records.`, - level: 'info', - }) -} - -async function logFailure( - context: ActionExecutionContext, - jobId: string, - count: number, - error: unknown -): Promise { - const errorMessage = error instanceof Error ? error.message : String(error) - await api.jobs.log(jobId, { - message: `Failed to export ${count} records. Error: ${errorMessage}`, - level: 'error', - }) -} - -async function updateJobProgress( - context: ActionExecutionContext, - jobId: string, - exported: number, - total: number -): Promise { - await api.jobs.update(jobId, { - status: 'active', - progress: { - completed: exported, - total: total, - }, - }) -} - -async function completeJob( - context: ActionExecutionContext, - jobId: string, - totalExported: number, - failedRecords: FlatfileRecord[], - successfulBatches: number, - failedBatches: number -): Promise { - const outcome = - failedRecords.length > 0 - ? { - message: `Exported ${totalExported} records. Failed to export ${failedRecords.length} records.`, - failedRecords: failedRecords.map((record) => record.id), - successfulBatches, - failedBatches, - } - : { - message: `Successfully exported ${totalExported} records to the external API.`, - successfulBatches, - } - - await api.jobs.complete(jobId, { outcome }) -} - -async function sendUserNotification( - context: ActionExecutionContext, - jobId: string, - successCount: number, - failureCount: number, - errorMessage?: string -): Promise { - const message = errorMessage - ? `Export job ${jobId} failed. Error: ${errorMessage}` - : `Export job ${jobId} completed. Successfully exported ${successCount} records. Failed to export ${failureCount} records.` - - await api.jobs.log(jobId, { - message: `User notification: ${message}`, - level: 'info', - }) - - // Implement actual user notification logic here (e.g., email, webhook) -} - -async function handleExportError( - context: ActionExecutionContext, - jobId: string, - error: unknown -): Promise { - const errorMessage = error instanceof Error ? error.message : String(error) - console.error('Error during export:', errorMessage) - await api.jobs.fail(jobId, { - outcome: { - message: 'Failed to export records to the external API.', - error: errorMessage, - }, - }) - await sendUserNotification(context, jobId, 0, 0, errorMessage) -} From 7f58c34c651f6b7d3ea0bf5445a2151eba4d1fe8 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Tue, 22 Oct 2024 10:24:04 -0600 Subject: [PATCH 3/6] feat: remove dataMapping functionality --- export/external-api/README.MD | 15 +++--------- .../external-api/src/external.api.plugin.ts | 23 ++++++------------- export/external-api/src/external.api.utils.ts | 15 ------------ export/external-api/src/index.ts | 2 +- 4 files changed, 11 insertions(+), 44 deletions(-) diff --git a/export/external-api/README.MD b/export/external-api/README.MD index 769282533..c872feb51 100644 --- a/export/external-api/README.MD +++ b/export/external-api/README.MD @@ -10,7 +10,6 @@ This plugin for Flatfile enables exporting data to an external API with advanced ## Features -- Data mapping from Flatfile fields to external API fields - Batch processing for efficient exports - Configurable batch size - Retry logic with configurable max retries and delay @@ -30,9 +29,6 @@ The URL of the external API endpoint to send data to. #### `authToken` - `string` - (required) The authentication token for the external API. -#### `dataMapping` - `object` - (required) -A mapping of Flatfile field names to external API field names. - #### `batchSize` - `number` - (required) The number of records to process in each batch. @@ -48,24 +44,19 @@ The delay (in milliseconds) between retry attempts. To install the plugin, use npm: ```bash -npm install @flatfile/plugin-external-api-export +npm install @flatfile/plugin-export-external-api ``` ## Example Usage ```javascript -import externalApiExportPlugin from '@flatfile/plugin-external-api-export'; +import { exportToExternalAPIPlugin } from '@flatfile/plugin-export-external-api'; export default function (listener) { listener.use( - externalApiExportPlugin({ + exportToExternalAPIPlugin({ apiEndpoint: 'https://api.example.com/import', authToken: 'your-api-token', - dataMapping: { - 'First Name': 'firstName', - 'Last Name': 'lastName', - 'Email': 'emailAddress' - }, batchSize: 100, maxRetries: 3, retryDelay: 1000 diff --git a/export/external-api/src/external.api.plugin.ts b/export/external-api/src/external.api.plugin.ts index bb9611125..f029a72ca 100644 --- a/export/external-api/src/external.api.plugin.ts +++ b/export/external-api/src/external.api.plugin.ts @@ -2,25 +2,18 @@ import { type Flatfile } from '@flatfile/api' import { type FlatfileEvent, type FlatfileListener } from '@flatfile/listener' import { jobHandler } from '@flatfile/plugin-job-handler' import { getSheetLength, Simplified } from '@flatfile/util-common' -import { - exportToExternalAPI, - processRecord, - retryOperation, -} from './external.api.utils' +import { exportToExternalAPI, retryOperation } from './external.api.utils' export interface PluginConfig { job: string apiEndpoint: string authToken: string - dataMapping: { - readonly [key: string]: string - } batchSize: number maxRetries: number retryDelay: number } -export const externalApiExportPlugin = (config: PluginConfig) => { +export const exportToExternalAPIPlugin = (config: PluginConfig) => { return (listener: FlatfileListener) => { listener.use( jobHandler( @@ -49,26 +42,22 @@ export const externalApiExportPlugin = (config: PluginConfig) => { pageNumber, }) - const processedBatch = records.map((record) => - processRecord(record, config.dataMapping) - ) - try { await retryOperation( () => exportToExternalAPI( - processedBatch, + records, config.apiEndpoint, config.authToken ), config.maxRetries, config.retryDelay ) - totalExported += processedBatch.length + totalExported += records.length successfulBatches++ } catch (error) { console.error('Failed to export batch after retries:', error) - failedRecords += processedBatch.length + failedRecords += records.length } const progress = (successfulBatches / batchCount) * 100 @@ -91,3 +80,5 @@ export const externalApiExportPlugin = (config: PluginConfig) => { ) } } + +export default exportToExternalAPIPlugin diff --git a/export/external-api/src/external.api.utils.ts b/export/external-api/src/external.api.utils.ts index fafdf9d5c..3f45cd2b4 100644 --- a/export/external-api/src/external.api.utils.ts +++ b/export/external-api/src/external.api.utils.ts @@ -1,19 +1,4 @@ -import { type SimpleRecord } from '@flatfile/util-common' import fetch from 'cross-fetch' -import { type PluginConfig } from './external.api.plugin' - -export function processRecord( - record: SimpleRecord, - mapping: PluginConfig['dataMapping'] -): Record { - return Object.entries(mapping).reduce( - (acc, [from, to]) => { - acc[to] = record[from] - return acc - }, - {} as Record - ) -} export async function exportToExternalAPI( data: Record[], diff --git a/export/external-api/src/index.ts b/export/external-api/src/index.ts index 8e0c2cf49..376eee838 100644 --- a/export/external-api/src/index.ts +++ b/export/external-api/src/index.ts @@ -1 +1 @@ -export { externalApiExportPlugin } from './external.api.plugin' +export * from './external.api.plugin' From af5556a02fe06103cfd9a58b5bd1da278fc92a9e Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Tue, 22 Oct 2024 10:24:25 -0600 Subject: [PATCH 4/6] chore: changeset --- .changeset/short-papayas-share.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/short-papayas-share.md diff --git a/.changeset/short-papayas-share.md b/.changeset/short-papayas-share.md new file mode 100644 index 000000000..c40ebab36 --- /dev/null +++ b/.changeset/short-papayas-share.md @@ -0,0 +1,5 @@ +--- +'@flatfile/plugin-export-external-api': minor +--- + +Initial Release From 01dc296ad9976386b290e9303a6509310a8ea20d Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Tue, 22 Oct 2024 12:23:23 -0500 Subject: [PATCH 5/6] fix bug --- .../external-api/src/external.api.plugin.ts | 3 +- flatfilers/sandbox/src/index.ts | 37 ++++++++++--------- package-lock.json | 5 +-- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/export/external-api/src/external.api.plugin.ts b/export/external-api/src/external.api.plugin.ts index f029a72ca..367ed5857 100644 --- a/export/external-api/src/external.api.plugin.ts +++ b/export/external-api/src/external.api.plugin.ts @@ -17,7 +17,7 @@ export const exportToExternalAPIPlugin = (config: PluginConfig) => { return (listener: FlatfileListener) => { listener.use( jobHandler( - { operation: config.job }, + `sheet:${config.job}`, async ( event: FlatfileEvent, tick: ( @@ -65,6 +65,7 @@ export const exportToExternalAPIPlugin = (config: PluginConfig) => { progress, `Exported ${totalExported} records. Failed to export ${failedRecords} records.` ) + pageNumber++ } return { diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index d8f6d0935..2322432a2 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -1,14 +1,15 @@ import type { FlatfileListener } from '@flatfile/listener' -import { pivotTablePlugin } from '@flatfile/plugin-export-pivot-table' +import { exportToExternalAPIPlugin } from '@flatfile/plugin-export-external-api' import { configureSpace } from '@flatfile/plugin-space-configure' - export default async function (listener: FlatfileListener) { listener.use( - pivotTablePlugin({ - pivotColumn: 'product', - aggregateColumn: 'salesAmount', - aggregationMethod: 'sum', - groupByColumn: 'region', + exportToExternalAPIPlugin({ + job: 'export-external-api', + apiEndpoint: 'http://localhost:5678/api/import', + authToken: 'your-api-token', + batchSize: 100, + maxRetries: 3, + retryDelay: 1000, }) ) listener.use( @@ -47,17 +48,17 @@ export default async function (listener: FlatfileListener) { label: 'Sales Amount', }, ], - }, - ], - actions: [ - { - operation: 'generatePivotTable', - label: 'Generate Pivot Table', - description: - 'This custom action code generates a pivot table from the records in the People sheet.', - primary: false, - mode: 'foreground', - type: 'string', + actions: [ + { + operation: 'export-external-api', + label: 'Export to External API', + description: + 'This custom action code exports the records in the Sales sheet to an external API.', + primary: false, + mode: 'foreground', + type: 'string', + }, + ], }, ], }, diff --git a/package-lock.json b/package-lock.json index 8a466082a..a65d20c29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -210,15 +210,14 @@ "cross-fetch": "^4.0.0" }, "devDependencies": { + "@flatfile/api": "^1.9.15", + "@flatfile/listener": "^1.0.5", "@flatfile/rollup-config": "^0.1.1" }, "engines": { "node": ">= 16" }, "peerDependencies": { - "@flatfile/api": "^1.9.15", - "@flatfile/hooks": "^1.5.0", - "@flatfile/listener": "^1.0.5", "@flatfile/util-common": "^1.4.1" } }, From f78601e5488aa2f5e765551066509f20df400f2f Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Tue, 22 Oct 2024 21:39:48 -0500 Subject: [PATCH 6/6] improve secret retrieval --- export/external-api/README.MD | 6 +++--- export/external-api/src/external.api.plugin.ts | 18 +++++++----------- flatfilers/sandbox/src/index.ts | 2 +- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/export/external-api/README.MD b/export/external-api/README.MD index c872feb51..4e5c6164d 100644 --- a/export/external-api/README.MD +++ b/export/external-api/README.MD @@ -26,8 +26,8 @@ The type of job to create. #### `apiEndpoint` - `string` - (required) The URL of the external API endpoint to send data to. -#### `authToken` - `string` - (required) -The authentication token for the external API. +#### `secretName` - `string` - (required) +The name of the Flatfile Secret that contains the authentication token for the external API. #### `batchSize` - `number` - (required) The number of records to process in each batch. @@ -56,7 +56,7 @@ export default function (listener) { listener.use( exportToExternalAPIPlugin({ apiEndpoint: 'https://api.example.com/import', - authToken: 'your-api-token', + secretName: 'YOUR_SECRET_NAME', batchSize: 100, maxRetries: 3, retryDelay: 1000 diff --git a/export/external-api/src/external.api.plugin.ts b/export/external-api/src/external.api.plugin.ts index 367ed5857..69065c28d 100644 --- a/export/external-api/src/external.api.plugin.ts +++ b/export/external-api/src/external.api.plugin.ts @@ -1,5 +1,5 @@ -import { type Flatfile } from '@flatfile/api' -import { type FlatfileEvent, type FlatfileListener } from '@flatfile/listener' +import type { Flatfile } from '@flatfile/api' +import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener' import { jobHandler } from '@flatfile/plugin-job-handler' import { getSheetLength, Simplified } from '@flatfile/util-common' import { exportToExternalAPI, retryOperation } from './external.api.utils' @@ -7,7 +7,7 @@ import { exportToExternalAPI, retryOperation } from './external.api.utils' export interface PluginConfig { job: string apiEndpoint: string - authToken: string + secretName: string batchSize: number maxRetries: number retryDelay: number @@ -27,6 +27,8 @@ export const exportToExternalAPIPlugin = (config: PluginConfig) => { ) => { const { sheetId } = event.context + const authToken = await event.secrets(config.secretName) + try { let totalExported = 0 let failedRecords = 0 @@ -45,11 +47,7 @@ export const exportToExternalAPIPlugin = (config: PluginConfig) => { try { await retryOperation( () => - exportToExternalAPI( - records, - config.apiEndpoint, - config.authToken - ), + exportToExternalAPI(records, config.apiEndpoint, authToken), config.maxRetries, config.retryDelay ) @@ -60,7 +58,7 @@ export const exportToExternalAPIPlugin = (config: PluginConfig) => { failedRecords += records.length } - const progress = (successfulBatches / batchCount) * 100 + const progress = ((pageNumber - 1) / batchCount) * 100 await tick( progress, `Exported ${totalExported} records. Failed to export ${failedRecords} records.` @@ -81,5 +79,3 @@ export const exportToExternalAPIPlugin = (config: PluginConfig) => { ) } } - -export default exportToExternalAPIPlugin diff --git a/flatfilers/sandbox/src/index.ts b/flatfilers/sandbox/src/index.ts index 2322432a2..dcf57a69c 100644 --- a/flatfilers/sandbox/src/index.ts +++ b/flatfilers/sandbox/src/index.ts @@ -6,7 +6,7 @@ export default async function (listener: FlatfileListener) { exportToExternalAPIPlugin({ job: 'export-external-api', apiEndpoint: 'http://localhost:5678/api/import', - authToken: 'your-api-token', + secretName: 'EXTERNAL_API_AUTH_SECRET', batchSize: 100, maxRetries: 3, retryDelay: 1000,