exporter: external api publisher

.changeset/short-papayas-share.md

@@ -0,0 +1,5 @@
+---
+'@flatfile/plugin-export-external-api': minor
+---
+
+Initial Release

export/external-api/README.MD

@@ -0,0 +1,66 @@
+<!-- START_INFOCARD -->
+
+# @flatfile/plugin-export-external-api
+
+This plugin for Flatfile enables exporting data to an external API with advanced features such as batching, error handling, retry logic, and user notifications. It processes records from Flatfile, maps them to the desired format, and exports them in batches to a specified external API endpoint.
+
+**Event Type:** `job:ready`
+
+<!-- END_INFOCARD -->
+
+## Features
+
+- Batch processing for efficient exports
+- Configurable batch size
+- Retry logic with configurable max retries and delay
+- Detailed progress tracking and job status updates
+- Error handling and logging
+- User notifications for export completion or failure
+- Support for custom record validation
+
+## Parameters
+
+#### `job` - `string` - (required)
+The type of job to create.
+
+#### `apiEndpoint` - `string` - (required)
+The URL of the external API endpoint to send data to.
+
+#### `secretName` - `string` - (required)
+The name of the Flatfile Secret that contains the authentication token for the external API.
+
+#### `batchSize` - `number` - (required)
+The number of records to process in each batch.
+
+#### `maxRetries` - `number` - (required)
+The maximum number of retry attempts for failed API calls.
+
+#### `retryDelay` - `number` - (required)
+The delay (in milliseconds) between retry attempts.
+
+
+## Installation
+
+To install the plugin, use npm:
+
+```bash
+npm install @flatfile/plugin-export-external-api
+```
+
+## Example Usage
+
+```javascript
+import { exportToExternalAPIPlugin } from '@flatfile/plugin-export-external-api';
+
+export default function (listener) {
+  listener.use(
+    exportToExternalAPIPlugin({
+      apiEndpoint: 'https://api.example.com/import',
+      secretName: 'YOUR_SECRET_NAME',
+      batchSize: 100,
+      maxRetries: 3,
+      retryDelay: 1000
+    })
+  );
+}
+```

export/external-api/jest.config.js

@@ -0,0 +1,16 @@
+module.exports = {
+  testEnvironment: 'node',
+
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest',
+  },
+  setupFiles: ['../../test/dotenv-config.js'],
+  setupFilesAfterEnv: [
+    '../../test/betterConsoleLog.js',
+    '../../test/unit.cleanup.js',
+  ],
+  testTimeout: 60_000,
+  globalSetup: '../../test/setup-global.js',
+  forceExit: true,
+  passWithNoTests: true,
+}

export/external-api/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@flatfile/plugin-export-external-api",
+  "version": "0.0.0",
+  "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/export/external-api",
+  "description": "A Flatfile plugin for exporting data to an external API with batching, error handling, and retry logic",
+  "registryMetadata": {
+    "category": "export"
+  },
+  "engines": {
+    "node": ">= 16"
+  },
+  "browserslist": [
+    "> 0.5%",
+    "last 2 versions",
+    "not dead"
+  ],
+  "browser": {
+    "./dist/index.js": "./dist/index.browser.js",
+    "./dist/index.mjs": "./dist/index.browser.mjs"
+  },
+  "exports": {
+    "types": "./dist/index.d.ts",
+    "node": {
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "browser": {
+      "require": "./dist/index.browser.js",
+      "import": "./dist/index.browser.mjs"
+    },
+    "default": "./dist/index.mjs"
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "source": "./src/index.ts",
+  "files": [
+    "dist/**"
+  ],
+  "scripts": {
+    "build": "rollup -c",
+    "build:watch": "rollup -c --watch",
+    "build:prod": "NODE_ENV=production rollup -c",
+    "check": "tsc ./**/*.ts --noEmit --esModuleInterop",
+    "test": "jest src/*.spec.ts --detectOpenHandles",
+    "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles",
+    "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles"
+  },
+  "keywords": [
+    "flatfile-plugins",
+    "category-export"
+  ],
+  "author": "Flatfile, Inc.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/FlatFilers/flatfile-plugins.git",
+    "directory": "export/external-api"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@flatfile/plugin-job-handler": "^0.6.1",
+    "cross-fetch": "^4.0.0"
+  },
+  "peerDependencies": {
+    "@flatfile/util-common": "^1.4.1"
+  },
+  "devDependencies": {
+    "@flatfile/api": "^1.9.15",
+    "@flatfile/listener": "^1.0.5",
+    "@flatfile/rollup-config": "^0.1.1"
+  }
+}

export/external-api/rollup.config.mjs

@@ -0,0 +1,5 @@
+import { buildConfig } from '@flatfile/rollup-config'
+
+const config = buildConfig({})
+
+export default config

export/external-api/src/external.api.plugin.ts

@@ -0,0 +1,81 @@
+import type { Flatfile } from '@flatfile/api'
+import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener'
+import { jobHandler } from '@flatfile/plugin-job-handler'
+import { getSheetLength, Simplified } from '@flatfile/util-common'
+import { exportToExternalAPI, retryOperation } from './external.api.utils'
+
+export interface PluginConfig {
+  job: string
+  apiEndpoint: string
+  secretName: string
+  batchSize: number
+  maxRetries: number
+  retryDelay: number
+}
+
+export const exportToExternalAPIPlugin = (config: PluginConfig) => {
+  return (listener: FlatfileListener) => {
+    listener.use(
+      jobHandler(
+        `sheet:${config.job}`,
+        async (
+          event: FlatfileEvent,
+          tick: (
+            progress: number,
+            message?: string
+          ) => Promise<Flatfile.JobResponse>
+        ) => {
+          const { sheetId } = event.context
+
+          const authToken = await event.secrets(config.secretName)
+
+          try {
+            let totalExported = 0
+            let failedRecords = 0
+            let successfulBatches = 0
+
+            const sheetLength = await getSheetLength(sheetId)
+            const batchCount = Math.ceil(sheetLength / config.batchSize)
+
+            let pageNumber = 1
+            while (pageNumber <= batchCount) {
+              const records = await Simplified.getAllRecords(sheetId, {
+                pageSize: config.batchSize,
+                pageNumber,
+              })
+
+              try {
+                await retryOperation(
+                  () =>
+                    exportToExternalAPI(records, config.apiEndpoint, authToken),
+                  config.maxRetries,
+                  config.retryDelay
+                )
+                totalExported += records.length
+                successfulBatches++
+              } catch (error) {
+                console.error('Failed to export batch after retries:', error)
+                failedRecords += records.length
+              }
+
+              const progress = ((pageNumber - 1) / batchCount) * 100
+              await tick(
+                progress,
+                `Exported ${totalExported} records. Failed to export ${failedRecords} records.`
+              )
+              pageNumber++
+            }
+
+            return {
+              info: `Exported ${totalExported} records. Failed to export ${failedRecords} records.`,
+            }
+          } catch (error) {
+            console.error('Error during export:', (error as Error).message)
+
+            throw new Error('An error occurred during export.')
+          }
+        }
+      )
+    )
+  }
+}

export/external-api/src/external.api.utils.ts

@@ -0,0 +1,37 @@
+import fetch from 'cross-fetch'
+
+export async function exportToExternalAPI(
+  data: Record<string, unknown>[],
+  apiEndpoint: string,
+  authToken: string
+): Promise<void> {
+  const response = await fetch(apiEndpoint, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${authToken}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(data),
+  })
+
+  if (!response.ok) {
+    throw new Error(`API request failed: ${response.statusText}`)
+  }
+}
+
+export async function retryOperation<T>(
+  operation: () => Promise<T>,
+  maxRetries: number,
+  delay: number
+): Promise<T> {
+  let lastError: Error | null = null
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      return await operation()
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error))
+      await new Promise((resolve) => setTimeout(resolve, delay))
+    }
+  }
+  throw lastError || new Error('Operation failed after max retries')
+}

export/external-api/src/index.ts

@@ -0,0 +1 @@
+export * from './external.api.plugin'

flatfilers/sandbox/src/index.ts

@@ -1,14 +1,15 @@
 import type { FlatfileListener } from '@flatfile/listener'
-import { pivotTablePlugin } from '@flatfile/plugin-export-pivot-table'
+import { exportToExternalAPIPlugin } from '@flatfile/plugin-export-external-api'
 import { configureSpace } from '@flatfile/plugin-space-configure'
-
 export default async function (listener: FlatfileListener) {
   listener.use(
-    pivotTablePlugin({
-      pivotColumn: 'product',
-      aggregateColumn: 'salesAmount',
-      aggregationMethod: 'sum',
-      groupByColumn: 'region',
+    exportToExternalAPIPlugin({
+      job: 'export-external-api',
+      apiEndpoint: 'http://localhost:5678/api/import',
+      secretName: 'EXTERNAL_API_AUTH_SECRET',
+      batchSize: 100,
+      maxRetries: 3,
+      retryDelay: 1000,
     })
   )
   listener.use(
@@ -47,17 +48,17 @@ export default async function (listener: FlatfileListener) {
                   label: 'Sales Amount',
                 },
               ],
-            },
-          ],
-          actions: [
-            {
-              operation: 'generatePivotTable',
-              label: 'Generate Pivot Table',
-              description:
-                'This custom action code generates a pivot table from the records in the People sheet.',
-              primary: false,
-              mode: 'foreground',
-              type: 'string',
+              actions: [
+                {
+                  operation: 'export-external-api',
+                  label: 'Export to External API',
+                  description:
+                    'This custom action code exports the records in the Sales sheet to an external API.',
+                  primary: false,
+                  mode: 'foreground',
+                  type: 'string',
+                },
+              ],
             },
           ],
         },