foreign db refactor (#852)

* foreign db refactor

* add changeset

* update message

Author: carlbrugger

.changeset/dry-bags-join.md

@@ -0,0 +1,5 @@
+---
+'@flatfile/plugin-foreign-db-extractor': minor
+---
+
+This release reflects the foreign db refactor in platform

flatfilers/sandbox/src/index.ts

@@ -4,8 +4,10 @@ import { contacts } from './sheets/contacts'
 import '@flatfile/http-logger/init'
 import api, { type Flatfile } from '@flatfile/api'
 import { exportWorkbookPlugin } from '@flatfile/plugin-export-workbook'
+import { foreignDBExtractor } from '@flatfile/plugin-foreign-db-extractor'
 
 export default async function (listener: FlatfileListener) {
+  listener.use(foreignDBExtractor())
   // Cache the sheets to avoid multiple API calls
   listener.on('job:ready', { job: 'workbook:downloadWorkbook' }, async (e) => {
     await e.cache.init('sheets-schema', async () => {

package-lock.json

@@ -3,10 +3,8 @@ import type { FlatfileEvent, FlatfileListener } from '@flatfile/listener'
 import type sql from 'mssql'
 import { pollDatabaseStatus } from './database.poll.status'
 import { restoreDatabase } from './database.restore'
-import type { DBUser } from './database.user'
-import { pollForUser } from './database.user'
 import { s3Upload } from './s3.upload'
-import { generateSheets } from './sheets.generator'
+import { configureWorkbook } from './workbook.configure'
 
 const api = new FlatfileClient()
 
@@ -83,29 +81,8 @@ export const foreignDBExtractor = () => {
           await pollDatabaseStatus(connectionConfig.database)
 
           // Step 2.5: Retrieve user credentials for the database
-          await tick(85, 'Retrieving database user credentials')
-          const user = (await pollForUser(connectionConfig.database)) as DBUser
-          connectionConfig.user = user.username
-          connectionConfig.password = user.password
-
-          // Step 2.6: Create a Workbook
-          // Get column names for all tables, loop through them and create Sheets for each table
-          await tick(90, 'Creating workbook')
-          const sheets = await generateSheets(connectionConfig)
-
-          // Sheets need to be added to the workbook before adding the connectionType to ensure the correct ephemeral
-          // driver is used
-          await api.workbooks.update(workbook.id, {
-            sheets,
-          })
-
-          // Update workbook with connection config
-          await api.workbooks.update(workbook.id, {
-            metadata: {
-              connectionType: 'FOREIGN_MSSQL',
-              connectionConfig,
-            },
-          })
+          await tick(85, 'Configuring workbook')
+          await configureWorkbook(workbook.id)
 
           // Step 2.7: Update file with workbookId
           await tick(95, 'Updating file')

plugins/foreign-db-extractor/src/database.user.ts

@@ -0,0 +1,34 @@
+import type { Flatfile } from '@flatfile/api'
+import fetch from 'cross-fetch'
+
+/**
+ * The configureWorkbook function is responsible for initiating the workbook configuration. It sends a GET request to
+ * the foreigndb API to configure the workbook. This must be called after the database restore is complete.
+ *
+ * @param workbookId
+ * @returns
+ */
+export async function configureWorkbook(
+  workbookId: string
+): Promise<Flatfile.SuccessData | Error> {
+  const response = await fetch(
+    `${process.env.AGENT_INTERNAL_URL}/v1/foreigndb/${workbookId}/configure`,
+    {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `Bearer ${process.env.FLATFILE_BEARER_TOKEN}`,
+      },
+    }
+  )
+
+  const jsonResponse = await response.json()
+
+  if (response.status !== 200) {
+    throw new Error(
+      `Status: ${response.status} Message: ${jsonResponse.message}`
+    )
+  }
+
+  return jsonResponse
+}