From 5b9d2bcc9d5a354e87bf9a7859a42edadc03e40d Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 15:12:27 +0100 Subject: [PATCH 01/24] Replace PouchDB with native CouchDB _replicate endpoint - Replication: use CouchDB _replicate with design document filter instead of PouchDB's source.replicate.to() - DatabaseImpl: replace PouchDB dump/load with allDocs/bulkDocs, createIndex/deleteIndex with nano/CouchDB _index API - Remove pouchdb, @budibase/pouchdb-replication-stream, pouchdb-adapter-memory dependencies - Remove replication:true config from server/worker db init - Remove pouchdb from esbuild externals - Clean up Replication class: remove close(), appReplicateOpts(), add ReplicateOpts interface --- packages/backend-core/package.json | 4 - packages/backend-core/src/db/Replication.ts | 195 ++++++++++------- .../backend-core/src/db/couch/DatabaseImpl.ts | 104 +++++++-- packages/backend-core/src/db/couch/index.ts | 1 - packages/backend-core/src/db/couch/pouchDB.ts | 92 -------- packages/server/build/pouchdb.js | 9 - packages/server/package.json | 2 - .../src/api/controllers/deploy/index.ts | 206 +++++++++--------- .../server/src/api/controllers/table/index.ts | 41 ++-- packages/server/src/db/index.ts | 4 +- .../server/src/sdk/dev/devRevertProcessor.ts | 2 - .../src/sdk/workspace/workspaces/sync.ts | 5 +- packages/types/src/sdk/db.ts | 2 - packages/worker/package.json | 1 - packages/worker/src/db/index.ts | 4 +- scripts/build.js | 1 - 16 files changed, 325 insertions(+), 348 deletions(-) delete mode 100644 packages/backend-core/src/db/couch/pouchDB.ts delete mode 100644 packages/server/build/pouchdb.js diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 931f148e..486f9532 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -35,7 +35,6 @@ "@aws-sdk/lib-storage": "3.1041.0", "@smithy/node-http-handler": "4.1.1", "nano": "11.0.3", - "@budibase/pouchdb-replication-stream": "1.2.11", "@supertoolmake/shared-core": "*", "@supertoolmake/types": "*", "@techpass/passport-openidconnect": "1.0.0", @@ -62,7 +61,6 @@ "passport-oauth2-refresh": "2.1.0", "pino": "8.11.0", "pino-http": "8.3.3", - "pouchdb": "9.0.0", "redlock": "4.2.0", "rotating-file-stream": "3.1.0", "sanitize-s3-objectkey": "0.0.1", @@ -78,7 +76,6 @@ "@types/koa": "3.0.0", "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", - "@types/pouchdb": "6.4.2", "@types/redlock": "4.0.7", "@types/semver": "7.3.7", "@types/tar-fs": "2.0.4", @@ -86,7 +83,6 @@ "ioredis-mock": "8.13.1", "nock": "^13.5.6", "pino-pretty": "10.0.0", - "pouchdb-adapter-memory": "9.0.0", "timekeeper": "2.2.0" }, "nx": { diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 326a2776..fdf915a2 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,25 +1,30 @@ import { type Document, DocumentType } from "@supertoolmake/types" -import PouchDB from "pouchdb" import { DesignDocuments, SEPARATOR, USER_METADATA_PREFIX } from "../constants" -import { closePouchDB, getPouchDB } from "./couch" +import { newid } from "../docIds/newid" +import { directCouchCall, directCouchQuery } from "./couch" +import { getDB } from "./db" -const _PouchDB = PouchDB // Keep Prettier from removing import +const FILTER_DESIGN_ID = `_design/${newid()}` enum ReplicationDirection { TO_PRODUCTION = "toProduction", TO_DEV = "toDev", } -type DocumentWithID = Omit & { _id: string } +interface ReplicateOpts { + isCreation?: boolean + tablesToSync?: string[] | "all" + filter?: (doc: Document) => boolean +} class Replication { - source: PouchDB.Database - target: PouchDB.Database + sourceName: string + targetName: string direction: ReplicationDirection | undefined constructor({ source, target }: { source: string; target: string }) { - this.source = getPouchDB(source) - this.target = getPouchDB(target) + this.sourceName = source + this.targetName = target if ( source.startsWith(DocumentType.WORKSPACE_DEV) && target.startsWith(DocumentType.WORKSPACE) @@ -33,44 +38,12 @@ class Replication { } } - async close() { - await Promise.all([closePouchDB(this.source), closePouchDB(this.target)]) - } - - replicate(opts: PouchDB.Replication.ReplicateOptions = {}) { - return new Promise>((resolve) => { - this.source.replicate - .to(this.target, opts) - .on("denied", (err) => { - // a document failed to replicate (e.g. due to permissions) - throw new Error(`Denied: Document failed to replicate ${err}`) - }) - .on("complete", (info) => resolve(info)) - .on("error", (err) => { - throw err - }) - }) - } - - appReplicateOpts( - opts: PouchDB.Replication.ReplicateOptions & { - isCreation?: boolean - tablesToSync?: string[] | "all" - } = {} - ): PouchDB.Replication.ReplicateOptions { - if (typeof opts.filter === "string") { - return opts - } - - const filter = opts.filter + async replicate(opts: ReplicateOpts = {}) { const direction = this.direction const toDev = direction === ReplicationDirection.TO_DEV - delete opts.filter - const isCreation = opts.isCreation const tablesToSync = opts.tablesToSync - delete opts.isCreation - delete opts.tablesToSync + const customFilter = opts.filter let syncAllTables = false, tableSyncList: string[] | undefined @@ -80,60 +53,122 @@ class Replication { tableSyncList = tablesToSync } - const startsWithID = (_id: string, documentType: string) => { - return _id?.startsWith(documentType + SEPARATOR) - } + let filterFunction: string + if (customFilter) { + const filterBody = customFilter.toString().replace(/\n/g, " ").replace(/"/g, '\\"') + filterFunction = `function (doc, req) { + var fn = (${filterBody}); + return fn(doc) ? 1 : 0; + }` + } else { + filterFunction = `function (doc, req) { + if (!req.query) { req.query = {}; } + var isCreation = req.query.isCreation === 'true'; + var toDev = req.query.toDev === 'true'; + var syncAllTables = req.query.syncAllTables === 'true'; + var tableSyncList = req.query.tableSyncList ? req.query.tableSyncList.split(',') : []; + + function startsWithID(_id, documentType) { + return _id && _id.indexOf(documentType + '${SEPARATOR}') === 0; + } - const isData = (_id: string) => - startsWithID(_id, DocumentType.ROW) || startsWithID(_id, DocumentType.LINK) + function isData(_id) { + return startsWithID(_id, '${DocumentType.ROW}') || startsWithID(_id, '${DocumentType.LINK}'); + } - return { - ...opts, - filter: (doc: DocumentWithID, params: any) => { - if (!isCreation && doc._id === DesignDocuments.MIGRATIONS) { - return false + if (!isCreation && doc._id === '${DesignDocuments.MIGRATIONS}') { + return false; } - // don't sync design documents - if (toDev && doc._id.startsWith("_design")) { - return false + if (toDev && doc._id.indexOf('_design') === 0) { + return false; } - // always replicate deleted documents if (doc._deleted) { - return true + return true; } - // always sync users from dev - if (startsWithID(doc._id, USER_METADATA_PREFIX)) { - return true + if (startsWithID(doc._id, '${USER_METADATA_PREFIX}')) { + return true; } - if ( - direction === ReplicationDirection.TO_PRODUCTION && - !isCreation && - startsWithID(doc._id, DocumentType.AUTO_COLUMN_STATE) - ) { - return false + if ('${direction}' === '${ReplicationDirection.TO_PRODUCTION}' && !isCreation && startsWithID(doc._id, '${DocumentType.AUTO_COLUMN_STATE}')) { + return false; } if (isData(doc._id)) { - return Boolean(tableSyncList?.find((id) => doc._id.includes(id))) || syncAllTables + if (syncAllTables) { return true; } + for (var i = 0; i < tableSyncList.length; i++) { + if (doc._id.indexOf(tableSyncList[i]) !== -1) { return true; } + } + return false; + } + if (startsWithID(doc._id, '${DocumentType.AUTOMATION_LOG}')) { + return false; } - if (startsWithID(doc._id, DocumentType.AUTOMATION_LOG)) { - return false + if (doc._id === '${DocumentType.WORKSPACE_METADATA}') { + return false; } - if (doc._id === DocumentType.WORKSPACE_METADATA) { - return false + return true; + }` + } + + const sourceDb = getDB(this.sourceName, { skip_setup: true }) + try { + await sourceDb.put({ + _id: FILTER_DESIGN_ID, + filters: { + replication: filterFunction, + }, + } as any) + } catch (err: any) { + if (err.status !== 409) { + throw err + } + const existing = await sourceDb.get(FILTER_DESIGN_ID) + await sourceDb.put({ + _id: FILTER_DESIGN_ID, + _rev: existing._rev, + filters: { + replication: filterFunction, + }, + } as any) + } + + try { + const queryParams: Record = {} + if (!customFilter) { + queryParams.isCreation = String(!!isCreation) + queryParams.toDev = String(!!toDev) + queryParams.syncAllTables = String(!!syncAllTables) + if (tableSyncList) { + queryParams.tableSyncList = tableSyncList.join(",") } - return filter ? filter(doc, params) : true - }, + } + + const replicateBody: any = { + source: this.sourceName, + target: this.targetName, + filter: `${FILTER_DESIGN_ID}/replication`, + query_params: queryParams, + create_target: true, + } + + const response = await directCouchCall("_replicate", "POST", replicateBody) + const result = await response.json() + + if (result.history?.find((h: any) => h.errors && h.errors.length > 0)) { + throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) + } + + return result + } finally { + try { + const existing = await sourceDb.get(FILTER_DESIGN_ID) + await sourceDb.remove(FILTER_DESIGN_ID, existing._rev) + } catch {} } } - /** - * Rollback the target DB back to the state of the source DB - */ async rollback() { - await this.target.destroy() - // Recreate the DB again - this.target = getPouchDB(this.target.name) - // take the opportunity to remove deleted tombstones + const targetDb = getDB(this.targetName, { skip_setup: true }) + await targetDb.destroy() + await directCouchQuery(`/${this.targetName}`, "PUT") await this.replicate() } } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 295cdfa4..ac336f12 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -5,6 +5,7 @@ import { type Database, type DatabaseCreateIndexOpts, type DatabaseDeleteIndexOpts, + type DatabaseDumpOpts, type DatabaseOpts, type DatabasePutOpts, type DatabaseQueryOpts, @@ -19,7 +20,6 @@ import { newid } from "../../docIds/newid" import { checkSlashesInUrl } from "../../helpers" import { DDInstrumentedDatabase } from "../instrumentation" import { getCouchInfo } from "./connections" -import { getPouchDB } from "./pouchDB" import { directCouchUrlCall } from "./utils" const DATABASE_NOT_FOUND = "Database does not exist." @@ -438,27 +438,103 @@ export class CouchDatabase implements Database { }) } - // All below functions are in-frequently called, just utilise PouchDB - // for them as it implements them better than we can - async dump(stream: WriteStream, opts?: { filter?: any }) { - const pouch = getPouchDB(this.name) - // @ts-expect-error - return pouch.dump(stream, opts) + async dump(stream: WriteStream, opts?: DatabaseDumpOpts) { + const batchSize = opts?.batch_size || 1000 + let startKey = "" + let hasMore = true + + const filterFn = opts?.filter + + while (hasMore) { + const params: DocumentListParams = { + include_docs: true, + limit: batchSize, + startkey: startKey || undefined, + } + + const response = await this.allDocs(params) + const rows = response.rows || [] + + if (rows.length === 0) { + hasMore = false + break + } + + for (const row of rows) { + const doc = row.doc as any + if (doc && (!filterFn || filterFn(doc))) { + stream.write(`${JSON.stringify(doc)}\n`) + } + startKey = row.key as string + } + + if (rows.length < batchSize) { + hasMore = false + } + } + + stream.end() } async load(stream: ReadStream) { - const pouch = getPouchDB(this.name) - // @ts-expect-error - return pouch.load(stream) + return new Promise<{ ok: boolean }>((resolve, reject) => { + let buffer = "" + + stream.on("data", async (chunk: string | Buffer) => { + buffer += chunk.toString() + const lines = buffer.split("\n") + // Keep the last incomplete line in the buffer + buffer = lines.pop() || "" + + const docs: any[] = [] + for (const line of lines) { + if (line.trim()) { + try { + docs.push(JSON.parse(line)) + } catch { + // skip malformed lines + } + } + } + + if (docs.length > 0) { + try { + await this.bulkDocs(docs) + } catch (err) { + reject(err) + return + } + } + }) + + stream.on("end", async () => { + // Process any remaining data in the buffer + if (buffer.trim()) { + try { + const doc = JSON.parse(buffer) + await this.bulkDocs([doc]) + } catch { + // skip malformed lines + } + } + resolve({ ok: true }) + }) + + stream.on("error", reject) + }) } async createIndex(opts: DatabaseCreateIndexOpts) { - const pouch = getPouchDB(this.name) - return pouch.createIndex(opts) + return this.performCall((db) => { + return () => db.createIndex(opts) + }) } async deleteIndex(opts: DatabaseDeleteIndexOpts) { - const pouch = getPouchDB(this.name) - return pouch.deleteIndex(opts) + await directCouchUrlCall({ + url: `${this.couchInfo.url}/${this.name}/_index/${opts.ddoc}/${opts.name}`, + method: "DELETE", + cookie: this.couchInfo.cookie, + }) } } diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 6ef237f1..a3e0f56e 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -1,4 +1,3 @@ export * from "./connections" export * from "./DatabaseImpl" -export { closePouchDB, getPouch, getPouchDB, init } from "./pouchDB" export * from "./utils" diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts deleted file mode 100644 index 6257b560..00000000 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { PouchOptions } from "@supertoolmake/types" -import PouchDB from "pouchdb" -import env from "../../environment" -import { getCouchInfo } from "./connections" - -let Pouch: any -let initialised = false - -/** - * Return a constructor for PouchDB. - * This should be rarely used outside of the main application config. - * Exposed for exceptional cases such as in-memory views. - */ -export const getPouch = (opts: PouchOptions = {}) => { - const { url, cookie } = getCouchInfo() - let POUCH_DB_DEFAULTS = { - prefix: url, - fetch: (url: string, opts: any) => { - // use a specific authorization cookie - be very explicit about how we authenticate - opts.headers.set("Authorization", cookie) - return PouchDB.fetch(url, opts) - }, - } - - if (opts.inMemory) { - const inMemory = require("pouchdb-adapter-memory") - PouchDB.plugin(inMemory) - POUCH_DB_DEFAULTS = { - // @ts-expect-error - adapter: "memory", - } - } - - if (opts.onDisk) { - POUCH_DB_DEFAULTS = { - // @ts-expect-error - adapter: "leveldb", - } - } - - if (opts.replication) { - const replicationStream = require("@budibase/pouchdb-replication-stream") - PouchDB.plugin(replicationStream.plugin) - // @ts-expect-error - PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) - } - - return PouchDB.defaults(POUCH_DB_DEFAULTS) -} - -export function init(opts?: PouchOptions) { - Pouch = getPouch(opts) - initialised = true -} - -const checkInitialised = () => { - if (!initialised) { - throw new Error("init has not been called") - } -} - -export function getPouchDB(dbName: string, opts?: any): PouchDB.Database { - checkInitialised() - const db = new Pouch(dbName, opts) - const dbPut = db.put - db.put = async (doc: any, options = {}) => { - if (!doc.createdAt) { - doc.createdAt = new Date().toISOString() - } - doc.updatedAt = new Date().toISOString() - return dbPut(doc, options) - } - db.exists = async () => { - const info = await db.info() - return !info.error - } - return db -} - -// use this function if you have called getPouchDB - close -// the databases you've opened once finished -export async function closePouchDB(db: PouchDB.Database) { - if (!db || env.isTest()) { - return - } - try { - // specifically await so that if there is an error, it can be ignored - return await db.close() - } catch { - // ignore error, already closed - } -} diff --git a/packages/server/build/pouchdb.js b/packages/server/build/pouchdb.js deleted file mode 100644 index 16942a7a..00000000 --- a/packages/server/build/pouchdb.js +++ /dev/null @@ -1,9 +0,0 @@ -function CouchDB() { - this.post = jest.fn() - this.allDocs = jest.fn(() => ({ rows: [] })) - this.put = jest.fn() - this.remove = jest.fn() - this.plugin = jest.fn() -} - -module.exports = CouchDB diff --git a/packages/server/package.json b/packages/server/package.json index 64d24c69..96ffe8b9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -86,7 +86,6 @@ "node-fetch": "2.6.7", "openapi-types": "9.3.1", "pg": "8.10.0", - "pouchdb": "9.0.0", "redis": "4", "semver": "7.5.4", "serialize-error": "7.0.1", @@ -114,7 +113,6 @@ "@types/lodash": "4.14.200", "@types/node-fetch": "2.6.4", "@types/pg": "8.6.6", - "@types/pouchdb": "6.4.2", "@types/semver": "7.3.7", "@types/swagger-jsdoc": "^6.0.4", "@types/tar": "6.1.5", diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 5bce8aca..3e24f18c 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -190,125 +190,115 @@ export const publishWorkspaceInternal = async (ctx: PublishContext) => { try { const publish = async () => { - let replication - try { - const devId = dbCore.getDevWorkspaceID(appId) - const prodId = dbCore.getProdWorkspaceID(appId) - - if (!(await sdk.workspaces.isWorkspacePublished(prodId))) { - const allWorkspaceApps = await sdk.workspaceApps.fetch() - for (const workspaceApp of allWorkspaceApps) { - if (workspaceApp.disabled !== undefined) { - continue - } - - await sdk.workspaceApps.update({ ...workspaceApp, disabled: true }) + const devId = dbCore.getDevWorkspaceID(appId) + const prodId = dbCore.getProdWorkspaceID(appId) + + if (!(await sdk.workspaces.isWorkspacePublished(prodId))) { + const allWorkspaceApps = await sdk.workspaceApps.fetch() + for (const workspaceApp of allWorkspaceApps) { + if (workspaceApp.disabled !== undefined) { + continue } - } - const isPublished = await sdk.workspaces.isWorkspacePublished(prodId) - const config = { - source: devId, - target: prodId, + await sdk.workspaceApps.update({ ...workspaceApp, disabled: true }) } - replication = new dbCore.Replication(config) - const devDb = context.getDevWorkspaceDB() - - await devDb.compact() - await replication.replicate( - replication.appReplicateOpts({ - isCreation: !isPublished, - tablesToSync, - checkpoint: false, - }) - ) - - const updatedProdTables = await applyPendingColumnRenames(prodId) - - // Keep development revs aligned with production after renaming, so the - // next publish doesn't see production ahead and delete / tombstone the doc. - if (updatedProdTables.length > 0) { - await context.doInWorkspaceContext(devId, async () => { - const devDb = context.getWorkspaceDB() - for (const prodTable of updatedProdTables) { - try { - const devTable = await devDb.tryGet(prodTable._id!) - if (!devTable) { - console.warn( - `Failed to get development table for table ${prodTable._id} when applying column renames` - ) - continue - } - - const prodRevNum = getRevisionNumber(prodTable._rev) - let devRevNum = getRevisionNumber(devTable._rev) - - let docForPut: Table = { - ...prodTable, - _rev: devTable._rev, - } - - // Bump dev revisions until dev is at least prod (avoids prod-ahead). - while (devRevNum < prodRevNum) { - const res = await devDb.put(docForPut) - docForPut = { ...docForPut, _rev: res.rev } - devRevNum = getRevisionNumber(res.rev) - } - - // Ensure final content is written with the latest dev rev (handles equal case). - await devDb.put(docForPut) - } catch (err) { + } + + const isPublished = await sdk.workspaces.isWorkspacePublished(prodId) + const config = { + source: devId, + target: prodId, + } + const replication = new dbCore.Replication(config) + const devDb = context.getDevWorkspaceDB() + + await devDb.compact() + await replication.replicate({ + isCreation: !isPublished, + tablesToSync, + }) + + const updatedProdTables = await applyPendingColumnRenames(prodId) + + // Keep development revs aligned with production after renaming, so the + // next publish doesn't see production ahead and delete / tombstone the doc. + if (updatedProdTables.length > 0) { + await context.doInWorkspaceContext(devId, async () => { + const devDb = context.getWorkspaceDB() + for (const prodTable of updatedProdTables) { + try { + const devTable = await devDb.tryGet
(prodTable._id!) + if (!devTable) { console.warn( - `Failed to update development table with production table - revision when applying column renames for table ${prodTable._id}: ${err}` + `Failed to get development table for table ${prodTable._id} when applying column renames` ) + continue } - } - }) - } - await clearPendingColumnRenames(devId) + const prodRevNum = getRevisionNumber(prodTable._rev) + let devRevNum = getRevisionNumber(devTable._rev) - const db = context.getProdWorkspaceDB() - const appDoc = await sdk.workspaces.metadata.tryGet({ - production: false, - }) - if (!appDoc) { - throw new Error("Unable to publish - cannot retrieve development app metadata") - } - const prodAppDoc = await sdk.workspaces.metadata.tryGet({ - production: true, + let docForPut: Table = { + ...prodTable, + _rev: devTable._rev, + } + + // Bump dev revisions until dev is at least prod (avoids prod-ahead). + while (devRevNum < prodRevNum) { + const res = await devDb.put(docForPut) + docForPut = { ...docForPut, _rev: res.rev } + devRevNum = getRevisionNumber(res.rev) + } + + // Ensure final content is written with the latest dev rev (handles equal case). + await devDb.put(docForPut) + } catch (err) { + console.warn( + `Failed to update development table with production table + revision when applying column renames for table ${prodTable._id}: ${err}` + ) + } + } }) - if (prodAppDoc) { - appDoc._rev = prodAppDoc._rev - } else { - delete appDoc._rev - } + } - deployment.appUrl = appDoc.url - appDoc.appId = prodId - appDoc.instance._id = prodId - const [workspaceApps, tables] = await Promise.all([ - sdk.workspaceApps.fetch(), - sdk.tables.getAllInternalTables(), - ]) - const workspaceAppIds = workspaceApps.map((app) => app._id!) - const tableIds = tables.map((table) => table._id!) - const fullMap = [...(workspaceAppIds ?? []), ...(tableIds ?? [])] - appDoc.resourcesPublishedAt = { - ...prodAppDoc?.resourcesPublishedAt, - ...Object.fromEntries(fullMap.map((id) => [id, new Date().toISOString()])), - } - delete appDoc.automationErrors - await db.put(appDoc) - await cache.workspace.invalidateWorkspaceMetadata(prodId) - - return { app: appDoc, prodWorkspaceId: prodId } - } finally { - if (replication) { - await replication.close() - } + await clearPendingColumnRenames(devId) + + const db = context.getProdWorkspaceDB() + const appDoc = await sdk.workspaces.metadata.tryGet({ + production: false, + }) + if (!appDoc) { + throw new Error("Unable to publish - cannot retrieve development app metadata") + } + const prodAppDoc = await sdk.workspaces.metadata.tryGet({ + production: true, + }) + if (prodAppDoc) { + appDoc._rev = prodAppDoc._rev + } else { + delete appDoc._rev } + + deployment.appUrl = appDoc.url + appDoc.appId = prodId + appDoc.instance._id = prodId + const [workspaceApps, tables] = await Promise.all([ + sdk.workspaceApps.fetch(), + sdk.tables.getAllInternalTables(), + ]) + const workspaceAppIds = workspaceApps.map((app) => app._id!) + const tableIds = tables.map((table) => table._id!) + const fullMap = [...(workspaceAppIds ?? []), ...(tableIds ?? [])] + appDoc.resourcesPublishedAt = { + ...prodAppDoc?.resourcesPublishedAt, + ...Object.fromEntries(fullMap.map((id) => [id, new Date().toISOString()])), + } + delete appDoc.automationErrors + await db.put(appDoc) + await cache.workspace.invalidateWorkspaceMetadata(prodId) + + return { app: appDoc, prodWorkspaceId: prodId } } await publish() } catch (error: unknown) { diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 13c83e9f..39c2db97 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -255,28 +255,25 @@ export async function publish(ctx: UserCtx { - const _id = doc?._id as string - if (!_id || _id.startsWith("_design")) { - return false - } - if (_id.startsWith(DocumentType.AUTOMATION_LOG)) { - return false - } - if (_id.startsWith(DocumentType.WORKSPACE_METADATA)) { - return false - } - if (!matchesTable(_id)) { - return false - } - return true - }, - }) - ) + await replication.replicate({ + tablesToSync: undefined, + filter: (doc: any) => { + const _id = doc?._id as string + if (!_id || _id.startsWith("_design")) { + return false + } + if (_id.startsWith(DocumentType.AUTOMATION_LOG)) { + return false + } + if (_id.startsWith(DocumentType.WORKSPACE_METADATA)) { + return false + } + if (!matchesTable(_id)) { + return false + } + return true + }, + }) const metadata = await sdk.workspaces.metadata.tryGet({ production: true, diff --git a/packages/server/src/db/index.ts b/packages/server/src/db/index.ts index ad734e01..39d40e88 100644 --- a/packages/server/src/db/index.ts +++ b/packages/server/src/db/index.ts @@ -2,9 +2,7 @@ import * as core from "@supertoolmake/backend-core" import env from "../environment" export function init() { - const dbConfig: any = { - replication: true, - } + const dbConfig: any = {} if (env.isTest() && !env.COUCH_DB_URL) { dbConfig.inMemory = true diff --git a/packages/server/src/sdk/dev/devRevertProcessor.ts b/packages/server/src/sdk/dev/devRevertProcessor.ts index 3ec4693d..4ce23b40 100644 --- a/packages/server/src/sdk/dev/devRevertProcessor.ts +++ b/packages/server/src/sdk/dev/devRevertProcessor.ts @@ -59,8 +59,6 @@ class DevRevertProcessor extends queue.QueuedProcessor { return { message: "Reverted changes successfully." } } catch (err) { throw new Error(`Unable to revert. ${err}`, { cause: err }) - } finally { - await replication.close() } } } diff --git a/packages/server/src/sdk/workspace/workspaces/sync.ts b/packages/server/src/sdk/workspace/workspaces/sync.ts index c7c68824..2298e5d3 100644 --- a/packages/server/src/sdk/workspace/workspaces/sync.ts +++ b/packages/server/src/sdk/workspace/workspaces/sync.ts @@ -173,12 +173,9 @@ export async function syncWorkspace(workspaceId: string): Promise<{ message: str target: workspaceId, }) try { - const replOpts = replication.appReplicateOpts() - await replication.replicate(replOpts) + await replication.replicate() } catch (err) { error = err - } finally { - await replication.close() } } diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index fd48f2b9..d7630253 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -152,8 +152,6 @@ export interface Database { ): Promise> destroy(): Promise compact(): Promise - // these are all PouchDB related functions that are rarely used - in future - // should be replaced by better typed/non-pouch implemented methods dump(stream: Writable, opts?: DatabaseDumpOpts): Promise load(stream: ReadStream): Promise } diff --git a/packages/worker/package.json b/packages/worker/package.json index 33632586..b1814b99 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -62,7 +62,6 @@ "nodemailer": "9.0.1", "passport-google-oauth": "2.0.0", "passport-local": "1.0.0", - "pouchdb": "7.3.0", "undici": "7.28.0", "yaml": "2.8.3" }, diff --git a/packages/worker/src/db/index.ts b/packages/worker/src/db/index.ts index ad734e01..39d40e88 100644 --- a/packages/worker/src/db/index.ts +++ b/packages/worker/src/db/index.ts @@ -2,9 +2,7 @@ import * as core from "@supertoolmake/backend-core" import env from "../environment" export function init() { - const dbConfig: any = { - replication: true, - } + const dbConfig: any = {} if (env.isTest() && !env.COUCH_DB_URL) { dbConfig.inMemory = true diff --git a/scripts/build.js b/scripts/build.js index 32adc887..2240971b 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -107,7 +107,6 @@ async function runBuild(entry, outfile) { "mock-aws-s3", "nock", "bull", - "pouchdb", "bcrypt", "bcryptjs", "graphql/*", From 591baf886ae2add9c60e5b578e8d2537ddd3e9e8 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 16:45:17 +0100 Subject: [PATCH 02/24] fix: replace broken design doc filter with post-replication cleanup CouchDB 3.5.1 _replicate fails with design doc filters over HTTP URLs (changes_req_failed,400). Replaced with replicate-all + cleanup approach: 1. _replicate all docs (no filter) 2. cleanupTarget() reads target docs and bulk-deletes those not matching the filter logic (ported from JS filter to TypeScript shouldKeepDoc) 3. 17 end-to-end integration tests (no couch mocking) verifying direction, replication, filtering, and rollback behavior --- packages/backend-core/src/db/Replication.ts | 213 +++--- .../src/db/tests/Replication.spec.ts | 608 +++++++----------- 2 files changed, 353 insertions(+), 468 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index fdf915a2..0f879b67 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,11 +1,8 @@ import { type Document, DocumentType } from "@supertoolmake/types" import { DesignDocuments, SEPARATOR, USER_METADATA_PREFIX } from "../constants" -import { newid } from "../docIds/newid" -import { directCouchCall, directCouchQuery } from "./couch" +import { directCouchCall, directCouchQuery, getCouchInfo } from "./couch" import { getDB } from "./db" -const FILTER_DESIGN_ID = `_design/${newid()}` - enum ReplicationDirection { TO_PRODUCTION = "toProduction", TO_DEV = "toDev", @@ -38,6 +35,74 @@ class Replication { } } + private shouldKeepDoc( + doc: { _id: string; _deleted?: boolean }, + opts: { + isCreation?: boolean + toDev: boolean + syncAllTables: boolean + tableSyncList?: string[] + customFilter?: (doc: Document) => boolean + } + ): boolean { + const { isCreation, toDev, syncAllTables, tableSyncList, customFilter } = opts + const toProduction = !toDev + + const startsWithType = (id: string, docType: string) => + id && id.indexOf(docType + SEPARATOR) === 0 + + const isData = (id: string) => + startsWithType(id, DocumentType.ROW) || startsWithType(id, DocumentType.LINK) + + if (doc._deleted) { + return true + } + + if (!isCreation && doc._id === DesignDocuments.MIGRATIONS) { + return false + } + + if (toDev && doc._id.indexOf("_design") === 0) { + return false + } + + if (doc._id && doc._id.indexOf(USER_METADATA_PREFIX) === 0) { + return true + } + + if (toProduction && !isCreation && startsWithType(doc._id, DocumentType.AUTO_COLUMN_STATE)) { + return false + } + + if (isData(doc._id)) { + if (syncAllTables) { + return true + } + if (tableSyncList) { + for (const tableId of tableSyncList) { + if (doc._id.indexOf(tableId) !== -1) { + return true + } + } + } + return false + } + + if (startsWithType(doc._id, DocumentType.AUTOMATION_LOG)) { + return false + } + + if (doc._id === DocumentType.WORKSPACE_METADATA) { + return false + } + + if (customFilter) { + return customFilter(doc as Document) + } + + return true + } + async replicate(opts: ReplicateOpts = {}) { const direction = this.direction const toDev = direction === ReplicationDirection.TO_DEV @@ -53,115 +118,51 @@ class Replication { tableSyncList = tablesToSync } - let filterFunction: string - if (customFilter) { - const filterBody = customFilter.toString().replace(/\n/g, " ").replace(/"/g, '\\"') - filterFunction = `function (doc, req) { - var fn = (${filterBody}); - return fn(doc) ? 1 : 0; - }` - } else { - filterFunction = `function (doc, req) { - if (!req.query) { req.query = {}; } - var isCreation = req.query.isCreation === 'true'; - var toDev = req.query.toDev === 'true'; - var syncAllTables = req.query.syncAllTables === 'true'; - var tableSyncList = req.query.tableSyncList ? req.query.tableSyncList.split(',') : []; - - function startsWithID(_id, documentType) { - return _id && _id.indexOf(documentType + '${SEPARATOR}') === 0; - } + const { url, auth } = getCouchInfo() + const parsed = new URL(url) + const internalUrl = `http://${auth.username}:${auth.password}@${parsed.hostname}:5984` + const replicateBody: any = { + source: `${internalUrl}/${this.sourceName}`, + target: `${internalUrl}/${this.targetName}`, + create_target: true, + } - function isData(_id) { - return startsWithID(_id, '${DocumentType.ROW}') || startsWithID(_id, '${DocumentType.LINK}'); - } + const response = await directCouchCall("_replicate", "POST", replicateBody) + const result = await response.json() - if (!isCreation && doc._id === '${DesignDocuments.MIGRATIONS}') { - return false; - } - if (toDev && doc._id.indexOf('_design') === 0) { - return false; - } - if (doc._deleted) { - return true; - } - if (startsWithID(doc._id, '${USER_METADATA_PREFIX}')) { - return true; - } - if ('${direction}' === '${ReplicationDirection.TO_PRODUCTION}' && !isCreation && startsWithID(doc._id, '${DocumentType.AUTO_COLUMN_STATE}')) { - return false; - } - if (isData(doc._id)) { - if (syncAllTables) { return true; } - for (var i = 0; i < tableSyncList.length; i++) { - if (doc._id.indexOf(tableSyncList[i]) !== -1) { return true; } - } - return false; - } - if (startsWithID(doc._id, '${DocumentType.AUTOMATION_LOG}')) { - return false; - } - if (doc._id === '${DocumentType.WORKSPACE_METADATA}') { - return false; - } - return true; - }` - } - - const sourceDb = getDB(this.sourceName, { skip_setup: true }) - try { - await sourceDb.put({ - _id: FILTER_DESIGN_ID, - filters: { - replication: filterFunction, - }, - } as any) - } catch (err: any) { - if (err.status !== 409) { - throw err - } - const existing = await sourceDb.get(FILTER_DESIGN_ID) - await sourceDb.put({ - _id: FILTER_DESIGN_ID, - _rev: existing._rev, - filters: { - replication: filterFunction, - }, - } as any) - } - - try { - const queryParams: Record = {} - if (!customFilter) { - queryParams.isCreation = String(!!isCreation) - queryParams.toDev = String(!!toDev) - queryParams.syncAllTables = String(!!syncAllTables) - if (tableSyncList) { - queryParams.tableSyncList = tableSyncList.join(",") - } - } + if (result.history?.find((h: any) => h.errors && h.errors.length > 0)) { + throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) + } - const replicateBody: any = { - source: this.sourceName, - target: this.targetName, - filter: `${FILTER_DESIGN_ID}/replication`, - query_params: queryParams, - create_target: true, - } + const filterOpts = { isCreation, toDev, syncAllTables, tableSyncList, customFilter } + await this.cleanupTarget(filterOpts) + + return result + } - const response = await directCouchCall("_replicate", "POST", replicateBody) - const result = await response.json() + private async cleanupTarget(opts: { + isCreation?: boolean + toDev: boolean + syncAllTables: boolean + tableSyncList?: string[] + customFilter?: (doc: Document) => boolean + }) { + const targetDb = getDB(this.targetName, { skip_setup: true }) + const allDocs = await targetDb.allDocs({ include_docs: true }) + const docsToDelete: { _id: string; _rev: string; _deleted: true }[] = [] - if (result.history?.find((h: any) => h.errors && h.errors.length > 0)) { - throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) + for (const row of allDocs.rows) { + const doc = row.doc as any + if (!doc || !doc._id) { + continue + } + if (!this.shouldKeepDoc(doc, opts)) { + docsToDelete.push({ _id: doc._id, _rev: doc._rev, _deleted: true }) } + } - return result - } finally { - try { - const existing = await sourceDb.get(FILTER_DESIGN_ID) - await sourceDb.remove(FILTER_DESIGN_ID, existing._rev) - } catch {} + if (docsToDelete.length > 0) { + await targetDb.bulkDocs(docsToDelete) } } diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index ec39aed7..050a2a9b 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -1,20 +1,25 @@ -import { DocumentType, InternalTable, SEPARATOR } from "@supertoolmake/types" +import { DocumentType } from "@supertoolmake/types" +import { getDB } from "../db" import { structures } from "../../../tests" -import { getPouchDB, init as initPouch } from "../couch" import Replication from "../Replication" -const USER_METADATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}` - -beforeAll(() => { - initPouch() -}) +const ensureDb = async (dbName: string) => { + const db = getDB(dbName) + const initId = `test_init_${structures.db.id()}` + await db.put({ _id: initId, init: true }) + await db.remove(initId, (await db.get(initId))._rev) +} const makeDoc = (id: string, extra: Record = {}) => ({ _id: id, - _rev: "1-abc", ...extra, }) +const getAllDocIds = async (dbName: string) => { + const allDocs = await getDB(dbName).allDocs({ include_docs: false }) + return allDocs.rows.map((r: any) => r.id) +} + describe("Replication", () => { describe("constructor", () => { it("sets direction to TO_PRODUCTION when source is workspace dev and target is workspace", () => { @@ -22,6 +27,8 @@ describe("Replication", () => { const target = `${DocumentType.WORKSPACE}_test` const rep = new Replication({ source, target }) expect(rep.direction).toBe("toProduction") + expect(rep.sourceName).toBe(source) + expect(rep.targetName).toBe(target) }) it("sets direction to TO_DEV when source is workspace and target is workspace dev", () => { @@ -39,396 +46,273 @@ describe("Replication", () => { }) }) - describe("appReplicateOpts", () => { - let rep: Replication - - beforeEach(() => { - const source = `${DocumentType.WORKSPACE_DEV}_test` - const target = `${DocumentType.WORKSPACE}_test` - rep = new Replication({ source, target }) - }) + describe("replicate", () => { + it("replicates documents from source to target", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) - it("returns opts as-is if filter is a string", () => { - const opts = { filter: "my_design/my_filter" } - const result = rep.appReplicateOpts(opts) - expect(result).toEqual(opts) - }) + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) - it("removes isCreation and tablesToSync from returned opts", () => { - const result = rep.appReplicateOpts({ - isCreation: true, - tablesToSync: ["ta_table1"], - }) - expect(result.isCreation).toBeUndefined() - expect(result.tablesToSync).toBeUndefined() - }) + const rep = new Replication({ source, target }) + await rep.replicate() - describe("filter function", () => { - const getFilter = (direction: "toProduction" | "toDev") => { - const source = - direction === "toDev" - ? `${DocumentType.WORKSPACE}_test` - : `${DocumentType.WORKSPACE_DEV}_test` - const target = - direction === "toDev" - ? `${DocumentType.WORKSPACE_DEV}_test` - : `${DocumentType.WORKSPACE}_test` - const r = new Replication({ source, target }) - return r.appReplicateOpts({}) - } - - const filterDoc = ( - direction: "toProduction" | "toDev", - doc: any, - extra: Record = {} - ) => { - const opts = getFilter(direction) - return (opts.filter as Function)(doc, extra) - } - - it("excludes design/migrations document when not creating", () => { - const doc = makeDoc("_design/migrations") - expect(filterDoc("toProduction", doc, { isCreation: false })).toBe(false) - }) + const doc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + expect(doc).toMatchObject({ name: "admin" }) + }, 30000) - it("includes design/migrations document when creating", () => { - const doc = makeDoc("_design/migrations") - const opts = rep.appReplicateOpts({ isCreation: true }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(true) - }) + it("replicates multiple documents", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) - it("excludes all design documents when replicating to dev", () => { - const doc = makeDoc("_design/some_view") - expect(filterDoc("toDev", doc)).toBe(false) - }) + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + makeDoc(`${DocumentType.ROLE}_user`, { name: "user" }), + makeDoc(`${DocumentType.DATASOURCE}_ds1`, { type: "postgres" }), + ]) - it("includes design documents when replicating to production", () => { - const doc = makeDoc("_design/some_view") - expect(filterDoc("toProduction", doc)).toBe(true) - }) + const rep = new Replication({ source, target }) + await rep.replicate() - it("includes deleted documents", () => { - const doc = makeDoc(`${DocumentType.ROW}_abc`, { _deleted: true }) - expect(filterDoc("toProduction", doc)).toBe(true) - expect(filterDoc("toDev", doc)).toBe(true) - }) + expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) + expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) + expect(await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`)).toMatchObject({ type: "postgres" }) + }, 30000) - it("includes user metadata documents", () => { - const doc = makeDoc(`${USER_METADATA_PREFIX}_user123`) - expect(filterDoc("toProduction", doc)).toBe(true) - expect(filterDoc("toDev", doc)).toBe(true) - }) + it("excludes automation log documents", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) - it("excludes automation log documents", () => { - const doc = makeDoc(`${DocumentType.AUTOMATION_LOG}_abc`) - expect(filterDoc("toProduction", doc)).toBe(false) - expect(filterDoc("toDev", doc)).toBe(false) - }) + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.AUTOMATION_LOG}_log1`, { data: "log" }), + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + ]) - it("excludes workspace metadata document", () => { - const doc = makeDoc(DocumentType.WORKSPACE_METADATA) - expect(filterDoc("toProduction", doc)).toBe(false) - expect(filterDoc("toDev", doc)).toBe(false) - }) + const rep = new Replication({ source, target }) + await rep.replicate() - it("excludes auto column state when not creating and replicating to production", () => { - const doc = makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_abc`) - expect(filterDoc("toProduction", doc)).toBe(false) - }) + const ids = await getAllDocIds(target) + expect(ids).not.toContain(`${DocumentType.AUTOMATION_LOG}_log1`) + expect(ids).toContain(`${DocumentType.ROLE}_admin`) + }, 30000) - it("includes auto column state when creating and replicating to production", () => { - const doc = makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_abc`) - const opts = rep.appReplicateOpts({ isCreation: true }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(true) - }) + it("does not replicate data documents by default (no tablesToSync)", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) - it("includes auto column state when replicating to dev", () => { - const doc = makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_abc`) - expect(filterDoc("toDev", doc)).toBe(true) - }) + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROW}_ta_table1_abc`, { value: 1 }), + makeDoc(`${DocumentType.LINK}_ta_table1_def`, { link: true }), + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + ]) - describe("data documents (rows and links)", () => { - it("includes row documents when replicating all tables", () => { - const doc = makeDoc(`${DocumentType.ROW}_ta_table1_abc`) - const opts = rep.appReplicateOpts({ tablesToSync: "all" }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(true) - }) - - it("includes link documents when replicating all tables", () => { - const doc = makeDoc(`${DocumentType.LINK}_ta_table1_abc`) - const opts = rep.appReplicateOpts({ tablesToSync: "all" }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(true) - }) - - it("excludes data documents when no tables specified", () => { - const doc = makeDoc(`${DocumentType.ROW}_ta_table1_abc`) - const opts = rep.appReplicateOpts({}) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(false) - }) - - it("excludes data documents when tablesToSync does not match", () => { - const doc = makeDoc(`${DocumentType.ROW}_ta_table1_abc`) - const opts = rep.appReplicateOpts({ tablesToSync: ["ta_table2"] }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(false) - }) - - it("includes data documents when tablesToSync matches", () => { - const doc = makeDoc(`${DocumentType.ROW}_ta_table1_abc`) - const opts = rep.appReplicateOpts({ tablesToSync: ["ta_table1"] }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(true) - }) - }) + const rep = new Replication({ source, target }) + await rep.replicate() - describe("non-data documents", () => { - it("includes app metadata documents", () => { - const doc = makeDoc(`${DocumentType.WORKSPACE}_test`) - expect(filterDoc("toProduction", doc)).toBe(true) - }) - - it("includes role documents", () => { - const doc = makeDoc(`${DocumentType.ROLE}_admin`) - expect(filterDoc("toProduction", doc)).toBe(true) - }) - - it("includes datasource documents", () => { - const doc = makeDoc(`${DocumentType.DATASOURCE}_abc`) - expect(filterDoc("toProduction", doc)).toBe(true) - }) - - it("includes screen documents", () => { - const doc = makeDoc(`${DocumentType.SCREEN}_abc`) - expect(filterDoc("toProduction", doc)).toBe(true) - }) - - it("includes layout documents", () => { - const doc = makeDoc(`${DocumentType.LAYOUT}_abc`) - expect(filterDoc("toProduction", doc)).toBe(true) - }) - }) + const ids = await getAllDocIds(target) + expect(ids).not.toContain(`${DocumentType.ROW}_ta_table1_abc`) + expect(ids).not.toContain(`${DocumentType.LINK}_ta_table1_def`) + expect(ids).toContain(`${DocumentType.ROLE}_admin`) + }, 30000) - it("delegates to custom filter when one is provided", () => { - const customFilter = jest.fn().mockReturnValue(true) - const doc = makeDoc(`${DocumentType.ROLE}_admin`) - const opts = rep.appReplicateOpts({ filter: customFilter }) - const filter = opts.filter as Function - const result = filter(doc, { query: {} }) - expect(result).toBe(true) - expect(customFilter).toHaveBeenCalledWith(doc, { query: {} }) - }) + it("replicates all rows and links when tablesToSync is 'all'", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) - it("custom filter can reject documents", () => { - const customFilter = jest.fn().mockReturnValue(false) - const doc = makeDoc(`${DocumentType.ROLE}_admin`) - const opts = rep.appReplicateOpts({ filter: customFilter }) - const filter = opts.filter as Function - expect(filter(doc, {})).toBe(false) - }) + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROW}_ta_table1_abc`, { value: 1 }), + makeDoc(`${DocumentType.LINK}_ta_table1_def`, { link: true }), + makeDoc(`${DocumentType.ROW}_ta_table2_xyz`, { value: 2 }), + ]) - it("passes through non-data docs to custom filter", () => { - const customFilter = jest.fn().mockReturnValue(true) - const doc = makeDoc(`${DocumentType.ROLE}_admin`) - const opts = rep.appReplicateOpts({ filter: customFilter }) - const filter = opts.filter as Function - filter(doc, {}) - expect(customFilter).toHaveBeenCalled() - }) + const rep = new Replication({ source, target }) + await rep.replicate({ tablesToSync: "all" }) - it("does not pass data docs to custom filter when tablesToSync is set", () => { - const customFilter = jest.fn().mockReturnValue(true) - const doc = makeDoc(`${DocumentType.ROW}_ta_table1_abc`) - const opts = rep.appReplicateOpts({ - filter: customFilter, - tablesToSync: "all", - }) - const filter = opts.filter as Function - const result = filter(doc, {}) - expect(result).toBe(true) - expect(customFilter).not.toHaveBeenCalled() - }) - }) - }) + const ids = await getAllDocIds(target) + expect(ids).toContain(`${DocumentType.ROW}_ta_table1_abc`) + expect(ids).toContain(`${DocumentType.LINK}_ta_table1_def`) + expect(ids).toContain(`${DocumentType.ROW}_ta_table2_xyz`) + }, 30000) - describe("replicate", () => { - it("replicates documents from source to target", async () => { + it("replicates only matching rows when tablesToSync is a list", async () => { const source = structures.db.id() const target = structures.db.id() - const sourceDB = getPouchDB(source) - const targetDB = getPouchDB(target) - - try { - await sourceDB.put({ - _id: `${DocumentType.ROLE}_admin`, - name: "admin", - }) - - const rep = new Replication({ source, target }) - const result = await rep.replicate() - - expect(result.ok).toBe(true) - expect(result.docs_written).toBeGreaterThan(0) - - const doc = await targetDB.get(`${DocumentType.ROLE}_admin`) - expect(doc.name).toBe("admin") - } finally { - await sourceDB.destroy() - await targetDB.destroy() - } - }) + await ensureDb(source) + await ensureDb(target) - it("replicates multiple documents", async () => { - const source = structures.db.id() - const target = structures.db.id() - const sourceDB = getPouchDB(source) - const targetDB = getPouchDB(target) - - try { - await sourceDB.bulkDocs([ - { _id: `${DocumentType.ROLE}_admin`, name: "admin" }, - { _id: `${DocumentType.ROLE}_user`, name: "user" }, - { _id: `${DocumentType.DATASOURCE}_ds1`, type: "postgres" }, - ]) - - const rep = new Replication({ source, target }) - const result = await rep.replicate() - - expect(result.ok).toBe(true) - expect(result.docs_written).toBeGreaterThanOrEqual(3) - - const adminDoc = await targetDB.get(`${DocumentType.ROLE}_admin`) - expect(adminDoc.name).toBe("admin") - const userDoc = await targetDB.get(`${DocumentType.ROLE}_user`) - expect(userDoc.name).toBe("user") - const dsDoc = await targetDB.get(`${DocumentType.DATASOURCE}_ds1`) - expect(dsDoc.type).toBe("postgres") - } finally { - await sourceDB.destroy() - await targetDB.destroy() - } - }) + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROW}_ta_table1_abc`, { value: 1 }), + makeDoc(`${DocumentType.ROW}_ta_table2_xyz`, { value: 2 }), + ]) + + const rep = new Replication({ source, target }) + await rep.replicate({ tablesToSync: ["ta_table1"] }) - it("replicates with filter options", async () => { + const ids = await getAllDocIds(target) + expect(ids).toContain(`${DocumentType.ROW}_ta_table1_abc`) + expect(ids).not.toContain(`${DocumentType.ROW}_ta_table2_xyz`) + }, 30000) + + it("replicates with custom filter", async () => { const source = structures.db.id() const target = structures.db.id() - const sourceDB = getPouchDB(source) - const targetDB = getPouchDB(target) - - try { - await sourceDB.bulkDocs([ - { _id: `${DocumentType.ROLE}_admin`, name: "admin" }, - { _id: `${DocumentType.ROLE}_user`, name: "user" }, - ]) - - const rep = new Replication({ source, target }) - const opts = { - filter: (doc: any) => doc._id.includes("admin"), - } - await rep.replicate(opts) - - const adminDoc = await targetDB.get(`${DocumentType.ROLE}_admin`) - expect(adminDoc.name).toBe("admin") - - try { - await targetDB.get(`${DocumentType.ROLE}_user`) - fail("Expected doc not found") - } catch (err: any) { - expect(err.status).toBe(404) - } - } finally { - await sourceDB.destroy() - await targetDB.destroy() - } - }) + await ensureDb(source) + await ensureDb(target) + + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + makeDoc(`${DocumentType.ROLE}_user`, { name: "user" }), + ]) + + const rep = new Replication({ source, target }) + await rep.replicate({ filter: (doc: any) => doc._id.includes("admin") }) + + const ids = await getAllDocIds(target) + expect(ids).toContain(`${DocumentType.ROLE}_admin`) + expect(ids).not.toContain(`${DocumentType.ROLE}_user`) + }, 30000) + + it("excludes design documents when replicating to dev (TO_DEV)", async () => { + const source = `${DocumentType.WORKSPACE}_${structures.db.id()}` + const target = `${DocumentType.WORKSPACE_DEV}_${structures.db.id()}` + await ensureDb(source) + await ensureDb(target) + + await getDB(source).bulkDocs([ + makeDoc("_design/some_view", { views: {} }), + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + ]) + + const rep = new Replication({ source, target }) + await rep.replicate() + + const ids = await getAllDocIds(target) + expect(ids).not.toContain("_design/some_view") + expect(ids).toContain(`${DocumentType.ROLE}_admin`) + }, 30000) + + it("includes design documents when replicating to production (TO_PRODUCTION)", async () => { + const source = `${DocumentType.WORKSPACE_DEV}_${structures.db.id()}` + const target = `${DocumentType.WORKSPACE}_${structures.db.id()}` + await ensureDb(source) + await ensureDb(target) + + await getDB(source).bulkDocs([ + makeDoc("_design/some_view", { views: {} }), + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + ]) + + const rep = new Replication({ source, target }) + await rep.replicate() + + const ids = await getAllDocIds(target) + expect(ids).toContain("_design/some_view") + expect(ids).toContain(`${DocumentType.ROLE}_admin`) + }, 30000) + + it("excludes auto column state when replicating to production (TO_PRODUCTION, default)", async () => { + const source = `${DocumentType.WORKSPACE_DEV}_${structures.db.id()}` + const target = `${DocumentType.WORKSPACE}_${structures.db.id()}` + await ensureDb(source) + await ensureDb(target) + + await getDB(source).put(makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true })) + + const rep = new Replication({ source, target }) + await rep.replicate() + + const ids = await getAllDocIds(target) + expect(ids).not.toContain(`${DocumentType.AUTO_COLUMN_STATE}_state1`) + }, 30000) + + it("replicates auto column state when isCreation is true (TO_PRODUCTION)", async () => { + const source = `${DocumentType.WORKSPACE_DEV}_${structures.db.id()}` + const target = `${DocumentType.WORKSPACE}_${structures.db.id()}` + await ensureDb(source) + await ensureDb(target) + + await getDB(source).put(makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true })) + + const rep = new Replication({ source, target }) + await rep.replicate({ isCreation: true }) + + const doc = await getDB(target).get(`${DocumentType.AUTO_COLUMN_STATE}_state1`) + expect(doc).toMatchObject({ active: true }) + }, 30000) + + it("replicates user metadata documents", async () => { + const source = `${DocumentType.WORKSPACE_DEV}_${structures.db.id()}` + const target = `${DocumentType.WORKSPACE}_${structures.db.id()}` + await ensureDb(source) + await ensureDb(target) + + const userMetaId = `ro_ta_users_${structures.db.id()}` + await getDB(source).put(makeDoc(userMetaId, { email: "test@example.com" })) + + const rep = new Replication({ source, target }) + await rep.replicate() + + const doc = await getDB(target).get(userMetaId) + expect(doc).toMatchObject({ email: "test@example.com" }) + }, 30000) }) describe("rollback", () => { it("restores target to source state", async () => { const source = structures.db.id() const target = structures.db.id() - const sourceDB = getPouchDB(source) - const targetDB = getPouchDB(target) - - try { - await sourceDB.put({ - _id: `${DocumentType.ROLE}_admin`, - name: "admin", - }) - - const rep = new Replication({ source, target }) - await rep.replicate() - - const beforeDoc = await targetDB.get(`${DocumentType.ROLE}_admin`) - expect(beforeDoc.name).toBe("admin") - - await targetDB.put({ - _id: `${DocumentType.ROLE}_admin`, - name: "modified", - _rev: beforeDoc._rev, - }) - - const modifiedDoc = await targetDB.get(`${DocumentType.ROLE}_admin`) - expect(modifiedDoc.name).toBe("modified") - - await rep.rollback() - - const afterDoc = await targetDB.get(`${DocumentType.ROLE}_admin`) - expect(afterDoc.name).toBe("admin") - } finally { - await sourceDB.destroy() - await targetDB.destroy() - } - }) + await ensureDb(source) + await ensureDb(target) - it("removes extra documents from target", async () => { - const source = structures.db.id() - const target = structures.db.id() - const sourceDB = getPouchDB(source) - const targetDB = getPouchDB(target) - - try { - await sourceDB.put({ - _id: `${DocumentType.ROLE}_admin`, - name: "admin", - }) - - const rep = new Replication({ source, target }) - await rep.replicate() - - await targetDB.put({ - _id: `${DocumentType.ROLE}_extra`, - name: "extra", - }) - - const extraDoc = await targetDB.get(`${DocumentType.ROLE}_extra`) - expect(extraDoc.name).toBe("extra") - - await rep.rollback() - - try { - await targetDB.get(`${DocumentType.ROLE}_extra`) - fail("Expected doc not found") - } catch (err: any) { - expect(err.status).toBe(404) - } - } finally { - await sourceDB.destroy() - await targetDB.destroy() - } - }) - }) + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + + const rep = new Replication({ source, target }) + await rep.replicate() + + const beforeDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + expect(beforeDoc.name).toBe("admin") + + await getDB(target).put({ + _id: `${DocumentType.ROLE}_admin`, + _rev: beforeDoc._rev, + name: "modified", + }) + + const modifiedDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + expect(modifiedDoc.name).toBe("modified") - describe("close", () => { - it("closes both source and target databases", async () => { + await rep.rollback() + + const afterDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + expect(afterDoc.name).toBe("admin") + }, 60000) + + it("re-replicates all source documents after destroy", async () => { const source = structures.db.id() const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) + + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + makeDoc(`${DocumentType.ROLE}_user`, { name: "user" }), + ]) + const rep = new Replication({ source, target }) - await expect(rep.close()).resolves.not.toThrow() - }) + await rep.replicate() + await rep.rollback() + + expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) + expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) + }, 60000) }) }) From a6c8496321cf0ba0efb9f4b5c0ac9d60f9787b7b Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 16:45:49 +0100 Subject: [PATCH 03/24] chore: biome formatting fixes --- packages/backend-core/src/db/tests/Replication.spec.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index 050a2a9b..11816f78 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -1,6 +1,6 @@ import { DocumentType } from "@supertoolmake/types" -import { getDB } from "../db" import { structures } from "../../../tests" +import { getDB } from "../db" import Replication from "../Replication" const ensureDb = async (dbName: string) => { @@ -79,7 +79,9 @@ describe("Replication", () => { expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) - expect(await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`)).toMatchObject({ type: "postgres" }) + expect(await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`)).toMatchObject({ + type: "postgres", + }) }, 30000) it("excludes automation log documents", async () => { From 0ca3be1e903f410575f42e64b8b2dacf116d44a2 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 16:52:53 +0100 Subject: [PATCH 04/24] fix: add DOM lib to tsconfig for HTMLInputElement in @supertoolmake/types --- tsconfig.build.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index ce41ed86..29cc8f09 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es6", "module": "commonjs", - "lib": ["es2022"], + "lib": ["es2022", "dom"], "strict": true, "noImplicitAny": true, "esModuleInterop": true, From 8c29676adb7bae0cdb69e3f474cd848f3238581a Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 16:57:40 +0100 Subject: [PATCH 05/24] fix: make backend-core init no-op after PouchDB removal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit db.init(opts.db) called the removed PouchDB initialization function. Worker/server db/index.ts still call core.init() — make it a no-op since nano/CouchDB needs no client-side initialization. --- packages/backend-core/src/index.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index e59d1290..35d25b52 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -55,8 +55,6 @@ export * from "./errors" export * from "./warnings" // expose package init function -import * as db from "./db" - -export const init = (opts: any = {}) => { - db.init(opts.db) +export const init = (_opts: any = {}) => { + // PouchDB removed — no initialization needed } From 74c7c86baaf62966b36da96fbbb825cdd458cc6d Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:02:19 +0100 Subject: [PATCH 06/24] fix: remove dead PouchDB init function and all callers --- packages/backend-core/src/index.ts | 5 ----- packages/server/src/app.ts | 4 ---- packages/server/src/db/index.ts | 13 ------------- .../src/middleware/tests/currentWorkspace.spec.ts | 3 --- .../server/src/tests/utilities/TestConfiguration.ts | 3 --- packages/server/src/threads/utils.ts | 2 -- packages/worker/src/db/index.ts | 13 ------------- packages/worker/src/index.ts | 3 --- packages/worker/src/tests/TestConfiguration.ts | 3 --- 9 files changed, 49 deletions(-) delete mode 100644 packages/server/src/db/index.ts delete mode 100644 packages/worker/src/db/index.ts diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 35d25b52..0b71a14b 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -53,8 +53,3 @@ export * from "./constants" export * from "./errors" // expose warning classes directly export * from "./warnings" - -// expose package init function -export const init = (_opts: any = {}) => { - // PouchDB removed — no initialization needed -} diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 14d2cc89..07f04e1f 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,7 +1,3 @@ -import * as db from "./db" - -db.init() - import { env as coreEnv } from "@supertoolmake/backend-core" import { ServiceType } from "@supertoolmake/types" diff --git a/packages/server/src/db/index.ts b/packages/server/src/db/index.ts deleted file mode 100644 index 39d40e88..00000000 --- a/packages/server/src/db/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as core from "@supertoolmake/backend-core" -import env from "../environment" - -export function init() { - const dbConfig: any = {} - - if (env.isTest() && !env.COUCH_DB_URL) { - dbConfig.inMemory = true - dbConfig.allDbs = true - } - - core.init({ db: dbConfig }) -} diff --git a/packages/server/src/middleware/tests/currentWorkspace.spec.ts b/packages/server/src/middleware/tests/currentWorkspace.spec.ts index 1af7c87c..1a44fefc 100644 --- a/packages/server/src/middleware/tests/currentWorkspace.spec.ts +++ b/packages/server/src/middleware/tests/currentWorkspace.spec.ts @@ -1,6 +1,3 @@ -import * as db from "../../db" - -db.init() mockAuthWithNoCookie() mockWorker() diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 7ed6bae7..5c5cbe31 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -37,7 +37,6 @@ import { } from "@supertoolmake/types" import jwt, { type Secret } from "jsonwebtoken" import supertest from "supertest" -import { init as dbInit } from "../../db" import { generateUserMetadataID } from "../../db/utils" import env from "../../environment" import { startup } from "../../startup" @@ -63,8 +62,6 @@ import { const newid = utils.newid -dbInit() - export interface TableToBuild extends Omit { sourceId?: string sourceType?: TableSourceType diff --git a/packages/server/src/threads/utils.ts b/packages/server/src/threads/utils.ts index 0653b5c0..72f34a18 100644 --- a/packages/server/src/threads/utils.ts +++ b/packages/server/src/threads/utils.ts @@ -1,5 +1,4 @@ import { db as dbCore, redis } from "@supertoolmake/backend-core" -import * as db from "../db" import env from "../environment" import * as jsRunner from "../jsRunner" import type { QueryVariable } from "./definitions" @@ -33,7 +32,6 @@ export function threadSetup() { console.debug(`[${env.FORKED_PROCESS_NAME}] thread setup running`) jsRunner.init() - db.init() } export async function getCachedVariable(queryId: string, variable: string) { diff --git a/packages/worker/src/db/index.ts b/packages/worker/src/db/index.ts deleted file mode 100644 index 39d40e88..00000000 --- a/packages/worker/src/db/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -import * as core from "@supertoolmake/backend-core" -import env from "../environment" - -export function init() { - const dbConfig: any = {} - - if (env.isTest() && !env.COUCH_DB_URL) { - dbConfig.inMemory = true - dbConfig.allDbs = true - } - - core.init({ db: dbConfig }) -} diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index 70bc9d1e..76412981 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -17,11 +17,8 @@ import Application, { type Middleware } from "koa" import koaBody, { HttpMethodEnum } from "koa-body" import api from "./api" import { loadTemplateConfig } from "./constants/templates" -import * as db from "./db" import env from "./environment" -db.init() - const koaSession = require("koa-session").default import { userAgent } from "koa-useragent" diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 9434a5f9..0d0ca6df 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -1,9 +1,6 @@ -import * as dbConfig from "../db" import env from "../environment" import * as controllers from "./controllers" -dbConfig.init() - import type http from "node:http" import { constants, From dd2773c5d6af7ee758189e98835b0ec7e5fc12e1 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:02:52 +0100 Subject: [PATCH 07/24] lint --- packages/worker/src/tests/TestConfiguration.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index 0d0ca6df..f4b962ab 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -1,6 +1,3 @@ -import env from "../environment" -import * as controllers from "./controllers" - import type http from "node:http" import { constants, @@ -24,7 +21,9 @@ import { import jwt, { type Secret } from "jsonwebtoken" import supertest from "supertest" import { Config } from "../constants" +import env from "../environment" import API from "./api" +import * as controllers from "./controllers" import structures, { CSRF_TOKEN } from "./structures" class TestConfiguration { From 4039d3545fd303fac61c5d4356cc3589b137d6c4 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:08:03 +0100 Subject: [PATCH 08/24] fix: remove optional chaining on doc._id to match filter return type --- packages/server/src/api/controllers/role.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 44656973..6e7176b0 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -171,7 +171,7 @@ export async function save(ctx: UserCtx) { }) await replication.replicate({ filter: (doc) => { - return doc._id?.startsWith("role_") + return doc._id.startsWith("role_") }, }) } From c9f41ba4a113629f9d3af8c151a94ce1162604ea Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:09:19 +0100 Subject: [PATCH 09/24] fix: accept boolean|undefined in ReplicateOpts.filter Document._id is optional, so filter callbacks using optional chaining return boolean|undefined. Accept that in the filter type and coerce to boolean in shouldKeepDoc. --- packages/backend-core/src/db/Replication.ts | 4 ++-- packages/server/src/api/controllers/role.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 0f879b67..1805f4ac 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -11,7 +11,7 @@ enum ReplicationDirection { interface ReplicateOpts { isCreation?: boolean tablesToSync?: string[] | "all" - filter?: (doc: Document) => boolean + filter?: (doc: Document) => boolean | undefined } class Replication { @@ -97,7 +97,7 @@ class Replication { } if (customFilter) { - return customFilter(doc as Document) + return !!customFilter(doc as Document) } return true diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 6e7176b0..44656973 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -171,7 +171,7 @@ export async function save(ctx: UserCtx) { }) await replication.replicate({ filter: (doc) => { - return doc._id.startsWith("role_") + return doc._id?.startsWith("role_") }, }) } From 01e7537f7eec94040102eea4cb829ad759334617 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:10:55 +0100 Subject: [PATCH 10/24] fix: match customFilter type in all Replication.ts opts --- packages/backend-core/src/db/Replication.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 1805f4ac..6be22fad 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -42,7 +42,7 @@ class Replication { toDev: boolean syncAllTables: boolean tableSyncList?: string[] - customFilter?: (doc: Document) => boolean + customFilter?: (doc: Document) => boolean | undefined } ): boolean { const { isCreation, toDev, syncAllTables, tableSyncList, customFilter } = opts @@ -145,7 +145,7 @@ class Replication { toDev: boolean syncAllTables: boolean tableSyncList?: string[] - customFilter?: (doc: Document) => boolean + customFilter?: (doc: Document) => boolean | undefined }) { const targetDb = getDB(this.targetName, { skip_setup: true }) const allDocs = await targetDb.allDocs({ include_docs: true }) From 274a33a4dcfb3d136874d0c1fbed86d9dde29696 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:16:05 +0100 Subject: [PATCH 11/24] fix: cast sharp toBuffer result to Buffer for type compat --- packages/server/src/api/controllers/static/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 5e573613..1b222910 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -393,13 +393,13 @@ export const uploadExternalFile = async (ctx: Ctx) => { const image = sharp(buffer) const metadata = await image.metadata() if (!compress.maxWidth) { - buffer = await image.avif({ quality }).toBuffer() + buffer = (await image.avif({ quality }).toBuffer()) as Buffer } if (metadata.width && metadata.width > compress.maxWidth) { - buffer = await image + buffer = (await image .resize(compress.maxWidth, null, { withoutEnlargement: true }) .avif({ quality }) - .toBuffer() + .toBuffer()) as Buffer } uploadKey = key.replace(/\.[^.]+$/, ".avif") } From 73e832a36112953629ec9e9d2fb1c1acc9e6653d Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 17:28:23 +0100 Subject: [PATCH 12/24] type fixes --- package.json | 2 +- .../backend-core/src/queue/inMemoryQueue.ts | 2 - .../src/api/controllers/static/index.ts | 14 +- yarn.lock | 722 +----------------- 4 files changed, 21 insertions(+), 719 deletions(-) diff --git a/package.json b/package.json index 8de5e5f1..e8fdb6ea 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "@swc/core": "1.13.5", "@swc/jest": "0.2.39", "@types/jest": "30.0.0", - "@types/node": "22.18.0", + "@types/node": "25.5.0", "@types/proper-lockfile": "4.1.4", "audit-ci": "7.1.0", "esbuild": "0.28.1", diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 19e0d7ca..88fcda34 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -282,13 +282,11 @@ export class InMemoryQueue implements Partial> { } on(event: string, callback: (...args: any[]) => void): Queue { - // @ts-expect-error - this callback can be one of many types this._emitter.on(event, callback) return this as unknown as Queue } off(event: string, callback: (...args: any[]) => void): Queue { - // @ts-expect-error - this callback can be one of many types this._emitter.off(event, callback) return this as unknown as Queue } diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 1b222910..65552a74 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -385,7 +385,7 @@ export const uploadExternalFile = async (ctx: Ctx) => { }) const filePath = (file as BBUploadedFile).filepath - let buffer = await fs.promises.readFile(filePath) + let buffer: Buffer = await fs.promises.readFile(filePath) let uploadKey = key if (compress && (file as BBUploadedFile).mimetype?.startsWith("image/")) { @@ -393,13 +393,15 @@ export const uploadExternalFile = async (ctx: Ctx) => { const image = sharp(buffer) const metadata = await image.metadata() if (!compress.maxWidth) { - buffer = (await image.avif({ quality }).toBuffer()) as Buffer + buffer = Buffer.from(await image.avif({ quality }).toBuffer()) } if (metadata.width && metadata.width > compress.maxWidth) { - buffer = (await image - .resize(compress.maxWidth, null, { withoutEnlargement: true }) - .avif({ quality }) - .toBuffer()) as Buffer + buffer = Buffer.from( + await image + .resize(compress.maxWidth, null, { withoutEnlargement: true }) + .avif({ quality }) + .toBuffer() + ) } uploadKey = key.replace(/\.[^.]+$/, ".avif") } diff --git a/yarn.lock b/yarn.lock index 0743dbea..5733d7a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1883,17 +1883,6 @@ typeof-article "^0.1.1" uuid "^11.1.1" -"@budibase/pouchdb-replication-stream@1.2.11": - version "1.2.11" - resolved "https://registry.yarnpkg.com/@budibase/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.11.tgz#b437584471f2cf6976fa690c338f432a3ecd24c9" - integrity sha512-gHeMczCCIcxiv4EmG4uosHZvGIgInGwP8SaWKGY3g+m7XPdlM5UgRYhyPHqmOU9fdfTtR3LgAqED6NRWFYmqyw== - dependencies: - inherits "^2.0.3" - ndjson "^1.4.3" - pouch-stream "^0.4.0" - pouchdb-promise "^6.0.4" - through2 "^2.0.0" - "@budibase/vm-browserify@1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262" @@ -5238,13 +5227,6 @@ "@types/d3-interpolate" "*" "@types/d3-selection" "*" -"@types/debug@*": - version "4.1.13" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.13.tgz#22d1cc9d542d3593caea764f974306ab36286ee7" - integrity sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw== - dependencies: - "@types/ms" "*" - "@types/deep-eql@*": version "4.0.2" resolved "https://registry.yarnpkg.com/@types/deep-eql/-/deep-eql-4.0.2.tgz#334311971d3a07121e7eb91b684a605e7eea9cbd" @@ -5443,11 +5425,6 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.5.tgz#ec10755e871497bcd83efe927e43ec46e8c0747e" integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag== -"@types/ms@*": - version "2.1.0" - resolved "https://registry.yarnpkg.com/@types/ms/-/ms-2.1.0.tgz#052aa67a48eccc4309d7f0191b7e41434b90bb78" - integrity sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA== - "@types/mssql@12.3.0": version "12.3.0" resolved "https://registry.yarnpkg.com/@types/mssql/-/mssql-12.3.0.tgz#77ae34e7a258f3b40208d4e0e1b46737d0f6a3c8" @@ -5465,20 +5442,13 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>=10.0.0", "@types/node@>=18": +"@types/node@*", "@types/node@25.5.0", "@types/node@>=10.0.0", "@types/node@>=18": version "25.5.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-25.5.0.tgz#5c99f37c443d9ccc4985866913f1ed364217da31" integrity sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw== dependencies: undici-types "~7.18.0" -"@types/node@22.18.0": - version "22.18.0" - resolved "https://registry.yarnpkg.com/@types/node/-/node-22.18.0.tgz#9e4709be4f104e3568f7dd1c71e2949bf147a47b" - integrity sha512-m5ObIqwsUp6BZzyiy4RdZpzWGub9bqLJMvZDD0QMXhxjqMHMENlj+SqF5QxoUwaQNFe+8kz8XM8ZQhqkQPTgMQ== - dependencies: - undici-types "~6.21.0" - "@types/node@^18.11.18": version "18.19.130" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.130.tgz#da4c6324793a79defb7a62cba3947ec5add00d59" @@ -5512,152 +5482,6 @@ pg-protocol "*" pg-types "^2.2.0" -"@types/pouchdb-adapter-cordova-sqlite@*": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-cordova-sqlite/-/pouchdb-adapter-cordova-sqlite-1.0.4.tgz#c16847ba4e7bf284d520999ca39942fffbc72642" - integrity sha512-1MGjmAMux3OIyJ+iXfhJ5hNIzS+KjGJ05O3bF5Gen5TiJUFNK1bOp3VVV9SxXgz+hGwnBruBAWdAqhbB6ZHhSA== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-fruitdown@*": - version "6.1.6" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-fruitdown/-/pouchdb-adapter-fruitdown-6.1.6.tgz#16ec19d9629bce623c8e2654bc15c64d6951e845" - integrity sha512-KaFB29hUI97eTtJI6pjv7EQcqhZ63qHWovKgyiE+HZF5fVmdrBbTmnIrbR87AJXcXKy47+oQFJ7rzxY8TalpLQ== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-http@*": - version "6.1.6" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-http/-/pouchdb-adapter-http-6.1.6.tgz#dbb80b2c1366f114bbd7002e62999b526750ef3d" - integrity sha512-DJur1mt07GJXwGb5K+MOILoCOSgoQpsi7hybcTzRLeR3IO8Y8eq7TnhTkftAJdx9VHJGOiOXFjO+8BYM69j5yA== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-idb@*": - version "6.1.7" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-idb/-/pouchdb-adapter-idb-6.1.7.tgz#4fba402c72aa1697b7922fba6eebec131cbb8670" - integrity sha512-KwjkJ4fTNz5wPXYu20bUoWud7ty0t7tgdo4oc0AJvG+fcURAH7mI7uFmpE4dZIT+hUq5G61xu96AVq9b2q4T3g== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-leveldb@*": - version "6.1.6" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-leveldb/-/pouchdb-adapter-leveldb-6.1.6.tgz#ec0070a5747583c5989d060df699e694f961c3e0" - integrity sha512-mqeTpA2Ni2U4FA5ISRESy4WwhfUahXViUa3jQpXGdSpruaeHlhTLzZJPyz7/mGlvdAfAFv9Vd5d6ys3ASmMujw== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-localstorage@*": - version "6.1.6" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-localstorage/-/pouchdb-adapter-localstorage-6.1.6.tgz#b7557ee54e19c5896208a767c1419f5735131188" - integrity sha512-+HQBCpD80XkKJE64r7uLwzkNRgkvMnhDI5rIFLx3USxdrRph/R3awcEubRFndcgtxzcUaL9iYw9KetgFMUqPrg== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-memory@*": - version "6.1.6" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-memory/-/pouchdb-adapter-memory-6.1.6.tgz#7fd68b42ab85ef59ffbcfdf70a8a1a143af4d458" - integrity sha512-QCCtW561XuwFACzP/4zYySzs/a4em0EeuQdszen0YOaGV1/fRqJE0dOlmzh8do4sNJomLO6+MFtEzguGljnkgA== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-node-websql@*": - version "6.1.5" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-node-websql/-/pouchdb-adapter-node-websql-6.1.5.tgz#ab9f46e061e4dad34df6a169d56905ae8b5f6b50" - integrity sha512-yi68syUvHs4OM3mzKlh4zfpov64KITIAnxi387zgdby6SEfAJzWPC0dfH77iEVRDGCrKb3cKTNkl/UGHnphaow== - dependencies: - "@types/pouchdb-adapter-websql" "*" - "@types/pouchdb-core" "*" - -"@types/pouchdb-adapter-websql@*": - version "6.1.7" - resolved "https://registry.yarnpkg.com/@types/pouchdb-adapter-websql/-/pouchdb-adapter-websql-6.1.7.tgz#774754b14385dea2dd9c41d61772e16755768c5b" - integrity sha512-9oNkP5ZCGMkQALO9KmtbHXlkBq8i2hoCEE6/gWzRicAvL1y+WIKjEQiIIEamMhj5u5tARvW3n2/r+JXwLCyYgw== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-browser@*": - version "6.1.5" - resolved "https://registry.yarnpkg.com/@types/pouchdb-browser/-/pouchdb-browser-6.1.5.tgz#fb1d1fe9670dda05081dd690f4778b7c2e2d8eaa" - integrity sha512-f+HjxEjYFpgoYWXnMI9AQZZ+SIG8dBiBPrpfWWGsCl+48rumsP5BuBWHq/aXoB8SRKYO0XdP4TNvMBWM3UATCw== - dependencies: - "@types/pouchdb-adapter-http" "*" - "@types/pouchdb-adapter-idb" "*" - "@types/pouchdb-adapter-websql" "*" - "@types/pouchdb-core" "*" - "@types/pouchdb-mapreduce" "*" - "@types/pouchdb-replication" "*" - -"@types/pouchdb-core@*": - version "7.0.15" - resolved "https://registry.yarnpkg.com/@types/pouchdb-core/-/pouchdb-core-7.0.15.tgz#02bb1b480c37b6d84b2e3c8babee716f1a101814" - integrity sha512-gq1Qbqn9nCaAKRRv6fRHZ4/ER+QYEwSXBZlDQcxwdbPrtZO8EhIn2Bct0AlguaSEdFcABfbaxxyQwFINkNQ9dQ== - dependencies: - "@types/debug" "*" - "@types/pouchdb-find" "*" - -"@types/pouchdb-find@*": - version "7.3.3" - resolved "https://registry.yarnpkg.com/@types/pouchdb-find/-/pouchdb-find-7.3.3.tgz#599e388e2a1c4e57ee8aba0d5deca24d37b6978e" - integrity sha512-U7zXk67s9Ar+9Pwj5kSbuMnn8zif0AOOIPy4KRFeJ/S/Tk+mNS90soj+3OV21H8xyB7WTxjvS1JLablZC6C6ow== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-http@*": - version "6.1.5" - resolved "https://registry.yarnpkg.com/@types/pouchdb-http/-/pouchdb-http-6.1.5.tgz#edf3b4ba9bf2d71e03bf2d6dbeb332982361df15" - integrity sha512-9jGCAl6DUsXIl1vjuPu8tzGykAr84549P4IS0zYdrOKq5eXzQRUb/tb2hEVTmmTcYKXu2P1N55ABsdDNZvzGGA== - dependencies: - "@types/pouchdb-adapter-http" "*" - "@types/pouchdb-core" "*" - -"@types/pouchdb-mapreduce@*": - version "6.1.10" - resolved "https://registry.yarnpkg.com/@types/pouchdb-mapreduce/-/pouchdb-mapreduce-6.1.10.tgz#2668c32a79920dbaed4ac335fa0fc4b986a0cb38" - integrity sha512-AgYVqCnaA5D7cWkWyzZVuk0137N4yZsmIQTD/i3DmuMxYYoFrtWUoQu0tbA52SpTRGdL8ubQ7JFQXzA13fA6IQ== - dependencies: - "@types/pouchdb-core" "*" - -"@types/pouchdb-node@*": - version "6.1.7" - resolved "https://registry.yarnpkg.com/@types/pouchdb-node/-/pouchdb-node-6.1.7.tgz#b59900db97bd0f04408b24e1ece593a7b76d8fc8" - integrity sha512-hryc2eCtNB3GbLcHSwU8glLaY66gDMus1AYkcIYAAxufdnK2BAy1oxaRLmnwRn1A1vG41P/t0htFD161LUnfQw== - dependencies: - "@types/pouchdb-adapter-http" "*" - "@types/pouchdb-adapter-leveldb" "*" - "@types/pouchdb-core" "*" - "@types/pouchdb-mapreduce" "*" - "@types/pouchdb-replication" "*" - -"@types/pouchdb-replication@*": - version "6.4.7" - resolved "https://registry.yarnpkg.com/@types/pouchdb-replication/-/pouchdb-replication-6.4.7.tgz#09f668fa750e0c4f5fcfc01667be4b29d662f449" - integrity sha512-slB4zOwri3SAVHioFx/FWC/KqOzzb7nDFtV+qzaKzxkf+U5zTwCbK3uRHaj0d/XQk0DwVeajf1ni3Wiyq3j2OA== - dependencies: - "@types/pouchdb-core" "*" - "@types/pouchdb-find" "*" - -"@types/pouchdb@6.4.2": - version "6.4.2" - resolved "https://registry.yarnpkg.com/@types/pouchdb/-/pouchdb-6.4.2.tgz#54777533d86f4abd1a3989b272e085323623bbe1" - integrity sha512-YsI47rASdtzR+3V3JE2UKY58snhm0AglHBpyckQBkRYoCbTvGagXHtV0x5n8nzN04jQmvTG+Sm85cIzKT3KXBA== - dependencies: - "@types/pouchdb-adapter-cordova-sqlite" "*" - "@types/pouchdb-adapter-fruitdown" "*" - "@types/pouchdb-adapter-http" "*" - "@types/pouchdb-adapter-idb" "*" - "@types/pouchdb-adapter-leveldb" "*" - "@types/pouchdb-adapter-localstorage" "*" - "@types/pouchdb-adapter-memory" "*" - "@types/pouchdb-adapter-node-websql" "*" - "@types/pouchdb-adapter-websql" "*" - "@types/pouchdb-browser" "*" - "@types/pouchdb-core" "*" - "@types/pouchdb-http" "*" - "@types/pouchdb-mapreduce" "*" - "@types/pouchdb-node" "*" - "@types/pouchdb-replication" "*" - "@types/proper-lockfile@4.1.4": version "4.1.4" resolved "https://registry.yarnpkg.com/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz#cd9fab92bdb04730c1ada542c356f03620f84008" @@ -6126,54 +5950,13 @@ abbrev@^4.0.0: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-4.0.0.tgz#ec933f0e27b6cd60e89b5c6b2a304af42209bb05" integrity sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA== -abort-controller@3.0.0, abort-controller@^3.0.0: +abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== dependencies: event-target-shim "^5.0.0" -abstract-leveldown@^6.2.1: - version "6.3.0" - resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz#d25221d1e6612f820c35963ba4bd739928f6026a" - integrity sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ== - dependencies: - buffer "^5.5.0" - immediate "^3.2.3" - level-concat-iterator "~2.0.0" - level-supports "~1.0.0" - xtend "~4.0.0" - -abstract-leveldown@^7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-7.2.0.tgz#08d19d4e26fb5be426f7a57004851b39e1795a2e" - integrity sha512-DnhQwcFEaYsvYDnACLZhMmCWd3rkOeEvglpa4q5i/5Jlm3UIsWaxVzuXvDLFCSCWRO3yy2/+V/G7FusFgejnfQ== - dependencies: - buffer "^6.0.3" - catering "^2.0.0" - is-buffer "^2.0.5" - level-concat-iterator "^3.0.0" - level-supports "^2.0.1" - queue-microtask "^1.2.3" - -abstract-leveldown@~2.7.1: - version "2.7.2" - resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-2.7.2.tgz#87a44d7ebebc341d59665204834c8b7e0932cc93" - integrity sha512-+OVvxH2rHVEhWLdbudP6p0+dNMXu8JA1CbhP19T8paTYAcX7oJ4OVjT+ZUVpv7mITxXHqDMej+GdqXBmXkw09w== - dependencies: - xtend "~4.0.0" - -abstract-leveldown@~6.2.1, abstract-leveldown@~6.2.3: - version "6.2.3" - resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz#036543d87e3710f2528e47040bc3261b77a9a8eb" - integrity sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ== - dependencies: - buffer "^5.5.0" - immediate "^3.2.3" - level-concat-iterator "~2.0.0" - level-supports "~1.0.0" - xtend "~4.0.0" - accepts@^1.3.5, accepts@^1.3.7, accepts@^1.3.8, accepts@~1.3.4, accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" @@ -6354,11 +6137,6 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -argsarray@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/argsarray/-/argsarray-0.0.1.tgz#6e7207b4ecdb39b0af88303fa5ae22bda8df61cb" - integrity sha512-u96dg2GcAKtpTrBdDoFIM7PjcBA+6rSP0OR94MOReNRyUECL6MtQt5XXmRr4qrftYaef9+l5hcpO5te7sML1Cg== - aria-query@5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/aria-query/-/aria-query-5.3.0.tgz#650c569e41ad90b51b3d7df5e5eed1c7549c103e" @@ -6884,7 +6662,7 @@ buffer-equal-constant-time@^1.0.1: resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA== -buffer-from@1.1.2, buffer-from@^1.0.0: +buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ== @@ -6919,7 +6697,7 @@ buffer@6.0.3, buffer@^6.0.3: base64-js "^1.3.1" ieee754 "^1.2.1" -buffer@^5.1.0, buffer@^5.5.0, buffer@^5.6.0: +buffer@^5.1.0, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== @@ -7064,11 +6842,6 @@ caniuse-lite@^1.0.30001759: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001781.tgz#344b47c03eb8168b79c3c158b872bcfbdd02a400" integrity sha512-RdwNCyMsNBftLjW6w01z8bKEvT6e/5tpPVEgtn22TiLGlstHOVecsX2KHFkD5e/vRnIE4EGzpuIODb3mtswtkw== -catering@^2.0.0, catering@^2.1.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/catering/-/catering-2.1.1.tgz#66acba06ed5ee28d5286133982a927de9a04b510" - integrity sha512-K7Qy8O9p76sL3/3m7/zLKbRkyOlSZAgzEaLhyj2mXS8PsCud2Eo4hAb8aLtZqHh0QGqLcb9dlJSu6lHRVENm1w== - chai@^6.2.2: version "6.2.2" resolved "https://registry.yarnpkg.com/chai/-/chai-6.2.2.tgz#ae41b52c9aca87734505362717f3255facda360e" @@ -7264,11 +7037,6 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" -clone-buffer@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58" - integrity sha512-KLLTJWrvwIP+OPfMn0x2PheDEP20RPUcGXj/ERegTgdmPEZylALQldygiqrPPu8P45uNuPs7ckmReLY6v/iA5g== - clone-response@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.3.tgz#af2032aa47816399cf5f0a1d0db902f517abb8c3" @@ -8088,14 +7856,6 @@ defer-to-connect@^1.0.1: resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591" integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ== -deferred-leveldown@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz#27a997ad95408b61161aa69bd489b86c71b78058" - integrity sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw== - dependencies: - abstract-leveldown "~6.2.1" - inherits "^2.0.3" - define-data-property@^1.0.1, define-data-property@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" @@ -8338,11 +8098,6 @@ dotenv@~16.4.5: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.7.tgz#0e20c5b82950140aa99be360a8a5f52335f53c26" integrity sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ== -double-ended-queue@2.1.0-0: - version "2.1.0-0" - resolved "https://registry.yarnpkg.com/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz#103d3527fd31528f40188130c841efdd78264e5c" - integrity sha512-+BNfZ+deCo8hMNpDqDnvT+c0XpJ5cUa6mqYq89bho2Ifze4URTqRkcwR399hWoTrTkbZ/XJYDgP6rc7pRgffEQ== - downloadjs@1.4.7: version "1.4.7" resolved "https://registry.yarnpkg.com/downloadjs/-/downloadjs-1.4.7.tgz#f69f96f940e0d0553dac291139865a3cd0101e3c" @@ -8447,16 +8202,6 @@ encodeurl@^2.0.0, encodeurl@~2.0.0: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== -encoding-down@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/encoding-down/-/encoding-down-6.3.0.tgz#b1c4eb0e1728c146ecaef8e32963c549e76d082b" - integrity sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw== - dependencies: - abstract-leveldown "^6.2.1" - inherits "^2.0.3" - level-codec "^9.0.0" - level-errors "^2.0.0" - encoding-sniffer@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz#396ec97ac22ce5a037ba44af1992ac9d46a7b819" @@ -8479,13 +8224,6 @@ end-of-stream@^1.1.0, end-of-stream@^1.4.1: dependencies: once "^1.4.0" -end-stream@~0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/end-stream/-/end-stream-0.1.0.tgz#32003f3f438a2b0143168137f8fa6e9866c81ed5" - integrity sha512-Brl10T8kYnc75IepKizW6Y9liyW8ikz1B7n/xoHrJxoVSSjoqPn30sb7XVFfQERK4QfUMYRGs9dhWwtt2eu6uA== - dependencies: - write-stream "~0.4.3" - engine.io-client@~6.6.1: version "6.6.5" resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.6.5.tgz#9baebe6358dc432fb174bb42ed74cfd17626d051" @@ -8555,7 +8293,7 @@ err-code@^2.0.2: resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9" integrity sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA== -errno@~0.1.1, errno@~0.1.7: +errno@~0.1.7: version "0.1.8" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.8.tgz#8bb3e9c7d463be4976ff888f76b4809ebc2e811f" integrity sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A== @@ -9076,21 +8814,6 @@ fengari@^0.1.4: sprintf-js "^1.1.3" tmp "^0.2.5" -fetch-cookie@0.11.0: - version "0.11.0" - resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-0.11.0.tgz#e046d2abadd0ded5804ce7e2cae06d4331c15407" - integrity sha512-BQm7iZLFhMWFy5CZ/162sAGjBfdNWb7a8LEqqnzsHFhxT/X/SVj/z2t2nu3aJvjlbQkrAlTUApplPRjWyH4mhA== - dependencies: - tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0" - -fetch-cookie@2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/fetch-cookie/-/fetch-cookie-2.2.0.tgz#01086b6b5b1c3e08f15ffd8647b02ca100377365" - integrity sha512-h9AgfjURuCgA2+2ISl8GbavpUdR+WGAM2McW/ovn4tVccegp8ZqCKWSBR8uRdM8dDNlx5WdKRWxBYUwteLDCNQ== - dependencies: - set-cookie-parser "^2.4.8" - tough-cookie "^4.0.0" - figures@3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -10081,16 +9804,6 @@ ignore@^7.0.5: resolved "https://registry.yarnpkg.com/ignore/-/ignore-7.0.5.tgz#4cb5f6cd7d4c7ab0365738c7aea888baa6d7efd9" integrity sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg== -immediate@3.3.0, immediate@^3.2.3: - version "3.3.0" - resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.3.0.tgz#1aef225517836bcdf7f2a2de2600c79ff0269266" - integrity sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q== - -immediate@~3.0.5: - version "3.0.6" - resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" - integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== - import-fresh@^3.1.0, import-fresh@^3.3.0: version "3.3.1" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.1.tgz#9cecb56503c0ada1f2741dbbd6546e4b13b57ccf" @@ -10336,11 +10049,6 @@ is-boolean-object@^1.2.1: call-bound "^1.0.3" has-tostringtag "^1.0.2" -is-buffer@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.5.tgz#ebc252e400d22ff8d77fa09888821a24a658c191" - integrity sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ== - is-callable@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" @@ -11760,116 +11468,6 @@ lerna@9.0.7: yargs "17.7.2" yargs-parser "21.1.1" -level-codec@9.0.2, level-codec@^9.0.0: - version "9.0.2" - resolved "https://registry.yarnpkg.com/level-codec/-/level-codec-9.0.2.tgz#fd60df8c64786a80d44e63423096ffead63d8cbc" - integrity sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ== - dependencies: - buffer "^5.6.0" - -level-concat-iterator@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/level-concat-iterator/-/level-concat-iterator-3.1.0.tgz#5235b1f744bc34847ed65a50548aa88d22e881cf" - integrity sha512-BWRCMHBxbIqPxJ8vHOvKUsaO0v1sLYZtjN3K2iZJsRBYtp+ONsY6Jfi6hy9K3+zolgQRryhIn2NRZjZnWJ9NmQ== - dependencies: - catering "^2.1.0" - -level-concat-iterator@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz#1d1009cf108340252cb38c51f9727311193e6263" - integrity sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw== - -level-errors@^2.0.0, level-errors@~2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/level-errors/-/level-errors-2.0.1.tgz#2132a677bf4e679ce029f517c2f17432800c05c8" - integrity sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw== - dependencies: - errno "~0.1.1" - -level-iterator-stream@~4.0.0: - version "4.0.2" - resolved "https://registry.yarnpkg.com/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz#7ceba69b713b0d7e22fcc0d1f128ccdc8a24f79c" - integrity sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q== - dependencies: - inherits "^2.0.4" - readable-stream "^3.4.0" - xtend "^4.0.2" - -level-js@^5.0.0: - version "5.0.2" - resolved "https://registry.yarnpkg.com/level-js/-/level-js-5.0.2.tgz#5e280b8f93abd9ef3a305b13faf0b5397c969b55" - integrity sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg== - dependencies: - abstract-leveldown "~6.2.3" - buffer "^5.5.0" - inherits "^2.0.3" - ltgt "^2.1.2" - -level-packager@^5.1.0: - version "5.1.1" - resolved "https://registry.yarnpkg.com/level-packager/-/level-packager-5.1.1.tgz#323ec842d6babe7336f70299c14df2e329c18939" - integrity sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ== - dependencies: - encoding-down "^6.3.0" - levelup "^4.3.2" - -level-supports@^2.0.1: - version "2.1.0" - resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-2.1.0.tgz#9af908d853597ecd592293b2fad124375be79c5f" - integrity sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA== - -level-supports@~1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/level-supports/-/level-supports-1.0.1.tgz#2f530a596834c7301622521988e2c36bb77d122d" - integrity sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg== - dependencies: - xtend "^4.0.2" - -level-write-stream@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/level-write-stream/-/level-write-stream-1.0.0.tgz#3f7fbb679a55137c0feb303dee766e12ee13c1dc" - integrity sha512-bBNKOEOMl8msO+uIM9YX/gUO6ckokZ/4pCwTm/lwvs46x6Xs8Zy0sn3Vh37eDqse4mhy4fOMIb/JsSM2nyQFtw== - dependencies: - end-stream "~0.1.0" - -level@6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/level/-/level-6.0.1.tgz#dc34c5edb81846a6de5079eac15706334b0d7cd6" - integrity sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw== - dependencies: - level-js "^5.0.0" - level-packager "^5.1.0" - leveldown "^5.4.0" - -leveldown@5.6.0, leveldown@^5.4.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-5.6.0.tgz#16ba937bb2991c6094e13ac5a6898ee66d3eee98" - integrity sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ== - dependencies: - abstract-leveldown "~6.2.1" - napi-macros "~2.0.0" - node-gyp-build "~4.1.0" - -leveldown@6.1.1: - version "6.1.1" - resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-6.1.1.tgz#0f0e480fa88fd807abf94c33cb7e40966ea4b5ce" - integrity sha512-88c+E+Eizn4CkQOBHwqlCJaTNEjGpaEIikn1S+cINc5E9HEvJ77bqY4JY/HxT5u0caWqsc3P3DcFIKBI1vHt+A== - dependencies: - abstract-leveldown "^7.2.0" - napi-macros "~2.0.0" - node-gyp-build "^4.3.0" - -levelup@4.4.0, levelup@^4.3.2: - version "4.4.0" - resolved "https://registry.yarnpkg.com/levelup/-/levelup-4.4.0.tgz#f89da3a228c38deb49c48f88a70fb71f01cafed6" - integrity sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ== - dependencies: - deferred-leveldown "~5.3.0" - level-errors "~2.0.0" - level-iterator-stream "~4.0.0" - level-supports "~1.0.0" - xtend "~4.0.0" - leven@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" @@ -11897,13 +11495,6 @@ libnpmpublish@11.1.2: sigstore "^4.0.0" ssri "^12.0.0" -lie@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" - integrity sha512-RiNhHysUjhrDQntfYSfY4MU24coXXdEOgw9WGcKHNeEwffDYbF//u87M1EWaMGzuFoSbqW0C9C6lEEhDOAswfw== - dependencies: - immediate "~3.0.5" - lightningcss-android-arm64@1.32.0: version "1.32.0" resolved "https://registry.yarnpkg.com/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz#f033885116dfefd9c6f54787523e3514b61e1968" @@ -12182,11 +11773,6 @@ lru.min@^1.1.0: resolved "https://registry.yarnpkg.com/lru.min/-/lru.min-1.1.4.tgz#6ea1737a8c1ba2300cc87ad46910a4bdffa0117b" integrity sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA== -ltgt@2.2.1, ltgt@^2.1.2, ltgt@~2.2.0: - version "2.2.1" - resolved "https://registry.yarnpkg.com/ltgt/-/ltgt-2.2.1.tgz#f35ca91c493f7b73da0e07495304f17b31f87ee5" - integrity sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA== - luxon@^3.2.1: version "3.7.2" resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.7.2.tgz#d697e48f478553cca187a0f8436aff468e3ba0ba" @@ -12348,18 +11934,6 @@ media-typer@^1.1.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-1.1.0.tgz#6ab74b8f2d3320f2064b2a87a38e7931ff3a5561" integrity sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw== -memdown@1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/memdown/-/memdown-1.4.1.tgz#b4e4e192174664ffbae41361aa500f3119efe215" - integrity sha512-iVrGHZB8i4OQfM155xx8akvG9FIj+ht14DX5CQkCTG4EHzZ3d3sgckIf/Lm9ivZalEsFuEVnWv2B2WZvbrro2w== - dependencies: - abstract-leveldown "~2.7.1" - functional-red-black-tree "^1.0.1" - immediate "^3.2.3" - inherits "~2.0.1" - ltgt "~2.2.0" - safe-buffer "~5.1.1" - memory-pager@^1.0.2: version "1.5.0" resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5" @@ -12775,11 +12349,6 @@ napi-build-utils@^2.0.0: resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-2.0.0.tgz#13c22c0187fcfccce1461844136372a47ddc027e" integrity sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA== -napi-macros@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b" - integrity sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg== - napi-postinstall@^0.3.0: version "0.3.4" resolved "https://registry.yarnpkg.com/napi-postinstall/-/napi-postinstall-0.3.4.tgz#7af256d6588b5f8e952b9190965d6b019653bbb9" @@ -12795,16 +12364,6 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== -ndjson@^1.4.3: - version "1.5.0" - resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8" - integrity sha512-hUPLuaziboGjNF7wHngkgVc0FOclR8dDk/HfEvTtDr/iUrqBWiRcRSTK3/nLOqKH33th714BrMmTPtObI9gZxQ== - dependencies: - json-stringify-safe "^5.0.1" - minimist "^1.2.0" - split2 "^2.1.0" - through2 "^2.0.3" - negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" @@ -12855,7 +12414,7 @@ node-addon-api@^8.3.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-8.6.0.tgz#b22497201b465cd0a92ef2c01074ee5068c79a6d" integrity sha512-gBVjCaqDlRUk0EwoPNKzIr9KkS9041G/q31IBShPs1Xz6UTA+EXdZADbzqAJQrpDRq71CIMnOP5VMut3SL0z5Q== -node-fetch@2.6.7, node-fetch@2.6.9, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^3.3.0, node-fetch@^3.3.2: +node-fetch@2.6.7, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^3.3.0, node-fetch@^3.3.2: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== @@ -12874,16 +12433,11 @@ node-gyp-build-optional-packages@5.2.2: dependencies: detect-libc "^2.0.1" -node-gyp-build@^4.3.0, node-gyp-build@^4.8.4: +node-gyp-build@^4.8.4: version "4.8.4" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.4.tgz#8a70ee85464ae52327772a90d66c6077a900cfc8" integrity sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ== -node-gyp-build@~4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.1.1.tgz#d7270b5d86717068d114cc57fff352f96d745feb" - integrity sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ== - node-gyp@^12.1.0: version "12.2.0" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-12.2.0.tgz#ff73f6f509e33d8b7e768f889ffc9738ad117b07" @@ -14064,187 +13618,6 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" -pouch-stream@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/pouch-stream/-/pouch-stream-0.4.1.tgz#0c6d8475c9307677627991a2f079b301c3b89bdd" - integrity sha512-RAWFhsGDbG4xZQpvrrQlhrITVUNVCKmglfe5WWDnJaDf1u9DMaRLHv//m65tBZevuo4QTGjwcyggwYxd7AGLsg== - dependencies: - inherits "^2.0.1" - readable-stream "^1.0.27-1" - -pouchdb-adapter-leveldb-core@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-adapter-leveldb-core/-/pouchdb-adapter-leveldb-core-9.0.0.tgz#cebb4fbae4ed55b946c051b1ea27c56a8b32327d" - integrity sha512-b3ZGPtVXyivGL5SK3AIDG7PrNsZdoDpGFkmTytDTtctkVhxOg71gnXXP+CrupENPqSNG/eGbKW4w+bbMpxy6aA== - dependencies: - double-ended-queue "2.1.0-0" - levelup "4.4.0" - pouchdb-adapter-utils "9.0.0" - pouchdb-binary-utils "9.0.0" - pouchdb-core "9.0.0" - pouchdb-errors "9.0.0" - pouchdb-json "9.0.0" - pouchdb-md5 "9.0.0" - pouchdb-merge "9.0.0" - pouchdb-utils "9.0.0" - sublevel-pouchdb "9.0.0" - through2 "3.0.2" - -pouchdb-adapter-memory@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-adapter-memory/-/pouchdb-adapter-memory-9.0.0.tgz#e91e8bf7c627dbaddbfadbf5648927d4e0088b9b" - integrity sha512-XbCwJ5f5U9dGdkiDikzYjTebdPHuA6Ghylx1Pq0lDe4y6l8R9xhjDSUy56pJ8G2F4Z+8QdB5FBY9EQoFlFSXWQ== - dependencies: - memdown "1.4.1" - pouchdb-adapter-leveldb-core "9.0.0" - -pouchdb-adapter-utils@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-adapter-utils/-/pouchdb-adapter-utils-9.0.0.tgz#9e95eec33baae82380acde655e826a2f111c8aa5" - integrity sha512-hmbm4ey0HL0vtoY1tRTPIt2FfYjvMh3DWoGGSxXDTS73qTFQ+Fhhi5I0AnN9PcD2omfKQAVXiYks4kkMvlAHqA== - dependencies: - pouchdb-binary-utils "9.0.0" - pouchdb-errors "9.0.0" - pouchdb-md5 "9.0.0" - pouchdb-merge "9.0.0" - pouchdb-utils "9.0.0" - -pouchdb-binary-utils@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-binary-utils/-/pouchdb-binary-utils-9.0.0.tgz#eafed32c21e92ef4b253456f9e53c4cf2cfd99fd" - integrity sha512-2OMtgDZi82vqs+zNDE0YiYjOaWkYCUcZJZKK3WkRr+XYRu+2B7umJrnygJFhUwoGedBbHSrlQBLhdNV3F1AX1A== - -pouchdb-changes-filter@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-changes-filter/-/pouchdb-changes-filter-9.0.0.tgz#24016be7a968b5935a81be394dc370a062c7bd25" - integrity sha512-ig0fo0WLgIjAniFJ19Uw1Y+oxiypqC+Skhd8BCETRVXOhLBzueRwEQR4thffyo0UayYVqldJfSR5wHSDvEVk/A== - dependencies: - pouchdb-errors "9.0.0" - pouchdb-selector-core "9.0.0" - pouchdb-utils "9.0.0" - -pouchdb-collate@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-collate/-/pouchdb-collate-9.0.0.tgz#654f6766927ada60603ba25b6b2ae533564fa302" - integrity sha512-TrnEDNZEmIIl+W3xKUO8h+geqVLQ90oZe5ujPkl8myUzpREULWXWQBnV5EzPXVEKDBpJlb8T3I6oy/zdWGQpdA== - -pouchdb-core@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-core/-/pouchdb-core-9.0.0.tgz#6204851233624d9d62e17f06c47b0bf4ca4e99db" - integrity sha512-98SJgs8bqXhr4gMGuOTR8yVeLlMYy797zlOtdlvlXIxIicvocyA8ColhVVhdBXPNOGxT2HwReIMywdIVAgibpg== - dependencies: - pouchdb-changes-filter "9.0.0" - pouchdb-errors "9.0.0" - pouchdb-fetch "9.0.0" - pouchdb-merge "9.0.0" - pouchdb-utils "9.0.0" - uuid "8.3.2" - -pouchdb-errors@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-errors/-/pouchdb-errors-9.0.0.tgz#f84269ce3327abef9455c0a90a51c26d7dca20c6" - integrity sha512-961PSMLhW0UqqdJ566g+CdLZ5pkBJRd6l4WWpCDdD0USvE4xYfYGzv43w7nZZBw1k3Xdy092yqPge7yX/tfnyw== - -pouchdb-fetch@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-fetch/-/pouchdb-fetch-9.0.0.tgz#a2cf407c75c9fc68a1924b08c9b574d28e1be7dd" - integrity sha512-TbE3cUcAJQrwb9kr44tDP0X+NAbcqgjsTvcL30L4xzBNJeCPTIRjukYX80s154SHJUXBxcWRiPsMmNqpXsjfCA== - dependencies: - fetch-cookie "2.2.0" - node-fetch "2.6.9" - -pouchdb-json@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-json/-/pouchdb-json-9.0.0.tgz#f0adcf73d0db5c175bfd4f2db7084254c0b01bb6" - integrity sha512-aI41mYVyI195GXuT1Ys7mLIB/Mvrz11ihoTP6km6hYqVgSuaUxuZcFUozlyTJiZXr7H5kdhNgclhlVnjir4JAA== - dependencies: - vuvuzela "1.0.3" - -pouchdb-md5@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-md5/-/pouchdb-md5-9.0.0.tgz#f67a2ba627309e65f8d1ce4d4baf6a5f29164617" - integrity sha512-58xUYBvW3/s+aH0j4uOhhN8yCk0LQ254cxBzI/gbKA9PrfwHpe4zrr0L/ia5ml3A30oH1f8aTnuVMwWDkFcuww== - dependencies: - pouchdb-binary-utils "9.0.0" - spark-md5 "3.0.2" - -pouchdb-merge@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-merge/-/pouchdb-merge-9.0.0.tgz#d56b12bee4f2c9a981cf11c5cd44774e5b12b4ac" - integrity sha512-Xh+TgOZCkGoZpI589btKf/cTiuQ5CsnPl9YpdW4h0cAPusniN6XNsR62F+/HbL9wirI6XTEPHUrk7MsQbk3S3A== - dependencies: - pouchdb-utils "9.0.0" - -pouchdb-promise@^6.0.4: - version "6.4.3" - resolved "https://registry.yarnpkg.com/pouchdb-promise/-/pouchdb-promise-6.4.3.tgz#74516f4acf74957b54debd0fb2c0e5b5a68ca7b3" - integrity sha512-ruJaSFXwzsxRHQfwNHjQfsj58LBOY1RzGzde4PM5CWINZwFjCQAhZwfMrch2o/0oZT6d+Xtt0HTWhq35p3b0qw== - dependencies: - lie "3.1.1" - -pouchdb-selector-core@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-9.0.0.tgz#6fee1df82cd5ecdbd0a034b38e6c604557d2e22a" - integrity sha512-ZYHYsdoedwm8j5tYofz+3+uUSK8i+7tRCBb01T0OuqDQb17+w5mzjHF8Ppi160xdPUPaWCo1Un+nLWGJzkmA3g== - dependencies: - pouchdb-collate "9.0.0" - pouchdb-utils "9.0.0" - -pouchdb-utils@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb-utils/-/pouchdb-utils-9.0.0.tgz#b68f3259add50163998201d1a6d16e6a35d5d57f" - integrity sha512-xWZE5c+nAslgmLC8JBZbky8AYgdz7pKtv7KTSi6CD2tuQD0WyNKib0YnhZndeE84dksTeZlqlg56RQHsHoB2LQ== - dependencies: - pouchdb-errors "9.0.0" - pouchdb-md5 "9.0.0" - uuid "8.3.2" - -pouchdb@7.3.0: - version "7.3.0" - resolved "https://registry.yarnpkg.com/pouchdb/-/pouchdb-7.3.0.tgz#440fbef12dfd8f9002320802528665e883a3b7f8" - integrity sha512-OwsIQGXsfx3TrU1pLruj6PGSwFH+h5k4hGNxFkZ76Um7/ZI8F5TzUHFrpldVVIhfXYi2vP31q0q7ot1FSLFYOw== - dependencies: - abort-controller "3.0.0" - argsarray "0.0.1" - buffer-from "1.1.2" - clone-buffer "1.0.0" - double-ended-queue "2.1.0-0" - fetch-cookie "0.11.0" - immediate "3.3.0" - inherits "2.0.4" - level "6.0.1" - level-codec "9.0.2" - level-write-stream "1.0.0" - leveldown "5.6.0" - levelup "4.4.0" - ltgt "2.2.1" - node-fetch "2.6.7" - readable-stream "1.1.14" - spark-md5 "3.0.2" - through2 "3.0.2" - uuid "8.3.2" - vuvuzela "1.0.3" - -pouchdb@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/pouchdb/-/pouchdb-9.0.0.tgz#569ee3941f7b03dd34b4b4e53132a9772981a35e" - integrity sha512-6wjFc/PzwaWz86rmMXoqdBlR/fBSkNoWO1mEJO7RZNS6n3xf+fhhXWAWtws741KpLKx84IkmmJ48tp+fhFzj4A== - dependencies: - double-ended-queue "2.1.0-0" - fetch-cookie "2.2.0" - level "6.0.1" - level-codec "9.0.2" - level-write-stream "1.0.0" - leveldown "6.1.1" - levelup "4.4.0" - ltgt "2.2.1" - node-fetch "2.6.9" - readable-stream "1.1.14" - spark-md5 "3.0.2" - through2 "3.0.2" - uuid "8.3.2" - vuvuzela "1.0.3" - prebuild-install@^7.1.3: version "7.1.3" resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.3.tgz#d630abad2b147443f20a212917beae68b8092eec" @@ -14535,11 +13908,6 @@ querystringify@^2.1.1: resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6" integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ== -queue-microtask@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" - integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== - quick-format-unescaped@^4.0.3: version "4.0.4" resolved "https://registry.yarnpkg.com/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz#93ef6dd8d3453cbc7970dd614fad4c5954d6b5a7" @@ -14638,17 +14006,7 @@ read@^4.0.0: dependencies: mute-stream "^2.0.0" -readable-stream@1.1.14, readable-stream@^1.0.27-1: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - integrity sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ== - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - -"readable-stream@2 || 3", readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: +readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.5.0, readable-stream@^3.6.0: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== @@ -14681,11 +14039,6 @@ readable-stream@^4.0.0, readable-stream@^4.2.0: process "^0.11.10" string_decoder "^1.3.0" -readable-stream@~0.0.2: - version "0.0.4" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-0.0.4.tgz#f32d76e3fb863344a548d79923007173665b3b8d" - integrity sha512-azrivNydKRYt7zwLV5wWUK7YzKTWs3q87xSmY6DlHapPrCvaT6ZrukvM5erV+yCSSPmZT8zkSdttOHQpWWm9zw== - readable-stream@~1.0.31: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" @@ -15261,11 +14614,6 @@ serve-static@~1.16.2: parseurl "~1.3.3" send "~0.19.1" -set-cookie-parser@^2.4.8: - version "2.7.2" - resolved "https://registry.yarnpkg.com/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz#ccd08673a9ae5d2e44ea2a2de25089e67c7edf68" - integrity sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw== - set-function-length@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" @@ -15576,11 +14924,6 @@ source-map@^0.7.4: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.6.tgz#a3658ab87e5b6429c8a1f3ba0083d4c61ca3ef02" integrity sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ== -spark-md5@3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc" - integrity sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw== - sparse-bitfield@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz#ff4ae6e68656056ba4b3e792ab3334d38273ca11" @@ -15632,13 +14975,6 @@ split-on-first@^1.0.0: resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" integrity sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw== -split2@^2.1.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/split2/-/split2-2.2.0.tgz#186b2575bcf83e85b7d18465756238ee4ee42493" - integrity sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw== - dependencies: - through2 "^2.0.2" - split2@^3.2.2: version "3.2.2" resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f" @@ -15973,15 +15309,6 @@ style-mod@^4.0.0, style-mod@^4.1.0: resolved "https://registry.yarnpkg.com/style-mod/-/style-mod-4.1.3.tgz#6e9012255bb799bdac37e288f7671b5d71bf9f73" integrity sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ== -sublevel-pouchdb@9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/sublevel-pouchdb/-/sublevel-pouchdb-9.0.0.tgz#e8c3ed95c08301c79e0cd840fb7c244ad5f78b6e" - integrity sha512-pX4r8+F7wuts0C81kUJ341h4bl2aRe7qV572FE8X1FMz9VkKlmi2nPD1vfeiOJXz5Y09I4MHjGULAbqvTfQZEQ== - dependencies: - level-codec "9.0.2" - ltgt "2.2.1" - readable-stream "1.1.14" - superagent@^10.1.1, superagent@^10.3.0: version "10.3.0" resolved "https://registry.yarnpkg.com/superagent/-/superagent-10.3.0.tgz#ff1e39e7976b63f8084291d65f5bfbbbbd156989" @@ -16383,15 +15710,7 @@ thread-stream@^2.0.0, thread-stream@^2.6.0: dependencies: real-require "^0.2.0" -through2@3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" - integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== - dependencies: - inherits "^2.0.4" - readable-stream "2 || 3" - -through2@^2.0.0, through2@^2.0.1, through2@^2.0.2, through2@^2.0.3: +through2@^2.0.0, through2@^2.0.1: version "2.0.5" resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd" integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ== @@ -16507,7 +15826,7 @@ touch@^3.1.0: resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694" integrity sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA== -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.4: +tough-cookie@4.1.3, tough-cookie@^4.1.4: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -16790,11 +16109,6 @@ undici-types@~5.26.4: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== -undici-types@~6.21.0: - version "6.21.0" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" - integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== - undici-types@~7.18.0: version "7.18.2" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.18.2.tgz#29357a89e7b7ca4aef3bf0fd3fd0cd73884229e9" @@ -16997,7 +16311,7 @@ uuid-random@^1.3.2: resolved "https://registry.yarnpkg.com/uuid-random/-/uuid-random-1.3.2.tgz#96715edbaef4e84b1dcf5024b00d16f30220e2d0" integrity sha512-UOzej0Le/UgkbWEO8flm+0y+G+ljUon1QWTEZOq1rnMAsxo2+SckbiZdKzAHHlVh6gJqI1TjC/xwgR50MuCrBQ== -uuid@11.1.1, uuid@8.0.0, uuid@8.3.2, uuid@^11.1.1, uuid@^8.3.0: +uuid@11.1.1, uuid@8.0.0, uuid@^11.1.1, uuid@^8.3.0: version "11.1.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.1.1.tgz#f6d81d2e1c65d00762e5e29b16c5d2d995e208ad" integrity sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ== @@ -17098,11 +16412,6 @@ vitest@4.1.9: vite "^6.0.0 || ^7.0.0 || ^8.0.0" why-is-node-running "^2.3.0" -vuvuzela@1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b" - integrity sha512-Tm7jR1xTzBbPW+6y1tknKiEhz04Wf/1iZkcTJjSFcpNko43+dFW6+OOeQe9taJIug3NdfUAjFKgUSyQrIKaDvQ== - w3c-keyname@^2.2.4: version "2.2.8" resolved "https://registry.yarnpkg.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz#7b17c8c6883d4e8b86ac8aba79d39e880f8869c5" @@ -17352,13 +16661,6 @@ write-file-atomic@^6.0.0: imurmurhash "^0.1.4" signal-exit "^4.0.1" -write-stream@~0.4.3: - version "0.4.3" - resolved "https://registry.yarnpkg.com/write-stream/-/write-stream-0.4.3.tgz#83cc8c0347d0af6057a93862b4e3ae01de5c81c1" - integrity sha512-IJrvkhbAnj89W/GAVdVgbnPiVw5Ntg/B4tc/MUCIEwj/g6JIww1DWJyB/yBMT3yw2/TkT6IUZ0+IYef3flEw8A== - dependencies: - readable-stream "~0.0.2" - ws@8.21.0, ws@^8.18.0, ws@~8.18.3, ws@~8.20.1: version "8.21.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.0.tgz#012e413fc07429945121b0c153158c4343086951" @@ -17402,7 +16704,7 @@ xmlhttprequest-ssl@~2.1.1: resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.1.2.tgz#e9e8023b3f29ef34b97a859f584c5e6c61418e23" integrity sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ== -xtend@^4.0.0, xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1: +xtend@^4.0.0, xtend@~4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== From ae5b25ca1f5b47df38d2f4e14a9995e153914360 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sat, 27 Jun 2026 19:57:37 +0100 Subject: [PATCH 13/24] Fix workspace import --- packages/backend-core/package.json | 2 +- .../backend-core/src/db/couch/DatabaseImpl.ts | 79 ++++++++----------- packages/server/package.json | 2 +- packages/types/package.json | 2 +- yarn.lock | 8 +- 5 files changed, 38 insertions(+), 55 deletions(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 486f9532..cb0ef20d 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -34,7 +34,7 @@ "@aws-sdk/client-s3": "3.1041.0", "@aws-sdk/lib-storage": "3.1041.0", "@smithy/node-http-handler": "4.1.1", - "nano": "11.0.3", + "nano": "11.0.5", "@supertoolmake/shared-core": "*", "@supertoolmake/types": "*", "@techpass/passport-openidconnect": "1.0.0", diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ac336f12..700910de 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -327,13 +327,16 @@ export class CouchDatabase implements Database { }) } - async bulkDocs(documents: AnyDocument[]) { + async bulkDocs(documents: AnyDocument[], opts?: { new_edits?: boolean }) { const now = new Date().toISOString() return this.performCallWithDBCreation((db) => { - return () => - db.bulk({ - docs: documents.map((d) => ({ createdAt: now, ...d, updatedAt: now })), - }) + const body: any = { + docs: documents.map((d) => ({ createdAt: now, ...d, updatedAt: now })), + } + if (opts?.new_edits !== undefined) { + body.new_edits = opts.new_edits + } + return () => db.bulk(body) }) } @@ -477,51 +480,31 @@ export class CouchDatabase implements Database { } async load(stream: ReadStream) { - return new Promise<{ ok: boolean }>((resolve, reject) => { - let buffer = "" - - stream.on("data", async (chunk: string | Buffer) => { - buffer += chunk.toString() - const lines = buffer.split("\n") - // Keep the last incomplete line in the buffer - buffer = lines.pop() || "" - - const docs: any[] = [] - for (const line of lines) { - if (line.trim()) { - try { - docs.push(JSON.parse(line)) - } catch { - // skip malformed lines - } - } - } - - if (docs.length > 0) { - try { - await this.bulkDocs(docs) - } catch (err) { - reject(err) - return - } - } - }) + const chunks: Buffer[] = [] + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + const content = Buffer.concat(chunks).toString() + const lines = content.split("\n") - stream.on("end", async () => { - // Process any remaining data in the buffer - if (buffer.trim()) { - try { - const doc = JSON.parse(buffer) - await this.bulkDocs([doc]) - } catch { - // skip malformed lines - } + const docs: any[] = [] + for (const line of lines) { + if (!line.trim()) continue + try { + const parsed = JSON.parse(line) + // PouchDB dump format: batch lines have a "docs" array + if (Array.isArray(parsed.docs)) { + docs.push(...parsed.docs) } - resolve({ ok: true }) - }) - - stream.on("error", reject) - }) + // Skip metadata headers (have "version"/"db_info") and seq tracking + } catch { + // skip malformed lines + } + } + if (docs.length > 0) { + await this.bulkDocs(docs, { new_edits: false }) + } + return { ok: true } } async createIndex(opts: DatabaseCreateIndexOpts) { diff --git a/packages/server/package.json b/packages/server/package.json index 96ffe8b9..db0c7549 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -82,7 +82,7 @@ "mongodb": "6.7.0", "mssql": "12.5.4", "mysql2": "3.9.8", - "nano": "11.0.3", + "nano": "11.0.5", "node-fetch": "2.6.7", "openapi-types": "9.3.1", "pg": "8.10.0", diff --git a/packages/types/package.json b/packages/types/package.json index f7d3a3a3..51b96dd9 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -15,7 +15,7 @@ "check:types": "tsc -p tsconfig.json --noEmit --paths null" }, "devDependencies": { - "nano": "11.0.3", + "nano": "11.0.5", "@types/json-schema": "^7.0.15", "@types/koa": "3.0.0", "@types/nodemailer": "8.0.0", diff --git a/yarn.lock b/yarn.lock index 5733d7a9..7452df04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12324,10 +12324,10 @@ nan@2.22.0, nan@^2.19.0, nan@^2.23.0: resolved "https://registry.yarnpkg.com/nan/-/nan-2.22.0.tgz#31bc433fc33213c97bad36404bb68063de604de3" integrity sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw== -nano@11.0.3: - version "11.0.3" - resolved "https://registry.yarnpkg.com/nano/-/nano-11.0.3.tgz#61dffb1133d9f844030550008964835098ccc8d2" - integrity sha512-bOiBWTW5sA92Rp2Tuu5LmujUe0rre5AAKuiXuhjD4tBAVRZRrKWK0AJyEYWFm9PN8TRktsbUfUdDb//c5Cr3Dg== +nano@11.0.5: + version "11.0.5" + resolved "https://registry.yarnpkg.com/nano/-/nano-11.0.5.tgz#4a6e5fe3e6b7824e3fdd75b6b51fa9ae08680dba" + integrity sha512-yLYmFH7S33OWYPAG12wNqMR6l9hC3qOWLbr7hZPqKBVKO8BYhAaLlWENeJEpPY8Rm12lGvzKWBp8LGEgWhEqmw== nanoid@^3.3.11: version "3.3.11" From e33198cdbcab6c532332daddec7e4d68678dbf2a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sat, 27 Jun 2026 20:41:21 +0100 Subject: [PATCH 14/24] fix: restore couchdb-native replication and export support --- packages/backend-core/src/db/Replication.ts | 228 ++++++++++-------- .../backend-core/src/db/couch/DatabaseImpl.ts | 21 +- .../src/db/couch/tests/DatabaseImpl.spec.ts | 101 ++++++++ .../src/db/tests/Replication.spec.ts | 42 ++++ .../server/src/api/controllers/table/index.ts | 43 ++-- packages/server/src/db/index.ts | 7 + 6 files changed, 317 insertions(+), 125 deletions(-) create mode 100644 packages/server/src/db/index.ts diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 6be22fad..e65db408 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,8 +1,11 @@ import { type Document, DocumentType } from "@supertoolmake/types" import { DesignDocuments, SEPARATOR, USER_METADATA_PREFIX } from "../constants" +import { newid } from "../docIds/newid" import { directCouchCall, directCouchQuery, getCouchInfo } from "./couch" import { getDB } from "./db" +const FILTER_NAME = "replication" + enum ReplicationDirection { TO_PRODUCTION = "toProduction", TO_DEV = "toDev", @@ -35,74 +38,6 @@ class Replication { } } - private shouldKeepDoc( - doc: { _id: string; _deleted?: boolean }, - opts: { - isCreation?: boolean - toDev: boolean - syncAllTables: boolean - tableSyncList?: string[] - customFilter?: (doc: Document) => boolean | undefined - } - ): boolean { - const { isCreation, toDev, syncAllTables, tableSyncList, customFilter } = opts - const toProduction = !toDev - - const startsWithType = (id: string, docType: string) => - id && id.indexOf(docType + SEPARATOR) === 0 - - const isData = (id: string) => - startsWithType(id, DocumentType.ROW) || startsWithType(id, DocumentType.LINK) - - if (doc._deleted) { - return true - } - - if (!isCreation && doc._id === DesignDocuments.MIGRATIONS) { - return false - } - - if (toDev && doc._id.indexOf("_design") === 0) { - return false - } - - if (doc._id && doc._id.indexOf(USER_METADATA_PREFIX) === 0) { - return true - } - - if (toProduction && !isCreation && startsWithType(doc._id, DocumentType.AUTO_COLUMN_STATE)) { - return false - } - - if (isData(doc._id)) { - if (syncAllTables) { - return true - } - if (tableSyncList) { - for (const tableId of tableSyncList) { - if (doc._id.indexOf(tableId) !== -1) { - return true - } - } - } - return false - } - - if (startsWithType(doc._id, DocumentType.AUTOMATION_LOG)) { - return false - } - - if (doc._id === DocumentType.WORKSPACE_METADATA) { - return false - } - - if (customFilter) { - return !!customFilter(doc as Document) - } - - return true - } - async replicate(opts: ReplicateOpts = {}) { const direction = this.direction const toDev = direction === ReplicationDirection.TO_DEV @@ -118,52 +53,143 @@ class Replication { tableSyncList = tablesToSync } - const { url, auth } = getCouchInfo() - const parsed = new URL(url) - const internalUrl = `http://${auth.username}:${auth.password}@${parsed.hostname}:5984` - const replicateBody: any = { - source: `${internalUrl}/${this.sourceName}`, - target: `${internalUrl}/${this.targetName}`, - create_target: true, - } + const filterDesignName = `replication_${newid()}` + const filterDesignId = `_design/${filterDesignName}` + const sourceDb = getDB(this.sourceName, { skip_setup: true }) + + await sourceDb.put({ + _id: filterDesignId, + filters: { + [FILTER_NAME]: this.buildFilterFunction({ + filterDesignId, + customFilter, + }), + }, + } as any) + + try { + const queryParams: Record = { + isCreation: String(!!isCreation), + toDev: String(!!toDev), + syncAllTables: String(!!syncAllTables), + } + if (tableSyncList) { + queryParams.tableSyncList = tableSyncList.join(",") + } - const response = await directCouchCall("_replicate", "POST", replicateBody) - const result = await response.json() + const { url, auth } = getCouchInfo() + const parsed = new URL(url) + const internalUrl = `http://${auth.username}:${auth.password}@${parsed.hostname}:5984` + const replicateBody: any = { + source: `${internalUrl}/${this.sourceName}`, + target: `${internalUrl}/${this.targetName}`, + filter: `${filterDesignName}/${FILTER_NAME}`, + query_params: queryParams, + create_target: true, + } - if (result.history?.find((h: any) => h.errors && h.errors.length > 0)) { - throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) - } + const response = await directCouchCall("_replicate", "POST", replicateBody) + const result = await response.json() - const filterOpts = { isCreation, toDev, syncAllTables, tableSyncList, customFilter } - await this.cleanupTarget(filterOpts) + if (result.history?.find((h: any) => h.errors && h.errors.length > 0)) { + throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) + } - return result + return result + } finally { + try { + const existing = await sourceDb.get(filterDesignId) + await sourceDb.remove(filterDesignId, existing._rev) + } catch {} + } } - private async cleanupTarget(opts: { - isCreation?: boolean - toDev: boolean - syncAllTables: boolean - tableSyncList?: string[] + private buildFilterFunction({ + filterDesignId, + customFilter, + }: { + filterDesignId: string customFilter?: (doc: Document) => boolean | undefined }) { - const targetDb = getDB(this.targetName, { skip_setup: true }) - const allDocs = await targetDb.allDocs({ include_docs: true }) - const docsToDelete: { _id: string; _rev: string; _deleted: true }[] = [] + const customFilterSource = customFilter ? customFilter.toString() : undefined - for (const row of allDocs.rows) { - const doc = row.doc as any + return `function (doc, req) { if (!doc || !doc._id) { - continue + return false; } - if (!this.shouldKeepDoc(doc, opts)) { - docsToDelete.push({ _id: doc._id, _rev: doc._rev, _deleted: true }) + if (doc._id === ${JSON.stringify(filterDesignId)}) { + return false; + } + if (!req.query) { + req.query = {}; } - } - if (docsToDelete.length > 0) { - await targetDb.bulkDocs(docsToDelete) - } + var isCreation = req.query.isCreation === "true"; + var toDev = req.query.toDev === "true"; + var syncAllTables = req.query.syncAllTables === "true"; + var tableSyncList = req.query.tableSyncList ? req.query.tableSyncList.split(",") : []; + var toProduction = !toDev; + + function startsWithType(id, docType) { + return id && id.indexOf(docType + ${JSON.stringify(SEPARATOR)}) === 0; + } + + function isData(id) { + return startsWithType(id, ${JSON.stringify(DocumentType.ROW)}) || + startsWithType(id, ${JSON.stringify(DocumentType.LINK)}); + } + + if (doc._deleted) { + return true; + } + + if (!isCreation && doc._id === ${JSON.stringify(DesignDocuments.MIGRATIONS)}) { + return false; + } + + if (toDev && doc._id.indexOf("_design") === 0) { + return false; + } + + if (doc._id.indexOf(${JSON.stringify(USER_METADATA_PREFIX)}) === 0) { + return true; + } + + if ( + toProduction && + !isCreation && + startsWithType(doc._id, ${JSON.stringify(DocumentType.AUTO_COLUMN_STATE)}) + ) { + return false; + } + + if (isData(doc._id)) { + if (syncAllTables) { + return true; + } + for (var i = 0; i < tableSyncList.length; i++) { + if (doc._id.indexOf(tableSyncList[i]) !== -1) { + return true; + } + } + return false; + } + + if (startsWithType(doc._id, ${JSON.stringify(DocumentType.AUTOMATION_LOG)})) { + return false; + } + + if (doc._id === ${JSON.stringify(DocumentType.WORKSPACE_METADATA)}) { + return false; + } + + ${ + customFilterSource + ? `var customFilter = (${customFilterSource}); + return !!customFilter(doc);` + : "return true;" + } + }` } async rollback() { diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 700910de..ae176e19 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -352,9 +352,12 @@ export class CouchDatabase implements Database { params: DocumentListParams ): Promise> { return this.performCall((db) => { + const cleanParams = Object.fromEntries( + Object.entries(params).filter(([, value]) => value !== undefined) + ) as DocumentListParams return async () => { try { - return (await db.list(params)) as AllDocsResponse + return (await db.list(cleanParams)) as AllDocsResponse } catch (err: any) { if (err.reason === DATABASE_NOT_FOUND) { return { @@ -443,7 +446,7 @@ export class CouchDatabase implements Database { async dump(stream: WriteStream, opts?: DatabaseDumpOpts) { const batchSize = opts?.batch_size || 1000 - let startKey = "" + let startKey: string | undefined let hasMore = true const filterFn = opts?.filter @@ -452,7 +455,8 @@ export class CouchDatabase implements Database { const params: DocumentListParams = { include_docs: true, limit: batchSize, - startkey: startKey || undefined, + startkey: startKey, + skip: startKey ? 1 : undefined, } const response = await this.allDocs(params) @@ -488,21 +492,28 @@ export class CouchDatabase implements Database { const lines = content.split("\n") const docs: any[] = [] + const pouchDumpDocs: any[] = [] for (const line of lines) { if (!line.trim()) continue try { const parsed = JSON.parse(line) // PouchDB dump format: batch lines have a "docs" array if (Array.isArray(parsed.docs)) { - docs.push(...parsed.docs) + pouchDumpDocs.push(...parsed.docs) + } else if (parsed._id) { + const { _rev, ...doc } = parsed + docs.push(doc) } // Skip metadata headers (have "version"/"db_info") and seq tracking } catch { // skip malformed lines } } + if (pouchDumpDocs.length > 0) { + await this.bulkDocs(pouchDumpDocs, { new_edits: false }) + } if (docs.length > 0) { - await this.bulkDocs(docs, { new_edits: false }) + await this.bulkDocs(docs) } return { ok: true } } diff --git a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts index 2a2ff6b5..5d07144e 100644 --- a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts +++ b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts @@ -1,3 +1,4 @@ +import { Readable, Writable } from "node:stream" import tk from "timekeeper" import { generator, structures } from "../../../../tests" import { CouchDatabase } from ".." @@ -177,4 +178,104 @@ describe("DatabaseImpl", () => { } }) }) + + describe("dump", () => { + async function collectDump(database: CouchDatabase, opts?: Parameters[1]) { + const chunks: string[] = [] + const stream = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(chunk.toString()) + callback() + }, + }) + + await database.dump(stream as any, opts) + + return chunks + .join("") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + } + + it("dumps documents across multiple batches", async () => { + const dumpDb = new CouchDatabase(structures.db.id()) + await dumpDb.bulkDocs([ + { _id: "dump_doc_1", value: 1 }, + { _id: "dump_doc_2", value: 2 }, + { _id: "dump_doc_3", value: 3 }, + ]) + + const docs = await collectDump(dumpDb, { batch_size: 2 }) + + expect(docs.map((doc) => doc._id)).toEqual(["dump_doc_1", "dump_doc_2", "dump_doc_3"]) + }) + + it("applies filters while dumping across multiple batches", async () => { + const dumpDb = new CouchDatabase(structures.db.id()) + await dumpDb.bulkDocs([ + { _id: "dump_filter_1", include: true }, + { _id: "dump_filter_2", include: false }, + { _id: "dump_filter_3", include: true }, + ]) + + const docs = await collectDump(dumpDb, { + batch_size: 2, + filter: (doc: any) => doc.include, + }) + + expect(docs.map((doc) => doc._id)).toEqual(["dump_filter_1", "dump_filter_3"]) + }) + }) + + describe("load", () => { + it("loads JSON-lines document dumps", async () => { + const loadDb = new CouchDatabase(structures.db.id()) + const stream = Readable.from( + [ + JSON.stringify({ _id: "load_doc_1", _rev: "1-source", value: 1 }), + JSON.stringify({ _id: "load_doc_2", _rev: "1-source", value: 2 }), + ].join("\n") + ) + + await loadDb.load(stream as any) + + expect(await loadDb.get("load_doc_1")).toMatchObject({ + _id: "load_doc_1", + _rev: expect.any(String), + value: 1, + }) + expect(await loadDb.get("load_doc_2")).toMatchObject({ + _id: "load_doc_2", + _rev: expect.any(String), + value: 2, + }) + }) + + it("loads PouchDB batch dumps", async () => { + const loadDb = new CouchDatabase(structures.db.id()) + const stream = Readable.from( + `${JSON.stringify({ + docs: [ + { + _id: "load_pouch_doc_1", + _rev: "1-967a00dff5e02add41819138abb3284d", + value: 1, + _revisions: { + start: 1, + ids: ["967a00dff5e02add41819138abb3284d"], + }, + }, + ], + })}\n` + ) + + await loadDb.load(stream as any) + + expect(await loadDb.get("load_pouch_doc_1")).toMatchObject({ + _id: "load_pouch_doc_1", + value: 1, + }) + }) + }) }) diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index 11816f78..f8b8b558 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -183,6 +183,48 @@ describe("Replication", () => { expect(ids).not.toContain(`${DocumentType.ROLE}_user`) }, 30000) + it("does not remove existing target documents excluded by a custom filter", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) + + await getDB(source).bulkDocs([ + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }), + makeDoc(`${DocumentType.ROLE}_user`, { name: "user" }), + ]) + await getDB(target).bulkDocs([ + makeDoc(`${DocumentType.ROLE}_existing`, { name: "existing" }), + makeDoc(`${DocumentType.DATASOURCE}_existing`, { type: "postgres" }), + ]) + + const rep = new Replication({ source, target }) + await rep.replicate({ filter: (doc: any) => doc._id.includes("admin") }) + + const ids = await getAllDocIds(target) + expect(ids).toContain(`${DocumentType.ROLE}_admin`) + expect(ids).toContain(`${DocumentType.ROLE}_existing`) + expect(ids).toContain(`${DocumentType.DATASOURCE}_existing`) + expect(ids).not.toContain(`${DocumentType.ROLE}_user`) + }, 30000) + + it("cleans up temporary replication filter design documents", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) + + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + + const rep = new Replication({ source, target }) + await rep.replicate() + + const sourceIds = await getAllDocIds(source) + const targetIds = await getAllDocIds(target) + expect(sourceIds.some((id: string) => id.startsWith("_design/replication_"))).toBe(false) + expect(targetIds.some((id: string) => id.startsWith("_design/replication_"))).toBe(false) + }, 30000) + it("excludes design documents when replicating to dev (TO_DEV)", async () => { const source = `${DocumentType.WORKSPACE}_${structures.db.id()}` const target = `${DocumentType.WORKSPACE_DEV}_${structures.db.id()}` diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 39c2db97..fee9dba4 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -246,33 +246,38 @@ export async function publish(ctx: UserCtx - _id === tableId || _id.endsWith(`${SEPARATOR}${tableId}`) || _id.includes(tableSegment) - const replication = new dbCore.Replication({ source: dbCore.getDevWorkspaceID(appId), target: prodWorkspaceId, }) - await replication.replicate({ - tablesToSync: undefined, - filter: (doc: any) => { - const _id = doc?._id as string - if (!_id || _id.startsWith("_design")) { - return false - } - if (_id.startsWith(DocumentType.AUTOMATION_LOG)) { - return false + const filter = new Function( + "doc", + ` + var _id = doc && doc._id; + var tableId = ${JSON.stringify(tableId)}; + var separator = ${JSON.stringify(SEPARATOR)}; + var tableSegment = separator + tableId + separator; + var tableSuffix = separator + tableId; + + if (!_id || _id.indexOf("_design") === 0) { + return false; } - if (_id.startsWith(DocumentType.WORKSPACE_METADATA)) { - return false + if (_id.indexOf(${JSON.stringify(DocumentType.AUTOMATION_LOG)}) === 0) { + return false; } - if (!matchesTable(_id)) { - return false + if (_id.indexOf(${JSON.stringify(DocumentType.WORKSPACE_METADATA)}) === 0) { + return false; } - return true - }, + return _id === tableId || + _id.lastIndexOf(tableSuffix) === _id.length - tableSuffix.length || + _id.indexOf(tableSegment) !== -1; + ` + ) as (doc: any) => boolean + + await replication.replicate({ + tablesToSync: undefined, + filter, }) const metadata = await sdk.workspaces.metadata.tryGet({ diff --git a/packages/server/src/db/index.ts b/packages/server/src/db/index.ts new file mode 100644 index 00000000..db264ed9 --- /dev/null +++ b/packages/server/src/db/index.ts @@ -0,0 +1,7 @@ +import { db as dbCore } from "@supertoolmake/backend-core" + +export * from "./utils" + +export function init() { + dbCore.CouchDatabase.init() +} From f524a3ea32f1a39252f8c65dfe4fcb06c0500f4b Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sat, 27 Jun 2026 20:55:28 +0100 Subject: [PATCH 15/24] fix: reduce publish path work --- packages/backend-core/src/db/Replication.ts | 92 +++++++++++++++---- .../src/db/tests/Replication.spec.ts | 51 +++++++--- .../src/api/controllers/deploy/index.ts | 2 - .../server/src/api/controllers/table/index.ts | 3 +- 4 files changed, 116 insertions(+), 32 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index e65db408..b3f75507 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -5,6 +5,8 @@ import { directCouchCall, directCouchQuery, getCouchInfo } from "./couch" import { getDB } from "./db" const FILTER_NAME = "replication" +const FILTER_DESIGN_NAME = "super_replication" +const FILTER_DESIGN_ID = `_design/${FILTER_DESIGN_NAME}` enum ReplicationDirection { TO_PRODUCTION = "toProduction", @@ -17,6 +19,34 @@ interface ReplicateOpts { filter?: (doc: Document) => boolean | undefined } +type ReplicationFilterDoc = Document & { + filters: Record +} + +type ReplicationResponse = { + history?: Array<{ + errors?: unknown[] + }> +} + +type ReplicationBody = { + source: string + target: string + filter: string + query_params: Record + create_target: boolean +} + +function isNotFoundError(err: unknown) { + return ( + typeof err === "object" && + err !== null && + ("status" in err || "statusCode" in err) && + ((err as { status?: unknown }).status === 404 || + (err as { statusCode?: unknown }).statusCode === 404) + ) +} + class Replication { sourceName: string targetName: string @@ -53,19 +83,15 @@ class Replication { tableSyncList = tablesToSync } - const filterDesignName = `replication_${newid()}` - const filterDesignId = `_design/${filterDesignName}` + const filterDesignName = customFilter ? `replication_${newid()}` : FILTER_DESIGN_NAME + const filterDesignId = customFilter ? `_design/${filterDesignName}` : FILTER_DESIGN_ID const sourceDb = getDB(this.sourceName, { skip_setup: true }) + const filterFunction = this.buildFilterFunction({ + filterDesignId, + customFilter, + }) - await sourceDb.put({ - _id: filterDesignId, - filters: { - [FILTER_NAME]: this.buildFilterFunction({ - filterDesignId, - customFilter, - }), - }, - } as any) + await this.ensureFilter(sourceDb, filterDesignId, filterFunction) try { const queryParams: Record = { @@ -80,7 +106,7 @@ class Replication { const { url, auth } = getCouchInfo() const parsed = new URL(url) const internalUrl = `http://${auth.username}:${auth.password}@${parsed.hostname}:5984` - const replicateBody: any = { + const replicateBody: ReplicationBody = { source: `${internalUrl}/${this.sourceName}`, target: `${internalUrl}/${this.targetName}`, filter: `${filterDesignName}/${FILTER_NAME}`, @@ -89,18 +115,48 @@ class Replication { } const response = await directCouchCall("_replicate", "POST", replicateBody) - const result = await response.json() + const result = (await response.json()) as ReplicationResponse - if (result.history?.find((h: any) => h.errors && h.errors.length > 0)) { + if (result.history?.find((h) => h.errors && h.errors.length > 0)) { throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) } return result } finally { - try { - const existing = await sourceDb.get(filterDesignId) - await sourceDb.remove(filterDesignId, existing._rev) - } catch {} + if (customFilter) { + try { + const existing = await sourceDb.get(filterDesignId) + await sourceDb.remove(filterDesignId, existing._rev) + } catch {} + } + } + } + + private async ensureFilter( + sourceDb: ReturnType, + filterDesignId: string, + filterFunction: string + ) { + const doc = { + _id: filterDesignId, + filters: { + [FILTER_NAME]: filterFunction, + }, + } satisfies ReplicationFilterDoc + try { + const existing = await sourceDb.get(filterDesignId) + if (existing.filters?.[FILTER_NAME] === filterFunction) { + return + } + await sourceDb.put({ + ...doc, + _rev: existing._rev, + }) + } catch (err: unknown) { + if (!isNotFoundError(err)) { + throw err + } + await sourceDb.put(doc) } } diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index f8b8b558..fc584a2d 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -1,23 +1,29 @@ -import { DocumentType } from "@supertoolmake/types" +import { type Document, DocumentType, type RowValue } from "@supertoolmake/types" import { structures } from "../../../tests" import { getDB } from "../db" import Replication from "../Replication" +type TestDocument = Document & Record +type NamedDocument = Document & { name: string } +type ReplicationFilterDoc = Document & { filters: Record } + +const adminFilter = (doc: Document) => Boolean(doc._id && doc._id.includes("admin")) + const ensureDb = async (dbName: string) => { const db = getDB(dbName) const initId = `test_init_${structures.db.id()}` await db.put({ _id: initId, init: true }) - await db.remove(initId, (await db.get(initId))._rev) + await db.remove(initId, (await db.get(initId))._rev) } -const makeDoc = (id: string, extra: Record = {}) => ({ +const makeDoc = (id: string, extra: Record = {}): TestDocument => ({ _id: id, ...extra, }) const getAllDocIds = async (dbName: string) => { - const allDocs = await getDB(dbName).allDocs({ include_docs: false }) - return allDocs.rows.map((r: any) => r.id) + const allDocs = await getDB(dbName).allDocs({ include_docs: false }) + return allDocs.rows.map((row) => row.id) } describe("Replication", () => { @@ -176,7 +182,7 @@ describe("Replication", () => { ]) const rep = new Replication({ source, target }) - await rep.replicate({ filter: (doc: any) => doc._id.includes("admin") }) + await rep.replicate({ filter: adminFilter }) const ids = await getAllDocIds(target) expect(ids).toContain(`${DocumentType.ROLE}_admin`) @@ -199,7 +205,7 @@ describe("Replication", () => { ]) const rep = new Replication({ source, target }) - await rep.replicate({ filter: (doc: any) => doc._id.includes("admin") }) + await rep.replicate({ filter: adminFilter }) const ids = await getAllDocIds(target) expect(ids).toContain(`${DocumentType.ROLE}_admin`) @@ -208,7 +214,7 @@ describe("Replication", () => { expect(ids).not.toContain(`${DocumentType.ROLE}_user`) }, 30000) - it("cleans up temporary replication filter design documents", async () => { + it("reuses a stable replication filter design document", async () => { const source = structures.db.id() const target = structures.db.id() await ensureDb(source) @@ -218,6 +224,29 @@ describe("Replication", () => { const rep = new Replication({ source, target }) await rep.replicate() + const firstFilterDoc = await getDB(source).get( + "_design/super_replication" + ) + await rep.replicate() + const secondFilterDoc = await getDB(source).get( + "_design/super_replication" + ) + + const targetIds = await getAllDocIds(target) + expect(secondFilterDoc._rev).toEqual(firstFilterDoc._rev) + expect(targetIds).not.toContain("_design/super_replication") + }, 30000) + + it("cleans up temporary replication filter design documents", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) + + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + + const rep = new Replication({ source, target }) + await rep.replicate({ filter: adminFilter }) const sourceIds = await getAllDocIds(source) const targetIds = await getAllDocIds(target) @@ -322,7 +351,7 @@ describe("Replication", () => { const rep = new Replication({ source, target }) await rep.replicate() - const beforeDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + const beforeDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) expect(beforeDoc.name).toBe("admin") await getDB(target).put({ @@ -331,12 +360,12 @@ describe("Replication", () => { name: "modified", }) - const modifiedDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + const modifiedDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) expect(modifiedDoc.name).toBe("modified") await rep.rollback() - const afterDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + const afterDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) expect(afterDoc.name).toBe("admin") }, 60000) diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 3e24f18c..7be52c1e 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -210,9 +210,7 @@ export const publishWorkspaceInternal = async (ctx: PublishContext) => { target: prodId, } const replication = new dbCore.Replication(config) - const devDb = context.getDevWorkspaceDB() - await devDb.compact() await replication.replicate({ isCreation: !isPublished, tablesToSync, diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index fee9dba4..d5768bfd 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -11,6 +11,7 @@ import { type CsvToJsonRequest, type CsvToJsonResponse, type DeleteTableResponse, + type Document, DocumentType, type FetchTablesResponse, FieldType, @@ -273,7 +274,7 @@ export async function publish(ctx: UserCtx boolean + ) as (doc: Document) => boolean await replication.replicate({ tablesToSync: undefined, From 6455a0fa771bdf190b958ed3bfa8dd1faecc3d39 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sat, 27 Jun 2026 21:21:36 +0100 Subject: [PATCH 16/24] fix: keep local CouchDB replication --- globalSetup.ts | 21 +++++++++++++++++++- hosting/docker-compose.dev.yaml | 1 + hosting/single/couch/10-docker-default.ini | 2 +- packages/backend-core/src/db/Replication.ts | 22 +++++++++++++++------ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/globalSetup.ts b/globalSetup.ts index 62560d9c..c3f64225 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -92,6 +92,9 @@ export default async function setup() { [httpd] socket_options = [{nodelay, true}] + [chttpd] + bind_address = 0.0.0.0 + [couchdb] single_node = true @@ -129,7 +132,23 @@ export default async function setup() { Wait.forHttp("/minio/health/ready", 9000).withStartupTimeout(10000) ) - await Promise.all([couchdb.start(), minio.start()]) + const [startedCouchdb] = await Promise.all([couchdb.start(), minio.start()]) + const couchPort = startedCouchdb.getMappedPort(5984) + const couchAuth = Buffer.from("budibase:budibase").toString("base64") + const response = await fetch( + `http://127.0.0.1:${couchPort}/_node/nonode@nohost/_config/chttpd/bind_address`, + { + method: "PUT", + headers: { + Authorization: `Basic ${couchAuth}`, + "Content-Type": "application/json", + }, + body: JSON.stringify("0.0.0.0"), + } + ) + if (!response.ok) { + throw new Error("Unable to configure CouchDB bind address") + } } finally { lockfile.unlockSync(lockPath) } diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 388c1e74..9aec7a02 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -48,6 +48,7 @@ services: - "${COUCH_DB_PORT}:5984" volumes: - couchdb_data:/data + - ./single/couch/10-docker-default.ini:/opt/couchdb/etc/local.d/10-docker-default.ini redis-service: container_name: super-tool-redis-dev diff --git a/hosting/single/couch/10-docker-default.ini b/hosting/single/couch/10-docker-default.ini index 1aa633cb..6f4e1db0 100644 --- a/hosting/single/couch/10-docker-default.ini +++ b/hosting/single/couch/10-docker-default.ini @@ -5,4 +5,4 @@ ; overwritten on server upgrade. [chttpd] -bind_address = any +bind_address = 0.0.0.0 diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index b3f75507..d9bb9c08 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,7 +1,7 @@ import { type Document, DocumentType } from "@supertoolmake/types" import { DesignDocuments, SEPARATOR, USER_METADATA_PREFIX } from "../constants" import { newid } from "../docIds/newid" -import { directCouchCall, directCouchQuery, getCouchInfo } from "./couch" +import { directCouchCall, directCouchQuery } from "./couch" import { getDB } from "./db" const FILTER_NAME = "replication" @@ -29,6 +29,11 @@ type ReplicationResponse = { }> } +type ReplicationErrorResponse = { + error?: string + reason?: string +} + type ReplicationBody = { source: string target: string @@ -103,18 +108,23 @@ class Replication { queryParams.tableSyncList = tableSyncList.join(",") } - const { url, auth } = getCouchInfo() - const parsed = new URL(url) - const internalUrl = `http://${auth.username}:${auth.password}@${parsed.hostname}:5984` const replicateBody: ReplicationBody = { - source: `${internalUrl}/${this.sourceName}`, - target: `${internalUrl}/${this.targetName}`, + source: this.sourceName, + target: this.targetName, filter: `${filterDesignName}/${FILTER_NAME}`, query_params: queryParams, create_target: true, } const response = await directCouchCall("_replicate", "POST", replicateBody) + if (!response.ok) { + const error = (await response.json()) as ReplicationErrorResponse + throw new Error( + `Replication failed: ${error.error || response.status} - ${ + error.reason || response.statusText + }` + ) + } const result = (await response.json()) as ReplicationResponse if (result.history?.find((h) => h.errors && h.errors.length > 0)) { From c383a2a518db51cc2a3ca61f7e1d88806a84f417 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sat, 27 Jun 2026 21:22:01 +0100 Subject: [PATCH 17/24] chore: format couch db test --- .../backend-core/src/db/couch/tests/DatabaseImpl.spec.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts index 5d07144e..141de753 100644 --- a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts +++ b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts @@ -180,7 +180,10 @@ describe("DatabaseImpl", () => { }) describe("dump", () => { - async function collectDump(database: CouchDatabase, opts?: Parameters[1]) { + async function collectDump( + database: CouchDatabase, + opts?: Parameters[1] + ) { const chunks: string[] = [] const stream = new Writable({ write(chunk, _encoding, callback) { From b8e4e140b0893ddcb4a5281467d218868b4b18b0 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sat, 27 Jun 2026 21:22:21 +0100 Subject: [PATCH 18/24] fix: satisfy biome in replication tests --- packages/backend-core/src/db/tests/Replication.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index fc584a2d..0c898f67 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -7,7 +7,7 @@ type TestDocument = Document & Record type NamedDocument = Document & { name: string } type ReplicationFilterDoc = Document & { filters: Record } -const adminFilter = (doc: Document) => Boolean(doc._id && doc._id.includes("admin")) +const adminFilter = (doc: Document) => Boolean(doc._id?.includes("admin")) const ensureDb = async (dbName: string) => { const db = getDB(dbName) From 7230723d636fffd92b10259d5987ebdcd9b63bfa Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sun, 28 Jun 2026 18:56:18 +0100 Subject: [PATCH 19/24] Improve CouchDB replication batching --- CLAUDE.md | 1 + packages/backend-core/src/db/Replication.ts | 428 ++++++++++-------- .../src/db/tests/Replication.spec.ts | 97 ++-- 3 files changed, 308 insertions(+), 218 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4610f3bc..a4feaf64 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,7 @@ against, provided there are no type errors. - Avoid adding nested ternary statements. - When writing tests involving a URL, use example.com as the domain. +- Provide a termination condition instead of using while (true) ## Test style - packages/server diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index d9bb9c08..3ef242b3 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,32 +1,32 @@ -import { type Document, DocumentType } from "@supertoolmake/types" +import { + type AnyDocument, + type Document, + DocumentType, +} from "@supertoolmake/types" import { DesignDocuments, SEPARATOR, USER_METADATA_PREFIX } from "../constants" -import { newid } from "../docIds/newid" -import { directCouchCall, directCouchQuery } from "./couch" -import { getDB } from "./db" - -const FILTER_NAME = "replication" -const FILTER_DESIGN_NAME = "super_replication" -const FILTER_DESIGN_ID = `_design/${FILTER_DESIGN_NAME}` +import { directCouchCall } from "./couch" enum ReplicationDirection { TO_PRODUCTION = "toProduction", TO_DEV = "toDev", } +const REPLICATION_BATCH_SIZE = 200 + interface ReplicateOpts { isCreation?: boolean tablesToSync?: string[] | "all" filter?: (doc: Document) => boolean | undefined } -type ReplicationFilterDoc = Document & { - filters: Record -} - type ReplicationResponse = { - history?: Array<{ - errors?: unknown[] - }> + ok: true + docs_read: number + docs_written: number + doc_write_failures: number + start_time: string + end_time: string + last_seq?: unknown } type ReplicationErrorResponse = { @@ -34,21 +34,55 @@ type ReplicationErrorResponse = { reason?: string } -type ReplicationBody = { - source: string - target: string - filter: string - query_params: Record - create_target: boolean +type ChangesResponse = { + results?: Array<{ + id: string + changes?: Array<{ rev: string }> + deleted?: boolean + doc?: AnyDocument + seq?: unknown + }> + last_seq?: unknown + pending?: number +} + +type RevsDiffResponse = Record + +type BulkGetResponse = { + results?: Array<{ + docs?: Array<{ + ok?: AnyDocument + error?: ReplicationErrorResponse + }> + }> +} + +type BulkDocsResponse = Array<{ + id?: string + rev?: string + ok?: boolean + error?: string + reason?: string +}> + +async function readJson( + response: Awaited> +): Promise { + return (await response.json()) as T } -function isNotFoundError(err: unknown) { - return ( - typeof err === "object" && - err !== null && - ("status" in err || "statusCode" in err) && - ((err as { status?: unknown }).status === 404 || - (err as { statusCode?: unknown }).statusCode === 404) +async function throwReplicationError( + response: Awaited>, + action: string +) { + let error: ReplicationErrorResponse = {} + try { + error = await readJson(response) + } catch {} + throw new Error( + `Replication ${action} failed: ${error.error || response.status} - ${ + error.reason || response.statusText + }` ) } @@ -74,194 +108,212 @@ class Replication { } async replicate(opts: ReplicateOpts = {}) { - const direction = this.direction - const toDev = direction === ReplicationDirection.TO_DEV - const isCreation = opts.isCreation - const tablesToSync = opts.tablesToSync - const customFilter = opts.filter - - let syncAllTables = false, - tableSyncList: string[] | undefined - if (typeof tablesToSync === "string" && tablesToSync === "all") { - syncAllTables = true - } else if (tablesToSync) { - tableSyncList = tablesToSync - } - - const filterDesignName = customFilter ? `replication_${newid()}` : FILTER_DESIGN_NAME - const filterDesignId = customFilter ? `_design/${filterDesignName}` : FILTER_DESIGN_ID - const sourceDb = getDB(this.sourceName, { skip_setup: true }) - const filterFunction = this.buildFilterFunction({ - filterDesignId, - customFilter, - }) - - await this.ensureFilter(sourceDb, filterDesignId, filterFunction) - - try { - const queryParams: Record = { - isCreation: String(!!isCreation), - toDev: String(!!toDev), - syncAllTables: String(!!syncAllTables), - } - if (tableSyncList) { - queryParams.tableSyncList = tableSyncList.join(",") + const startTime = new Date().toISOString() + await this.ensureTarget() + + let since: unknown = 0 + let docsRead = 0 + let docsWritten = 0 + let docWriteFailures = 0 + let lastSeq: unknown + let hasMoreChanges = true + + while (hasMoreChanges) { + const changes = await this.getChanges(since) + const results = changes.results || [] + if (!results.length) { + lastSeq = changes.last_seq ?? lastSeq + hasMoreChanges = false + continue } - const replicateBody: ReplicationBody = { - source: this.sourceName, - target: this.targetName, - filter: `${filterDesignName}/${FILTER_NAME}`, - query_params: queryParams, - create_target: true, + const filteredChanges = results.filter(change => { + const doc = + change.doc || + ({ _id: change.id, _deleted: change.deleted } as Document) + return this.shouldReplicateDoc(doc, opts) + }) + const revsDiff = await this.getMissingRevisions(filteredChanges) + const docs = await this.getMissingDocs(revsDiff) + + docsRead += docs.length + if (docs.length) { + const response = await this.writeDocs(docs) + docsWritten += response.filter(row => row.ok).length + docWriteFailures += response.filter(row => row.error).length } - const response = await directCouchCall("_replicate", "POST", replicateBody) - if (!response.ok) { - const error = (await response.json()) as ReplicationErrorResponse - throw new Error( - `Replication failed: ${error.error || response.status} - ${ - error.reason || response.statusText - }` - ) - } - const result = (await response.json()) as ReplicationResponse + since = changes.last_seq + lastSeq = changes.last_seq + hasMoreChanges = changes.pending !== 0 && changes.last_seq !== undefined + } - if (result.history?.find((h) => h.errors && h.errors.length > 0)) { - throw new Error(`Replication failed: ${JSON.stringify(result.history)}`) - } + return { + ok: true, + docs_read: docsRead, + docs_written: docsWritten, + doc_write_failures: docWriteFailures, + start_time: startTime, + end_time: new Date().toISOString(), + last_seq: lastSeq, + } satisfies ReplicationResponse + } - return result - } finally { - if (customFilter) { - try { - const existing = await sourceDb.get(filterDesignId) - await sourceDb.remove(filterDesignId, existing._rev) - } catch {} - } + private async ensureTarget() { + const response = await directCouchCall(this.targetName, "PUT") + if (![201, 202, 412].includes(response.status)) { + await throwReplicationError(response, "target creation") } } - private async ensureFilter( - sourceDb: ReturnType, - filterDesignId: string, - filterFunction: string - ) { - const doc = { - _id: filterDesignId, - filters: { - [FILTER_NAME]: filterFunction, - }, - } satisfies ReplicationFilterDoc - try { - const existing = await sourceDb.get(filterDesignId) - if (existing.filters?.[FILTER_NAME] === filterFunction) { - return - } - await sourceDb.put({ - ...doc, - _rev: existing._rev, - }) - } catch (err: unknown) { - if (!isNotFoundError(err)) { - throw err + private async getChanges(since: unknown): Promise { + const response = await directCouchCall( + `${this.sourceName}/_changes`, + "POST", + { + since, + limit: REPLICATION_BATCH_SIZE, + style: "all_docs", + include_docs: true, } - await sourceDb.put(doc) + ) + if (!response.ok) { + await throwReplicationError(response, "changes read") } + return readJson(response) } - private buildFilterFunction({ - filterDesignId, - customFilter, - }: { - filterDesignId: string - customFilter?: (doc: Document) => boolean | undefined - }) { - const customFilterSource = customFilter ? customFilter.toString() : undefined - - return `function (doc, req) { - if (!doc || !doc._id) { - return false; - } - if (doc._id === ${JSON.stringify(filterDesignId)}) { - return false; - } - if (!req.query) { - req.query = {}; - } - - var isCreation = req.query.isCreation === "true"; - var toDev = req.query.toDev === "true"; - var syncAllTables = req.query.syncAllTables === "true"; - var tableSyncList = req.query.tableSyncList ? req.query.tableSyncList.split(",") : []; - var toProduction = !toDev; - - function startsWithType(id, docType) { - return id && id.indexOf(docType + ${JSON.stringify(SEPARATOR)}) === 0; - } - - function isData(id) { - return startsWithType(id, ${JSON.stringify(DocumentType.ROW)}) || - startsWithType(id, ${JSON.stringify(DocumentType.LINK)}); - } - - if (doc._deleted) { - return true; + private async getMissingRevisions( + changes: ChangesResponse["results"] + ): Promise { + const revsById: Record = {} + for (const change of changes || []) { + const revs = + change.changes?.map(change => change.rev).filter(Boolean) || [] + if (revs.length) { + revsById[change.id] = revs } + } + if (!Object.keys(revsById).length) { + return {} + } - if (!isCreation && doc._id === ${JSON.stringify(DesignDocuments.MIGRATIONS)}) { - return false; - } + const response = await directCouchCall( + `${this.targetName}/_revs_diff`, + "POST", + revsById + ) + if (!response.ok) { + await throwReplicationError(response, "revision diff") + } + return readJson(response) + } - if (toDev && doc._id.indexOf("_design") === 0) { - return false; - } + private async getMissingDocs( + revsDiff: RevsDiffResponse + ): Promise { + const docsToFetch = Object.entries(revsDiff).flatMap(([id, diff]) => + (diff.missing || []).map(rev => ({ id, rev })) + ) + if (!docsToFetch.length) { + return [] + } - if (doc._id.indexOf(${JSON.stringify(USER_METADATA_PREFIX)}) === 0) { - return true; + const response = await directCouchCall( + `${this.sourceName}/_bulk_get?revs=true`, + "POST", + { + docs: docsToFetch, } + ) + if (!response.ok) { + await throwReplicationError(response, "bulk get") + } - if ( - toProduction && - !isCreation && - startsWithType(doc._id, ${JSON.stringify(DocumentType.AUTO_COLUMN_STATE)}) - ) { - return false; - } + const body = await readJson(response) + return (body.results || []) + .flatMap(result => result.docs || []) + .map(result => result.ok) + .filter((doc): doc is AnyDocument => !!doc) + } - if (isData(doc._id)) { - if (syncAllTables) { - return true; - } - for (var i = 0; i < tableSyncList.length; i++) { - if (doc._id.indexOf(tableSyncList[i]) !== -1) { - return true; - } - } - return false; + private async writeDocs(docs: AnyDocument[]): Promise { + const response = await directCouchCall( + `${this.targetName}/_bulk_docs`, + "POST", + { + docs, + new_edits: false, } + ) + if (!response.ok) { + await throwReplicationError(response, "bulk write") + } + return readJson(response) + } - if (startsWithType(doc._id, ${JSON.stringify(DocumentType.AUTOMATION_LOG)})) { - return false; - } + private shouldReplicateDoc(doc: Document, opts: ReplicateOpts) { + if (!doc?._id) { + return false + } - if (doc._id === ${JSON.stringify(DocumentType.WORKSPACE_METADATA)}) { - return false; - } + const isCreation = opts.isCreation + const toDev = this.direction === ReplicationDirection.TO_DEV + const toProduction = !toDev + const startsWithType = (id: string, docType: string) => + id.indexOf(`${docType}${SEPARATOR}`) === 0 + const isData = (id: string) => + startsWithType(id, DocumentType.ROW) || + startsWithType(id, DocumentType.LINK) + + let syncAllTables = false + let tableSyncList: string[] | undefined + if (opts.tablesToSync === "all") { + syncAllTables = true + } else if (opts.tablesToSync) { + tableSyncList = opts.tablesToSync + } - ${ - customFilterSource - ? `var customFilter = (${customFilterSource}); - return !!customFilter(doc);` - : "return true;" + if (doc._deleted) { + return true + } + if (!isCreation && doc._id === DesignDocuments.MIGRATIONS) { + return false + } + if (toDev && doc._id.indexOf("_design") === 0) { + return false + } + if (doc._id.indexOf(USER_METADATA_PREFIX) === 0) { + return true + } + if ( + toProduction && + !isCreation && + startsWithType(doc._id, DocumentType.AUTO_COLUMN_STATE) + ) { + return false + } + if (isData(doc._id)) { + if (syncAllTables) { + return true } - }` + return Boolean(tableSyncList?.some(id => doc._id?.includes(id))) + } + if (startsWithType(doc._id, DocumentType.AUTOMATION_LOG)) { + return false + } + if (doc._id === DocumentType.WORKSPACE_METADATA) { + return false + } + return opts.filter ? !!opts.filter(doc) : true } async rollback() { - const targetDb = getDB(this.targetName, { skip_setup: true }) - await targetDb.destroy() - await directCouchQuery(`/${this.targetName}`, "PUT") + const response = await directCouchCall(this.targetName, "DELETE") + if (![200, 202, 404].includes(response.status)) { + await throwReplicationError(response, "target deletion") + } + await this.ensureTarget() await this.replicate() } } diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index 0c898f67..a2188fde 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -1,11 +1,14 @@ -import { type Document, DocumentType, type RowValue } from "@supertoolmake/types" +import { + type Document, + DocumentType, + type RowValue, +} from "@supertoolmake/types" import { structures } from "../../../tests" import { getDB } from "../db" import Replication from "../Replication" type TestDocument = Document & Record type NamedDocument = Document & { name: string } -type ReplicationFilterDoc = Document & { filters: Record } const adminFilter = (doc: Document) => Boolean(doc._id?.includes("admin")) @@ -16,14 +19,17 @@ const ensureDb = async (dbName: string) => { await db.remove(initId, (await db.get(initId))._rev) } -const makeDoc = (id: string, extra: Record = {}): TestDocument => ({ +const makeDoc = ( + id: string, + extra: Record = {} +): TestDocument => ({ _id: id, ...extra, }) const getAllDocIds = async (dbName: string) => { const allDocs = await getDB(dbName).allDocs({ include_docs: false }) - return allDocs.rows.map((row) => row.id) + return allDocs.rows.map(row => row.id) } describe("Replication", () => { @@ -59,7 +65,9 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + await getDB(source).put( + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) + ) const rep = new Replication({ source, target }) await rep.replicate() @@ -83,9 +91,15 @@ describe("Replication", () => { const rep = new Replication({ source, target }) await rep.replicate() - expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) - expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) - expect(await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`)).toMatchObject({ + expect( + await getDB(target).get(`${DocumentType.ROLE}_admin`) + ).toMatchObject({ name: "admin" }) + expect( + await getDB(target).get(`${DocumentType.ROLE}_user`) + ).toMatchObject({ name: "user" }) + expect( + await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`) + ).toMatchObject({ type: "postgres", }) }, 30000) @@ -214,26 +228,23 @@ describe("Replication", () => { expect(ids).not.toContain(`${DocumentType.ROLE}_user`) }, 30000) - it("reuses a stable replication filter design document", async () => { + it("does not create replication filter design documents", async () => { const source = structures.db.id() const target = structures.db.id() await ensureDb(source) await ensureDb(target) - await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + await getDB(source).put( + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) + ) const rep = new Replication({ source, target }) await rep.replicate() - const firstFilterDoc = await getDB(source).get( - "_design/super_replication" - ) await rep.replicate() - const secondFilterDoc = await getDB(source).get( - "_design/super_replication" - ) + const sourceIds = await getAllDocIds(source) const targetIds = await getAllDocIds(target) - expect(secondFilterDoc._rev).toEqual(firstFilterDoc._rev) + expect(sourceIds).not.toContain("_design/super_replication") expect(targetIds).not.toContain("_design/super_replication") }, 30000) @@ -243,15 +254,21 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + await getDB(source).put( + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) + ) const rep = new Replication({ source, target }) await rep.replicate({ filter: adminFilter }) const sourceIds = await getAllDocIds(source) const targetIds = await getAllDocIds(target) - expect(sourceIds.some((id: string) => id.startsWith("_design/replication_"))).toBe(false) - expect(targetIds.some((id: string) => id.startsWith("_design/replication_"))).toBe(false) + expect( + sourceIds.some((id: string) => id.startsWith("_design/replication_")) + ).toBe(false) + expect( + targetIds.some((id: string) => id.startsWith("_design/replication_")) + ).toBe(false) }, 30000) it("excludes design documents when replicating to dev (TO_DEV)", async () => { @@ -298,7 +315,9 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put(makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true })) + await getDB(source).put( + makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true }) + ) const rep = new Replication({ source, target }) await rep.replicate() @@ -313,12 +332,16 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put(makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true })) + await getDB(source).put( + makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true }) + ) const rep = new Replication({ source, target }) await rep.replicate({ isCreation: true }) - const doc = await getDB(target).get(`${DocumentType.AUTO_COLUMN_STATE}_state1`) + const doc = await getDB(target).get( + `${DocumentType.AUTO_COLUMN_STATE}_state1` + ) expect(doc).toMatchObject({ active: true }) }, 30000) @@ -329,7 +352,9 @@ describe("Replication", () => { await ensureDb(target) const userMetaId = `ro_ta_users_${structures.db.id()}` - await getDB(source).put(makeDoc(userMetaId, { email: "test@example.com" })) + await getDB(source).put( + makeDoc(userMetaId, { email: "test@example.com" }) + ) const rep = new Replication({ source, target }) await rep.replicate() @@ -346,12 +371,16 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) + await getDB(source).put( + makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) + ) const rep = new Replication({ source, target }) await rep.replicate() - const beforeDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + const beforeDoc = await getDB(target).get( + `${DocumentType.ROLE}_admin` + ) expect(beforeDoc.name).toBe("admin") await getDB(target).put({ @@ -360,12 +389,16 @@ describe("Replication", () => { name: "modified", }) - const modifiedDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + const modifiedDoc = await getDB(target).get( + `${DocumentType.ROLE}_admin` + ) expect(modifiedDoc.name).toBe("modified") await rep.rollback() - const afterDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) + const afterDoc = await getDB(target).get( + `${DocumentType.ROLE}_admin` + ) expect(afterDoc.name).toBe("admin") }, 60000) @@ -384,8 +417,12 @@ describe("Replication", () => { await rep.replicate() await rep.rollback() - expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) - expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) + expect( + await getDB(target).get(`${DocumentType.ROLE}_admin`) + ).toMatchObject({ name: "admin" }) + expect( + await getDB(target).get(`${DocumentType.ROLE}_user`) + ).toMatchObject({ name: "user" }) }, 60000) }) }) From f9a0478ec211d15779e54496e281aab9f62caaa4 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sun, 28 Jun 2026 18:56:31 +0100 Subject: [PATCH 20/24] Apply formatting to replication fix --- packages/backend-core/src/db/Replication.ts | 88 ++++++------------- .../src/db/tests/Replication.spec.ts | 85 +++++------------- 2 files changed, 49 insertions(+), 124 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 3ef242b3..4bca9206 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,8 +1,4 @@ -import { - type AnyDocument, - type Document, - DocumentType, -} from "@supertoolmake/types" +import { type AnyDocument, type Document, DocumentType } from "@supertoolmake/types" import { DesignDocuments, SEPARATOR, USER_METADATA_PREFIX } from "../constants" import { directCouchCall } from "./couch" @@ -65,9 +61,7 @@ type BulkDocsResponse = Array<{ reason?: string }> -async function readJson( - response: Awaited> -): Promise { +async function readJson(response: Awaited>): Promise { return (await response.json()) as T } @@ -127,10 +121,8 @@ class Replication { continue } - const filteredChanges = results.filter(change => { - const doc = - change.doc || - ({ _id: change.id, _deleted: change.deleted } as Document) + const filteredChanges = results.filter((change) => { + const doc = change.doc || ({ _id: change.id, _deleted: change.deleted } as Document) return this.shouldReplicateDoc(doc, opts) }) const revsDiff = await this.getMissingRevisions(filteredChanges) @@ -139,8 +131,8 @@ class Replication { docsRead += docs.length if (docs.length) { const response = await this.writeDocs(docs) - docsWritten += response.filter(row => row.ok).length - docWriteFailures += response.filter(row => row.error).length + docsWritten += response.filter((row) => row.ok).length + docWriteFailures += response.filter((row) => row.error).length } since = changes.last_seq @@ -167,16 +159,12 @@ class Replication { } private async getChanges(since: unknown): Promise { - const response = await directCouchCall( - `${this.sourceName}/_changes`, - "POST", - { - since, - limit: REPLICATION_BATCH_SIZE, - style: "all_docs", - include_docs: true, - } - ) + const response = await directCouchCall(`${this.sourceName}/_changes`, "POST", { + since, + limit: REPLICATION_BATCH_SIZE, + style: "all_docs", + include_docs: true, + }) if (!response.ok) { await throwReplicationError(response, "changes read") } @@ -188,8 +176,7 @@ class Replication { ): Promise { const revsById: Record = {} for (const change of changes || []) { - const revs = - change.changes?.map(change => change.rev).filter(Boolean) || [] + const revs = change.changes?.map((change) => change.rev).filter(Boolean) || [] if (revs.length) { revsById[change.id] = revs } @@ -198,54 +185,40 @@ class Replication { return {} } - const response = await directCouchCall( - `${this.targetName}/_revs_diff`, - "POST", - revsById - ) + const response = await directCouchCall(`${this.targetName}/_revs_diff`, "POST", revsById) if (!response.ok) { await throwReplicationError(response, "revision diff") } return readJson(response) } - private async getMissingDocs( - revsDiff: RevsDiffResponse - ): Promise { + private async getMissingDocs(revsDiff: RevsDiffResponse): Promise { const docsToFetch = Object.entries(revsDiff).flatMap(([id, diff]) => - (diff.missing || []).map(rev => ({ id, rev })) + (diff.missing || []).map((rev) => ({ id, rev })) ) if (!docsToFetch.length) { return [] } - const response = await directCouchCall( - `${this.sourceName}/_bulk_get?revs=true`, - "POST", - { - docs: docsToFetch, - } - ) + const response = await directCouchCall(`${this.sourceName}/_bulk_get?revs=true`, "POST", { + docs: docsToFetch, + }) if (!response.ok) { await throwReplicationError(response, "bulk get") } const body = await readJson(response) return (body.results || []) - .flatMap(result => result.docs || []) - .map(result => result.ok) + .flatMap((result) => result.docs || []) + .map((result) => result.ok) .filter((doc): doc is AnyDocument => !!doc) } private async writeDocs(docs: AnyDocument[]): Promise { - const response = await directCouchCall( - `${this.targetName}/_bulk_docs`, - "POST", - { - docs, - new_edits: false, - } - ) + const response = await directCouchCall(`${this.targetName}/_bulk_docs`, "POST", { + docs, + new_edits: false, + }) if (!response.ok) { await throwReplicationError(response, "bulk write") } @@ -263,8 +236,7 @@ class Replication { const startsWithType = (id: string, docType: string) => id.indexOf(`${docType}${SEPARATOR}`) === 0 const isData = (id: string) => - startsWithType(id, DocumentType.ROW) || - startsWithType(id, DocumentType.LINK) + startsWithType(id, DocumentType.ROW) || startsWithType(id, DocumentType.LINK) let syncAllTables = false let tableSyncList: string[] | undefined @@ -286,18 +258,14 @@ class Replication { if (doc._id.indexOf(USER_METADATA_PREFIX) === 0) { return true } - if ( - toProduction && - !isCreation && - startsWithType(doc._id, DocumentType.AUTO_COLUMN_STATE) - ) { + if (toProduction && !isCreation && startsWithType(doc._id, DocumentType.AUTO_COLUMN_STATE)) { return false } if (isData(doc._id)) { if (syncAllTables) { return true } - return Boolean(tableSyncList?.some(id => doc._id?.includes(id))) + return Boolean(tableSyncList?.some((id) => doc._id?.includes(id))) } if (startsWithType(doc._id, DocumentType.AUTOMATION_LOG)) { return false diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index a2188fde..d58d70f3 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -1,8 +1,4 @@ -import { - type Document, - DocumentType, - type RowValue, -} from "@supertoolmake/types" +import { type Document, DocumentType, type RowValue } from "@supertoolmake/types" import { structures } from "../../../tests" import { getDB } from "../db" import Replication from "../Replication" @@ -19,17 +15,14 @@ const ensureDb = async (dbName: string) => { await db.remove(initId, (await db.get(initId))._rev) } -const makeDoc = ( - id: string, - extra: Record = {} -): TestDocument => ({ +const makeDoc = (id: string, extra: Record = {}): TestDocument => ({ _id: id, ...extra, }) const getAllDocIds = async (dbName: string) => { const allDocs = await getDB(dbName).allDocs({ include_docs: false }) - return allDocs.rows.map(row => row.id) + return allDocs.rows.map((row) => row.id) } describe("Replication", () => { @@ -65,9 +58,7 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put( - makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) - ) + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) const rep = new Replication({ source, target }) await rep.replicate() @@ -91,15 +82,9 @@ describe("Replication", () => { const rep = new Replication({ source, target }) await rep.replicate() - expect( - await getDB(target).get(`${DocumentType.ROLE}_admin`) - ).toMatchObject({ name: "admin" }) - expect( - await getDB(target).get(`${DocumentType.ROLE}_user`) - ).toMatchObject({ name: "user" }) - expect( - await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`) - ).toMatchObject({ + expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) + expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) + expect(await getDB(target).get(`${DocumentType.DATASOURCE}_ds1`)).toMatchObject({ type: "postgres", }) }, 30000) @@ -234,9 +219,7 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put( - makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) - ) + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) const rep = new Replication({ source, target }) await rep.replicate() @@ -254,21 +237,15 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put( - makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) - ) + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) const rep = new Replication({ source, target }) await rep.replicate({ filter: adminFilter }) const sourceIds = await getAllDocIds(source) const targetIds = await getAllDocIds(target) - expect( - sourceIds.some((id: string) => id.startsWith("_design/replication_")) - ).toBe(false) - expect( - targetIds.some((id: string) => id.startsWith("_design/replication_")) - ).toBe(false) + expect(sourceIds.some((id: string) => id.startsWith("_design/replication_"))).toBe(false) + expect(targetIds.some((id: string) => id.startsWith("_design/replication_"))).toBe(false) }, 30000) it("excludes design documents when replicating to dev (TO_DEV)", async () => { @@ -315,9 +292,7 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put( - makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true }) - ) + await getDB(source).put(makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true })) const rep = new Replication({ source, target }) await rep.replicate() @@ -332,16 +307,12 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put( - makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true }) - ) + await getDB(source).put(makeDoc(`${DocumentType.AUTO_COLUMN_STATE}_state1`, { active: true })) const rep = new Replication({ source, target }) await rep.replicate({ isCreation: true }) - const doc = await getDB(target).get( - `${DocumentType.AUTO_COLUMN_STATE}_state1` - ) + const doc = await getDB(target).get(`${DocumentType.AUTO_COLUMN_STATE}_state1`) expect(doc).toMatchObject({ active: true }) }, 30000) @@ -352,9 +323,7 @@ describe("Replication", () => { await ensureDb(target) const userMetaId = `ro_ta_users_${structures.db.id()}` - await getDB(source).put( - makeDoc(userMetaId, { email: "test@example.com" }) - ) + await getDB(source).put(makeDoc(userMetaId, { email: "test@example.com" })) const rep = new Replication({ source, target }) await rep.replicate() @@ -371,16 +340,12 @@ describe("Replication", () => { await ensureDb(source) await ensureDb(target) - await getDB(source).put( - makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" }) - ) + await getDB(source).put(makeDoc(`${DocumentType.ROLE}_admin`, { name: "admin" })) const rep = new Replication({ source, target }) await rep.replicate() - const beforeDoc = await getDB(target).get( - `${DocumentType.ROLE}_admin` - ) + const beforeDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) expect(beforeDoc.name).toBe("admin") await getDB(target).put({ @@ -389,16 +354,12 @@ describe("Replication", () => { name: "modified", }) - const modifiedDoc = await getDB(target).get( - `${DocumentType.ROLE}_admin` - ) + const modifiedDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) expect(modifiedDoc.name).toBe("modified") await rep.rollback() - const afterDoc = await getDB(target).get( - `${DocumentType.ROLE}_admin` - ) + const afterDoc = await getDB(target).get(`${DocumentType.ROLE}_admin`) expect(afterDoc.name).toBe("admin") }, 60000) @@ -417,12 +378,8 @@ describe("Replication", () => { await rep.replicate() await rep.rollback() - expect( - await getDB(target).get(`${DocumentType.ROLE}_admin`) - ).toMatchObject({ name: "admin" }) - expect( - await getDB(target).get(`${DocumentType.ROLE}_user`) - ).toMatchObject({ name: "user" }) + expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) + expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) }, 60000) }) }) From 0b4a7270662e9e66c1ca168ca26114c526da4b91 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sun, 28 Jun 2026 18:59:15 +0100 Subject: [PATCH 21/24] Revert "fix: keep local CouchDB replication" This reverts commit 6455a0fa771bdf190b958ed3bfa8dd1faecc3d39. --- globalSetup.ts | 21 +-------------------- hosting/docker-compose.dev.yaml | 1 - hosting/single/couch/10-docker-default.ini | 2 +- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/globalSetup.ts b/globalSetup.ts index c3f64225..62560d9c 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -92,9 +92,6 @@ export default async function setup() { [httpd] socket_options = [{nodelay, true}] - [chttpd] - bind_address = 0.0.0.0 - [couchdb] single_node = true @@ -132,23 +129,7 @@ export default async function setup() { Wait.forHttp("/minio/health/ready", 9000).withStartupTimeout(10000) ) - const [startedCouchdb] = await Promise.all([couchdb.start(), minio.start()]) - const couchPort = startedCouchdb.getMappedPort(5984) - const couchAuth = Buffer.from("budibase:budibase").toString("base64") - const response = await fetch( - `http://127.0.0.1:${couchPort}/_node/nonode@nohost/_config/chttpd/bind_address`, - { - method: "PUT", - headers: { - Authorization: `Basic ${couchAuth}`, - "Content-Type": "application/json", - }, - body: JSON.stringify("0.0.0.0"), - } - ) - if (!response.ok) { - throw new Error("Unable to configure CouchDB bind address") - } + await Promise.all([couchdb.start(), minio.start()]) } finally { lockfile.unlockSync(lockPath) } diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 9aec7a02..388c1e74 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -48,7 +48,6 @@ services: - "${COUCH_DB_PORT}:5984" volumes: - couchdb_data:/data - - ./single/couch/10-docker-default.ini:/opt/couchdb/etc/local.d/10-docker-default.ini redis-service: container_name: super-tool-redis-dev diff --git a/hosting/single/couch/10-docker-default.ini b/hosting/single/couch/10-docker-default.ini index 6f4e1db0..1aa633cb 100644 --- a/hosting/single/couch/10-docker-default.ini +++ b/hosting/single/couch/10-docker-default.ini @@ -5,4 +5,4 @@ ; overwritten on server upgrade. [chttpd] -bind_address = 0.0.0.0 +bind_address = any From 0afa17374a955b1cfa3ea65a8b9834847247ef0f Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sun, 28 Jun 2026 19:15:08 +0100 Subject: [PATCH 22/24] Fix rollback replication metadata --- packages/backend-core/src/db/Replication.ts | 19 ++++++++++++++----- .../src/db/tests/Replication.spec.ts | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 4bca9206..5cb50b5f 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -13,6 +13,7 @@ interface ReplicateOpts { isCreation?: boolean tablesToSync?: string[] | "all" filter?: (doc: Document) => boolean | undefined + useAppFilters?: boolean } type ReplicationResponse = { @@ -121,10 +122,7 @@ class Replication { continue } - const filteredChanges = results.filter((change) => { - const doc = change.doc || ({ _id: change.id, _deleted: change.deleted } as Document) - return this.shouldReplicateDoc(doc, opts) - }) + const filteredChanges = results.filter((change) => this.shouldReplicateChange(change, opts)) const revsDiff = await this.getMissingRevisions(filteredChanges) const docs = await this.getMissingDocs(revsDiff) @@ -225,6 +223,17 @@ class Replication { return readJson(response) } + private shouldReplicateChange( + change: NonNullable[number], + opts: ReplicateOpts + ) { + const doc = change.doc || ({ _id: change.id, _deleted: change.deleted } as Document) + if (opts.useAppFilters === false) { + return Boolean(doc._id) + } + return this.shouldReplicateDoc(doc, opts) + } + private shouldReplicateDoc(doc: Document, opts: ReplicateOpts) { if (!doc?._id) { return false @@ -282,7 +291,7 @@ class Replication { await throwReplicationError(response, "target deletion") } await this.ensureTarget() - await this.replicate() + await this.replicate({ useAppFilters: false }) } } diff --git a/packages/backend-core/src/db/tests/Replication.spec.ts b/packages/backend-core/src/db/tests/Replication.spec.ts index d58d70f3..c2d49756 100644 --- a/packages/backend-core/src/db/tests/Replication.spec.ts +++ b/packages/backend-core/src/db/tests/Replication.spec.ts @@ -381,5 +381,21 @@ describe("Replication", () => { expect(await getDB(target).get(`${DocumentType.ROLE}_admin`)).toMatchObject({ name: "admin" }) expect(await getDB(target).get(`${DocumentType.ROLE}_user`)).toMatchObject({ name: "user" }) }, 60000) + + it("replicates workspace metadata during rollback", async () => { + const source = structures.db.id() + const target = structures.db.id() + await ensureDb(source) + await ensureDb(target) + + await getDB(source).put(makeDoc(DocumentType.WORKSPACE_METADATA, { appId: source })) + + const rep = new Replication({ source, target }) + await rep.rollback() + + expect(await getDB(target).get(DocumentType.WORKSPACE_METADATA)).toMatchObject({ + appId: source, + }) + }, 60000) }) }) From 23105e8aa02bfb6eaddce28e74d9bfbdba41bf9d Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Sun, 28 Jun 2026 19:37:05 +0100 Subject: [PATCH 23/24] Fix Couch DB dump and restore behavior --- .../backend-core/src/db/couch/DatabaseImpl.ts | 34 ++++++-- .../src/db/couch/tests/DatabaseImpl.spec.ts | 80 +++++++++++++++++++ 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ae176e19..3ba44cd2 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -1,4 +1,7 @@ -import type { ReadStream, WriteStream } from "node:fs" +import { once } from "node:events" +import type { ReadStream } from "node:fs" +import type { Writable } from "node:stream" +import { finished } from "node:stream/promises" import { type AllDocsResponse, type AnyDocument, @@ -36,6 +39,12 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise type DBCallback = (db: Nano.DocumentScope) => Promise> | DBCall +async function writeToStream(stream: Writable, chunk: string) { + if (!stream.write(chunk)) { + await once(stream, "drain") + } +} + class CouchDBError extends Error implements DBError { status: number statusCode: number @@ -340,6 +349,18 @@ export class CouchDatabase implements Database { }) } + private async bulkDocsRaw(documents: AnyDocument[], opts?: { new_edits?: boolean }) { + return this.performCallWithDBCreation((db) => { + const body: any = { + docs: documents, + } + if (opts?.new_edits !== undefined) { + body.new_edits = opts.new_edits + } + return () => db.bulk(body) + }) + } + async find(params: Nano.MangoQuery): Promise> { return this.performCall((db) => { return async () => { @@ -444,12 +465,14 @@ export class CouchDatabase implements Database { }) } - async dump(stream: WriteStream, opts?: DatabaseDumpOpts) { + async dump(stream: Writable, opts?: DatabaseDumpOpts) { const batchSize = opts?.batch_size || 1000 let startKey: string | undefined let hasMore = true const filterFn = opts?.filter + const streamFinished = finished(stream) + streamFinished.catch(() => undefined) while (hasMore) { const params: DocumentListParams = { @@ -470,7 +493,7 @@ export class CouchDatabase implements Database { for (const row of rows) { const doc = row.doc as any if (doc && (!filterFn || filterFn(doc))) { - stream.write(`${JSON.stringify(doc)}\n`) + await writeToStream(stream, `${JSON.stringify(doc)}\n`) } startKey = row.key as string } @@ -481,6 +504,7 @@ export class CouchDatabase implements Database { } stream.end() + await streamFinished } async load(stream: ReadStream) { @@ -510,10 +534,10 @@ export class CouchDatabase implements Database { } } if (pouchDumpDocs.length > 0) { - await this.bulkDocs(pouchDumpDocs, { new_edits: false }) + await this.bulkDocsRaw(pouchDumpDocs, { new_edits: false }) } if (docs.length > 0) { - await this.bulkDocs(docs) + await this.bulkDocsRaw(docs) } return { ok: true } } diff --git a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts index 141de753..113e8839 100644 --- a/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts +++ b/packages/backend-core/src/db/couch/tests/DatabaseImpl.spec.ts @@ -229,6 +229,28 @@ describe("DatabaseImpl", () => { expect(docs.map((doc) => doc._id)).toEqual(["dump_filter_1", "dump_filter_3"]) }) + + it("waits for the dump stream to finish before resolving", async () => { + const dumpDb = new CouchDatabase(structures.db.id()) + await dumpDb.bulkDocs([{ _id: "dump_finish_doc", value: 1 }]) + + let finalised = false + const stream = new Writable({ + write(_chunk, _encoding, callback) { + callback() + }, + final(callback) { + setTimeout(() => { + finalised = true + callback() + }, 10) + }, + }) + + await dumpDb.dump(stream as any) + + expect(finalised).toBe(true) + }) }) describe("load", () => { @@ -255,6 +277,31 @@ describe("DatabaseImpl", () => { }) }) + it("preserves timestamps from JSON-lines document dumps", async () => { + const loadDb = new CouchDatabase(structures.db.id()) + const createdAt = "2023-01-01T00:00:00.000Z" + const updatedAt = "2023-01-02T00:00:00.000Z" + tk.travel(new Date("2024-01-01T00:00:00.000Z")) + const stream = Readable.from( + JSON.stringify({ + _id: "load_timestamp_doc", + _rev: "1-source", + createdAt, + updatedAt, + value: 1, + }) + ) + + await loadDb.load(stream as any) + + expect(await loadDb.get("load_timestamp_doc")).toMatchObject({ + _id: "load_timestamp_doc", + createdAt, + updatedAt, + value: 1, + }) + }) + it("loads PouchDB batch dumps", async () => { const loadDb = new CouchDatabase(structures.db.id()) const stream = Readable.from( @@ -280,5 +327,38 @@ describe("DatabaseImpl", () => { value: 1, }) }) + + it("preserves timestamps from PouchDB batch dumps", async () => { + const loadDb = new CouchDatabase(structures.db.id()) + const createdAt = "2023-01-01T00:00:00.000Z" + const updatedAt = "2023-01-02T00:00:00.000Z" + tk.travel(new Date("2024-01-01T00:00:00.000Z")) + const stream = Readable.from( + `${JSON.stringify({ + docs: [ + { + _id: "load_pouch_timestamp_doc", + _rev: "1-967a00dff5e02add41819138abb3284d", + createdAt, + updatedAt, + value: 1, + _revisions: { + start: 1, + ids: ["967a00dff5e02add41819138abb3284d"], + }, + }, + ], + })}\n` + ) + + await loadDb.load(stream as any) + + expect(await loadDb.get("load_pouch_timestamp_doc")).toMatchObject({ + _id: "load_pouch_timestamp_doc", + createdAt, + updatedAt, + value: 1, + }) + }) }) }) From 274f50509e85d73703044477f2f3f4b86592e8b3 Mon Sep 17 00:00:00 2001 From: mohagan9 Date: Sun, 28 Jun 2026 20:19:49 +0100 Subject: [PATCH 24/24] Remove unused pendingColumnRenames feature - Remove pendingColumnRenames from Table type - Remove applyPendingColumnRenames, clearPendingColumnRenames, getRevisionNumber functions - Remove dev-production rev sync block in publish (only used for renames) - Remove unused mergePendingColumnRenames helper --- .../src/api/controllers/deploy/index.ts | 118 ------------------ .../server/src/api/controllers/table/utils.ts | 6 - .../src/documents/workspace/table/table.ts | 1 - 3 files changed, 125 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 7be52c1e..e4c3aa97 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -9,7 +9,6 @@ import { type PublishTableResponse, type PublishWorkspaceRequest, type PublishWorkspaceResponse, - type Table, type UserCtx, } from "@supertoolmake/types" import { DocumentType } from "../../../db/utils" @@ -68,77 +67,6 @@ async function storeDeploymentHistory(deployment: Deployment) { return deployment } -async function applyPendingColumnRenames(workspaceId: string): Promise { - return await context.doInWorkspaceContext(workspaceId, async () => { - const db = context.getWorkspaceDB() - const tables = await sdk.tables.getAllInternalTables() - const updatedTables: Table[] = [] - - for (let table of tables) { - if (table._deleted) { - continue - } - const pendingColumnRenames = table.pendingColumnRenames - if (!pendingColumnRenames?.length) { - continue - } - - for (const rename of pendingColumnRenames) { - const tableToUpdate: Table = { - ...table, - schema: { ...table.schema }, - } - - const existingNew = tableToUpdate.schema[rename.updated] - const existingOld = tableToUpdate.schema[rename.old] - - // If the updatd column schema doesn't exist (replication left the old schema), - // use the old column schema to the new name so the rename can complete. - if (!existingNew && existingOld) { - tableToUpdate.schema[rename.updated] = { - ...existingOld, - name: rename.updated, - } - delete tableToUpdate.schema[rename.old] - } - - await sdk.tables.update(tableToUpdate, rename) - table = await sdk.tables.getTable(table._id!) - } - - const updatedTable: Table = { ...table, pendingColumnRenames: [] } - const putResult = await db.put(updatedTable) - const persistedTable: Table = { ...updatedTable, _rev: putResult.rev } - updatedTables.push(persistedTable) - } - - return updatedTables - }) -} - -const getRevisionNumber = (rev?: string) => parseInt(rev?.split("-")?.[0] || "0", 10) - -async function clearPendingColumnRenames(workspaceId: string) { - await context.doInWorkspaceContext(workspaceId, async () => { - const db = context.getWorkspaceDB() - const tables = await sdk.tables.getAllInternalTables() - - for (const table of tables) { - if (table._deleted) { - continue - } - if (!table.pendingColumnRenames?.length) { - continue - } - const updatedTable: Table = { - ...table, - pendingColumnRenames: [], - } - await db.put(updatedTable) - } - }) -} - export async function fetchDeployments(ctx: UserCtx) { try { const db = context.getWorkspaceDB() @@ -216,52 +144,6 @@ export const publishWorkspaceInternal = async (ctx: PublishContext) => { tablesToSync, }) - const updatedProdTables = await applyPendingColumnRenames(prodId) - - // Keep development revs aligned with production after renaming, so the - // next publish doesn't see production ahead and delete / tombstone the doc. - if (updatedProdTables.length > 0) { - await context.doInWorkspaceContext(devId, async () => { - const devDb = context.getWorkspaceDB() - for (const prodTable of updatedProdTables) { - try { - const devTable = await devDb.tryGet
(prodTable._id!) - if (!devTable) { - console.warn( - `Failed to get development table for table ${prodTable._id} when applying column renames` - ) - continue - } - - const prodRevNum = getRevisionNumber(prodTable._rev) - let devRevNum = getRevisionNumber(devTable._rev) - - let docForPut: Table = { - ...prodTable, - _rev: devTable._rev, - } - - // Bump dev revisions until dev is at least prod (avoids prod-ahead). - while (devRevNum < prodRevNum) { - const res = await devDb.put(docForPut) - docForPut = { ...docForPut, _rev: res.rev } - devRevNum = getRevisionNumber(res.rev) - } - - // Ensure final content is written with the latest dev rev (handles equal case). - await devDb.put(docForPut) - } catch (err) { - console.warn( - `Failed to update development table with production table - revision when applying column renames for table ${prodTable._id}: ${err}` - ) - } - } - }) - } - - await clearPendingColumnRenames(devId) - const db = context.getProdWorkspaceDB() const appDoc = await sdk.workspaces.metadata.tryGet({ production: false, diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 10fc17c9..b6eb41fc 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -260,12 +260,6 @@ class TableSaveFunctions { } } -export function mergePendingColumnRenames(existing: RenameColumn[], renaming: RenameColumn) { - const pending = existing ? [...existing] : [] - pending.push(renaming) - return pending -} - export function generateForeignKey(column: RelationshipFieldMetadata, relatedTable: Table) { return `fk_${relatedTable.name}_${column.fieldName}` } diff --git a/packages/types/src/documents/workspace/table/table.ts b/packages/types/src/documents/workspace/table/table.ts index 72caa389..43ac64b6 100644 --- a/packages/types/src/documents/workspace/table/table.ts +++ b/packages/types/src/documents/workspace/table/table.ts @@ -25,7 +25,6 @@ export interface Table extends Document { created?: boolean rowHeight?: number aiGenerated?: boolean - pendingColumnRenames?: RenameColumn[] } export interface TableRequest extends Table {