diff --git a/DESIGN.md b/DESIGN.md index af2447a86..77888f31b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -32,6 +32,12 @@ The mitigations live in three places: When adding a new commit-handler early-return path: reset `write.skipped = false` at the top of the handler if you don't already, then set `write.skipped = true` immediately before the `return`. Decide first whether the audit log will reference the blob (via `auditRecordToStore`) — if it does, leave `skipped` unset. `cleanupOrphans` is the periodic safety net; don't rely on it for transactional correctness. +## Opening a source LMDB DBI for migration must thread through `compression` + +When `migrateOnStart` opens a source LMDB primary store to read records out for the RocksDB copy, it constructs an `OpenDBIObject` and calls `sourceRootStore.openDB(key, dbiInit)`. Critically, the per-attribute `compression` setting from the corresponding `__dbis__` entry must be assigned onto `dbiInit` before that call — `dbiInit.compression = attribute.compression`. Without it, lmdb-js doesn't install its decompression layer; every read on the DBI returns raw compressed bytes. msgpackr then misreads bytes in the `0x40–0x7F` range as shared-structure refs, calls `loadStructures` → decodes the (also compressed) structures buffer → finds more bytes in that range → recurses → stack overflow. + +Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match. + ## Schema migration and `runIndexing` internals (`databases.ts`) When `table()` is called with an attribute newly marked `indexed: true` (or with any change that requires re-building the secondary index), `runIndexing` is launched asynchronously and `Table.indexingOperation` is set to its promise. While running: diff --git a/bin/copyDb.ts b/bin/copyDb.ts index 2166e08be..b71a7f46b 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -306,8 +306,6 @@ export async function migrateOnStart() { const rootPath = get(CONFIG_PARAMS.ROOTPATH); const databases = getDatabases(); - updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false); - try { let databaseNames = Object.keys(databases); // system is a dontenum property, so we have to manually add it @@ -362,6 +360,9 @@ export async function migrateOnStart() { } } + // Only clear the flag after all databases have migrated successfully + updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false); + try { resetDatabases(); } catch (err) { @@ -404,8 +405,13 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath targetDbisDb.put(key, attribute); if (!(isPrimary || attribute.indexed)) continue; - // Open source LMDB dbi with default encoding so values are decoded + // Open source LMDB dbi with default encoding so values are decoded. + // Compression must be passed through from the attribute descriptor so lmdb-js + // installs its decompression layer; without it, compressed record/structure bytes + // are interpreted as raw msgpack, which on records that reference shared structures + // triggers infinite getStructures recursion → "Maximum call stack size exceeded". const dbiInit = new OpenDBIObject(!isPrimary, isPrimary); + dbiInit.compression = attribute.compression; const sourceDbi = sourceRootStore.openDB(key, dbiInit); let targetDbi; @@ -453,7 +459,8 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath async function copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction) { let recordsCopied = 0; let skippedRecord = 0; - let retries = 1000000; + const MAX_RETRIES = 1000; + let retries = MAX_RETRIES; let start = null; while (retries-- > 0) { try { @@ -537,7 +544,12 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath } console.log('finish migrating, copied', recordsCopied, 'entries, skipped', skippedRecord, 'delete records'); return; - } catch { + } catch (err) { + const retriesLeft = retries; + console.error( + `Error iterating dbi for ${sourceDatabase} near key ${JSON.stringify(start)}, retrying (${retriesLeft} retries left):`, + err + ); if (typeof start === 'string') { if (start === 'z') { return console.error('Reached end of dbi', start, 'for', sourceDatabase); @@ -547,5 +559,10 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath else return console.error('Unknown key type', start, 'for', sourceDatabase); } } + // Fail loudly so migrateOnStart's try/catch preserves the migrateOnStart flag and + // skips moving the LMDB files to backup, instead of leaving a partial copy. + throw new Error( + `Migration of ${sourceDatabase} exceeded ${MAX_RETRIES} retries, giving up at key ${JSON.stringify(start)}` + ); } }