fix(migration): pass compression to OpenDBIObject (root cause of LMDB→RocksDB hang)

DESIGN.md

@@ -32,6 +32,12 @@ The mitigations live in three places:
 
 When adding a new commit-handler early-return path: reset `write.skipped = false` at the top of the handler if you don't already, then set `write.skipped = true` immediately before the `return`. Decide first whether the audit log will reference the blob (via `auditRecordToStore`) — if it does, leave `skipped` unset. `cleanupOrphans` is the periodic safety net; don't rely on it for transactional correctness.
 
+## Opening a source LMDB DBI for migration must thread through `compression`
+
+When `migrateOnStart` opens a source LMDB primary store to read records out for the RocksDB copy, it constructs an `OpenDBIObject` and calls `sourceRootStore.openDB(key, dbiInit)`. Critically, the per-attribute `compression` setting from the corresponding `__dbis__` entry must be assigned onto `dbiInit` before that call — `dbiInit.compression = attribute.compression`. Without it, lmdb-js doesn't install its decompression layer; every read on the DBI returns raw compressed bytes. msgpackr then misreads bytes in the `0x40–0x7F` range as shared-structure refs, calls `loadStructures` → decodes the (also compressed) structures buffer → finds more bytes in that range → recurses → stack overflow.
+
+Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match.
+
 ## Schema migration and `runIndexing` internals (`databases.ts`)
 
 When `table()` is called with an attribute newly marked `indexed: true` (or with any change that requires re-building the secondary index), `runIndexing` is launched asynchronously and `Table.indexingOperation` is set to its promise. While running:

bin/copyDb.ts

@@ -306,8 +306,6 @@ export async function migrateOnStart() {
 	const rootPath = get(CONFIG_PARAMS.ROOTPATH);
 	const databases = getDatabases();
 
-	updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false);
-
 	try {
 		let databaseNames = Object.keys(databases);
 		// system is a dontenum property, so we have to manually add it
@@ -362,6 +360,9 @@ export async function migrateOnStart() {
 			}
 		}
 
+		// Only clear the flag after all databases have migrated successfully
+		updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false);
+
 		try {
 			resetDatabases();
 		} catch (err) {
@@ -404,8 +405,13 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath
 			targetDbisDb.put(key, attribute);
 			if (!(isPrimary || attribute.indexed)) continue;
 
-			// Open source LMDB dbi with default encoding so values are decoded
+			// Open source LMDB dbi with default encoding so values are decoded.
+			// Compression must be passed through from the attribute descriptor so lmdb-js
+			// installs its decompression layer; without it, compressed record/structure bytes
+			// are interpreted as raw msgpack, which on records that reference shared structures
+			// triggers infinite getStructures recursion → "Maximum call stack size exceeded".
 			const dbiInit = new OpenDBIObject(!isPrimary, isPrimary);
+			dbiInit.compression = attribute.compression;
 			const sourceDbi = sourceRootStore.openDB(key, dbiInit);
 
 			let targetDbi;
@@ -453,7 +459,8 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath
 	async function copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction) {
 		let recordsCopied = 0;
 		let skippedRecord = 0;
-		let retries = 1000000;
+		const MAX_RETRIES = 1000;
+		let retries = MAX_RETRIES;
 		let start = null;
 		while (retries-- > 0) {
 			try {
@@ -537,7 +544,12 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath
 				}
 				console.log('finish migrating, copied', recordsCopied, 'entries, skipped', skippedRecord, 'delete records');
 				return;
-			} catch {
+			} catch (err) {
+				const retriesLeft = retries;
+				console.error(
+					`Error iterating dbi for ${sourceDatabase} near key ${JSON.stringify(start)}, retrying (${retriesLeft} retries left):`,
+					err
+				);
 				if (typeof start === 'string') {
 					if (start === 'z') {
 						return console.error('Reached end of dbi', start, 'for', sourceDatabase);
@@ -547,5 +559,10 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath
 				else return console.error('Unknown key type', start, 'for', sourceDatabase);
 			}
 		}
+		// Fail loudly so migrateOnStart's try/catch preserves the migrateOnStart flag and
+		// skips moving the LMDB files to backup, instead of leaving a partial copy.
+		throw new Error(
+			`Migration of ${sourceDatabase} exceeded ${MAX_RETRIES} retries, giving up at key ${JSON.stringify(start)}`
+		);
 	}
 }