From 883230774d062b7c0136faefa940aa87c7286aa8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 22 May 2026 15:44:19 -0600 Subject: [PATCH 1/4] =?UTF-8?q?fix:=20lmdb=E2=86=92rocksdb=20migration=20h?= =?UTF-8?q?ang=20on=20shared-structures=20decode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit migrateOnStart silently pinned ~94% CPU when migrating a table whose shared msgpack structures hadn't been loaded into memory before the migration ran. lmdb-js's internal saveStructures stores the structures buffer via put() with no version argument; on a useVersions=true store that emits an 8-byte float64 version=0 prefix (8 zero bytes). The LMDB branch of RecordEncoder.decode only recognized the ordered-binary timestamp prefix (first byte 0x02) so it mis-read the 8 zeros as a 2-byte legacy metadata header, leaving 6 junk bytes ahead of the msgpack payload. msgpackr then threw "Data read, but end of buffer not reached", the iterator surfaced the throw, and copyDbiToRocks's outer 1,000,000-retry loop swallowed it without logging. - RecordEncoder.decode: also strip an 8-byte all-zero prefix on the LMDB path. The full 8-byte zero check disambiguates from legacy 2-byte metadata records (e.g. HAS_RESIDENCY_ID alone encodes as [00 01 ...]). - copyStructures: strip the 8-byte version prefix before writing the structures buffer to RocksDB so the RocksDB decoder doesn't see it as a metadata record. - migrateOnStart: clear STORAGE_MIGRATEONSTART only after every database migrates successfully, so a crash mid-migration is retried on the next start rather than leaving a half-migrated state with the flag already off. - copyDbiToRocks: cap retries at 1000 (was 1,000,000), log per-retry context, and throw at exhaustion so migrateOnStart's catch keeps the flag set and skips moving the LMDB files to backup. Co-Authored-By: Claude Opus 4.7 --- bin/copyDb.ts | 25 +++++++-- resources/RecordEncoder.ts | 18 ++++++- unitTests/resources/recordEncoder.test.js | 65 +++++++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 unitTests/resources/recordEncoder.test.js diff --git a/bin/copyDb.ts b/bin/copyDb.ts index 2166e08be..3856ccf78 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -306,8 +306,6 @@ export async function migrateOnStart() { const rootPath = get(CONFIG_PARAMS.ROOTPATH); const databases = getDatabases(); - updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false); - try { let databaseNames = Object.keys(databases); // system is a dontenum property, so we have to manually add it @@ -362,6 +360,9 @@ export async function migrateOnStart() { } } + // Only clear the flag after all databases have migrated successfully + updateConfigValue(CONFIG_PARAMS.STORAGE_MIGRATEONSTART, false); + try { resetDatabases(); } catch (err) { @@ -389,7 +390,10 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath const copyStructures = (sourceDbi, storeName: string) => { const buffer = sourceDbi.getBinary?.(STRUCTURES_KEY); if (buffer) { - targetRootStore.putSync([STRUCTURES_KEY, storeName], asBinary(buffer)); + // lmdb-js prepends an 8-byte version prefix when useVersions=true and no version is given. + // Strip it before storing in RocksDB, which doesn't use this prefix. + const msgpackBuffer = buffer.length > 8 && buffer[0] === 0 ? buffer.subarray(8) : buffer; + targetRootStore.putSync([STRUCTURES_KEY, storeName], asBinary(msgpackBuffer)); } }; @@ -453,7 +457,8 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath async function copyDbiToRocks(sourceDbi, targetDbi, isPrimary, transaction) { let recordsCopied = 0; let skippedRecord = 0; - let retries = 1000000; + const MAX_RETRIES = 1000; + let retries = MAX_RETRIES; let start = null; while (retries-- > 0) { try { @@ -537,7 +542,12 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath } console.log('finish migrating, copied', recordsCopied, 'entries, skipped', skippedRecord, 'delete records'); return; - } catch { + } catch (err) { + const retriesLeft = retries; + console.error( + `Error iterating dbi for ${sourceDatabase} near key ${JSON.stringify(start)}, retrying (${retriesLeft} retries left):`, + err + ); if (typeof start === 'string') { if (start === 'z') { return console.error('Reached end of dbi', start, 'for', sourceDatabase); @@ -547,5 +557,10 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath else return console.error('Unknown key type', start, 'for', sourceDatabase); } } + // Fail loudly so migrateOnStart's try/catch preserves the migrateOnStart flag and + // skips moving the LMDB files to backup, instead of leaving a partial copy. + throw new Error( + `Migration of ${sourceDatabase} exceeded ${MAX_RETRIES} retries, giving up at key ${JSON.stringify(start)}` + ); } } diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index 81aa3cfcc..8a6c2e228 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -251,7 +251,23 @@ export class RecordEncoder extends Encoder { position += 8; localTime = TIMESTAMP_VIEW.getFloat64(0); nextByte = buffer[position]; - } else if (nextByte === 2) { + } else if ( + nextByte === 2 || + // lmdb-js prepends an 8-byte float64 version prefix on stores with + // useVersions=true; version=0 (used by lmdb-js's internal saveStructures) + // is 8 zero bytes. We require the full 8-byte zero prefix to disambiguate + // from legacy 2-byte metadata records that can also start with 0x00 (e.g. + // HAS_RESIDENCY_ID=32 alone encodes as [00 01 ...]). + (nextByte === 0 && + end >= start + 8 && + buffer[start + 1] === 0 && + buffer[start + 2] === 0 && + buffer[start + 3] === 0 && + buffer[start + 4] === 0 && + buffer[start + 5] === 0 && + buffer[start + 6] === 0 && + buffer[start + 7] === 0) + ) { if (buffer.copy) { buffer.copy(TIMESTAMP_HOLDER, 0, position); position += 8; diff --git a/unitTests/resources/recordEncoder.test.js b/unitTests/resources/recordEncoder.test.js new file mode 100644 index 000000000..0663757b4 --- /dev/null +++ b/unitTests/resources/recordEncoder.test.js @@ -0,0 +1,65 @@ +require('../testUtils'); +const assert = require('assert'); +const { pack } = require('msgpackr'); +const { RecordEncoder } = require('#src/resources/RecordEncoder'); + +describe('RecordEncoder.decode', () => { + describe('LMDB version prefix handling', () => { + // lmdb-js prepends an 8-byte float64 version header to every value on stores opened + // with `useVersions: true`. When `put(key, value)` is called without an explicit + // version (as in lmdb-js's internal `saveStructures`), the version defaults to 0, + // producing a buffer that begins with 8 zero bytes. The decoder must skip this + // prefix instead of misreading it as Harper metadata flags. + it('decodes a buffer with an 8-byte version=0 prefix (e.g. shared structures)', () => { + const encoder = new RecordEncoder({ name: 'test' }); + const payload = [['field1', 'field2'], ['fieldA']]; + const msgpackData = pack(payload); + const lmdbBuffer = Buffer.concat([Buffer.alloc(8), msgpackData]); + + const decoded = encoder.decode(lmdbBuffer); + + assert.deepEqual(decoded, payload); + }); + + it('still decodes a buffer with Harper timestamp prefix (first byte 0x02)', () => { + const encoder = new RecordEncoder({ name: 'test' }); + const payload = ['hello', 'world']; + const msgpackData = pack(payload); + // First byte 0x02 matches the ordered-binary encoded float64 prefix Harper + // produces for recent millisecond timestamps (0x42 XOR 0x40 = 0x02). + const prefix = Buffer.from([0x02, 0x78, 0xd5, 0xa0, 0x00, 0x00, 0x00, 0x00]); + const lmdbBuffer = Buffer.concat([prefix, msgpackData]); + + const decoded = encoder.decode(lmdbBuffer); + + assert.deepEqual(decoded, payload); + }); + + it('decodes a buffer with no metadata prefix (first byte >= 32) as plain msgpack', () => { + const encoder = new RecordEncoder({ name: 'test' }); + const payload = ['plain', 'record']; + const msgpackData = pack(payload); + + const decoded = encoder.decode(msgpackData); + + assert.deepEqual(decoded, payload); + }); + + it('does not mistake legacy 2-byte metadata (first byte 0, second byte non-zero) for a version prefix', () => { + // Legacy 2-byte metadata records exist where the low 5 flag bits are 0 but + // higher bits are set. Example: HAS_RESIDENCY_ID alone (= 32) encodes as + // [0x00, 0x01, <4 bytes residency_id>, ...msgpack...]. The decoder must + // NOT treat this as an lmdb-js version=0 prefix and skip 8 bytes. + const encoder = new RecordEncoder({ name: 'test' }); + const payload = ['after', 'metadata']; + const msgpackData = pack(payload); + // 2-byte metadata = [0x00, 0x01] = HAS_RESIDENCY_ID; then 4 bytes residency_id. + const legacyPrefix = Buffer.from([0x00, 0x01, 0x00, 0x00, 0x00, 0x07]); + const buffer = Buffer.concat([legacyPrefix, msgpackData]); + + const decoded = encoder.decode(buffer); + + assert.deepEqual(decoded, payload); + }); + }); +}); From e5c3b09cb3b9c377fa4704299b56eae007112b01 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 22 May 2026 15:45:58 -0600 Subject: [PATCH 2/4] docs: document lmdb-js version prefix on shared structures buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note the gotcha that caused the v5 RocksDB migration hang — the 8-byte version=0 prefix that lmdb-js prepends to the structures buffer and the implications for decode and cross-engine copy paths. Co-Authored-By: Claude Opus 4.7 --- DESIGN.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/DESIGN.md b/DESIGN.md index af2447a86..c2b1d1cdb 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -32,6 +32,17 @@ The mitigations live in three places: When adding a new commit-handler early-return path: reset `write.skipped = false` at the top of the handler if you don't already, then set `write.skipped = true` immediately before the `return`. Decide first whether the audit log will reference the blob (via `auditRecordToStore`) — if it does, leave `skipped` unset. `cleanupOrphans` is the periodic safety net; don't rely on it for transactional correctness. +## Shared structures buffer has an lmdb-js version prefix on `useVersions=true` stores + +lmdb-js's internal `saveStructures()` (in `setupSharedStructures` in `lmdb/open.js`) stores the msgpack structures array via `this.put(sharedStructuresKey, structures)` with no version argument. On a store opened with `useVersions: true` (every Harper primary store — see `OpenDBIObject`), lmdb-js defaults version=0 and prepends 8 zero bytes (raw float64 `0.0`, big-endian) to the stored value. `getBinary(sharedStructuresKey)` returns those 8 bytes as part of the buffer. + +When wiring code that reads or copies the structures buffer: + +- **Decoding the buffer (`RecordEncoder.decode`)**: handle a leading 8-byte all-zero prefix as a version header to skip. Don't fall through to the legacy 2-byte metadata reader — checking only `nextByte === 0` is ambiguous because legacy metadata records with `HAS_RESIDENCY_ID` / `HAS_NODE_ID` / `HAS_ADDITIONAL_AUDIT_REFS` set (and the low 5 flag bits clear) also encode `[00 ...]`. Require the full 8 zero bytes before treating it as the lmdb version prefix. +- **Copying the buffer to RocksDB (`copyDb.ts` `copyStructures`)**: strip the 8-byte version prefix before writing to RocksDB. RocksDB has no equivalent version prefix; if not stripped, the RocksDB-side decoder mis-reads it as a metadata record (`isRocksDB && nextByte < 32` enters the metadata branch with `localTime = 0`), returns the `lastMetadata` wrapper instead of the structures array, and every subsequent structures lookup fails. + +Symptom when missed: a table whose structures aren't warm in memory hits `getStructures()` → fails to decode → `msgpackr` throws `"Data read, but end of buffer not reached"`. In `migrateOnStart`'s `copyDbiToRocks`, that throw propagates out of the per-record try/catch into the outer retry loop, which used to swallow it silently and spin at ~94% CPU. + ## Schema migration and `runIndexing` internals (`databases.ts`) When `table()` is called with an attribute newly marked `indexed: true` (or with any change that requires re-building the secondary index), `runIndexing` is launched asynchronously and `Table.indexingOperation` is set to its promise. While running: From ca18eb912a422f04b99294a25bdbbe00e0a98ba1 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 22 May 2026 18:07:28 -0600 Subject: [PATCH 3/4] fix(migration): pass compression option when opening source LMDB dbi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the actual root cause of the silent-hang bug reported against v5.0.21 migrateOnStart. The migration was opening each source primary store via `new OpenDBIObject(!isPrimary, isPrimary)` but never copying the per-attribute compression setting onto it. Without compression configured, lmdb-js doesn't install its decompression layer; record buffers come back as raw compressed bytes. When msgpackr then tries to decode those bytes, it interprets bytes in the 0x40-0x7F range as shared-structure references → calls loadStructures → decodes the structures buffer → finds more bytes in that range → recurses → stack overflow. Verified end-to-end against the bug-report tarball (linode.mdb with MonthlyReport records containing 32-element nested orgBreakdown arrays): migrated database digest to RocksDB migrated database linode to RocksDB migrated database oauth to RocksDB migrated database opmodel to RocksDB migrated database scheduler to RocksDB migrated database system to RocksDB Harper 5.1.0 successfully started Post-migration `SELECT * FROM linode.MonthlyReport` returns all 8 records correctly. Matches Harper's normal `databases.ts` path which sets `dbiInit.compression = primaryKeyAttribute.compression` at line 963. Co-Authored-By: Claude Opus 4.7 --- bin/copyDb.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/copyDb.ts b/bin/copyDb.ts index 3856ccf78..007341bb2 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -408,8 +408,13 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath targetDbisDb.put(key, attribute); if (!(isPrimary || attribute.indexed)) continue; - // Open source LMDB dbi with default encoding so values are decoded + // Open source LMDB dbi with default encoding so values are decoded. + // Compression must be passed through from the attribute descriptor so lmdb-js + // installs its decompression layer; without it, compressed record/structure bytes + // are interpreted as raw msgpack, which on records that reference shared structures + // triggers infinite getStructures recursion → "Maximum call stack size exceeded". const dbiInit = new OpenDBIObject(!isPrimary, isPrimary); + dbiInit.compression = attribute.compression; const sourceDbi = sourceRootStore.openDB(key, dbiInit); let targetDbi; From 293d08608cc018f2e72f439cdba91fa6901bd44d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 22 May 2026 18:37:10 -0600 Subject: [PATCH 4/4] revert: drop lmdb-js version=0 prefix handling (not a real case) Earlier commits in this PR added a defensive zero-byte-prefix branch to RecordEncoder.decode and an 8-byte-prefix strip to copyStructures, on the theory that lmdb-js's internal saveStructures emits an 8-byte float64 version=0 prefix on useVersions=true stores. Empirical check across all 23 primary stores in the bug-report tarball shows zero structures buffers begin with [00...]; every one starts with 0x82 (fixmap) directly. v4 RecordEncoder ships with the same `nextByte === 2` check and has worked in production for years on this same data shape, so there is no concrete pathway to a zero-prefix case here. Revert to match v4. Also drops the now-orphaned recordEncoder.test.js (the legacy-2-byte regression guard was only relevant against the reverted branch). Co-Authored-By: Claude Opus 4.7 --- DESIGN.md | 11 ++-- bin/copyDb.ts | 5 +- resources/RecordEncoder.ts | 18 +------ unitTests/resources/recordEncoder.test.js | 65 ----------------------- 4 files changed, 5 insertions(+), 94 deletions(-) delete mode 100644 unitTests/resources/recordEncoder.test.js diff --git a/DESIGN.md b/DESIGN.md index c2b1d1cdb..77888f31b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -32,16 +32,11 @@ The mitigations live in three places: When adding a new commit-handler early-return path: reset `write.skipped = false` at the top of the handler if you don't already, then set `write.skipped = true` immediately before the `return`. Decide first whether the audit log will reference the blob (via `auditRecordToStore`) — if it does, leave `skipped` unset. `cleanupOrphans` is the periodic safety net; don't rely on it for transactional correctness. -## Shared structures buffer has an lmdb-js version prefix on `useVersions=true` stores +## Opening a source LMDB DBI for migration must thread through `compression` -lmdb-js's internal `saveStructures()` (in `setupSharedStructures` in `lmdb/open.js`) stores the msgpack structures array via `this.put(sharedStructuresKey, structures)` with no version argument. On a store opened with `useVersions: true` (every Harper primary store — see `OpenDBIObject`), lmdb-js defaults version=0 and prepends 8 zero bytes (raw float64 `0.0`, big-endian) to the stored value. `getBinary(sharedStructuresKey)` returns those 8 bytes as part of the buffer. +When `migrateOnStart` opens a source LMDB primary store to read records out for the RocksDB copy, it constructs an `OpenDBIObject` and calls `sourceRootStore.openDB(key, dbiInit)`. Critically, the per-attribute `compression` setting from the corresponding `__dbis__` entry must be assigned onto `dbiInit` before that call — `dbiInit.compression = attribute.compression`. Without it, lmdb-js doesn't install its decompression layer; every read on the DBI returns raw compressed bytes. msgpackr then misreads bytes in the `0x40–0x7F` range as shared-structure refs, calls `loadStructures` → decodes the (also compressed) structures buffer → finds more bytes in that range → recurses → stack overflow. -When wiring code that reads or copies the structures buffer: - -- **Decoding the buffer (`RecordEncoder.decode`)**: handle a leading 8-byte all-zero prefix as a version header to skip. Don't fall through to the legacy 2-byte metadata reader — checking only `nextByte === 0` is ambiguous because legacy metadata records with `HAS_RESIDENCY_ID` / `HAS_NODE_ID` / `HAS_ADDITIONAL_AUDIT_REFS` set (and the low 5 flag bits clear) also encode `[00 ...]`. Require the full 8 zero bytes before treating it as the lmdb version prefix. -- **Copying the buffer to RocksDB (`copyDb.ts` `copyStructures`)**: strip the 8-byte version prefix before writing to RocksDB. RocksDB has no equivalent version prefix; if not stripped, the RocksDB-side decoder mis-reads it as a metadata record (`isRocksDB && nextByte < 32` enters the metadata branch with `localTime = 0`), returns the `lastMetadata` wrapper instead of the structures array, and every subsequent structures lookup fails. - -Symptom when missed: a table whose structures aren't warm in memory hits `getStructures()` → fails to decode → `msgpackr` throws `"Data read, but end of buffer not reached"`. In `migrateOnStart`'s `copyDbiToRocks`, that throw propagates out of the per-record try/catch into the outer retry loop, which used to swallow it silently and spin at ~94% CPU. +Harper's normal `databases.ts` path already does this (search for `dbiInit.compression = primaryKeyAttribute.compression`); the migration path in `bin/copyDb.ts` has to match. ## Schema migration and `runIndexing` internals (`databases.ts`) diff --git a/bin/copyDb.ts b/bin/copyDb.ts index 007341bb2..b71a7f46b 100644 --- a/bin/copyDb.ts +++ b/bin/copyDb.ts @@ -390,10 +390,7 @@ async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath const copyStructures = (sourceDbi, storeName: string) => { const buffer = sourceDbi.getBinary?.(STRUCTURES_KEY); if (buffer) { - // lmdb-js prepends an 8-byte version prefix when useVersions=true and no version is given. - // Strip it before storing in RocksDB, which doesn't use this prefix. - const msgpackBuffer = buffer.length > 8 && buffer[0] === 0 ? buffer.subarray(8) : buffer; - targetRootStore.putSync([STRUCTURES_KEY, storeName], asBinary(msgpackBuffer)); + targetRootStore.putSync([STRUCTURES_KEY, storeName], asBinary(buffer)); } }; diff --git a/resources/RecordEncoder.ts b/resources/RecordEncoder.ts index 8a6c2e228..81aa3cfcc 100644 --- a/resources/RecordEncoder.ts +++ b/resources/RecordEncoder.ts @@ -251,23 +251,7 @@ export class RecordEncoder extends Encoder { position += 8; localTime = TIMESTAMP_VIEW.getFloat64(0); nextByte = buffer[position]; - } else if ( - nextByte === 2 || - // lmdb-js prepends an 8-byte float64 version prefix on stores with - // useVersions=true; version=0 (used by lmdb-js's internal saveStructures) - // is 8 zero bytes. We require the full 8-byte zero prefix to disambiguate - // from legacy 2-byte metadata records that can also start with 0x00 (e.g. - // HAS_RESIDENCY_ID=32 alone encodes as [00 01 ...]). - (nextByte === 0 && - end >= start + 8 && - buffer[start + 1] === 0 && - buffer[start + 2] === 0 && - buffer[start + 3] === 0 && - buffer[start + 4] === 0 && - buffer[start + 5] === 0 && - buffer[start + 6] === 0 && - buffer[start + 7] === 0) - ) { + } else if (nextByte === 2) { if (buffer.copy) { buffer.copy(TIMESTAMP_HOLDER, 0, position); position += 8; diff --git a/unitTests/resources/recordEncoder.test.js b/unitTests/resources/recordEncoder.test.js deleted file mode 100644 index 0663757b4..000000000 --- a/unitTests/resources/recordEncoder.test.js +++ /dev/null @@ -1,65 +0,0 @@ -require('../testUtils'); -const assert = require('assert'); -const { pack } = require('msgpackr'); -const { RecordEncoder } = require('#src/resources/RecordEncoder'); - -describe('RecordEncoder.decode', () => { - describe('LMDB version prefix handling', () => { - // lmdb-js prepends an 8-byte float64 version header to every value on stores opened - // with `useVersions: true`. When `put(key, value)` is called without an explicit - // version (as in lmdb-js's internal `saveStructures`), the version defaults to 0, - // producing a buffer that begins with 8 zero bytes. The decoder must skip this - // prefix instead of misreading it as Harper metadata flags. - it('decodes a buffer with an 8-byte version=0 prefix (e.g. shared structures)', () => { - const encoder = new RecordEncoder({ name: 'test' }); - const payload = [['field1', 'field2'], ['fieldA']]; - const msgpackData = pack(payload); - const lmdbBuffer = Buffer.concat([Buffer.alloc(8), msgpackData]); - - const decoded = encoder.decode(lmdbBuffer); - - assert.deepEqual(decoded, payload); - }); - - it('still decodes a buffer with Harper timestamp prefix (first byte 0x02)', () => { - const encoder = new RecordEncoder({ name: 'test' }); - const payload = ['hello', 'world']; - const msgpackData = pack(payload); - // First byte 0x02 matches the ordered-binary encoded float64 prefix Harper - // produces for recent millisecond timestamps (0x42 XOR 0x40 = 0x02). - const prefix = Buffer.from([0x02, 0x78, 0xd5, 0xa0, 0x00, 0x00, 0x00, 0x00]); - const lmdbBuffer = Buffer.concat([prefix, msgpackData]); - - const decoded = encoder.decode(lmdbBuffer); - - assert.deepEqual(decoded, payload); - }); - - it('decodes a buffer with no metadata prefix (first byte >= 32) as plain msgpack', () => { - const encoder = new RecordEncoder({ name: 'test' }); - const payload = ['plain', 'record']; - const msgpackData = pack(payload); - - const decoded = encoder.decode(msgpackData); - - assert.deepEqual(decoded, payload); - }); - - it('does not mistake legacy 2-byte metadata (first byte 0, second byte non-zero) for a version prefix', () => { - // Legacy 2-byte metadata records exist where the low 5 flag bits are 0 but - // higher bits are set. Example: HAS_RESIDENCY_ID alone (= 32) encodes as - // [0x00, 0x01, <4 bytes residency_id>, ...msgpack...]. The decoder must - // NOT treat this as an lmdb-js version=0 prefix and skip 8 bytes. - const encoder = new RecordEncoder({ name: 'test' }); - const payload = ['after', 'metadata']; - const msgpackData = pack(payload); - // 2-byte metadata = [0x00, 0x01] = HAS_RESIDENCY_ID; then 4 bytes residency_id. - const legacyPrefix = Buffer.from([0x00, 0x01, 0x00, 0x00, 0x00, 0x07]); - const buffer = Buffer.concat([legacyPrefix, msgpackData]); - - const decoded = encoder.decode(buffer); - - assert.deepEqual(decoded, payload); - }); - }); -});