From bb1c6656a9c2db487e53ab9ba584b32447ae0553 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 01:36:03 +0300 Subject: [PATCH 01/16] feat(kernel): harden durability, safety, and API contracts - refuse non-LibreDB files with a WAL magic header; never truncate foreign bytes (headerless v0.1.x files keep opening via a legacy read path) - classify recovery failures: torn tails truncate (reported through the new onRecovery option), mid-log corruption refuses to open - validate record payload structure during replay - reject async transact() callbacks before commit - latch the database after a failed append/fsync (fsyncgate) - copy keys and values at the transaction boundary (no aliasing) - snapshot getRange so delete-while-scanning visits every entry - name close()-during-transaction instead of surfacing raw EBADF - exclusive open lock via the FileSystem seam: pid/host/nonce lock file with stale-lock reclaim; CLI --force verifies holder liveness - node-fs: fd-based positional reads, parent-directory fsync on create, fsync after recovery truncation - opfs: loop reads so a short read cannot masquerade as a torn tail - recovery treats an incomplete read as an IO fault, never truncation - typed LibreDbError with stable codes on every kernel failure --- .size-limit.json | 11 +- src/adapter/node-fs.test.ts | 239 ++++++++++++++++++ src/adapter/node-fs.ts | 197 ++++++++++++++- src/adapter/opfs.ts | 14 +- src/browser.ts | 3 +- src/cli/lock.test.ts | 87 ------- src/cli/lock.ts | 67 ----- src/cli/readonly-fs.test.ts | 15 +- src/cli/readonly-fs.ts | 5 +- src/cli/run.test.ts | 43 +++- src/cli/run.ts | 35 +-- src/core.hardening.test.ts | 489 ++++++++++++++++++++++++++++++++++++ src/core.recovery.test.ts | 6 +- src/core.ts | 432 +++++++++++++++++++++++++++---- src/index.ts | 4 +- src/sim/dst.test.ts | 71 +++--- 16 files changed, 1439 insertions(+), 279 deletions(-) create mode 100644 src/adapter/node-fs.test.ts delete mode 100644 src/cli/lock.test.ts delete mode 100644 src/cli/lock.ts create mode 100644 src/core.hardening.test.ts diff --git a/.size-limit.json b/.size-limit.json index cb0dded..f940e8a 100644 --- a/.size-limit.json +++ b/.size-limit.json @@ -2,12 +2,17 @@ { "name": "public entry (min+brotli)", "path": "dist/index.js", - "ignore": ["node:fs", "node:os", "node:path"], - "limit": "4 kB" + "ignore": [ + "node:crypto", + "node:fs", + "node:os", + "node:path" + ], + "limit": "5 kB" }, { "name": "browser entry (min+brotli)", "path": "dist/browser.js", - "limit": "4 kB" + "limit": "5 kB" } ] diff --git a/src/adapter/node-fs.test.ts b/src/adapter/node-fs.test.ts new file mode 100644 index 0000000..b410bbf --- /dev/null +++ b/src/adapter/node-fs.test.ts @@ -0,0 +1,239 @@ +/** + * node-fs.test.ts — the default node:fs adapter: fd-based IO, directory fsync, + * and the exclusive open lock. + * + * The adapter is an edge (no durability logic of its own), but it carries two + * platform facts the kernel's guarantees stand on: a created file's directory + * entry must be fsync'd to be durable, and the `.lock` file is what turns + * a second writer into a loud LOCKED error. Both are pinned here, alongside the + * read/append/truncate mechanics the WAL drives. + */ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { hostname, tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, expect, test } from "bun:test"; + +import { LibreDbError } from "../core.ts"; +import { forceUnlock, fsyncDirectoryOf, isStaleLock, LOCK_SENTINEL, nodeFileSystem } from "./node-fs.ts"; + +const dirs: string[] = []; +const tempPath = (name: string): string => { + const dir = mkdtempSync(join(tmpdir(), "libredb-nodefs-")); + dirs.push(dir); + return join(dir, name); +}; + +afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop() as string, { recursive: true, force: true }); +}); + +/** A pid above every default Linux pid_max: guaranteed not alive. */ +const DEAD_PID = 4194304; +const liveLock = (): string => `${LOCK_SENTINEL}\n${process.pid}\n${hostname()}\nnonce\n`; +const deadLock = (): string => `${LOCK_SENTINEL}\n${DEAD_PID}\n${hostname()}\nnonce\n`; +const otherHostLock = (): string => `${LOCK_SENTINEL}\n${DEAD_PID}\nsome-other-host\nnonce\n`; + +// --- file IO mechanics --- + +test("read() honors offset and length through the file descriptor", () => { + const path = tempPath("io"); + const file = nodeFileSystem().open(path); + file.append(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8])); + expect(file.read(2, 4)).toEqual(new Uint8Array([3, 4, 5, 6])); + expect(file.size()).toBe(8); + file.close(); +}); + +test("read() past the end returns only the bytes that exist", () => { + const path = tempPath("short"); + const file = nodeFileSystem().open(path); + file.append(new Uint8Array([1, 2, 3])); + expect(file.read(1, 100)).toEqual(new Uint8Array([2, 3])); + file.close(); +}); + +test("truncate() shrinks the file through the same descriptor", () => { + const path = tempPath("trunc"); + const file = nodeFileSystem().open(path); + file.append(new Uint8Array([1, 2, 3, 4])); + file.truncate(2); + file.fsync(); + expect(file.size()).toBe(2); + expect(file.read(0, 2)).toEqual(new Uint8Array([1, 2])); + file.close(); + expect(new Uint8Array(readFileSync(path))).toEqual(new Uint8Array([1, 2])); +}); + +test("fsyncDirectoryOf tolerates a directory that cannot be opened", () => { + // Platforms without directory fsync (Windows) throw on the open; the helper + // must swallow that, since directory-entry durability is best-effort there. + expect(() => fsyncDirectoryOf(join(tmpdir(), "libredb-no-such-dir-xyz", "file"))).not.toThrow(); +}); + +test("fsyncDirectoryOf fsyncs an existing parent directory without error", () => { + const path = tempPath("synced"); + writeFileSync(path, "x"); + expect(() => fsyncDirectoryOf(path)).not.toThrow(); +}); + +// --- the exclusive open lock --- + +test("lock() creates .lock and the release function removes it", () => { + const path = tempPath("db"); + const fs = nodeFileSystem(); + const release = fs.lock?.(path) as () => void; + expect(existsSync(`${path}.lock`)).toBe(true); + const contents = readFileSync(`${path}.lock`, "utf8"); + expect(contents.startsWith(LOCK_SENTINEL)).toBe(true); + expect(contents).toContain(String(process.pid)); + release(); + expect(existsSync(`${path}.lock`)).toBe(false); +}); + +test("a second lock() against a live holder throws LOCKED", () => { + const path = tempPath("db"); + const fs = nodeFileSystem(); + const release = fs.lock?.(path) as () => void; + let caught: unknown; + try { + fs.lock?.(path); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(LibreDbError); + expect((caught as LibreDbError).code).toBe("LOCKED"); + release(); +}); + +test("a stale lock (dead pid) is reclaimed and locking proceeds", () => { + const path = tempPath("db"); + writeFileSync(`${path}.lock`, deadLock()); + const release = nodeFileSystem().lock?.(path) as () => void; + expect(readFileSync(`${path}.lock`, "utf8")).toContain(String(process.pid)); + release(); +}); + +test("a legacy sentinel-only lock (no owner recorded) is reclaimed", () => { + const path = tempPath("db"); + writeFileSync(`${path}.lock`, `${LOCK_SENTINEL}\n`); // v0.1.x CLI format + const release = nodeFileSystem().lock?.(path) as () => void; + release(); + expect(existsSync(`${path}.lock`)).toBe(false); +}); + +test("a lock held on another host is not reclaimed (liveness unverifiable)", () => { + const path = tempPath("db"); + writeFileSync(`${path}.lock`, otherHostLock()); + let caught: unknown; + try { + nodeFileSystem().lock?.(path); + } catch (error) { + caught = error; + } + expect((caught as LibreDbError).code).toBe("LOCKED"); + expect(existsSync(`${path}.lock`)).toBe(true); +}); + +test("a foreign file named .lock is never treated as a stale lock", () => { + const path = tempPath("db"); + writeFileSync(`${path}.lock`, "user data that merely shares the name"); + let caught: unknown; + try { + nodeFileSystem().lock?.(path); + } catch (error) { + caught = error; + } + expect((caught as LibreDbError).code).toBe("LOCKED"); + expect(readFileSync(`${path}.lock`, "utf8")).toBe("user data that merely shares the name"); +}); + +test("a non-EEXIST failure creating the lock surfaces unchanged (not LOCKED)", () => { + const bogus = join(tmpdir(), "libredb-no-such-dir-xyz", "db"); + let caught: unknown; + try { + nodeFileSystem().lock?.(bogus); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as { code?: string }).code).toBe("ENOENT"); +}); + +test("release() does not delete a lock someone else re-acquired after a force", () => { + const path = tempPath("db"); + const fs = nodeFileSystem(); + const release = fs.lock?.(path) as () => void; + // Simulate: our lock was force-removed and another writer locked the file. + const theirs = `${LOCK_SENTINEL}\n${DEAD_PID}\n${hostname()}\ntheir-nonce\n`; + writeFileSync(`${path}.lock`, theirs); + release(); // must notice the nonce mismatch and leave their lock alone + expect(readFileSync(`${path}.lock`, "utf8")).toBe(theirs); +}); + +test("release() tolerates the lock file already being gone", () => { + const path = tempPath("db"); + const release = nodeFileSystem().lock?.(path) as () => void; + rmSync(`${path}.lock`); + expect(() => release()).not.toThrow(); +}); + +// --- staleness rules, pinned directly --- + +test("isStaleLock: a vanished lock file counts as stale (retryable)", () => { + expect(isStaleLock(join(tmpdir(), "libredb-vanished-xyz.lock"))).toBe(true); +}); + +test("isStaleLock: empty stray, dead holder, and legacy sentinel are stale; live and foreign are not", () => { + const path = tempPath("db"); + const lockPath = `${path}.lock`; + writeFileSync(lockPath, ""); + expect(isStaleLock(lockPath)).toBe(true); + writeFileSync(lockPath, deadLock()); + expect(isStaleLock(lockPath)).toBe(true); + writeFileSync(lockPath, `${LOCK_SENTINEL}\n`); + expect(isStaleLock(lockPath)).toBe(true); + writeFileSync(lockPath, liveLock()); + expect(isStaleLock(lockPath)).toBe(false); + writeFileSync(lockPath, otherHostLock()); + expect(isStaleLock(lockPath)).toBe(false); + writeFileSync(lockPath, "not a lock at all"); + expect(isStaleLock(lockPath)).toBe(false); +}); + +// --- forceUnlock: the CLI's --force --- + +test("forceUnlock removes dead-holder and unverifiable locks, refuses live and foreign ones", () => { + const path = tempPath("db"); + const lockPath = `${path}.lock`; + + forceUnlock(path); // no lock at all: a quiet no-op + + writeFileSync(lockPath, deadLock()); + forceUnlock(path); + expect(existsSync(lockPath)).toBe(false); + + writeFileSync(lockPath, otherHostLock()); + forceUnlock(path); // unverifiable holder: force takes it, risk on the caller + expect(existsSync(lockPath)).toBe(false); + + writeFileSync(lockPath, liveLock()); + let live: unknown; + try { + forceUnlock(path); + } catch (error) { + live = error; + } + expect((live as LibreDbError).code).toBe("LOCKED"); + expect(existsSync(lockPath)).toBe(true); + + writeFileSync(lockPath, "the user's own bytes"); + let foreign: unknown; + try { + forceUnlock(path); + } catch (error) { + foreign = error; + } + expect((foreign as LibreDbError).code).toBe("LOCKED"); + expect(readFileSync(lockPath, "utf8")).toBe("the user's own bytes"); +}); diff --git a/src/adapter/node-fs.ts b/src/adapter/node-fs.ts index 8cc91fd..b511e0e 100644 --- a/src/adapter/node-fs.ts +++ b/src/adapter/node-fs.ts @@ -9,28 +9,174 @@ * `node:fs` into its import graph. The default Node entry (`index.ts`) wires this * adapter in as the default `fs`, so production behaviour is unchanged. * - * Each method is the obvious synchronous syscall, so the adapter adds an - * interface boundary, not behaviour. Appends go through one append-mode - * descriptor (creating the file if missing); reads, size and truncate work by - * path, matching how the WAL has always reached the disk. + * Everything runs on one file descriptor opened in append mode: positional + * reads, appends, fsync and truncate all address the same inode, so a path + * swapped out from under a live database cannot split reads from writes. Two + * durability details live here because they are platform facts, not kernel + * logic: creating the file fsyncs the PARENT DIRECTORY (POSIX does not make a + * new directory entry durable until then), and the exclusive {@link + * FileSystem.lock} is a `.lock` file so a second writer — same process or + * another one — fails loudly instead of silently corrupting the log. */ -import { closeSync, fsyncSync, openSync, readFileSync, statSync, truncateSync, writeSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { + closeSync, + existsSync, + fstatSync, + fsyncSync, + ftruncateSync, + openSync, + readFileSync, + readSync, + rmSync, + writeSync, +} from "node:fs"; +import { hostname } from "node:os"; +import { dirname } from "node:path"; -import type { FileSystem } from "../core.ts"; +import { LibreDbError, type FileSystem } from "../core.ts"; + +/** The first line of every LibreDB lock file, so tooling (and `--force`) can + * tell a real lock from an unrelated file that merely shares the name. */ +export const LOCK_SENTINEL = "libredb-lock"; + +/** What a lock file records about its holder. `pid`/`host` let a later opener + * detect a stale lock (the holder died); `nonce` proves ownership on release. */ +interface LockOwner { + readonly pid: number; + readonly host: string; + readonly nonce: string; +} + +/** Parse a lock file's contents. Returns undefined for an empty file or a + * legacy sentinel-only lock (both are LibreDB strays with no liveness info) and + * throws nothing — a FOREIGN file (no sentinel) returns null. */ +function parseLock(contents: string): LockOwner | undefined | null { + if (contents === "") return undefined; // a crashed create left an empty file + if (!contents.startsWith(LOCK_SENTINEL)) return null; // not ours + const [, pid, host, nonce] = contents.split("\n"); + if (pid === undefined || host === undefined || nonce === undefined || nonce === "") { + return undefined; // sentinel-only legacy lock: ours, but anonymous + } + return { pid: Number(pid), host, nonce }; +} + +/** Can the process behind `owner` be probed on THIS host, and is it alive? + * "verified dead" means same host and signal-0 says the pid is gone; a holder + * on another host is never verifiable either way. */ +function livenessOf(owner: LockOwner): "alive" | "dead" | "unverifiable" { + if (owner.host !== hostname()) return "unverifiable"; + try { + process.kill(owner.pid, 0); // signal 0: existence probe, no signal sent + return "alive"; + } catch (error) { + // ESRCH: no such process (dead). EPERM: exists but not ours (alive). + return (error as { code?: string }).code === "EPERM" ? "alive" : "dead"; + } +} + +/** + * Is the lock at `lockPath` stale — held by a LibreDB process that verifiably + * no longer exists? Foreign files are never stale (they are not locks to + * steal), and a holder that cannot be probed (another host) counts as live: + * auto-reclaim must never race a writer that might still be running. `--force` + * (see {@link forceUnlock}) is the explicit escape hatch for that case. + */ +export function isStaleLock(lockPath: string): boolean { + let contents: string; + try { + contents = readFileSync(lockPath, "utf8"); + } catch { + return true; // vanished between the failed create and this read: retry + } + const owner = parseLock(contents); + if (owner === null) return false; // foreign file: refuse to touch it + if (owner === undefined) return true; // anonymous LibreDB stray: reclaim it + return livenessOf(owner) === "dead"; +} + +/** One attempt to create `lockPath` exclusively. Returns false when it already + * exists; any other failure (missing directory, permissions) propagates. */ +function tryCreateLock(lockPath: string, contents: string): boolean { + let fd: number; + try { + fd = openSync(lockPath, "wx"); // "wx": exclusive create, fails if it exists + } catch (error) { + if ((error as { code?: string }).code !== "EEXIST") throw error; + return false; + } + try { + writeSync(fd, contents); + } finally { + closeSync(fd); + } + return true; +} + +/** + * Remove a lock file with `--force` semantics: a LibreDb lock is removed + * unless its holder is VERIFIABLY alive (same host, pid exists); a foreign + * file is always refused. An unverifiable holder (another host) is removed — + * that is exactly the case force exists for — with the risk on the caller. + * Exported for the CLI, which offers this as its `--force` flag. + */ +export function forceUnlock(path: string): void { + const lockPath = `${path}.lock`; + if (!existsSync(lockPath)) return; // nothing to remove + const owner = parseLock(readFileSync(lockPath, "utf8")); + if (owner === null) { + throw new LibreDbError("LOCKED", `refusing to remove ${lockPath}: not a libredb lock file`); + } + if (owner !== undefined && livenessOf(owner) === "alive") { + throw new LibreDbError("LOCKED", `refusing to remove ${lockPath}: holder (pid ${owner.pid}) is alive`); + } + rmSync(lockPath, { force: true }); +} + +/** + * Fsync the directory containing `path`, making a just-created file's directory + * entry durable. POSIX leaves a new entry volatile until the directory itself + * is fsync'd — without this, a freshly created database (and every commit in + * it) can vanish wholesale on power loss. Exported for the tests that pin it. + */ +export function fsyncDirectoryOf(path: string): void { + try { + const dirFd = openSync(dirname(path), "r"); + try { + fsyncSync(dirFd); + } finally { + closeSync(dirFd); + } + } catch { + // Platforms without directory fsync (Windows) throw on the open or the + // fsync; directory-entry durability is the OS's best effort there. + } +} /** Build the default node:fs-backed {@link FileSystem}. */ export function nodeFileSystem(): FileSystem { return { open(path) { - const fd = openSync(path, "a"); // append-only; creates the file if missing + const creating = !existsSync(path); + const fd = openSync(path, "a+"); // read + append-only writes; creates if missing + if (creating) fsyncDirectoryOf(path); return { size() { - return statSync(path).size; + return fstatSync(fd).size; }, read(offset, length) { - // A fresh Uint8Array so the returned slice is an independent copy, not - // a view aliasing a shared Buffer pool. - return new Uint8Array(readFileSync(path)).subarray(offset, offset + length); + // Positional reads on the WAL's own descriptor, looped because a + // single readSync may legally return fewer bytes than asked. Fewer + // bytes than the file holds would otherwise read as a torn tail and + // truncate committed data — the kernel treats that as an IO fault. + const out = new Uint8Array(length); + let filled = 0; + while (filled < length) { + const count = readSync(fd, out, filled, length - filled, offset + filled); + if (count === 0) break; // end of file + filled += count; + } + return filled === length ? out : out.subarray(0, filled); }, append(bytes) { for (let written = 0; written < bytes.length; ) { @@ -41,12 +187,39 @@ export function nodeFileSystem(): FileSystem { fsyncSync(fd); }, truncate(length) { - truncateSync(path, length); + ftruncateSync(fd, length); }, close() { closeSync(fd); }, }; }, + lock(path) { + const lockPath = `${path}.lock`; + const nonce = randomBytes(8).toString("hex"); + const contents = `${LOCK_SENTINEL}\n${process.pid}\n${hostname()}\n${nonce}\n`; + // Two attempts: the second runs only after a stale lock (a crashed + // holder's leftover) was reclaimed. A live holder never yields. + for (let attempt = 0; attempt < 2; attempt++) { + if (tryCreateLock(lockPath, contents)) { + return () => { + // Release only OUR lock: if someone force-removed it and locked + // again, deleting theirs would let a third writer in. + try { + if (parseLock(readFileSync(lockPath, "utf8"))?.nonce !== nonce) return; + } catch { + return; // already gone + } + rmSync(lockPath, { force: true }); + }; + } + if (!isStaleLock(lockPath)) break; + rmSync(lockPath, { force: true }); // reclaim the stale lock, then retry + } + throw new LibreDbError( + "LOCKED", + `${path} is locked (${lockPath}); another writer holds it — close it first, or use --force in the CLI`, + ); + }, }; } diff --git a/src/adapter/opfs.ts b/src/adapter/opfs.ts index c79bd8a..68709fb 100644 --- a/src/adapter/opfs.ts +++ b/src/adapter/opfs.ts @@ -52,9 +52,19 @@ export function opfsFileSystem(handle: SyncAccessHandle): FileSystem { return handle.getSize(); }, read(offset, length) { + // read() may return fewer bytes than asked even when more exist (the + // returned count exists precisely because short reads are legal), so + // loop until the request is filled or the handle reports end-of-file. + // Without the loop a transient short read during recovery would look + // like a torn tail and truncate committed data. Mirrors append(). const buffer = new Uint8Array(length); - const read = handle.read(buffer, { at: offset }); - return buffer.subarray(0, read); + let filled = 0; + while (filled < length) { + const count = handle.read(buffer.subarray(filled), { at: offset + filled }); + if (count === 0) break; // end of file + filled += count; + } + return filled === length ? buffer : buffer.subarray(0, filled); }, append(data) { // write() may write fewer bytes than asked (hence the returned count), diff --git a/src/browser.ts b/src/browser.ts index a9496d4..3703e80 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -13,7 +13,8 @@ */ import { open as openKernel, type Database, type FileSystem } from "./core.ts"; -export { version } from "./core.ts"; +export { version, LibreDbError } from "./core.ts"; +export type { ErrorCode, RecoveryInfo } from "./core.ts"; // OpenOptions (the kernel's permissive type, fs optional) is intentionally NOT // re-exported here: the browser `open` is typed with BrowserOpenOptions, where fs // is required alongside a path, so exposing OpenOptions would advertise a diff --git a/src/cli/lock.test.ts b/src/cli/lock.test.ts deleted file mode 100644 index d6eb9c7..0000000 --- a/src/cli/lock.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -/** - * lock.test.ts — the CLI's advisory write lock. - * - * LibreDB is single-process with no file locking of its own, so two concurrent - * writers would corrupt a file. Write commands take an advisory `.lock` - * to make a second writer fail loudly instead. The lock is advisory, not a - * kernel guarantee: `--force` overrides a stale one. - */ -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { afterEach, expect, test } from "bun:test"; - -import { acquireLock } from "./lock.ts"; - -const dirs: string[] = []; -const tempPath = (): string => { - const dir = mkdtempSync(join(tmpdir(), "libredb-lock-")); - dirs.push(dir); - return join(dir, "db"); -}; - -afterEach(() => { - while (dirs.length > 0) rmSync(dirs.pop() as string, { recursive: true, force: true }); -}); - -test("acquires a lock file and releases it", () => { - const path = tempPath(); - const lock = acquireLock(path, false); - expect(existsSync(`${path}.lock`)).toBe(true); - lock.release(); - expect(existsSync(`${path}.lock`)).toBe(false); -}); - -test("refuses when a lock is already held", () => { - const path = tempPath(); - writeFileSync(`${path}.lock`, ""); // another writer holds it - expect(() => acquireLock(path, false)).toThrow(/locked/i); -}); - -test("--force drops a real libredb lock and re-acquires", () => { - const path = tempPath(); - acquireLock(path, false); // a prior writer's lock, left behind (e.g. it crashed) - const lock = acquireLock(path, true); // force clears it and takes a fresh one - expect(existsSync(`${path}.lock`)).toBe(true); - lock.release(); - expect(existsSync(`${path}.lock`)).toBe(false); -}); - -test("--force with no existing lock simply acquires", () => { - const path = tempPath(); - const lock = acquireLock(path, true); - expect(existsSync(`${path}.lock`)).toBe(true); - lock.release(); -}); - -test("a non-lock IO error is surfaced, not misreported as locked", () => { - // A lock path inside a directory that does not exist fails with ENOENT; that - // must not be reported as "locked" (which would send the user down --force). - const bogus = join(tmpdir(), "libredb-no-such-dir-xyz", "db"); - let caught: unknown; - try { - acquireLock(bogus, false); - } catch (error) { - caught = error; - } - expect(caught).toBeInstanceOf(Error); - expect((caught as Error).message).not.toMatch(/locked/i); -}); - -test("--force clears an empty stray lock (e.g. a crash before the sentinel was written)", () => { - const path = tempPath(); - writeFileSync(`${path}.lock`, ""); // empty stray, not foreign user data - const lock = acquireLock(path, true); - expect(existsSync(`${path}.lock`)).toBe(true); - lock.release(); - expect(existsSync(`${path}.lock`)).toBe(false); -}); - -test("--force refuses to delete a file that is not a libredb lock", () => { - const path = tempPath(); - writeFileSync(`${path}.lock`, "this is the user's own data, not a lock"); - expect(() => acquireLock(path, true)).toThrow(/not a libredb lock/i); - // The user's file is left intact. - expect(existsSync(`${path}.lock`)).toBe(true); -}); diff --git a/src/cli/lock.ts b/src/cli/lock.ts deleted file mode 100644 index 32b4547..0000000 --- a/src/cli/lock.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** - * cli/lock.ts — an advisory write lock for the CLI. - * - * LibreDB is single-process and does no file locking itself, so two writers to - * one file would corrupt it. Write commands take a `.lock` first: an - * exclusive create that fails if the file already exists, turning a concurrent - * writer into a loud error instead of silent corruption. It is advisory only — - * `--force` drops a stale lock and proceeds. The lock is always released after - * the write (see withWriteDb in run.ts). - */ -import { closeSync, existsSync, openSync, readFileSync, rmSync, writeSync } from "node:fs"; - -/** A held lock. Call {@link Lock.release} once the write is done. */ -export interface Lock { - release(): void; -} - -// Written into every lock file so a forced acquire can tell a real libredb lock -// from an unrelated file that merely happens to be named .lock, and refuse -// to delete the latter. -const SENTINEL = "libredb-lock\n"; - -/** Acquire the advisory lock for `path`. With `force`, an existing libredb lock - * is dropped first; otherwise an existing lock makes this throw. */ -export function acquireLock(path: string, force: boolean): Lock { - const lockPath = `${path}.lock`; - if (force) dropOwnLock(lockPath); - try { - const fd = openSync(lockPath, "wx"); // "wx": exclusive create, fails if it exists - try { - writeSync(fd, SENTINEL); - } finally { - closeSync(fd); // always release the descriptor, even if the write throws - } - } catch (error) { - // A failed sentinel write (e.g. ENOSPC) leaves an EMPTY .lock behind; - // that stray is recoverable because dropOwnLock treats an empty file as ours. - // Only an existing lock file (EEXIST) means "locked". Surface any other IO - // error (missing directory, permissions, ...) unchanged, so a real problem - // is not misreported as a held lock. - if ((error as { code?: string }).code !== "EEXIST") throw error; - throw new Error( - `libredb: ${path} is locked (${lockPath}); another writer may be active — use --force to override`, - { cause: error }, - ); - } - return { - release() { - rmSync(lockPath, { force: true }); - }, - }; -} - -/** Remove an existing libredb lock so a forced acquire can proceed. A lock this - * tool created carries the SENTINEL; an empty file is a stray from a crash or a - * failed sentinel write (also ours) and is safe to drop. Any other content means - * the file is not our lock, so `--force` refuses it rather than delete unrelated - * user data that happens to share the `.lock` name. A read error other than "no - * such file" (e.g. EACCES) propagates instead of being silently swallowed. */ -function dropOwnLock(lockPath: string): void { - if (!existsSync(lockPath)) return; // nothing to drop - const contents = readFileSync(lockPath, "utf8"); - if (contents !== "" && !contents.startsWith(SENTINEL)) { - throw new Error(`libredb: refusing to remove ${lockPath} with --force: not a libredb lock file`); - } - rmSync(lockPath, { force: true }); -} diff --git a/src/cli/readonly-fs.test.ts b/src/cli/readonly-fs.test.ts index 1478234..3ce13f6 100644 --- a/src/cli/readonly-fs.test.ts +++ b/src/cli/readonly-fs.test.ts @@ -37,14 +37,25 @@ test("size and read reflect the file on disk", () => { file.close(); }); -test("append and fsync refuse, so a read can never write", () => { +test("append refuses, so a read can never write", () => { const path = tempFile(new Uint8Array([1])); const file = readonlyFileSystem().open(path); expect(() => file.append(new Uint8Array([9]))).toThrow(/read-only/i); - expect(() => file.fsync()).toThrow(/read-only/i); file.close(); }); +test("fsync is a harmless no-op (recovery fsyncs after its no-op truncate)", () => { + // The commit path appends BEFORE it fsyncs, and append refuses above, so a + // no-op fsync can never silently acknowledge a write. It exists because + // recovery fsyncs after truncating a torn tail — a no-op here, too. + const original = new Uint8Array([1, 2, 3]); + const path = tempFile(original); + const file = readonlyFileSystem().open(path); + expect(() => file.fsync()).not.toThrow(); + file.close(); + expect(new Uint8Array(readFileSync(path))).toEqual(original); +}); + test("truncate is a no-op: the file on disk is left untouched", () => { const original = new Uint8Array([1, 2, 3, 4]); const path = tempFile(original); diff --git a/src/cli/readonly-fs.ts b/src/cli/readonly-fs.ts index 12b85d7..a30c67a 100644 --- a/src/cli/readonly-fs.ts +++ b/src/cli/readonly-fs.ts @@ -29,7 +29,10 @@ export function readonlyFileSystem(): FileSystem { throw new Error("libredb: read-only database; refusing to write"); }, fsync() { - throw new Error("libredb: read-only database; refusing to write"); + // Deliberate no-op (not a throw): recovery fsyncs after truncating a + // torn tail, and this adapter's truncate is itself a no-op — there is + // nothing to flush. A real WRITE can never reach this fsync, because + // the commit path appends first and append refuses above. }, truncate() { // Deliberate no-op: a read must not alter the file. Recovery still diff --git a/src/cli/run.test.ts b/src/cli/run.test.ts index 23b3e7f..70e365b 100644 --- a/src/cli/run.test.ts +++ b/src/cli/run.test.ts @@ -7,16 +7,16 @@ * stats, get, scan) against real .libredb files, plus usage and error handling. */ import { appendFileSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, test } from "bun:test"; +import { LOCK_SENTINEL } from "../adapter/node-fs.ts"; import { open } from "../index.ts"; import { doc } from "../lens/document.ts"; import { kv } from "../lens/kv.ts"; import { table } from "../lens/relational.ts"; -import { acquireLock } from "./lock.ts"; import { run } from "./run.ts"; const dirs: string[] = []; @@ -242,22 +242,53 @@ test("import rejects malformed JSON as a usage error (exit 2)", () => { expect(r.err.join("\n")).toMatch(/json/i); }); -test("a write refuses when the database is locked", () => { +/** A lock file naming a live holder: this very test process. */ +const liveLock = (): string => `${LOCK_SENTINEL}\n${process.pid}\n${hostname()}\nnonce\n`; + +test("a write refuses when a live writer holds the lock", () => { const path = fixture(); - writeFileSync(`${path}.lock`, ""); // another writer holds the lock + writeFileSync(`${path}.lock`, liveLock()); // a live holder (this process) const r = cli("set", path, "k", "v"); expect(r.code).toBe(1); expect(r.err.join("\n")).toMatch(/locked/i); }); -test("--force overrides a stale libredb lock", () => { +test("a stale lock (dead holder) is reclaimed automatically, no --force needed", () => { const path = fixture(); - acquireLock(path, false); // a prior writer's lock, left behind (e.g. it crashed) + // A crashed writer's leftover: an empty lock file carries no live holder. + writeFileSync(`${path}.lock`, ""); + const r = cli("set", path, "k", "v"); + expect(r.code).toBe(0); + expect(cli("get", path, "k").out).toEqual(["v"]); + expect(existsSync(`${path}.lock`)).toBe(false); +}); + +test("--force refuses to remove a live holder's lock", () => { + const path = fixture(); + writeFileSync(`${path}.lock`, liveLock()); + const r = cli("set", path, "k", "v", "--force"); + expect(r.code).toBe(1); + expect(r.err.join("\n")).toMatch(/alive/i); +}); + +test("--force removes a lock from another host (liveness unverifiable)", () => { + const path = fixture(); + writeFileSync(`${path}.lock`, `${LOCK_SENTINEL}\n99999\nsome-other-host\nnonce\n`); + expect(cli("set", path, "k", "v").code).toBe(1); // without --force: locked const r = cli("set", path, "k", "v", "--force"); expect(r.code).toBe(0); expect(cli("get", path, "k").out).toEqual(["v"]); }); +test("--force refuses to delete a file that is not a libredb lock", () => { + const path = fixture(); + writeFileSync(`${path}.lock`, "this is the user's own data, not a lock"); + const r = cli("set", path, "k", "v", "--force"); + expect(r.code).toBe(1); + expect(r.err.join("\n")).toMatch(/not a libredb lock/i); + expect(existsSync(`${path}.lock`)).toBe(true); // the user's file is intact +}); + test("set refuses to write a reserved key", () => { const r = cli("set", fixture(), "\u0000libredb:catalog:people", "x"); expect(r.code).toBe(2); diff --git a/src/cli/run.ts b/src/cli/run.ts index ff990ab..da92c26 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -9,17 +9,18 @@ * This is open-edge tooling over the public API, not kernel code: it adds no * durability logic. Read commands (inspect/stats/get/scan) open through the * read-only filesystem adapter so inspecting a file never mutates it. Write - * commands (set/delete/import) take an advisory lock first, and import commits - * all keys in one transaction so a bulk load is atomic. + * commands (set/delete/import) rely on the kernel's exclusive open lock (a + * second writer fails loudly; --force clears a lock whose holder is gone), and + * import commits all keys in one transaction so a bulk load is atomic. */ import { readFileSync, statSync } from "node:fs"; import { parseArgs } from "node:util"; -import type { Database } from "../core.ts"; +import { forceUnlock } from "../adapter/node-fs.ts"; +import { LibreDbError, type Database } from "../core.ts"; import { open } from "../index.ts"; import { catalog, isReservedKey } from "../lens/catalog.ts"; import { kv } from "../lens/kv.ts"; -import { acquireLock } from "./lock.ts"; import { readonlyFileSystem } from "./readonly-fs.ts"; /** Where the CLI writes its output. One call is one line; the sink adds newlines. */ @@ -53,7 +54,7 @@ const USAGE = [ " libredb import Bulk-set keys from a JSON object (one atomic commit)", "", "Options:", - " --force Override an existing write lock", + " --force Remove a write lock whose holder is no longer alive", ].join("\n"); /** Open `path` read-only, run `fn`, and always close — so a read leaves the file @@ -67,19 +68,23 @@ const withReadDb = (path: string, fn: (db: Database) => T): T => { } }; -/** Take the advisory lock, open `path` for writing, run `fn`, then always close - * and release — so a crash mid-write cannot leave the lock stranded. */ +/** Open `path` for writing (the kernel takes the exclusive lock), run `fn`, + * then always close — which releases the lock. With `force`, a LOCKED open + * removes the lock first when its holder is not verifiably alive; a live + * holder still refuses, so --force cannot create two live writers. */ const withWriteDb = (path: string, force: boolean, fn: (db: Database) => T): T => { - const lock = acquireLock(path, force); + let db: Database; try { - const db = open({ path }); - try { - return fn(db); - } finally { - db.close(); - } + db = open({ path }); + } catch (error) { + if (!force || !(error instanceof LibreDbError) || error.code !== "LOCKED") throw error; + forceUnlock(path); + db = open({ path }); + } + try { + return fn(db); } finally { - lock.release(); + db.close(); } }; diff --git a/src/core.hardening.test.ts b/src/core.hardening.test.ts new file mode 100644 index 0000000..342e02c --- /dev/null +++ b/src/core.hardening.test.ts @@ -0,0 +1,489 @@ +/** + * core.hardening.test.ts — the kernel's failure-mode contract. + * + * The crash model (append-only + CRC + fsync-before-visible) is pinned by + * core.recovery.test.ts and the DST suite. THIS suite pins everything that sits + * outside that model — the pre-announcement audit findings: + * + * - a foreign file is refused, never truncated (the WAL magic header) + * - mid-log corruption refuses to open instead of silently truncating + * - an async transact() body is rejected before it can half-commit + * - a failed append/fsync latches the database (fsyncgate) + * - keys and values are copied at the API boundary (no aliasing) + * - getRange snapshots, so delete-while-scanning visits every entry + * - close() during a transaction is a named error, not a raw EBADF + * - double-open of one file is a loud LOCKED error + * + * Every error carries a stable LibreDbError code — asserted here so the codes + * are contract, not decoration. + */ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { hostname, tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, expect, test } from "bun:test"; + +import { LibreDbError, open, type FileSystem, type WalFile } from "./core.ts"; +import { open as openNode } from "./index.ts"; + +const bytes = (...b: number[]): Uint8Array => new Uint8Array(b); + +const dirs: string[] = []; +const tempPath = (name: string): string => { + const dir = mkdtempSync(join(tmpdir(), "libredb-hard-")); + dirs.push(dir); + return join(dir, name); +}; + +afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop() as string, { recursive: true, force: true }); +}); + +/** Grab the LibreDbError a thunk throws, asserting it threw at all. */ +const errorFrom = (thunk: () => unknown): LibreDbError => { + try { + thunk(); + } catch (error) { + expect(error).toBeInstanceOf(LibreDbError); + return error as LibreDbError; + } + throw new Error("expected the thunk to throw"); +}; + +// --- a fault-injectable in-memory filesystem for the kernel-level tests --- + +interface FaultFile { + data: number[]; + /** When set, the next append persists only this many bytes, then throws. */ + failAppendAfter: number | undefined; + /** When true, the next fsync throws (the bytes stay in `data`). */ + failNextFsync: boolean | undefined; + /** When set, the next read returns this many bytes fewer than asked. */ + shortReadBy: number | undefined; +} + +function faultFs(): { fs: FileSystem; file: FaultFile } { + const file: FaultFile = { data: [], failAppendAfter: undefined, failNextFsync: undefined, shortReadBy: undefined }; + const fs: FileSystem = { + open(): WalFile { + return { + size: () => file.data.length, + read(offset, length) { + let end = Math.min(offset + length, file.data.length); + if (file.shortReadBy !== undefined) { + end -= file.shortReadBy; + file.shortReadBy = undefined; + } + return Uint8Array.from(file.data.slice(offset, end)); + }, + append(b) { + if (file.failAppendAfter !== undefined) { + const kept = file.failAppendAfter; + file.failAppendAfter = undefined; + for (const byte of b.subarray(0, kept)) file.data.push(byte); + throw new Error("injected: ENOSPC"); + } + for (const byte of b) file.data.push(byte); + }, + fsync() { + if (file.failNextFsync === true) { + file.failNextFsync = undefined; + throw new Error("injected: EIO on fsync"); + } + }, + truncate(length) { + file.data.length = length; + }, + close() {}, + }; + }, + }; + return { fs, file }; +} + +/** CRC-32 (IEEE), duplicated here so the tests can hand-craft on-disk records + * without borrowing the kernel's implementation (which they judge). */ +function crc32(data: Uint8Array): number { + let crc = 0xffffffff; + for (let i = 0; i < data.length; i++) { + crc ^= data[i] as number; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return (crc ^ 0xffffffff) >>> 0; +} + +const u32 = (n: number): number[] => [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff]; + +/** Encode one legacy (headerless, v0.1.x) record that sets `key` to `value`. */ +function legacyRecord(key: number[], value: number[]): number[] { + const payload = [1, ...u32(key.length), ...key, ...u32(value.length), ...value]; + return [...u32(payload.length), ...u32(crc32(Uint8Array.from(payload))), ...payload]; +} + +// --- LibreDbError: the typed error contract --- + +test("kernel errors are LibreDbError instances with a stable code and libredb: prefix", () => { + const db = open(); + db.close(); + const error = errorFrom(() => db.transact(() => 0)); + expect(error.name).toBe("LibreDbError"); + expect(error.code).toBe("CLOSED"); + expect(error.message).toMatch(/^libredb: /); +}); + +test("a nested transact carries the NESTED_TRANSACTION code", () => { + const db = open(); + expect(errorFrom(() => db.transact(() => db.transact(() => 0))).code).toBe("NESTED_TRANSACTION"); + db.close(); +}); + +test("an empty path and a missing filesystem carry INVALID_ARGUMENT", () => { + expect(errorFrom(() => open({ path: "" })).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => open({ path: "x" })).code).toBe("INVALID_ARGUMENT"); +}); + +// --- issue #16: async transact() bodies are rejected before commit --- + +test("an async transact() callback throws ASYNC_TRANSACTION and commits nothing", () => { + const path = tempPath("async"); + const db = openNode({ path }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + + const error = errorFrom(() => + db.transact(async (tx) => { + tx.set(bytes(2), bytes(20)); + await Promise.resolve(); + tx.set(bytes(3), bytes(30)); + }), + ); + expect(error.code).toBe("ASYNC_TRANSACTION"); + + // Nothing from the async body was committed — not even the pre-await write — + // in memory or on disk. + expect(db.transact((tx) => tx.get(bytes(2)))).toBeUndefined(); + db.close(); + const reopened = openNode({ path }); + expect(reopened.transact((tx) => tx.get(bytes(2)))).toBeUndefined(); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + reopened.close(); +}); + +test("a thenable (non-Promise) return value is rejected the same way", () => { + const db = open(); + const thenable = { then: () => {} }; + expect(errorFrom(() => db.transact(() => thenable)).code).toBe("ASYNC_TRANSACTION"); + db.close(); +}); + +test("returning a plain object or function from transact() still works", () => { + const db = open(); + expect(db.transact(() => ({ answer: 42 }))).toEqual({ answer: 42 }); + const fn = (): number => 7; + expect(db.transact(() => fn)).toBe(fn); + db.close(); +}); + +// --- issue #31: close() during a transaction is a named error --- + +test("close() inside a transaction throws CLOSE_IN_TRANSACTION and the db stays usable", () => { + const db = open(); + expect( + errorFrom(() => + db.transact(() => { + db.close(); + }), + ).code, + ).toBe("CLOSE_IN_TRANSACTION"); + // The database survived the misuse: it is neither closed nor wedged. + db.transact((tx) => tx.set(bytes(1), bytes(1))); + expect(db.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(1)); + db.close(); +}); + +// --- issue #13: foreign files are refused, never truncated --- + +test("open() on a non-LibreDB file throws NOT_A_DATABASE and leaves every byte in place", () => { + const path = tempPath("notes.txt"); + const contents = "these are the user's notes, not a database\n"; + writeFileSync(path, contents); + + expect(errorFrom(() => openNode({ path })).code).toBe("NOT_A_DATABASE"); + expect(readFileSync(path, "utf8")).toBe(contents); + // The refusal also released the open lock. + expect(existsSync(`${path}.lock`)).toBe(false); +}); + +test("a new database writes the LRDB file header with its first commit", () => { + const path = tempPath("fresh"); + const db = openNode({ path }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.close(); + + const disk = new Uint8Array(readFileSync(path)); + expect([...disk.subarray(0, 4)]).toEqual([0x4c, 0x52, 0x44, 0x42]); // "LRDB" + expect(((disk[4] as number) << 8) | (disk[5] as number)).toBe(1); // format version +}); + +test("a headerless v0.1.x database still opens, reads, and accepts writes", () => { + const path = tempPath("legacy"); + writeFileSync(path, Uint8Array.from([...legacyRecord([1], [10]), ...legacyRecord([2], [20])])); + + const db = openNode({ path }); + expect(db.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + db.transact((tx) => tx.set(bytes(3), bytes(30))); + db.close(); + + // The legacy file stays headerless (a header cannot be inserted mid-file); + // reopening replays all three records through the legacy read path. + const reopened = openNode({ path }); + expect(reopened.transact((tx) => tx.get(bytes(2)))).toEqual(bytes(20)); + expect(reopened.transact((tx) => tx.get(bytes(3)))).toEqual(bytes(30)); + reopened.close(); +}); + +test("a file written by a NEWER format version is refused with UNSUPPORTED_VERSION", () => { + const path = tempPath("future"); + writeFileSync(path, Uint8Array.from([0x4c, 0x52, 0x44, 0x42, 0, 2, 0, 0])); + expect(errorFrom(() => openNode({ path })).code).toBe("UNSUPPORTED_VERSION"); +}); + +test("a header torn mid-write (first commit interrupted) restarts the database from empty", () => { + const path = tempPath("torn-header"); + writeFileSync(path, Uint8Array.from([0x4c, 0x52])); // "LR": a magic prefix, cut short + const truncations: number[] = []; + const db = openNode({ path, onRecovery: (info) => truncations.push(info.truncatedBytes) }); + expect(truncations).toEqual([2]); // the torn header was reported, not silent + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.close(); + const reopened = openNode({ path }); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + reopened.close(); +}); + +// --- issue #22: corruption classification --- + +test("mid-log corruption refuses to open with CORRUPT_WAL and truncates nothing", () => { + const { fs, file } = faultFs(); + const db = open({ path: "wal", fs }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.transact((tx) => tx.set(bytes(2), bytes(20))); + db.close(); + + // Flip a payload byte of the FIRST record (offset: 8 header + 8 record + // header = 16). Intact record 2 sits after it, so this is damage to + // once-durable bytes, not a crash artifact. + file.data[16] = (file.data[16] as number) ^ 0xff; + const sizeBefore = file.data.length; + + expect(errorFrom(() => open({ path: "wal", fs })).code).toBe("CORRUPT_WAL"); + expect(file.data.length).toBe(sizeBefore); // refuse means refuse: no truncate +}); + +test("a CRC-valid record with a malformed payload is corruption, not ops", () => { + const path = tempPath("malformed"); + // A legacy-framed record whose payload has a valid checksum but a bogus op + // tag (7): structurally impossible output of this kernel, so it must refuse + // rather than misread garbage into the store. Probed at offset 0 with no + // valid record before it, the file is simply not a database we recognize. + const payload = [7, 0, 0, 0, 1, 0]; + const record = [...u32(payload.length), ...u32(crc32(Uint8Array.from(payload))), ...payload]; + writeFileSync(path, Uint8Array.from(record)); + expect(errorFrom(() => openNode({ path })).code).toBe("NOT_A_DATABASE"); + + // The same malformed record BEHIND a valid one (a recognized database) is + // named for what it is: a corrupt WAL. + const path2 = tempPath("malformed-2"); + writeFileSync(path2, Uint8Array.from([...legacyRecord([1], [10]), ...record])); + expect(errorFrom(() => openNode({ path: path2 })).code).toBe("CORRUPT_WAL"); +}); + +test("recovery reports a torn tail through onRecovery instead of dropping it silently", () => { + const path = tempPath("reported"); + const db = openNode({ path }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.close(); + // A crash mid-append: the header promises more bytes than follow. + const torn = Uint8Array.from([...u32(0xffff), ...u32(0), 1, 2, 3]); + writeFileSync(path, Uint8Array.from([...readFileSync(path), ...torn])); + + const truncations: number[] = []; + const reopened = openNode({ path, onRecovery: (info) => truncations.push(info.truncatedBytes) }); + expect(truncations).toEqual([torn.length]); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + reopened.close(); +}); + +// --- issue #23/#10: a short read is an IO fault, never a truncation --- + +test("a read that returns fewer bytes than the file holds throws INCOMPLETE_READ", () => { + const { fs, file } = faultFs(); + const db = open({ path: "wal", fs }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.close(); + + file.shortReadBy = 3; + expect(errorFrom(() => open({ path: "wal", fs })).code).toBe("INCOMPLETE_READ"); + // Transient fault: the next open (full read) succeeds. + const reopened = open({ path: "wal", fs }); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + reopened.close(); +}); + +// --- issue #19: a failed append/fsync latches the database (fsyncgate) --- + +test("a partially-written append latches the database; the torn tail cannot poison later commits", () => { + const { fs, file } = faultFs(); + const db = open({ path: "wal", fs }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); // commit A: durable + + file.failAppendAfter = 5; // commit B tears after 5 bytes, then ENOSPC + expect(() => db.transact((tx) => tx.set(bytes(2), bytes(20)))).toThrow(/ENOSPC/); + + // The database is latched: it refuses commit C outright instead of appending + // it after the torn bytes (where the next recovery would destroy it). + expect(errorFrom(() => db.transact((tx) => tx.set(bytes(3), bytes(30)))).code).toBe("FAILED"); + // Reads-in-memory are also refused: the instance is done until reopen. + expect(errorFrom(() => db.transact((tx) => tx.get(bytes(1)))).code).toBe("FAILED"); + db.close(); + + // Reopen repairs the tail: commit A survives, B and C never happened. + const reopened = open({ path: "wal", fs }); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + expect(reopened.transact((tx) => tx.get(bytes(2)))).toBeUndefined(); + reopened.close(); +}); + +test("after a failed fsync the database refuses further work until reopened", () => { + const { fs, file } = faultFs(); + const db = open({ path: "wal", fs }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + + file.failNextFsync = true; + expect(() => db.transact((tx) => tx.set(bytes(2), bytes(20)))).toThrow(/EIO/); + expect(errorFrom(() => db.transact((tx) => tx.set(bytes(3), bytes(30)))).code).toBe("FAILED"); + db.close(); + + // After fsyncgate, the un-acknowledged bytes of commit B may or may not have + // reached the disk — both are legal outcomes. What is NOT legal is losing + // commit A, whose transact() returned success. + const reopened = open({ path: "wal", fs }); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + reopened.close(); +}); + +// --- issue #14: keys and values are copied at the API boundary --- + +test("mutating a caller buffer after set() cannot corrupt the committed store", () => { + const path = tempPath("aliasing"); + const db = openNode({ path }); + const key = bytes(1); + const value = bytes(10); + db.transact((tx) => tx.set(key, value)); + + key[0] = 99; // the scratch-buffer-reuse pattern + value[0] = 88; + + expect(db.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + expect(db.transact((tx) => tx.get(bytes(99)))).toBeUndefined(); + db.close(); + // Disk agrees with memory: the journal held copies too. + const reopened = openNode({ path }); + expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + reopened.close(); +}); + +test("mutating a value returned by get() cannot corrupt the committed store", () => { + const db = open(); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + const returned = db.transact((tx) => tx.get(bytes(1))) as Uint8Array; + returned[0] = 77; + expect(db.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + db.close(); +}); + +test("mutating entries yielded by getRange() cannot corrupt the committed store", () => { + const db = open(); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.transact((tx) => { + for (const entry of tx.getRange(bytes(0), bytes(255))) { + (entry.key as Uint8Array)[0] = 66; + (entry.value as Uint8Array)[0] = 66; + } + }); + expect(db.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + db.close(); +}); + +test("a buffer reused across a whole loop leaves every committed key intact and sorted", () => { + const db = open(); + const scratch = new Uint8Array(1); + db.transact((tx) => { + for (const b of [5, 3, 9, 1]) { + scratch[0] = b; + tx.set(scratch, scratch); + } + }); + const keys = db.transact((tx) => [...tx.getRange(bytes(0), bytes(255))].map((e) => e.key[0])); + expect(keys).toEqual([1, 3, 5, 9]); // sorted invariant survived the reuse + db.close(); +}); + +// --- issue #17: getRange snapshots, so scans and writes compose --- + +test("delete-while-scanning visits and deletes every entry in the range", () => { + const db = open(); + db.transact((tx) => { + for (const b of [1, 2, 3, 4, 5]) tx.set(bytes(b), bytes(b)); + }); + const visited: number[] = []; + db.transact((tx) => { + for (const entry of tx.getRange(bytes(0), bytes(255))) { + visited.push(entry.key[0] as number); + tx.delete(entry.key); + } + }); + expect(visited).toEqual([1, 2, 3, 4, 5]); // no skips + expect(db.transact((tx) => [...tx.getRange(bytes(0), bytes(255))])).toEqual([]); + db.close(); +}); + +test("insert-while-scanning does not duplicate or surprise the running scan", () => { + const db = open(); + db.transact((tx) => { + tx.set(bytes(1), bytes(1)); + tx.set(bytes(3), bytes(3)); + }); + const visited: number[] = []; + db.transact((tx) => { + for (const entry of tx.getRange(bytes(0), bytes(255))) { + visited.push(entry.key[0] as number); + tx.set(bytes(2), bytes(2)); // lands between the two — invisible to THIS scan + } + }); + expect(visited).toEqual([1, 3]); + // ...but visible to the next scan, as documented. + const after = db.transact((tx) => [...tx.getRange(bytes(0), bytes(255))].map((e) => e.key[0])); + expect(after).toEqual([1, 2, 3]); + db.close(); +}); + +// --- issue #21: exclusive open lock --- + +test("a second open() of the same live path throws LOCKED; close() releases it", () => { + const path = tempPath("locked"); + const first = openNode({ path }); + expect(errorFrom(() => openNode({ path })).code).toBe("LOCKED"); + first.close(); + const second = openNode({ path }); // released: reopen succeeds + second.close(); +}); + +test("a stale lock from a dead process is reclaimed automatically", () => { + const path = tempPath("stale"); + // Pid 2^22 is above every default Linux pid_max, so it cannot be alive. + writeFileSync(`${path}.lock`, `libredb-lock\n4194304\n${hostname()}\nnonce\n`); + const db = openNode({ path }); + db.transact((tx) => tx.set(bytes(1), bytes(1))); + db.close(); + expect(existsSync(`${path}.lock`)).toBe(false); +}); diff --git a/src/core.recovery.test.ts b/src/core.recovery.test.ts index 4d97403..61f4872 100644 --- a/src/core.recovery.test.ts +++ b/src/core.recovery.test.ts @@ -76,7 +76,11 @@ test("committed writes survive a simulated crash (no graceful close)", () => { crashed.transact((tx) => tx.set(bytes(1), bytes(10))); crashed.transact((tx) => tx.set(bytes(2), bytes(20))); // Simulate a crash: the process dies without ever calling close(). Because - // each commit fsync'd before returning, the data is already durable. + // each commit fsync'd before returning, the data is already durable. In a + // real crash the holder's pid would be dead and the lock auto-reclaimed; + // in-process the "crashed" handle is still this live pid, so drop the lock + // by hand to complete the simulation. + rmSync(`${path}.lock`); const recovered = open({ path }); expect(dump(recovered)).toEqual([ diff --git a/src/core.ts b/src/core.ts index 0a3ec03..527654c 100644 --- a/src/core.ts +++ b/src/core.ts @@ -16,13 +16,59 @@ * * The boundaries are declared first as types — the contract every lens builds * on — followed by the implementation: an ordered key-value store, serializable - * transactions, and a write-ahead log for durability. Recovery replays that log - * and discards any record a crash left half-written. + * transactions, and a write-ahead log for durability. Recovery replays that log, + * discards a record a crash left half-written, and refuses to touch a file it + * does not recognize as a LibreDB database. */ /** The LibreDB package version. Kept in sync with package.json. */ export const version = "0.1.3"; +/** + * The stable failure codes of the kernel. Every error the kernel throws is a + * {@link LibreDbError} carrying one of these, so a caller can branch on `code` + * instead of matching message strings (which the package is free to reword). + * + * - CLOSED the database was closed; open a new one + * - NESTED_TRANSACTION transact() called inside a transaction body + * - ASYNC_TRANSACTION the transaction body returned a Promise/thenable + * - CLOSE_IN_TRANSACTION close() called inside a transaction body + * - FAILED a commit hit an IO error; reopen to recover + * - NOT_A_DATABASE the file at `path` is not a LibreDB database + * - UNSUPPORTED_VERSION the file was written by a newer format version + * - CORRUPT_WAL mid-log corruption; refusing to destroy data + * - INCOMPLETE_READ the filesystem returned fewer bytes than it holds + * - LOCKED another writer holds the database open + * - INVALID_ARGUMENT a malformed argument (empty path, missing fs) + */ +export type ErrorCode = + | "CLOSED" + | "NESTED_TRANSACTION" + | "ASYNC_TRANSACTION" + | "CLOSE_IN_TRANSACTION" + | "FAILED" + | "NOT_A_DATABASE" + | "UNSUPPORTED_VERSION" + | "CORRUPT_WAL" + | "INCOMPLETE_READ" + | "LOCKED" + | "INVALID_ARGUMENT"; + +/** + * The error type of the kernel (and of the adapters that implement its + * filesystem seam). The `code` is the stable contract; the message is for + * humans and may change between releases. + */ +export class LibreDbError extends Error { + readonly code: ErrorCode; + + constructor(code: ErrorCode, message: string, options?: { cause?: unknown }) { + super(`libredb: ${message}`, options); + this.name = "LibreDbError"; + this.code = code; + } +} + /** * A key in the kernel: an immutable sequence of bytes. * @@ -55,21 +101,29 @@ export interface Entry { * kernel forbids re-entrant transactions (the only way to overlap two) to keep * that guarantee real — see {@link Database.transact}. * + * Buffer ownership: the kernel COPIES every key and value that crosses this + * boundary, in both directions. A caller may freely reuse a scratch buffer + * after set()/delete(), and may freely mutate anything get()/getRange() + * returned — neither can corrupt the committed store. + * * Keys are ordered by unsigned byte-lexicographic comparison. That ordering is * the kernel's defining property: it is what makes `getRange` meaningful and * what later lenses encode their indexes against. */ export interface Transaction { - /** The value for `key`, or `undefined` if it is not set. */ + /** The value for `key` (a copy), or `undefined` if it is not set. */ get(key: Key): Value | undefined; - /** Set `key` to `value`, overwriting any existing value. */ + /** Set `key` to `value`, overwriting any existing value. Both are copied. */ set(key: Key, value: Value): void; /** Remove `key`. A no-op if it is not set. */ delete(key: Key): void; /** * Scan the half-open range `[start, end)` in ascending key order: - * `start` is included, `end` is excluded. Lazy by design — callers iterate - * without the kernel materializing the whole range. + * `start` is included, `end` is excluded. The matching entries are + * SNAPSHOTTED when iteration starts (at the first `next()`), so mutating the + * transaction while scanning — the delete-what-you-find pattern — visits + * every entry exactly once; the writes are visible to reads and to later + * scans, not to this one. */ getRange(start: Key, end: Key): Iterable; } @@ -88,9 +142,22 @@ export interface Database { * returns, the writes commit together and become durable; if `run` throws, * the transaction aborts and applies nothing. The return value of `run` is * passed through. + * + * `run` MUST be synchronous. An async callback returns a pending Promise — + * the kernel cannot see writes made after an `await`, so committing at that + * point would silently lose them. Any thenable return value therefore + * aborts the transaction with {@link ErrorCode} ASYNC_TRANSACTION. + * + * If a commit fails to reach the disk (the append or fsync throws), the + * database LATCHES into a failed state: every later transact() throws with + * code FAILED until the database is closed and reopened. Continuing to + * append after a torn write could let recovery silently discard later, + * acknowledged commits — refusing further writes is what keeps "a returned + * transact() is durable" true. */ transact(run: (tx: Transaction) => T): T; - /** Flush pending state and release resources. Safe to call once. */ + /** Flush pending state and release resources. Safe to call once. Throws if + * called from inside a transaction body. */ close(): void; } @@ -110,6 +177,15 @@ export interface FileSystem { /** Open the log file at `path` for reading and appending, creating it if it * is absent, and return a handle to it. */ open(path: string): WalFile; + /** + * Optional: take an exclusive advisory lock on the database at `path`, + * returning a function that releases it. A filesystem that implements this + * makes double-open loud: the kernel calls it before touching the file and + * expects a second concurrent lock of the same path to throw with code + * LOCKED. A filesystem without it (a read-only inspector, a test fake) + * simply opts out of the protection. + */ + lock?(path: string): () => void; } /** @@ -120,7 +196,9 @@ export interface FileSystem { export interface WalFile { /** The number of bytes currently in the file. */ size(): number; - /** Read `length` bytes starting at `offset`. */ + /** Read `length` bytes starting at `offset`. May return fewer only when the + * file itself ends early; the kernel treats a short read of bytes the file + * claims to hold as an IO fault, never as missing data. */ read(offset: number, length: number): Uint8Array; /** Append `bytes` to the end of the file. */ append(bytes: Uint8Array): void; @@ -133,6 +211,14 @@ export interface WalFile { close(): void; } +/** What recovery had to do to the log, reported through + * {@link OpenOptions.onRecovery} so a torn-tail truncation is never silent. */ +export interface RecoveryInfo { + /** Bytes discarded from the tail of the log: the remains of a commit a crash + * interrupted mid-append. Always > 0 when the callback fires. */ + readonly truncatedBytes: number; +} + /** * How to open a kernel instance. * @@ -152,6 +238,13 @@ export interface OpenOptions { * filesystem). */ readonly fs?: FileSystem; + /** + * Called when recovery discarded a torn tail — the bytes of a commit a crash + * interrupted, which were never acknowledged as durable. This is the expected + * crash-model outcome, not corruption (corruption refuses to open instead), + * but it should never be invisible; pass a callback to log or count it. + */ + readonly onRecovery?: (info: RecoveryInfo) => void; } /** The signature of the kernel's entry point (see {@link open} for the @@ -171,7 +264,8 @@ export type Open = (options?: OpenOptions) => Database; /** One stored key/value pair. Identical in shape to {@link Entry}; the distinct * name marks a value living in the committed store rather than one handed back - * to a caller. */ + * to a caller. The buffers are owned by the kernel: they were copied on the way + * in and are copied again on the way out, so no caller ever aliases them. */ interface StoredEntry { readonly key: Key; readonly value: Value; @@ -238,36 +332,61 @@ type Op = | { readonly kind: "set"; readonly key: Key; readonly value: Value } | { readonly kind: "delete"; readonly key: Key }; +/** Is `value` a thenable — the duck-typed shape `await` would latch onto? + * Used to reject async transaction bodies (see {@link Database.transact}). */ +function isThenable(value: unknown): boolean { + return ( + (typeof value === "object" || typeof value === "function") && + value !== null && + typeof (value as { then?: unknown }).then === "function" + ); +} + /** * A transaction backed by `working`, a mutable snapshot of the committed store. * Reads and writes hit the snapshot directly, which is what gives * read-your-writes; the caller commits or discards the snapshot as a whole. * Every mutation is also appended to `journal`, the redo record a durable * database writes to its log on commit (and ignores when purely in-memory). + * + * Every buffer is copied at this boundary — caller buffers on the way in, + * kernel buffers on the way out — so no caller mutation can ever reach the + * committed store or desynchronize memory from disk. */ function makeTransaction(working: StoredEntry[], journal: Op[]): Transaction { return { get(key) { const { found, index } = locate(working, key); - return found ? (working[index] as StoredEntry).value : undefined; + return found ? (working[index] as StoredEntry).value.slice() : undefined; }, set(key, value) { - applySet(working, key, value); - journal.push({ kind: "set", key, value }); + // One copy each, shared by the store and the journal: both treat entries + // as immutable, so sharing the copy is safe and halves the allocations. + const ownedKey = key.slice(); + const ownedValue = value.slice(); + applySet(working, ownedKey, ownedValue); + journal.push({ kind: "set", key: ownedKey, value: ownedValue }); }, delete(key) { - applyDelete(working, key); - journal.push({ kind: "delete", key }); + const ownedKey = key.slice(); + applyDelete(working, ownedKey); + journal.push({ kind: "delete", key: ownedKey }); }, *getRange(start, end) { + // Snapshot the matching entries when iteration starts (a generator body + // runs at the first next()). Walking the live array by index instead + // would let a delete-during-scan shift entries under the cursor and + // silently skip them — the classic delete-while-scanning bug. // locate() returns the first index whose key is >= start (the insertion // point), so the scan is naturally inclusive of start. It stops at the // first key that is not < end, making the range half-open [start, end). + const snapshot: Entry[] = []; for (let i = locate(working, start).index; i < working.length; i++) { const entry = working[i] as StoredEntry; if (compareKeys(entry.key, end) >= 0) break; - yield entry; + snapshot.push({ key: entry.key.slice(), value: entry.value.slice() }); } + yield* snapshot; }, }; } @@ -282,21 +401,35 @@ function makeTransaction(working: StoredEntry[], journal: Op[]): Transaction { // rebuild the store. This is the same mechanism real databases use, written // plainly so the file still teaches how durability works. // -// On-disk record = an 8-byte header followed by a payload: -// [u32 payloadLength][u32 crc32(payload)][payload] +// On-disk layout = an 8-byte file header followed by records: +// header: [4-byte magic "LRDB"][u16 formatVersion][u16 reserved] +// record: [u32 payloadLength][u32 crc32(payload)][payload] +// The magic is what lets open() refuse a file that is NOT a LibreDB database +// instead of misparsing arbitrary bytes (and destroying them); the version is +// what lets the format evolve without ambushing older readers. Files written +// by v0.1.x predate the header; recovery still reads them (see recover()). +// // The payload is the transaction's mutations back to back. Each op is: // set: [u8 1][u32 keyLength][key][u32 valueLength][value] // delete: [u8 0][u32 keyLength][key] // Integers are big-endian. Because the log is append-only and fsync'd, a crash // can only ever damage the LAST record, so recovery trusts every record up to -// the first one that is incomplete or fails its checksum, and truncates the -// rest away. +// the end of the file, truncates a torn tail away — and treats a bad record +// with intact records AFTER it as what it really is: corruption, which refuses +// the open rather than silently truncating committed data. // --------------------------------------------------------------------------- const OP_SET = 1; const OP_DELETE = 0; /** Bytes in a record header: the payload length and its checksum, both u32. */ const RECORD_HEADER = 8; +/** The file magic: "LRDB" in ASCII. A file that does not start with it (and + * does not parse as a headerless v0.1.x log) is refused, untouched. */ +const MAGIC = Uint8Array.of(0x4c, 0x52, 0x44, 0x42); +/** The on-disk format version this kernel writes and the newest it reads. */ +const FORMAT_VERSION = 1; +/** Bytes in the file header: magic, u16 version, u16 reserved. */ +const FILE_HEADER = 8; /** Write `value` as a big-endian u32 at `offset`. */ function writeU32(out: Uint8Array, offset: number, value: number): void { @@ -334,6 +467,16 @@ function crc32(data: Uint8Array): number { return (crc ^ 0xffffffff) >>> 0; } +/** Encode the 8-byte file header a new database starts with. */ +function encodeFileHeader(): Uint8Array { + const header = new Uint8Array(FILE_HEADER); + header.set(MAGIC, 0); + header[4] = (FORMAT_VERSION >>> 8) & 0xff; + header[5] = FORMAT_VERSION & 0xff; + // Bytes 6-7 are reserved and stay zero. + return header; +} + /** Encode one committed transaction's ops as a length-framed, checksummed * record ready to append to the log. */ function encodeRecord(ops: readonly Op[]): Uint8Array { @@ -366,19 +509,32 @@ function encodeRecord(ops: readonly Op[]): Uint8Array { return record; } -/** Replay one record's payload onto `entries`, in order, reconstructing the - * committed state the transaction produced. */ +/** + * Replay one record's payload onto `entries`, in order, reconstructing the + * committed state the transaction produced. Every length is validated against + * the payload's actual size first: a payload that passed its CRC but promises + * bytes it does not hold was never produced by this kernel, so it is corruption + * — not something to silently misread as ops. + */ function replayPayload(entries: StoredEntry[], payload: Uint8Array): void { + const corrupt = (): never => { + throw new LibreDbError("CORRUPT_WAL", "corrupt WAL record: malformed payload"); + }; let off = 0; while (off < payload.length) { const tag = payload[off++] as number; + if (tag !== OP_SET && tag !== OP_DELETE) corrupt(); + if (off + 4 > payload.length) corrupt(); const keyLength = readU32(payload, off); off += 4; + if (off + keyLength > payload.length) corrupt(); const key = payload.slice(off, off + keyLength); off += keyLength; if (tag === OP_SET) { + if (off + 4 > payload.length) corrupt(); const valueLength = readU32(payload, off); off += 4; + if (off + valueLength > payload.length) corrupt(); const value = payload.slice(off, off + valueLength); off += valueLength; applySet(entries, key, value); @@ -388,30 +544,150 @@ function replayPayload(entries: StoredEntry[], payload: Uint8Array): void { } } +/** What {@link recover} learned about the log. */ +interface Recovery { + entries: StoredEntry[]; + /** True for an empty file: the first append must write the file header. A + * non-empty headerless v0.1.x file keeps appending headerless records — a + * header cannot be inserted mid-file. */ + needsHeader: boolean; + /** Torn-tail bytes discarded, reported via {@link OpenOptions.onRecovery}. */ + truncatedBytes: number; +} + /** - * Rebuild the committed store from the log behind `file`, returning its entries. - * Replays every intact record and stops at the first record a crash left torn - * (header promises bytes that are not there) or corrupt (checksum mismatch), - * truncating that tail away so the next append starts from a clean boundary. + * Replay every record of `log` from `base` onto a fresh entry array. + * + * Failure classification is the heart of recovery: + * + * - A record whose promised end lies BEYOND the file, or a trailing fragment + * shorter than a record header, is a TORN TAIL: the append a crash + * interrupted. Only the tail can tear (the log is append-only and every + * earlier record was fsync'd), so it is truncated away. + * - A record that fails its checksum but ends exactly at the file's end is + * a DAMAGED TAIL — a half-flushed final block — and truncates the same way. + * - A record that fails its checksum with more data AFTER it cannot be a + * crash artifact: bytes after it mean a later append succeeded, which means + * this record was once durable and has since been damaged (bit rot, a + * partial copy, a second writer). That is corruption; recovery THROWS and + * leaves the file untouched rather than destroy the committed records + * behind the damage. */ -function recover(file: WalFile): StoredEntry[] { +function replayLog(log: Uint8Array, base: number): { entries: StoredEntry[]; tail: number; replayed: number } { const entries: StoredEntry[] = []; - const log = file.read(0, file.size()); - let offset = 0; + let offset = base; + let replayed = 0; while (offset + RECORD_HEADER <= log.length) { const size = readU32(log, offset); - const checksum = readU32(log, offset + 4); const start = offset + RECORD_HEADER; const end = start + size; - if (end > log.length) break; // torn: fewer bytes than the header promised + if (end > log.length) break; // torn tail: fewer bytes than the header promised const payload = log.subarray(start, end); - if (crc32(payload) !== checksum) break; // corrupt: record damaged mid-write + if (crc32(payload) !== readU32(log, offset + 4)) { + if (end < log.length) { + throw new LibreDbError( + "CORRUPT_WAL", + `corrupt WAL record at offset ${offset} with intact data after it; refusing to open`, + ); + } + break; // damaged tail: the final record half-flushed + } replayPayload(entries, payload); offset = end; + replayed++; } + return { entries, tail: offset, replayed }; +} - if (offset < log.length) file.truncate(offset); - return entries; +/** + * Rebuild the committed store from the log behind `file`. + * + * The file is recognized before it is touched: + * + * - An empty file is a new database; the header is written with the first + * commit (never at open, so a read-only open writes nothing). + * - A file starting with the magic is ours: records replay after the header. + * - A file that instead replays as headerless v0.1.x records (at least one + * valid record) is a legacy database and keeps working. + * - Anything else is NOT a LibreDB database: open() throws and the file is + * left byte-for-byte untouched. This is what makes a typo'd path an error + * instead of a destroyed file. + */ +function recover(file: WalFile): Recovery { + const size = file.size(); + const log = file.read(0, size); + if (log.length < size) { + // The file claims `size` bytes but the read returned fewer. Treating that + // as a torn tail would truncate committed data over a transient IO fault, + // so it is an error, never a recovery. + throw new LibreDbError("INCOMPLETE_READ", `WAL read returned ${log.length} of ${size} bytes`); + } + if (size === 0) return { entries: [], needsHeader: true, truncatedBytes: 0 }; + + const magicPrefix = Math.min(log.length, MAGIC.length); + const hasMagicPrefix = log.subarray(0, magicPrefix).every((byte, i) => byte === MAGIC[i]); + if (hasMagicPrefix) { + if (log.length < FILE_HEADER) { + // A torn header: the very first commit (header + record in one append) + // was interrupted before the header finished. Nothing was ever + // acknowledged, so start the database over from empty. + file.truncate(0); + file.fsync(); + return { entries: [], needsHeader: true, truncatedBytes: log.length }; + } + const fileVersion = ((log[4] as number) << 8) | (log[5] as number); + if (fileVersion !== FORMAT_VERSION) { + throw new LibreDbError( + "UNSUPPORTED_VERSION", + `database format version ${fileVersion} is newer than this library supports (${FORMAT_VERSION})`, + ); + } + return { ...finishReplay(file, log, FILE_HEADER), needsHeader: false }; + } + + // No magic: either a headerless v0.1.x database or a foreign file. Probe the + // FIRST record without side effects — if it does not replay cleanly, this is + // not a database we recognize, and the file must be left exactly as found. + // (Only the first record decides: once it proves the file is ours, a bad + // LATER record is judged by the normal torn-tail/corruption rules.) + if (!isLegacyLog(log)) { + throw new LibreDbError("NOT_A_DATABASE", "file is not a libredb database; refusing to touch it"); + } + return { ...finishReplay(file, log, 0), needsHeader: false }; +} + +/** Does `log` begin with one complete, checksummed, well-formed v0.1.x record? + * That is the recognition test for a headerless legacy database: real bytes + * from this kernel always start with one, foreign bytes essentially never do. */ +function isLegacyLog(log: Uint8Array): boolean { + if (log.length < RECORD_HEADER) return false; + const size = readU32(log, 0); + const end = RECORD_HEADER + size; + if (end > log.length) return false; + const payload = log.subarray(RECORD_HEADER, end); + if (crc32(payload) !== readU32(log, 4)) return false; + try { + replayPayload([], payload); // throwaway replay: structural validation only + } catch { + return false; + } + return true; +} + +/** Run the replay and apply its torn-tail truncation (fsync'd, so the clean + * boundary itself survives a crash) — shared by the headered and legacy paths. */ +function finishReplay( + file: WalFile, + log: Uint8Array, + base: number, +): { entries: StoredEntry[]; truncatedBytes: number } { + const { entries, tail } = replayLog(log, base); + const truncatedBytes = log.length - tail; + if (truncatedBytes > 0) { + file.truncate(tail); + file.fsync(); + } + return { entries, truncatedBytes }; } /** A durable backing store: append committed records, then release the file. */ @@ -426,14 +702,32 @@ interface Log { /** Open the log file at `path` on `fs`, recover the store from it, and return * the recovered entries together with a {@link Log} that appends future commits * to the same file. */ -function openLog(path: string, fs: FileSystem): { entries: StoredEntry[]; log: Log } { +function openLog( + path: string, + fs: FileSystem, + onRecovery: ((info: RecoveryInfo) => void) | undefined, +): { entries: StoredEntry[]; log: Log } { const file = fs.open(path); - const entries = recover(file); + const recovery = recover(file); + if (recovery.truncatedBytes > 0) onRecovery?.({ truncatedBytes: recovery.truncatedBytes }); + let needsHeader = recovery.needsHeader; return { - entries, + entries: recovery.entries, log: { append(ops) { - file.append(encodeRecord(ops)); + const record = encodeRecord(ops); + if (needsHeader) { + // First commit of a new database: header and record go down in ONE + // append, so a crash can only ever leave a recognizable prefix + // (handled by recover()) — never a headerless fragment. + const first = new Uint8Array(FILE_HEADER + record.length); + first.set(encodeFileHeader(), 0); + first.set(record, FILE_HEADER); + file.append(first); + needsHeader = false; + } else { + file.append(record); + } file.fsync(); // the durability point: bytes are on disk before we return }, close() { @@ -457,29 +751,45 @@ export const open: Open = (options) => { // are recovered from the log and `log` persists every later commit. let committed: StoredEntry[]; let log: Log | null; + let releaseLock: (() => void) | null = null; if (options?.path !== undefined) { // A path must actually name a file: reject the degenerate empty string here // with a clear error, rather than letting it reach the filesystem and // surface a raw, adapter-specific failure (e.g. node's ENOENT for ""). if (options.path === "") { - throw new Error("libredb: open({ path }) requires a non-empty path"); + throw new LibreDbError("INVALID_ARGUMENT", "open({ path }) requires a non-empty path"); } // The kernel is runtime-agnostic: it carries no default filesystem, so a // path-backed open MUST be given one. The default node:fs adapter lives at // the package edge (index.ts wires it in); the browser entry has none. A // pathless, in-memory open never reaches here and needs no filesystem. if (options.fs === undefined) { - throw new Error("libredb: open({ path }) requires a filesystem; none was provided"); + throw new LibreDbError("INVALID_ARGUMENT", "open({ path }) requires a filesystem; none was provided"); + } + // Exclusive access first: a second writer on the same file would recover an + // independent store and interleave appends — silent divergence. Filesystems + // that implement the lock make that a loud LOCKED error instead. + if (options.fs.lock !== undefined) releaseLock = options.fs.lock(options.path); + try { + const opened = openLog(options.path, options.fs, options.onRecovery); + committed = opened.entries; + log = opened.log; + } catch (error) { + releaseLock?.(); // recovery refused the file; do not hold its lock + throw error; } - const opened = openLog(options.path, options.fs); - committed = opened.entries; - log = opened.log; } else { committed = []; log = null; } let closed = false; + // Latched on the first commit that fails to reach the disk. A failed append + // can leave a torn record at the tail; appending MORE records after it would + // let the next recovery truncate them away even though their transact() + // returned success. Refusing all further writes until reopen (which repairs + // the tail) is what keeps an acknowledged commit durable. + let failed = false; // Guards against re-entrancy. The API is synchronous and single-threaded, so // a transaction body runs to completion before the next one begins — making // the schedule serial (hence serializable) by construction. The only way to @@ -491,21 +801,39 @@ export const open: Open = (options) => { return { transact(run) { - if (closed) throw new Error("libredb: database is closed"); + if (closed) throw new LibreDbError("CLOSED", "database is closed"); + if (failed) { + throw new LibreDbError("FAILED", "a previous commit failed to reach the disk; close and reopen to recover"); + } if (inTransaction) { - throw new Error("libredb: nested transactions are not supported"); + throw new LibreDbError("NESTED_TRANSACTION", "nested transactions are not supported"); } inTransaction = true; try { const working = committed.slice(); const journal: Op[] = []; const result = run(makeTransaction(working, journal)); + // An async body returns a pending Promise: any write after its first + // await would land in memory but never in the journal already written + // below — silent data loss on reopen. Refuse before committing. + if (isThenable(result)) { + throw new LibreDbError( + "ASYNC_TRANSACTION", + "transact() body must be synchronous; an async callback cannot commit correctly", + ); + } // Reached only if run() did not throw. Make the commit DURABLE before - // exposing it in memory: if the append fails, we throw with memory and - // disk still agreeing on the prior state. A read-only transaction has - // nothing to persist, so it skips the log entirely. The in-memory - // commit is then one atomic reference swap. - if (log !== null && journal.length > 0) log.append(journal); + // exposing it in memory. If the append or fsync fails, the tail of the + // file may hold a torn record — so the database latches (see `failed`) + // and memory keeps the prior state; reopening repairs the tail. + if (log !== null && journal.length > 0) { + try { + log.append(journal); + } catch (error) { + failed = true; + throw error; + } + } committed = working; return result; } finally { @@ -516,8 +844,14 @@ export const open: Open = (options) => { }, close() { if (closed) return; // idempotent: never double-close the underlying file + if (inTransaction) { + // Closing mid-transaction would rip the file out from under the commit + // path and surface a raw adapter error; name the misuse instead. + throw new LibreDbError("CLOSE_IN_TRANSACTION", "cannot close the database inside a transaction"); + } closed = true; if (log !== null) log.close(); + releaseLock?.(); committed = []; }, }; diff --git a/src/index.ts b/src/index.ts index 3bce1af..6fca142 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,8 +13,8 @@ import { open as openKernel, type Open } from "./core.ts"; import { nodeFileSystem } from "./adapter/node-fs.ts"; -export { version } from "./core.ts"; -export type { Database, FileSystem, OpenOptions, WalFile } from "./core.ts"; +export { version, LibreDbError } from "./core.ts"; +export type { Database, ErrorCode, FileSystem, OpenOptions, RecoveryInfo, WalFile } from "./core.ts"; /** * Open a LibreDB database on Node or Bun. Identical to the kernel's diff --git a/src/sim/dst.test.ts b/src/sim/dst.test.ts index 9065a67..1fd2d91 100644 --- a/src/sim/dst.test.ts +++ b/src/sim/dst.test.ts @@ -96,28 +96,30 @@ test("a torn in-flight append is discarded; committed state survives the crash", expect([...recovered.keys()].sort()).toEqual(["k1", "k2"]); }); -// --- explicit CRC-corruption case --- +// --- explicit corruption cases --- -test("corruption of the first record drops the whole log on recovery", () => { +/** The byte offset where records start: past the 8-byte file header. */ +const RECORDS_BASE = 8; + +test("a corrupted length field on the final record is indistinguishable from a torn tail and truncates", () => { const fs = new SimFS(3); - const steps: WorkloadStep[] = [ - { ops: [{ kind: "set", key: "k0", value: "a" }], abort: false }, - { ops: [{ kind: "set", key: "k1", value: "b" }], abort: false }, - { ops: [{ kind: "set", key: "k2", value: "c" }], abort: false }, - ]; + const steps: WorkloadStep[] = [{ ops: [{ kind: "set", key: "k0", value: "a" }], abort: false }]; const db = open({ path: WAL, fs }); runWorkload(db, steps); - // Flip a byte inside the FIRST record's payload (offset 8 = just past its - // 8-byte header). Its checksum then fails, so recovery stops at record 0. - fs.corrupt(WAL, 8); + // Flip a byte in the LAST record's length field. The promised end then lies + // beyond the file, which is byte-for-byte what a torn tail looks like — + // format v1 has no per-record header to tell them apart (see DESIGN.md), so + // recovery truncates from that record on. Every record before it survives; + // here there are none, so the store recovers empty. + fs.corrupt(WAL, RECORDS_BASE); const recovered = dump(open({ path: WAL, fs })); - expect(recovered.size).toBe(0); // nothing before the first record to keep + expect(recovered.size).toBe(0); expect(isCommittedPrefix(recovered, committedPrefixStates(steps))).toBe(true); }); -test("corruption mid-log keeps the valid prefix and drops the rest", () => { +test("mid-log payload corruption refuses to open instead of truncating committed data", () => { const fs = new SimFS(5); const steps: WorkloadStep[] = [ { ops: [{ kind: "set", key: "k0", value: "a" }], abort: false }, @@ -127,38 +129,45 @@ test("corruption mid-log keeps the valid prefix and drops the rest", () => { const db = open({ path: WAL, fs }); runWorkload(db, steps); - // Corrupt a byte inside the SECOND record's payload. Record 0 has payload - // length len0 (header at 0, payload at 8); record 1's header starts at - // 8 + len0 and its payload at (8 + len0) + 8. Same-shape rows => same size. + // Corrupt a byte inside the SECOND record's payload: record 0 spans + // [RECORDS_BASE, RECORDS_BASE+8+len0); record 1's payload starts 8 bytes + // after that. Its CRC then fails while INTACT data (record 2) sits after it + // — that is not a crash artifact (only the final append can tear), it is + // damage to once-durable bytes. Truncating would destroy record 2's + // committed transaction, so recovery must refuse the open and leave every + // byte in place. const durable = fs.durableBytes(WAL); - const len0 = readU32(durable, 0); - fs.corrupt(WAL, 8 + len0 + 8); - - const recovered = dump(open({ path: WAL, fs })); - // Record 0 survives; record 1 (corrupt) and everything after are dropped. - expect(mapEqual(recovered, new Map([["k0", "a"]]))).toBe(true); - expect(isCommittedPrefix(recovered, committedPrefixStates(steps))).toBe(true); + const len0 = readU32(durable, RECORDS_BASE); + const before = fs.durableBytes(WAL); + fs.corrupt(WAL, RECORDS_BASE + 8 + len0 + 8); + + expect(() => open({ path: WAL, fs })).toThrow(/corrupt/i); + // Refuse means refuse: the file was not truncated or rewritten (only the + // one deliberately-flipped byte differs). + const after = fs.durableBytes(WAL); + expect(after.length).toBe(before.length); }); // --- explicit short-read case --- -test("a short read during recovery still lands on a valid committed prefix", () => { +test("a short read during recovery is an IO fault, not license to truncate committed data", () => { const fs = new SimFS(11); const steps = generateWorkload(11); const db = open({ path: WAL, fs }); runWorkload(db, steps); // The next read (recovery's single read of the whole log) returns a seeded - // short prefix. Recovery must not fabricate state from the truncated bytes. + // short prefix of bytes the file actually holds. Treating that as a torn + // tail would truncate fsync'd commits over a transient fault, so recovery + // must fail loudly instead — and leave the file untouched for a retry. + const before = fs.durableBytes(WAL); fs.armShortRead(); - const recovered = dump(open({ path: WAL, fs })); + expect(() => open({ path: WAL, fs })).toThrow(/read returned/i); + expect(fs.durableBytes(WAL).length).toBe(before.length); - // A short read may lose a SUFFIX of committed records, but the result is - // still a valid committed prefix — never a torn or fabricated state. (It can - // even hold MORE keys than the final model if later deletes were cut off, so - // the only honest assertion is prefix membership.) - const states = committedPrefixStates(steps); - expect(isCommittedPrefix(recovered, states)).toBe(true); + // The fault was transient: the very next open succeeds with the full model. + const recovered = dump(open({ path: WAL, fs })); + expect(mapEqual(recovered, modelAfter(steps))).toBe(true); }); // --- the oracle helpers themselves --- From 8bd2e6eed36e741b73368b38f10f180a21f63022 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 01:44:50 +0300 Subject: [PATCH 02/16] feat(lens,cli,dst): namespace safety, validation guards, IO-fault DST profiles - reject ':' and empty collection/table names (namespace isolation) - reject lone-surrogate keys, ids, names, and kv values (UTF-8 round-trip) - reject NaN/Infinity in relational number columns - doc() refuses relational tables; table() refuses document collections - find()/where() reject explicitly-undefined predicate fields eagerly - CLI get/scan escape control characters by default; --raw opts out - SimFS gains armAppendError/armFsyncError; DST covers partial-append latch, fsync-fault latch, and multi-cycle crash-recover-write runs - seeded binary fuzz: full byte alphabet, empty/large payloads, and the sorted invariant asserted on recovered entries --- src/cli/run.test.ts | 17 +++ src/cli/run.ts | 38 +++++-- src/lens/catalog.test.ts | 4 +- src/lens/catalog.ts | 73 +++++++++++-- src/lens/document.ts | 51 ++++++++- src/lens/hardening.test.ts | 203 ++++++++++++++++++++++++++++++++++++ src/lens/kv.ts | 21 ++-- src/lens/relational.test.ts | 7 +- src/lens/relational.ts | 14 ++- src/sim/dst.test.ts | 86 +++++++++++++++ src/sim/dst.ts | 5 +- src/sim/fuzz.test.ts | 111 ++++++++++++++++++++ src/sim/simfs.ts | 32 ++++++ 13 files changed, 625 insertions(+), 37 deletions(-) create mode 100644 src/lens/hardening.test.ts create mode 100644 src/sim/fuzz.test.ts diff --git a/src/cli/run.test.ts b/src/cli/run.test.ts index 70e365b..8e294ca 100644 --- a/src/cli/run.test.ts +++ b/src/cli/run.test.ts @@ -309,6 +309,23 @@ test("import refuses a reserved key so it cannot corrupt the catalog", () => { expect(r.err.join("\n")).toMatch(/reserved key/i); }); +test("get and scan escape control characters so stored data cannot drive the terminal", () => { + const path = fixture(); + cli("set", path, "evil", "\u001b[2Jcleared\u0007bell"); + const got = cli("get", path, "evil"); + expect(got.code).toBe(0); + expect(got.out).toEqual(["\\x1b[2Jcleared\\x07bell"]); // no raw ESC/BEL reaches the sink + const scanned = cli("scan", path, "evil"); + expect(scanned.out).toEqual(["evil=\\x1b[2Jcleared\\x07bell"]); +}); + +test("--raw prints the stored bytes verbatim for callers that want them", () => { + const path = fixture(); + cli("set", path, "evil", "\u001b[31mred"); + const r = cli("get", path, "evil", "--raw"); + expect(r.out).toEqual(["\u001b[31mred"]); +}); + test("a read recovers a crash-torn file in memory without changing the bytes on disk", () => { const path = fixture(); // Simulate a crash mid-append: tack a partial/garbage record onto the WAL. diff --git a/src/cli/run.ts b/src/cli/run.ts index da92c26..ee97c1d 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -30,12 +30,27 @@ interface Io { } /** Everything a command handler needs: the file path, the command's positional - * arguments (everything after the path), the IO sink, and the `--force` flag. */ + * arguments (everything after the path), the IO sink, and the flags. */ interface Ctx { path: string; args: string[]; io: Io; force: boolean; + raw: boolean; +} + +/** + * Escape control characters for terminal output. A stored value is arbitrary + * user data; printed verbatim it could carry ANSI/OSC sequences that move the + * cursor, retitle the window, or write the clipboard of whoever inspects the + * file — the classic escape-injection gap in tools that dump untrusted bytes. + * Every C0 control (including newline — output here is line-oriented), DEL, + * and C1 control renders as its \xNN escape instead. `--raw` opts out. + */ +function sanitize(text: string, raw: boolean): string { + if (raw) return text; + // eslint-disable-next-line no-control-regex + return text.replace(/[\u0000-\u001f\u007f-\u009f]/g, (c) => `\\x${c.charCodeAt(0).toString(16).padStart(2, "0")}`); } const encoder = new TextEncoder(); @@ -55,6 +70,7 @@ const USAGE = [ "", "Options:", " --force Remove a write lock whose holder is no longer alive", + " --raw Print values verbatim (default escapes control characters)", ].join("\n"); /** Open `path` read-only, run `fn`, and always close — so a read leaves the file @@ -115,7 +131,7 @@ function stats({ path, io }: Ctx): number { }); } -function get({ path, args, io }: Ctx): number { +function get({ path, args, io, raw }: Ctx): number { const [key] = args; if (key === undefined) { io.err("missing "); @@ -127,19 +143,21 @@ function get({ path, args, io }: Ctx): number { io.err(`key not found: ${key}`); return 1; } - io.out(value); + io.out(sanitize(value, raw)); return 0; }); } -function scan({ path, args, io }: Ctx): number { +function scan({ path, args, io, raw }: Ctx): number { const [prefix] = args; if (prefix === undefined) { io.err("missing "); return 2; } return withReadDb(path, (db) => { - for (const entry of kv(db).prefix(prefix)) io.out(`${entry.key}=${entry.value}`); + for (const entry of kv(db).prefix(prefix)) { + io.out(`${sanitize(entry.key, raw)}=${sanitize(entry.value, raw)}`); + } return 0; }); } @@ -235,12 +253,16 @@ const commands = new Map number>([ export function run(argv: string[], io: Io): number { let positionals: string[]; - let values: { help?: boolean; force?: boolean }; + let values: { help?: boolean; force?: boolean; raw?: boolean }; try { const parsed = parseArgs({ args: argv, allowPositionals: true, - options: { help: { type: "boolean", short: "h" }, force: { type: "boolean" } }, + options: { + help: { type: "boolean", short: "h" }, + force: { type: "boolean" }, + raw: { type: "boolean" }, + }, }); positionals = parsed.positionals; values = parsed.values; @@ -268,7 +290,7 @@ export function run(argv: string[], io: Io): number { } try { - return handler({ path, args: positionals.slice(2), io, force: values.force === true }); + return handler({ path, args: positionals.slice(2), io, force: values.force === true, raw: values.raw === true }); } catch (error) { io.err(error instanceof Error ? error.message : String(error)); return 1; diff --git a/src/lens/catalog.test.ts b/src/lens/catalog.test.ts index 4162236..88b4d8c 100644 --- a/src/lens/catalog.test.ts +++ b/src/lens/catalog.test.ts @@ -41,7 +41,9 @@ describe("catalog reserved-prefix policy", () => { test("assertUserName accepts a normal name", () => { expect(() => assertUserName("users")).not.toThrow(); expect(() => assertUserName("orders_2026")).not.toThrow(); - expect(() => assertUserName("")).not.toThrow(); + // An empty name is no longer a user name: it cannot be isolated in the + // : key layout (issue #20), so it throws below instead. + expect(() => assertUserName("")).toThrow(/empty/i); }); test("assertUserName rejects a name starting with the reserved marker", () => { diff --git a/src/lens/catalog.ts b/src/lens/catalog.ts index b5d8dff..b5fe977 100644 --- a/src/lens/catalog.ts +++ b/src/lens/catalog.ts @@ -23,7 +23,7 @@ */ import { prefixRange } from "../query/range.ts"; import type { Store } from "../adapter/store.ts"; -import type { Transaction } from "../core.ts"; +import { LibreDbError, type Transaction } from "../core.ts"; import type { TableSchema } from "./relational.ts"; /** @@ -45,20 +45,67 @@ export const RESERVED_MARKER = "\x00"; export const CATALOG_PREFIX: string = `${RESERVED_MARKER}libredb:catalog:`; /** - * Reject a user namespace name that begins with the reserved marker — a loud - * error, the same class of correctness rule as the prefix-soundness checks the - * lenses already enforce. Such a name could place a user key inside the catalog - * namespace and break the isolation the catalog relies on, so it is forbidden - * outright rather than silently remapped. Called by `doc` and `table` when a - * handle is built; the raw kv lens is deliberately not guarded — it is the raw - * layer with full keyspace access (DESIGN.md section 6.3). + * Reject a string that is not well-formed UTF-16 (it contains a lone + * surrogate). Such a string cannot round-trip through the UTF-8 encoding the + * lenses store keys in — every lone surrogate encodes to the same replacement + * character, so two DISTINCT malformed strings silently collide on one key. + * Rejecting at the lens boundary keeps "distinct strings are distinct keys" + * true. Shared by the kv lens (keys) and the document lens (ids and names). + */ +export function assertWellFormedText(text: string, what: string): void { + if (!text.isWellFormed()) { + throw new LibreDbError( + "INVALID_ARGUMENT", + `${what} ${JSON.stringify(text)} contains a lone surrogate and cannot round-trip through UTF-8`, + ); + } +} + +/** + * Reject a user namespace name the key layout cannot isolate — a loud error, + * the same class of correctness rule as the prefix-soundness checks the lenses + * already enforce. Three shapes are forbidden outright rather than silently + * remapped, because each would let one namespace's keys collide with another's: + * + * - a name starting with the reserved marker would place user keys inside the + * catalog namespace; + * - a name containing ":" breaks the `:` byte boundary the lenses scan + * by — `doc(db, "tenant:1")` and id "x" would collide with `doc(db, + * "tenant")` id "1:x" (encode the tenant into the ID, not the name); + * - an empty name would make every id a bare `:` key shared by all + * empty-named namespaces. + * + * Called by `doc` and `table` when a handle is built; the raw kv lens is + * deliberately not guarded — it is the raw layer with full keyspace access + * (DESIGN.md section 6.3). */ export function assertUserName(name: string): void { if (name.startsWith(RESERVED_MARKER)) { - throw new Error( - `libredb: namespace name ${JSON.stringify(name)} may not start with the reserved catalog marker (U+0000)`, + throw new LibreDbError( + "INVALID_ARGUMENT", + `namespace name ${JSON.stringify(name)} may not start with the reserved catalog marker (U+0000)`, ); } + if (name === "") { + throw new LibreDbError("INVALID_ARGUMENT", "namespace name may not be empty"); + } + if (name.includes(":")) { + throw new LibreDbError( + "INVALID_ARGUMENT", + `namespace name ${JSON.stringify(name)} may not contain ":" (it delimits the namespace in the key ` + + `layout); encode variable parts into document ids instead`, + ); + } + assertWellFormedText(name, "namespace name"); +} + +/** The cataloged kind of `name`, or undefined when it is not cataloged. The + * lens entry points use this to route a name to the right lens: `doc()` refuses + * a relational table (its rows are schema-validated), `table()` refuses a + * document collection (its documents never were). */ +export function catalogKindOf(store: Store, name: string): CatalogEntry["kind"] | undefined { + const bytes = store.transact((tx) => tx.get(catalogKey(name))); + return bytes === undefined ? undefined : (JSON.parse(fromUtf8.decode(bytes)) as CatalogEntry).kind; } /** @@ -144,6 +191,12 @@ export function recordRelational(store: Store, name: string, schema: TableSchema return; } const persisted = JSON.parse(fromUtf8.decode(existing)) as CatalogEntry; + if (persisted.kind !== "relational") { + throw new LibreDbError( + "INVALID_ARGUMENT", + `${JSON.stringify(name)} is a ${persisted.kind} namespace; use its own lens instead of table()`, + ); + } if (persisted.schema === undefined || !schemasEqual(persisted.schema, schema)) { throw new Error( `libredb: table ${JSON.stringify(name)} was reopened with a schema that does not match ` + diff --git a/src/lens/document.ts b/src/lens/document.ts index d55894b..c12cb40 100644 --- a/src/lens/document.ts +++ b/src/lens/document.ts @@ -13,8 +13,9 @@ */ import { result, type Result, type WriteResult } from "./types.ts"; import { prefixRange } from "../query/range.ts"; -import { assertUserName, recordDocument } from "./catalog.ts"; +import { assertUserName, assertWellFormedText, catalogKindOf, recordDocument } from "./catalog.ts"; import type { Store } from "../adapter/store.ts"; +import { LibreDbError } from "../core.ts"; /** * Any value JSON can represent: the closure of the primitives under arrays and @@ -106,6 +107,25 @@ export function matches(document: Doc, predicate: Doc): boolean { return Object.keys(predicate).every((key) => deepEqual(document[key], predicate[key])); } +/** + * Reject a predicate carrying an explicit `undefined` field value. `undefined` + * is not a {@link JsonValue} — no stored document can hold it — but an untyped + * JS caller passing `{ status: maybeUndefined }` would otherwise match every + * document MISSING the field (deepEqual's undefined === undefined), silently + * inverting the query's meaning. Validated eagerly at find()/where() call time, + * so the mistake surfaces even against an empty collection. + */ +export function assertDefinedPredicate(predicate: Doc): void { + for (const key of Object.keys(predicate)) { + if (predicate[key] === undefined) { + throw new LibreDbError( + "INVALID_ARGUMENT", + `predicate field ${JSON.stringify(key)} is undefined — not a JSON value; omit the field to not filter by it`, + ); + } + } +} + /** * The kernel key for one document: `:`, UTF-8 encoded (DESIGN.md * section 6.1). Prefixing every id with the collection name is what scopes a @@ -158,13 +178,30 @@ export interface DocCollection { /** * Build a {@link DocCollection} handle scoped to `collection` over a * {@link Store} (the kernel's `Database` satisfies it, as does any object that - * can run a transaction). + * can run a transaction). Refuses a name the catalog records as a RELATIONAL + * table: its rows are schema-validated, and a doc() handle would write around + * that validation and break the catalog's faithful-view contract — use + * {@link import("./relational.ts").table} for it instead. */ export function doc(store: Store, collection: string): DocCollection { - // A collection name may not intrude on the reserved catalog namespace - // (DESIGN.md section 6.3) — reject it loudly before any key is derived. + // A collection name may not intrude on the reserved catalog namespace and + // must be isolatable in the key layout — reject it before any key is derived. assertUserName(collection); + if (catalogKindOf(store, collection) === "relational") { + throw new LibreDbError( + "INVALID_ARGUMENT", + `${JSON.stringify(collection)} is a relational table; use table() instead of doc()`, + ); + } + return collectionHandle(store, collection); +} +/** + * The unguarded collection builder behind {@link doc}. The relational lens uses + * it directly: a table IS this handle plus schema validation, so the "is this + * name relational?" guard that protects doc() callers must not apply there. + */ +export function collectionHandle(store: Store, collection: string): DocCollection { // The byte range covering every `:` key. prefixRange computes the // [start, end) bound on raw bytes so it agrees with the kernel's order, which // is what makes the colon a sound collection boundary (a sibling like "users2" @@ -197,6 +234,9 @@ export function doc(store: Store, collection: string): DocCollection { return { put(id, document) { + // An id with a lone surrogate cannot round-trip through the UTF-8 key + // encoding — two distinct malformed ids would silently share one key. + assertWellFormedText(id, "document id"); store.transact((tx) => { // Register this collection in the catalog on its first write (DESIGN.md // section 6.3). Idempotent and inside the write's own transaction, so the @@ -229,6 +269,9 @@ export function doc(store: Store, collection: string): DocCollection { return scan(() => true); }, find(predicate) { + // Validated eagerly, so `{ field: undefined }` fails at the call site + // instead of silently matching documents that LACK the field. + assertDefinedPredicate(predicate); return scan((document) => matches(document, predicate)); }, }; diff --git a/src/lens/hardening.test.ts b/src/lens/hardening.test.ts new file mode 100644 index 0000000..d5da3fb --- /dev/null +++ b/src/lens/hardening.test.ts @@ -0,0 +1,203 @@ +/** + * lens/hardening.test.ts — the lens boundary's rejection rules. + * + * The lenses' key layout is `:` over the kernel's byte order, and the + * catalog records each namespace's kind. Both only hold if the inputs cannot + * lie: a name containing ":" forges namespace boundaries, a lone surrogate + * collides distinct strings onto one key, a doc() handle on a relational table + * writes around its schema, and NaN validates as a "number" JSON cannot store. + * This suite pins the loud errors that keep those inputs out (pre-announcement + * audit findings B7 and section-2/4 lens items). + */ +import { expect, test } from "bun:test"; + +import { LibreDbError } from "../core.ts"; +import { open } from "../index.ts"; +import { catalog } from "./catalog.ts"; +import { doc } from "./document.ts"; +import { kv } from "./kv.ts"; +import { table } from "./relational.ts"; + +/** Grab the LibreDbError a thunk throws, asserting it threw at all. */ +const errorFrom = (thunk: () => unknown): LibreDbError => { + try { + thunk(); + } catch (error) { + expect(error).toBeInstanceOf(LibreDbError); + return error as LibreDbError; + } + throw new Error("expected the thunk to throw"); +}; + +const SCHEMA = { primaryKey: "id", columns: { id: "string", n: "number" } } as const; + +// --- issue #20: names that break the key layout are rejected --- + +test("a collection name containing ':' is rejected (namespace isolation)", () => { + const db = open(); + const error = errorFrom(() => doc(db, "tenant:42")); + expect(error.code).toBe("INVALID_ARGUMENT"); + expect(error.message).toMatch(/may not contain ":"/); + db.close(); +}); + +test("an empty collection or table name is rejected", () => { + const db = open(); + expect(errorFrom(() => doc(db, "")).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => table(db, "", SCHEMA)).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("a table name containing ':' is rejected before it can shadow the catalog", () => { + const db = open(); + expect(errorFrom(() => table(db, "users:admin", SCHEMA)).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("colliding names the old layout allowed are now impossible", () => { + const db = open(); + // Without the guard, doc(db, "tenant:1") with id "x" and doc(db, "tenant") + // with id "1:x" would share the kernel key "tenant:1:x". The first form is + // rejected; the safe encoding (tenant in the id) works and stays isolated. + const tenants = doc(db, "tenant"); + tenants.put("1:x", { from: "tenant 1" }); + tenants.put("2:x", { from: "tenant 2" }); + expect(tenants.get("1:x")).toEqual({ from: "tenant 1" }); + expect(tenants.all().toArray().length).toBe(2); + db.close(); +}); + +test("ids may contain ':' freely — only the namespace name is structural", () => { + const db = open(); + const logs = doc(db, "logs"); + logs.put("2026:07:03", { level: "info" }); + expect(logs.get("2026:07:03")).toEqual({ level: "info" }); + db.close(); +}); + +// --- issue #26: NaN and Infinity are not numbers a schema can store --- + +test("NaN and the infinities are rejected by number column validation", () => { + const db = open(); + const t = table(db, "metrics", SCHEMA); + for (const bad of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { + expect(() => t.insert({ id: "m1", n: bad })).toThrow(/expected number/); + } + // Nothing was stored by the rejected inserts. + expect(t.get("m1")).toBeUndefined(); + // Ordinary numbers (including zero and negatives) still pass. + t.insert({ id: "m2", n: -12.5 }); + expect(t.get("m2")).toEqual({ id: "m2", n: -12.5 }); + db.close(); +}); + +// --- issue #30: a name belongs to one lens --- + +test("doc() refuses a name cataloged as a relational table", () => { + const db = open(); + table(db, "accounts", SCHEMA).insert({ id: "a1", n: 1 }); + const error = errorFrom(() => doc(db, "accounts")); + expect(error.code).toBe("INVALID_ARGUMENT"); + expect(error.message).toMatch(/use table\(\)/); + // The catalog's faithful view survived: still relational, schema intact. + expect(catalog(db).get("accounts")).toEqual({ kind: "relational", schema: SCHEMA }); + db.close(); +}); + +test("table() refuses a name cataloged as a document collection", () => { + const db = open(); + doc(db, "notes").put("n1", { text: "hi" }); + const error = errorFrom(() => table(db, "notes", SCHEMA)); + expect(error.code).toBe("INVALID_ARGUMENT"); + expect(error.message).toMatch(/document namespace/); + db.close(); +}); + +test("the relational lens itself still reads and writes its rows (the guard is for outsiders)", () => { + const db = open(); + const t = table(db, "people", SCHEMA); + t.insert({ id: "p1", n: 7 }); + expect(t.get("p1")).toEqual({ id: "p1", n: 7 }); + expect(t.where({ n: 7 }).toArray()).toEqual([{ id: "p1", n: 7 }]); + t.delete("p1"); + expect(t.get("p1")).toBeUndefined(); + db.close(); +}); + +// --- issue #35: lone surrogates cannot round-trip through UTF-8 keys --- + +test("kv keys with a lone surrogate are rejected instead of silently colliding", () => { + const db = open(); + const store = kv(db); + const malformedA = "key-\ud800"; // lone high surrogate + const malformedB = "key-\udfff"; // lone low surrogate — distinct string, same UTF-8 + expect(errorFrom(() => store.set(malformedA, "a")).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => store.get(malformedB)).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => store.delete(malformedA)).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => store.prefix(malformedA).toArray()).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => store.range(malformedA, "z").toArray()).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("kv values with a lone surrogate are rejected (they would read back altered)", () => { + const db = open(); + expect(errorFrom(() => kv(db).set("k", "broken-\ud800")).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("document ids and namespace names with a lone surrogate are rejected", () => { + const db = open(); + expect(errorFrom(() => doc(db, "users").put("id-\ud800", {})).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => doc(db, "users-\ud800")).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("well-formed non-ASCII keys, ids, and values round-trip exactly", () => { + const db = open(); + const store = kv(db); + store.set("anahtar-ğüşöçİ", "değer-🚀"); + expect(store.get("anahtar-ğüşöçİ")).toBe("değer-🚀"); + const col = doc(db, "kayıtlar"); + col.put("belge-🎯", { başlık: "merhaba" }); + expect(col.get("belge-🎯")).toEqual({ başlık: "merhaba" }); + db.close(); +}); + +// --- issue #36: an undefined predicate value is an error, not match-nothing/-everything --- + +test("find() with an explicitly-undefined predicate field throws instead of inverting meaning", () => { + const db = open(); + const users = doc(db, "users"); + users.put("1", { name: "Ada" }); // no `status` field + const predicate = { status: undefined } as unknown as { [key: string]: never }; + const error = errorFrom(() => users.find(predicate)); + expect(error.code).toBe("INVALID_ARGUMENT"); + expect(error.message).toMatch(/undefined/); + db.close(); +}); + +test("find() validation is eager: it throws even against an empty collection", () => { + const db = open(); + const empty = doc(db, "empty"); + const predicate = { flag: undefined } as unknown as { [key: string]: never }; + expect(errorFrom(() => empty.find(predicate)).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("where() on a table rejects an undefined predicate value the same way", () => { + const db = open(); + const t = table(db, "rows", SCHEMA); + t.insert({ id: "r1", n: 1 }); + const predicate = { n: undefined } as unknown as { [key: string]: never }; + expect(errorFrom(() => t.where(predicate)).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("find({}) still matches every document (an empty predicate names no fields)", () => { + const db = open(); + const users = doc(db, "users"); + users.put("1", { name: "Ada" }); + users.put("2", { name: "Grace" }); + expect(users.find({}).toArray().length).toBe(2); + db.close(); +}); diff --git a/src/lens/kv.ts b/src/lens/kv.ts index 02dc5e4..9328d34 100644 --- a/src/lens/kv.ts +++ b/src/lens/kv.ts @@ -23,6 +23,7 @@ */ import { result, type Result, type WriteResult } from "./types.ts"; import { prefixRange } from "../query/range.ts"; +import { assertWellFormedText } from "./catalog.ts"; import type { Store } from "../adapter/store.ts"; /** One key/value pair from a range scan, decoded to strings. The kv-lens @@ -59,28 +60,36 @@ export interface Kv { const utf8 = new TextEncoder(); const fromUtf8 = new TextDecoder(); -const encode = (s: string): Uint8Array => utf8.encode(s); const decode = (b: Uint8Array): string => fromUtf8.decode(b); +/** Encode a string for storage, refusing one that UTF-8 cannot round-trip + * (a lone surrogate): distinct malformed strings would otherwise silently + * collide on the same bytes — colliding keys, or a value that reads back as a + * different string than was stored. */ +const encode = (s: string, what: string): Uint8Array => { + assertWellFormedText(s, what); + return utf8.encode(s); +}; + /** Build a {@link Kv} lens over a {@link Store} (the kernel's `Database` * satisfies it, as does any object that can run a transaction). */ export function kv(store: Store): Kv { return { get(key) { return store.transact((tx) => { - const value = tx.get(encode(key)); + const value = tx.get(encode(key, "key")); return value === undefined ? undefined : decode(value); }); }, set(key, value) { store.transact((tx) => { - tx.set(encode(key), encode(value)); + tx.set(encode(key, "key"), encode(value, "value")); }); return { changed: 1 }; }, delete(key) { const changed = store.transact((tx) => { - const k = encode(key); + const k = encode(key, "key"); const existed = tx.get(k) !== undefined; tx.delete(k); return existed ? 1 : 0; @@ -88,14 +97,14 @@ export function kv(store: Store): Kv { return { changed }; }, range(start, end) { - return scan(store, encode(start), encode(end)); + return scan(store, encode(start, "range start"), encode(end, "range end")); }, prefix(p) { // prefixRange computes the [start, end) bounds on bytes so they agree with // the kernel's order, and rejects a prefix with no finite end (an empty // string). It runs at call time, so a bad prefix fails here, not on a // later iteration; the scan itself stays lazy. - const { start, end } = prefixRange(encode(p)); + const { start, end } = prefixRange(encode(p, "prefix")); return scan(store, start, end); }, }; diff --git a/src/lens/relational.test.ts b/src/lens/relational.test.ts index 9dd8fec..3033a1b 100644 --- a/src/lens/relational.test.ts +++ b/src/lens/relational.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; // Persistence cases use a real path, so open through the Node entry (which // defaults the node:fs adapter); the kernel carries no default filesystem. import { open } from "../index.ts"; -import { doc, type Doc } from "./document.ts"; +import { collectionHandle, type Doc } from "./document.ts"; import { table, type TableSchema, type Row } from "./relational.ts"; // A schema exercising every column type, so type validation is checked across @@ -64,7 +64,8 @@ describe("relational insert-time row validation", () => { // the document storage scheme rather than inventing a parallel one. const db = open(); table(db, "users", userSchema).insert(validUser); - expect(doc(db, "users").get("u1")).toEqual(validUser as unknown as Doc); + // collectionHandle: doc() itself now refuses relational names (issue #30). + expect(collectionHandle(db, "users").get("u1")).toEqual(validUser as unknown as Doc); }); test("rejects a row missing a declared column", () => { @@ -476,7 +477,7 @@ describe("relational insert durability", () => { db.close(); const reopened = open({ path }); - expect(doc(reopened, "users").get("u1")).toEqual(validUser as unknown as Doc); + expect(collectionHandle(reopened, "users").get("u1")).toEqual(validUser as unknown as Doc); reopened.close(); }); }); diff --git a/src/lens/relational.ts b/src/lens/relational.ts index 95b9880..f6b7a6f 100644 --- a/src/lens/relational.ts +++ b/src/lens/relational.ts @@ -23,7 +23,7 @@ * equi-join via nested loop, producing rows with columns qualified as * `table.column`). The kernel is unchanged. */ -import { doc, matches, type Doc } from "./document.ts"; +import { assertDefinedPredicate, collectionHandle, matches, type Doc } from "./document.ts"; import { assertUserName, recordRelational } from "./catalog.ts"; import { result, type Result, type WriteResult } from "./types.ts"; import type { Store } from "../adapter/store.ts"; @@ -62,7 +62,10 @@ function matchesType(value: string | number | boolean | object, type: ColumnType case "string": return typeof value === "string"; case "number": - return typeof value === "number"; + // NaN and Infinity pass `typeof` but JSON cannot represent them — + // JSON.stringify writes null, so the value would round-trip as a + // schema-violating null. Only finite numbers are numbers here. + return typeof value === "number" && Number.isFinite(value); case "boolean": return typeof value === "boolean"; case "object": @@ -226,6 +229,9 @@ function query(name: string, source: () => Iterable): Query { return base.toArray(); }, where(predicate) { + // Same eager validation as the document lens's find(): an undefined + // predicate value would silently invert to "rows missing the column". + assertDefinedPredicate(predicate as unknown as Doc); return query(name, () => base.toArray().filter((row) => rowMatches(row, predicate))); }, select(...columns) { @@ -316,7 +322,9 @@ export function table(store: Store, name: string, schema: TableSchema): Table { // a row is a Doc whose fields happen to be the declared columns. The Row/Doc // casts are type-system artifacts (their value unions overlap but neither is a // subtype of the other); validation guarantees what crosses the boundary. - const rows = doc(store, name); + // collectionHandle, not doc(): doc() refuses relational names to protect + // OUTSIDE callers from writing around the schema — this lens IS the schema. + const rows = collectionHandle(store, name); // Every read starts from the full table scan re-wrapped as a chainable Query: // drop the document lens's DocEntry id wrapper (the pk lives in the row), then diff --git a/src/sim/dst.test.ts b/src/sim/dst.test.ts index 1fd2d91..4401ac9 100644 --- a/src/sim/dst.test.ts +++ b/src/sim/dst.test.ts @@ -25,6 +25,7 @@ import { SimFS } from "./simfs.ts"; import { committedPrefixStates, describeFailure, + injectTornTail, isCommittedPrefix, mapEqual, runSeed, @@ -170,6 +171,91 @@ test("a short read during recovery is an IO fault, not license to truncate commi expect(mapEqual(recovered, modelAfter(steps))).toBe(true); }); +// --- IO-error fault profiles (audit B3 / issue #25) --- + +const utf8 = new TextEncoder(); + +/** Apply `steps`' committed effects onto `model`, accumulating across cycles + * (modelAfter starts from empty; multi-cycle runs need the running total). */ +function applySteps(model: Map, steps: readonly WorkloadStep[]): void { + for (const step of steps) { + if (step.abort) continue; + for (const op of step.ops) { + if (op.kind === "set") model.set(op.key, op.value); + else model.delete(op.key); + } + } +} + +test("a partial-append fault latches the database and never destroys acknowledged commits", () => { + for (let seed = 0; seed < 10; seed++) { + const fs = new SimFS(seed); + const db = open({ path: WAL, fs }); + const steps = generateWorkload(seed, { steps: 30, abortRate: 0 }); + runWorkload(db, steps); + const expected = modelAfter(steps); + + // The next commit's append persists only a torn prefix, then throws. + fs.armAppendError(); + expect(() => db.transact((tx) => tx.set(utf8.encode("doomed"), utf8.encode("x")))).toThrow(/ENOSPC/); + // Error-then-continue: the database refuses further work (the latch) — + // appending after the torn bytes would poison every later commit. + expect(() => db.transact((tx) => tx.set(utf8.encode("later"), utf8.encode("y")))).toThrow(/reopen/); + + // Crash, reopen: every acknowledged commit survives; the doomed one never + // appears (its record is torn by construction). + fs.crash(); + const recovered = dump(open({ path: WAL, fs })); + expect(mapEqual(recovered, expected)).toBe(true); + } +}); + +test("a failed fsync latches the database; recovery lands on acknowledged state, at most plus the unacknowledged tail", () => { + for (let seed = 0; seed < 10; seed++) { + const fs = new SimFS(seed); + const db = open({ path: WAL, fs }); + const steps = generateWorkload(seed, { steps: 30, abortRate: 0 }); + runWorkload(db, steps); + const expected = modelAfter(steps); + + fs.armFsyncError(); + expect(() => db.transact((tx) => tx.set(utf8.encode("doomed"), utf8.encode("x")))).toThrow(/EIO/); + expect(() => db.transact((tx) => tx.set(utf8.encode("later"), utf8.encode("y")))).toThrow(/reopen/); + + // The doomed commit's bytes were fully appended but never fsync'd: a crash + // may keep any prefix of them. The honest durability contract is a LOWER + // bound — every acknowledged commit survives; the unacknowledged commit MAY + // also survive if its whole record reached the disk (the same contract real + // databases have for an errored commit). + fs.crash(); + const recovered = dump(open({ path: WAL, fs })); + const withDoomed = new Map(expected); + withDoomed.set("doomed", "x"); + expect(mapEqual(recovered, expected) || mapEqual(recovered, withDoomed)).toBe(true); + } +}); + +test("crash-recover-write cycles preserve every acknowledged commit across generations", () => { + for (let seed = 100; seed < 105; seed++) { + const fs = new SimFS(seed); + const cumulative = new Map(); + for (let cycle = 0; cycle < 5; cycle++) { + const db = open({ path: WAL, fs }); + // The durability lower bound: everything committed in EVERY earlier + // cycle is still here, exactly. + expect(mapEqual(dump(db), cumulative)).toBe(true); + const steps = generateWorkload(seed * 31 + cycle, { steps: 20 }); + runWorkload(db, steps); + applySteps(cumulative, steps); + // Crash without closing, sometimes with a torn in-flight append first. + if (cycle % 2 === 0) injectTornTail(fs); + fs.crash(); + } + const recovered = dump(open({ path: WAL, fs })); + expect(mapEqual(recovered, cumulative)).toBe(true); + } +}); + // --- the oracle helpers themselves --- test("committedPrefixStates enumerates the per-commit prefixes, skipping aborts", () => { diff --git a/src/sim/dst.ts b/src/sim/dst.ts index 1e5071a..9a8e53f 100644 --- a/src/sim/dst.ts +++ b/src/sim/dst.ts @@ -90,13 +90,14 @@ export interface SeedResult { * fsync, so the next {@link SimFS.crash} can tear it. The header promises 0xffff * payload bytes but only three are provided, so however the crash tears it the * record can never recover to a valid frame — exactly a commit interrupted by - * power loss before its fsync. Recovery must always discard it. + * power loss before its fsync. Recovery must always discard it. Exported so + * the test suite can compose it into multi-cycle crash schedules. * * Synchronous append+fsync means the kernel itself can never leave un-fsync'd * bytes between commits, so injecting this partial write is the honest way to * reproduce the one moment a crash can damage an append-only log: mid-record. */ -function injectTornTail(fs: SimFS): void { +export function injectTornTail(fs: SimFS): void { fs.open(WAL_PATH).append(Uint8Array.from([0, 0, 0xff, 0xff, 0, 0, 0, 0, 1, 2, 3])); } diff --git a/src/sim/fuzz.test.ts b/src/sim/fuzz.test.ts new file mode 100644 index 0000000..9b2df89 --- /dev/null +++ b/src/sim/fuzz.test.ts @@ -0,0 +1,111 @@ +/** + * fuzz.test.ts — seeded binary round-trip fuzz of the record codec and store. + * + * The DST workload (workload.ts) speaks 16 short ASCII keys — deliberately + * narrow so overwrites and deletes collide often. What it never exercises is + * the codec's full input space: binary keys with 0x00 separators (exactly what + * the lenses' composite keys embed), empty keys, empty values, high bytes that + * would betray a signed-byte comparison, and payloads big enough to cross any + * accidental buffer boundary. This suite drives seeded random byte workloads + * through commit, crash, and recovery, and checks the recovered store against + * an independent model — plus the SORTED invariant the kernel's binary search + * stands on, asserted directly on the recovered entries. + */ +import { expect, test } from "bun:test"; + +import { open, type Database } from "../core.ts"; +import { mulberry32 } from "./prng.ts"; +import { SimFS } from "./simfs.ts"; + +const WAL = "wal"; + +/** An upper bound above every key the fuzz can generate (keys cap at 24 bytes; + * this is 33 bytes of 0xff, which sorts after any shorter or equal-prefix key). */ +const KEY_CEILING = new Uint8Array(33).fill(0xff); + +/** A lossless string encoding of bytes, usable as a Map key for the model. */ +const encodeKey = (bytes: Uint8Array): string => bytes.join(","); +const decodeKey = (encoded: string): Uint8Array => + encoded === "" ? new Uint8Array(0) : Uint8Array.from(encoded.split(",").map(Number)); + +/** Unsigned byte-lexicographic comparison, written independently of the kernel + * (the invariant checker must not reuse the code it judges). */ +function compareBytes(a: Uint8Array, b: Uint8Array): number { + const shared = Math.min(a.length, b.length); + for (let i = 0; i < shared; i++) { + const delta = (a[i] as number) - (b[i] as number); + if (delta !== 0) return delta; + } + return a.length - b.length; +} + +/** Read the whole store as [key, value] byte pairs, asserting sortedness. */ +function dumpSorted(db: Database): [Uint8Array, Uint8Array][] { + return db.transact((tx) => { + const out: [Uint8Array, Uint8Array][] = []; + for (const entry of tx.getRange(new Uint8Array(0), KEY_CEILING)) { + out.push([entry.key, entry.value]); + } + for (let i = 1; i < out.length; i++) { + // Strictly ascending: sorted AND duplicate-free. + expect( + compareBytes((out[i - 1] as [Uint8Array, Uint8Array])[0], (out[i] as [Uint8Array, Uint8Array])[0]), + ).toBeLessThan(0); + } + return out; + }); +} + +test("seeded binary workloads round-trip through commit, crash, and recovery, sorted", () => { + for (let seed = 0; seed < 20; seed++) { + const rng = mulberry32(seed ^ 0x9e3779b9); + const fs = new SimFS(seed); + const model = new Map(); + + /** Random bytes over the FULL alphabet, length 0..maxLength inclusive. */ + const randomBytes = (maxLength: number): Uint8Array => { + const length = Math.floor(rng() * (maxLength + 1)); + const out = new Uint8Array(length); + for (let i = 0; i < length; i++) out[i] = Math.floor(rng() * 256); + return out; + }; + + let db = open({ path: WAL, fs }); + for (let cycle = 0; cycle < 3; cycle++) { + for (let step = 0; step < 15; step++) { + db.transact((tx) => { + const ops = 1 + Math.floor(rng() * 4); + for (let o = 0; o < ops; o++) { + const key = randomBytes(24); // includes the empty key + if (rng() < 0.75) { + // Values span empty to multi-KB (the large tail crosses any + // accidental 256/1024-byte assumption in the codec). + const value = rng() < 0.1 ? randomBytes(4096) : randomBytes(64); + tx.set(key, value); + model.set(encodeKey(key), value.slice()); + } else { + tx.delete(key); + model.delete(encodeKey(key)); + } + } + }); + } + // Crash without a close (every commit above was fsync'd), then recover. + fs.crash(); + db = open({ path: WAL, fs }); + const recovered = dumpSorted(db); + expect(recovered.length).toBe(model.size); + for (const [key, value] of recovered) { + expect(model.get(encodeKey(key))).toEqual(value); + } + } + } +}); + +test("the model dump helpers are lossless for the byte alphabet they encode", () => { + // The encodeKey/decodeKey pair is the model's foundation; a lossy encoding + // would make the whole fuzz vacuous, so pin it on the awkward inputs. + for (const bytes of [new Uint8Array(0), Uint8Array.from([0]), Uint8Array.from([0, 255, 1, 128])]) { + expect(decodeKey(encodeKey(bytes))).toEqual(bytes); + } +}); diff --git a/src/sim/simfs.ts b/src/sim/simfs.ts index 5415995..ee259a7 100644 --- a/src/sim/simfs.ts +++ b/src/sim/simfs.ts @@ -42,6 +42,12 @@ export class SimFS implements FileSystem { private readonly files = new Map(); /** When set, the NEXT read returns a seeded-short prefix, then disarms. */ private shortReadArmed = false; + /** When set, the NEXT append persists only a seeded STRICT prefix of its + * bytes, then throws — a partial write cut short by ENOSPC/EIO. */ + private appendErrorArmed = false; + /** When set, the NEXT fsync throws — the bytes stay pending (not durable), + * modelling a durability point that failed after the write. */ + private fsyncErrorArmed = false; constructor(seed: number) { this.random = mulberry32(seed); @@ -58,9 +64,21 @@ export class SimFS implements FileSystem { size: () => f.durable.length + f.pending.length, read: (offset, length) => this.readFrom(f, offset, length), append: (b) => { + if (this.appendErrorArmed) { + this.appendErrorArmed = false; + // A STRICT prefix (never the full record): the fault is "the write + // was cut short", so the record on disk must be torn. + const kept = Math.floor(this.random() * b.length); + for (const byte of b.subarray(0, kept)) f.pending.push(byte); + throw new Error("simfs: injected append fault (ENOSPC)"); + } for (const byte of b) f.pending.push(byte); }, fsync: () => { + if (this.fsyncErrorArmed) { + this.fsyncErrorArmed = false; + throw new Error("simfs: injected fsync fault (EIO)"); + } f.durable = f.durable.concat(f.pending); f.pending = []; }, @@ -71,6 +89,20 @@ export class SimFS implements FileSystem { }; } + /** Arm a one-shot append fault: the next {@link WalFile.append} persists a + * seeded strict prefix of its bytes and throws. The torn record this leaves + * is exactly the poisoned-tail scenario the kernel's failure latch exists + * for (audit finding B3 / fsyncgate). */ + armAppendError(): void { + this.appendErrorArmed = true; + } + + /** Arm a one-shot fsync fault: the next {@link WalFile.fsync} throws and the + * appended bytes stay in the un-fsync'd (crash-tearable) pending pool. */ + armFsyncError(): void { + this.fsyncErrorArmed = true; + } + /** * Simulate a process/power crash. For every open file the fsync'd bytes are * kept in full and the un-fsync'd tail is truncated to a seeded length in From 5f1e3d468ce2e7f18a52e316efbc73e3114da288 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 01:50:30 +0300 Subject: [PATCH 03/16] feat(ci,docs): supply-chain hardening and honest durability docs - npm publish with provenance (OIDC id-token); release tag must match package.json version; JSR CLI pinned to an exact version - Dependabot for actions, docker, and npm devDependencies - Node 22 smoke job exercises the built package (open, lenses, lock, reopen) so declared Node support is off the honor system - Docker runtime switches to distroless :nonroot - README/RELIABILITY state the durability contract precisely: the clean-crash guarantee plus every handled failure mode (IO-error latch, exclusive lock, foreign-file refusal, corruption refusal, short reads); status line updated to early beta 0.1.x - performance envelope and bulk-load guidance documented - CLI docs cover the new lock semantics, output escaping, --raw, and the backup/restore procedure - OPFS flush()'s weaker power-loss guarantee documented in the adapter and BROWSER.md - changeset for the whole hardening wave (minor) --- .changeset/pre-announcement-hardening.md | 39 ++++++++++++++ .github/dependabot.yml | 25 +++++++++ .github/workflows/ci.yml | 23 +++++++- .github/workflows/publish.yml | 27 ++++++++-- Dockerfile | 10 ++-- README.md | 56 ++++++++++++++------ docs/BROWSER.md | 11 ++++ docs/CLI.md | 44 +++++++++++++--- docs/RELIABILITY.md | 67 ++++++++++++++++++++---- scripts/node-smoke.mjs | 49 +++++++++++++++++ src/adapter/opfs.ts | 5 ++ 11 files changed, 315 insertions(+), 41 deletions(-) create mode 100644 .changeset/pre-announcement-hardening.md create mode 100644 .github/dependabot.yml create mode 100644 scripts/node-smoke.mjs diff --git a/.changeset/pre-announcement-hardening.md b/.changeset/pre-announcement-hardening.md new file mode 100644 index 0000000..31dfdfe --- /dev/null +++ b/.changeset/pre-announcement-hardening.md @@ -0,0 +1,39 @@ +--- +"@libredb/libredb": minor +--- + +Durability, safety, and API-contract hardening across the kernel, adapters, lenses, and CLI (the pre-announcement audit wave). + +On-disk format: new databases now begin with an 8-byte `LRDB` magic/version header. Files written by earlier releases (headerless) keep opening through a legacy read path; files written by this release are not readable by older releases. The header is what lets `open()` refuse a file that is not a LibreDB database with a clear error instead of destroying it. + +Kernel: + +- `open({ path })` on a non-LibreDB file throws `NOT_A_DATABASE` and leaves the file byte-for-byte untouched (previously the file was silently truncated to zero). +- Recovery classifies failures: a torn tail truncates (reported through the new `onRecovery` open option), while mid-log corruption throws `CORRUPT_WAL` and truncates nothing. Record payloads are structurally validated during replay. +- A failed append/fsync latches the database: every later `transact()` throws `FAILED` until reopen, so an IO error can never lead recovery to silently drop later acknowledged commits. +- `transact()` rejects async callbacks (`ASYNC_TRANSACTION`): writes after an `await` could never reach the log. +- Keys and values are copied at the transaction boundary in both directions — caller buffer reuse and mutation of returned buffers can no longer corrupt the store. +- `getRange` snapshots at first iteration, so delete-while-scanning visits every entry exactly once. +- `close()` inside a transaction throws `CLOSE_IN_TRANSACTION` instead of surfacing a raw file error. +- `open()` takes an exclusive per-file lock (`.lock`, pid/host/nonce): a second writer throws `LOCKED` instead of silently diverging; locks from verifiably dead holders are reclaimed automatically. `FileSystem` gains an optional `lock()` seam method. +- All kernel failures are now `LibreDbError` instances carrying a stable `code` (exported, with the `ErrorCode` and `RecoveryInfo` types). + +Adapters: + +- node-fs: creating a database fsyncs the parent directory (a fresh database can no longer vanish wholesale on power loss); recovery truncation is fsync'd; reads are positional on the WAL's own file descriptor instead of re-reading the whole file per call. +- OPFS: reads loop until filled, so a legal short read can no longer masquerade as a torn tail; recovery treats an incomplete read as an IO fault (`INCOMPLETE_READ`), never as license to truncate. + +Lenses: + +- Collection/table names may not be empty or contain `:` (both broke namespace isolation); ids keep full freedom. +- Strings that are not well-formed UTF-16 (lone surrogates) are rejected wherever they would become keys, ids, names, or kv values — distinct strings can no longer silently collide on one key. +- Relational `number` columns reject `NaN` and the infinities (JSON would store them as `null`). +- `doc()` refuses a name cataloged as a relational table (it would bypass schema validation); `table()` refuses a document collection's name. +- `find()`/`where()` reject a predicate field explicitly set to `undefined`, which previously matched documents *missing* the field. + +CLI: + +- Write commands rely on the kernel's exclusive lock; `--force` removes a lock only when its holder is not verifiably alive, and never deletes a file that is not a libredb lock. +- `get`/`scan` escape control characters by default so untrusted values cannot inject terminal escape sequences; `--raw` prints verbatim. + +Docker image now runs as a non-root user (distroless `:nonroot`, uid 65532). diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ea8091b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,25 @@ +# Dependabot keeps the SHA-pinned actions, the digest-pinned Docker bases, and +# the npm devDependencies from going stale: pinning without an update loop +# inverts over time (CVE fixes never arrive unless someone remembers to bump +# digests by hand). Weekly PRs preserve the pinning discipline — every bump is +# still a reviewed, pinned change. +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "npm" + directory: "/" + schedule: + interval: "weekly" + # Zero runtime dependencies is a design fact (license tripwire enforces + # it), so everything here is devDependencies; group the noise. + groups: + dev-dependencies: + patterns: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index debb09a..7f3ed9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,5 +60,26 @@ jobs: echo "| License tripwire (runtime deps) | ${{ steps.license.outcome }} |" echo "" echo "- Coverage: 100% line/function/statement (enforced by bunfig.toml)" - echo "- Bundle: ${KB} kB (min+brotli) / 4 kB budget" + echo "- Bundle: ${KB} kB (min+brotli) / 5 kB budget" } >> "$GITHUB_STEP_SUMMARY" + + node-smoke: + name: node 22 smoke + # package.json declares engines.node >= 22 and the docs advertise `npx + # libredb` and Node embedding, but the gate runs on Bun. This job keeps the + # Node half of the declared runtime surface off the honor system: build the + # real dist/, then open, write, lock, reopen, and read a file-backed + # database under Node, plus one pass of the CLI entry. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .bun-version + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "22" + - run: bun install --frozen-lockfile + - run: bun run build + - name: Smoke-test the built package under Node + run: node scripts/node-smoke.mjs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9ade7b4..bc2d315 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,8 +29,23 @@ jobs: # `latest` channel or Docker `:latest`. Every job carries the same guard. if: ${{ !github.event.release.prerelease }} runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # OIDC: signs the npm provenance attestation steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + # The release tag must name the version the tree actually carries: a tag + # created before `changeset:version` (or a typo'd tag) must fail loudly + # here, not ship binaries and images labeled with the wrong semver. + - name: Verify release tag matches package.json + env: + TAG: ${{ github.event.release.tag_name }} + run: | + PKG="v$(node -p 'require("./package.json").version')" + if [ "$TAG" != "$PKG" ]; then + echo "Release tag $TAG does not match package.json $PKG" >&2 + exit 1 + fi - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: bun-version-file: .bun-version @@ -45,9 +60,9 @@ jobs: node-version: "22" registry-url: "https://registry.npmjs.org" # npm publish runs prepublishOnly (build + attw + publint) automatically. - # Provenance is intentionally omitted while the repo is private; once it is - # public, add `--provenance` here and `id-token: write` to permissions. - - run: npm publish --access public + # --provenance attaches a signed attestation binding the tarball to this + # repo, commit, and workflow (verifiable via `npm audit signatures`). + - run: npm publish --access public --provenance env: NODE_AUTH_TOKEN: ${{ secrets.NPMJS_TOKEN }} # Run summary on the run page (no emoji, per repo convention). @@ -81,7 +96,11 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: "22" - - run: npx --yes jsr publish + # Pinned: an unpinned `jsr@latest` would hand the publish-capable OIDC + # token to whatever the registry serves at run time (supply-chain risk). + # Bump the version deliberately, like every other pin in this workflow + # (Dependabot does not track npx invocations). + - run: npx --yes jsr@0.14.3 publish - name: Job summary if: success() run: | diff --git a/Dockerfile b/Dockerfile index b1fb53b..ea78906 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,10 +24,12 @@ COPY src ./src RUN bun build --compile src/cli/main.ts --outfile /libredb # Stage 2: a minimal runtime carrying only the binary and the glibc/libstdc++ a -# bun-compiled executable links against. distroless/cc has exactly that. Pinned -# by digest because `cc-debian12` is a rolling tag (no version), so the digest is -# what makes the runtime reproducible; refresh it periodically for base updates. -FROM gcr.io/distroless/cc-debian12@sha256:d703b626ba455c4e6c6fbe5f36e6f427c85d51445598d564652a2f334179f96e +# bun-compiled executable links against. distroless/cc has exactly that. The +# :nonroot variant runs as uid 65532 so files the CLI creates in a bind-mounted +# host directory are not root-owned (and a container escape holds no root). +# Mount data writable by that uid, or pass `--user "$(id -u):$(id -g)"`. Pinned +# by digest because the tag is rolling; refresh it periodically for base updates. +FROM gcr.io/distroless/cc-debian12:nonroot@sha256:b0ae8e989418b458e0f25489bc3be523718938a2b70864cc0f6a00af1ddbd985 COPY --from=build /libredb /usr/local/bin/libredb WORKDIR /data ENTRYPOINT ["/usr/local/bin/libredb"] diff --git a/README.md b/README.md index 47ad99e..d87287f 100644 --- a/README.md +++ b/README.md @@ -247,10 +247,11 @@ the store only through one narrow `transact` port. For the full tour, read **Do not use it (yet) when you need:** -- A hardened production datastore at scale — it is **pre-alpha**; today's beachhead is test/dev. +- A hardened production datastore at scale — it is an **early beta**; today's beachhead is test/dev. - Secondary indexes or a query planner — queries are O(n) scans by design in v1 (on the roadmap). -- Concurrent multi-process access, replication, or a networked client/server — it is embedded and - in-process. +- Concurrent multi-process access, replication, or a networked client/server — it is embedded, + in-process, and strictly single-writer (a second `open()` on the same file is refused by an + exclusive lock rather than silently corrupting it). - SQL wire compatibility or an existing-driver ecosystem. These limits are deliberate v1 scope, not hidden gaps — LibreDB's strength comes from what it refuses. @@ -264,17 +265,36 @@ See the [Manifesto](./MANIFESTO.md). A transaction that returns has been written to a length-framed, CRC-32-checksummed write-ahead log and -`fsync`'d *before* the commit becomes visible — so a committed write survives a crash, and a crash can -only ever damage the last, un-fsync'd record (which recovery detects and truncates). This is not just -asserted: the kernel's crash/recovery path is proven by **deterministic simulation testing**, running -the real engine against a seeded in-memory filesystem that tears, corrupts, and crashes the log on -command, then checking that recovery is always a valid committed prefix. +`fsync`'d *before* the commit becomes visible — so on a healthy disk a committed write survives a +crash, and a crash can only ever damage the last, un-fsync'd record (which recovery detects, +truncates, and reports). The failure modes *outside* the clean-crash model are handled explicitly, not +assumed away: a failed append/fsync latches the database instead of writing past a torn record, a +second writer is refused by an exclusive open lock, a file that is not a LibreDB database is refused +untouched (the `LRDB` header), mid-log corruption refuses to open rather than silently truncating, and +a short read is an IO error, never data loss. This is not just asserted: the crash/recovery path is +proven by **deterministic simulation testing** — the real engine against a seeded in-memory filesystem +that tears, corrupts, errors, and crashes the log on command — plus a binary round-trip fuzz. ```sh -bun run test # includes a bounded 50-seed DST run +bun run test # includes a bounded 50-seed DST run and the fault-injection suites ``` -The full durability and DST walkthrough is in [`docs/RELIABILITY.md`](./docs/RELIABILITY.md). +The precise durability contract and the DST walkthrough are in +[`docs/RELIABILITY.md`](./docs/RELIABILITY.md). + +## Performance envelope + +Honesty about scale (comprehension is the budget in v1, not throughput): + +- **The whole store lives in memory** as one sorted array; the file on disk is the append-only log + that rebuilds it on open. The practical ceiling is data that comfortably fits in RAM — the test/dev + beachhead, not a server working set. +- **Each `transact()` copies the store** before applying writes, so a per-row auto-commit loop is + quadratic in store size and will look hung on large seeds. **Wrap bulk loads in one `transact()`** + (or use `libredb import`, which already does): one copy, one fsync, one record for the whole batch. +- **No secondary indexes**: a `find`/`where` is an O(n) scan by design in v1. +- **The log grows without bound** until compaction lands (tracked in + [#12](https://github.com/libredb/libredb/issues/12)); reopening replays the whole log. ## Documentation @@ -293,14 +313,18 @@ The full durability and DST walkthrough is in [`docs/RELIABILITY.md`](./docs/REL ## Project status & roadmap -LibreDB is **pre-alpha** (`0.0.x`). The architecture is in place and every line of the core is tested, -but the API may still change and it is not yet meant for production data. +LibreDB is an **early beta** (`0.1.x`). The architecture is in place, every line of the core is +tested, and the durability contract above is enforced — but the API may still change before 1.0, and +the recommended home is still test/dev data. -- **Done:** the ordered key-value kernel (transactions, WAL, crash recovery); the key-value, document, - and relational lenses; the self-describing catalog; the deterministic simulation testing harness; - 100% line/function/statement coverage on the core. +- **Done:** the ordered key-value kernel (transactions, WAL with a versioned on-disk header, crash + recovery that refuses corruption and foreign files, an IO-failure latch, an exclusive open lock); + the key-value, document, and relational lenses; the self-describing catalog; typed `LibreDbError` + codes; the DST harness with IO-fault injection and binary fuzz; 100% line/function/statement + coverage. - **Next:** secondary indexes and a richer query surface; more query operators; additional lenses; - production-hardening milestones (directory fsync on first create, WAL compaction/checkpointing). + WAL compaction/checkpointing ([#12](https://github.com/libredb/libredb/issues/12)); real-browser + OPFS verification ([#10](https://github.com/libredb/libredb/issues/10)). ## The LibreDB family diff --git a/docs/BROWSER.md b/docs/BROWSER.md index 4b39b00..6bb0367 100644 --- a/docs/BROWSER.md +++ b/docs/BROWSER.md @@ -57,6 +57,17 @@ maps onto an OPFS **sync access handle** (whose `read`/`write`/`getSize`/ with no async core. Sync access handles are only available **inside a dedicated Web Worker**, so durable LibreDB *must* live in a Worker. +One honest caveat on the word *durable*: the kernel's durability point maps to +the handle's `flush()`, and the OPFS specification does not promise that +`flush()` carries POSIX-`fsync` strength against **power loss** — the browser's +storage layer decides when bytes reach stable media. In practice a committed +write survives a tab crash, a page reload, and a browser restart; what a sudden +power cut can lose is browser-and-OS dependent. Treat OPFS durability as "as +strong as the browser's flush", not as a battery-backed guarantee (verifying +this per engine is tracked in +[#10](https://github.com/libredb/libredb/issues/10)). Storage may also be +evicted under pressure unless you request persistence — see the checklist below. + --- ## 3. In-memory: the 30-second start (main thread) diff --git a/docs/CLI.md b/docs/CLI.md index a47e736..4309d43 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -41,7 +41,8 @@ Usage: libredb import Bulk-set keys from a JSON object (one atomic commit) Options: - --force Override an existing write lock + --force Remove a write lock whose holder is no longer alive + --raw Print values verbatim (default escapes control characters) ``` --- @@ -141,11 +142,21 @@ The CLI touches real database files, so it is deliberately careful: (`inspect`/`stats`/`get`/`scan`) open through a **read-only filesystem adapter**: recovery drops a torn tail *in memory only*; the bytes on disk are left exactly as found. -- **Writes take an advisory lock.** LibreDB is single-process with no internal - file locking, so two concurrent writers would corrupt a file. `set`/`delete`/ - `import` create a `.lock` first; if one is already held the command fails - loudly. Pass `--force` only to clear a **stale** lock left by a crashed writer - — it will not delete a file that isn't a libredb lock. +- **A wrong path cannot destroy a file.** Opening a file that is not a LibreDB + database (a typo, a text file) fails with a clear error and leaves the file + byte-for-byte untouched — the on-disk `LRDB` header is checked before anything + is written. +- **Writes hold the exclusive open lock.** The library itself locks the database + on open (`.lock`, recording the holder's pid and host), so a second + writer — this CLI against a live app, or two CLI invocations — fails loudly + instead of silently corrupting the file. A lock whose holder is verifiably dead + is reclaimed automatically; `--force` additionally removes a lock that cannot + be verified (for example one from another machine), but refuses a verifiably + live holder and refuses to delete a file that is not a libredb lock. +- **Output is escaped by default.** `get`/`scan` print stored values with control + characters escaped (`\x1b`, `\x07`, ...), so a value containing terminal escape + sequences cannot clear your screen, retitle your terminal, or write your + clipboard when you inspect an untrusted file. Pass `--raw` for the exact bytes. - **Reserved keys are refused.** Writes reject keys in LibreDB's reserved namespace (the `\x00`-prefixed catalog space), so the CLI cannot corrupt the catalog or another lens's layout. @@ -153,6 +164,27 @@ The CLI touches real database files, so it is deliberately careful: --- +## Backup and restore + +The WAL **is** the database: one `.libredb` file holds everything, so backup is a +file copy — with one rule. + +- **Backup:** copy the file while **no writer has it open** (no `.lock` + present, or only your own closed session). A copy taken mid-write can split a + record in half; the copy would then open only up to the split. + + ```sh + cp app.libredb backup/app-$(date -u +%Y%m%d).libredb + ``` + +- **Restore:** copy the file back and open it — recovery replays it like any + reopen. Nothing else to do. +- **Export as text:** `libredb scan ""` is not supported (an empty prefix + is refused); scan per namespace prefix, or use the programmatic lenses for a + structured export. A first-class `export` command is on the roadmap. + +--- + ## Exit codes | Code | Meaning | Examples | diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md index 4cb063a..8ea0477 100644 --- a/docs/RELIABILITY.md +++ b/docs/RELIABILITY.md @@ -1,9 +1,45 @@ -# Reliability — deterministic simulation testing - -Durability is the trust-critical promise: a transaction that returns has been fsync'd, and a crash can -only ever damage the last, un-fsync'd record. LibreDB proves this with **deterministic simulation -testing (DST)** — the write-ahead log's crash/recovery path is tortured under a seeded, in-memory -simulated filesystem. +# Reliability — the durability contract, precisely + +Durability is the trust-critical promise, so it is stated here with its exact scope — the guarantee, +the failure modes outside it, and how each one is handled. The crash/recovery path is proven with +**deterministic simulation testing (DST)** — the write-ahead log tortured under a seeded, in-memory +simulated filesystem — plus IO-fault injection and a binary fuzz suite. + +## The contract + +**On a healthy disk, a `transact()` that returns has been fsync'd and survives a crash.** A crash can +only ever damage the last, un-fsync'd record; recovery detects it (length framing + CRC-32), truncates +it away, and reports the truncation through the `onRecovery` open option — never silently more than +that one record. + +The failure modes *outside* the clean-crash model are handled explicitly rather than assumed away: + +- **A failed append or fsync (ENOSPC, EIO) latches the database.** The tail of the log may hold a torn + record, so every later `transact()` throws (`code: "FAILED"`) until the file is closed and reopened — + reopening repairs the tail. Without the latch, later "successful" commits would sit behind the torn + record and be silently destroyed by the next recovery. Never assume a write after a failed fsync is + safe (the "fsyncgate" lesson). +- **A second writer is refused.** `open({ path })` takes an exclusive lock (`.lock`, holder + pid/host recorded); a second open — same process or another — throws `code: "LOCKED"` instead of + silently diverging two in-memory stores over one log. A lock whose holder is verifiably dead is + reclaimed automatically. +- **A file that is not a LibreDB database is refused, untouched.** New databases carry an `LRDB` + magic/version header; a foreign file (a typo'd path) throws `code: "NOT_A_DATABASE"` and is left + byte-for-byte intact. Headerless files written by v0.1.x still open through a legacy read path. +- **Mid-log corruption refuses to open.** A record that fails its checksum with intact records *after* + it cannot be a crash artifact (only the final append can tear) — it is damage to once-durable bytes + (bit rot, a partial copy). Recovery throws `code: "CORRUPT_WAL"` and truncates nothing, preserving + the evidence and the committed records behind it. One honest limitation: corruption that hits a + record's *length field* on the final record is byte-for-byte indistinguishable from a torn tail in + format v1 and truncates like one. +- **A short read is an IO fault, not missing data.** If the filesystem returns fewer bytes than the + file holds, recovery throws (`code: "INCOMPLETE_READ"`) instead of mistaking the cut for a torn tail + and truncating committed transactions. +- **Creating a database fsyncs the parent directory** (POSIX does not make a new directory entry + durable until then), and a recovery truncation is itself fsync'd. On platforms without directory + fsync (Windows), new-file entry durability is the OS's best effort. +- **In the browser (OPFS), the durability point is `flush()`**, which may be weaker than a POSIX + `fsync` against power loss — see [`BROWSER.md`](./BROWSER.md). ## How it works @@ -17,6 +53,13 @@ workload — every transaction that returned successfully, and never a torn or u independent committed-map model (sharing no code with the engine) is the oracle the recovered state is compared against. +Beyond clean crashes, the suite injects the IO faults the contract above names: an append that +persists only a torn prefix then throws, an fsync that throws, short reads, mid-log corruption, and +multi-cycle crash-recover-write schedules — asserting in each case that acknowledged commits survive +and the database latches instead of appending past damage. A separate seeded fuzz drives binary keys +and values (the full byte alphabet, empty and multi-KB payloads) through commit/crash/recovery and +asserts the recovered store is exactly the model and strictly sorted. + ## Running it The DST suite runs as part of the normal gate: @@ -51,13 +94,17 @@ not shipped code. ## The durability path, precisely -Each committed transaction is written as a length-framed, CRC-32-checksummed redo record appended to a +The on-disk file opens with an 8-byte header — the `LRDB` magic and a format version — which is what +lets `open()` refuse a file that is not a LibreDB database instead of misreading (and damaging) it. +Each committed transaction is then a length-framed, CRC-32-checksummed redo record appended to that single write-ahead log (there is no separate data file — the log *is* the database). The commit sequence is: encode the transaction's writes into one record, append it to the log, `fsync`, and only *then* swap the in-memory committed state to make the writes visible. The fsync happens before the -swap, so a `transact()` that has returned is on disk. On reopen the log is replayed into an in-memory -sorted array; a torn or corrupt tail record (a crash mid-append) fails its length/CRC check and is -truncated away, leaving exactly the committed prefix. +swap, so a `transact()` that has returned is on disk. If the append or fsync throws, the database +latches failed and the swap never happens — memory and disk stay on the prior state. On reopen the log +is replayed into an in-memory sorted array; a torn tail record (a crash mid-append) fails its +length/CRC check and is truncated away (and reported via `onRecovery`), leaving exactly the committed +prefix — while a bad record with intact records after it refuses the open as corruption. ```mermaid flowchart LR diff --git a/scripts/node-smoke.mjs b/scripts/node-smoke.mjs new file mode 100644 index 0000000..68c92e1 --- /dev/null +++ b/scripts/node-smoke.mjs @@ -0,0 +1,49 @@ +// scripts/node-smoke.mjs — the Node-runtime smoke test behind CI's node-smoke +// job. The gate runs on Bun; this is the minimum proof that the BUILT package +// actually works on the Node version package.json declares (engines >= 22): +// open a file-backed database, write through each lens, hit the exclusive +// lock, reopen, and read everything back. Run with `node scripts/node-smoke.mjs` +// after `bun run build`. +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const { open, kv, doc, table, LibreDbError } = await import("../dist/index.js"); + +const dir = mkdtempSync(join(tmpdir(), "libredb-node-smoke-")); +const path = join(dir, "smoke.libredb"); + +try { + // Open, write through every lens, and exercise the double-open lock. + const db = open({ path }); + kv(db).set("greeting", "hello from node"); + doc(db, "notes").put("n1", { text: "smoke" }); + table(db, "people", { primaryKey: "id", columns: { id: "string", name: "string" } }).insert({ + id: "p1", + name: "Ada", + }); + + let locked; + try { + open({ path }); + } catch (error) { + locked = error; + } + assert.ok(locked instanceof LibreDbError && locked.code === "LOCKED", "second open must throw LOCKED"); + db.close(); + + // Reopen: everything written above must have survived through the WAL. + const reopened = open({ path }); + assert.equal(kv(reopened).get("greeting"), "hello from node"); + assert.deepEqual(doc(reopened, "notes").get("n1"), { text: "smoke" }); + assert.deepEqual(table(reopened, "people", { + primaryKey: "id", + columns: { id: "string", name: "string" }, + }).get("p1"), { id: "p1", name: "Ada" }); + reopened.close(); + + console.log("node smoke: ok (open, lenses, lock, reopen all behaved under Node)"); +} finally { + rmSync(dir, { recursive: true, force: true }); +} diff --git a/src/adapter/opfs.ts b/src/adapter/opfs.ts index 68709fb..36e9b89 100644 --- a/src/adapter/opfs.ts +++ b/src/adapter/opfs.ts @@ -77,6 +77,11 @@ export function opfsFileSystem(handle: SyncAccessHandle): FileSystem { } }, fsync() { + // The strongest durability point OPFS offers. Note the honest gap: + // the spec does not promise flush() carries POSIX-fsync strength + // against power loss — a committed write survives a tab crash or + // browser restart, but a power cut is engine-dependent (see + // docs/BROWSER.md and issue #10). handle.flush(); }, truncate(length) { From 143b660c74bf1fc418a4156addcab616b475e205 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 02:28:09 +0300 Subject: [PATCH 04/16] fix(kernel,adapter): close adversarial-review findings on the hardening wave Review of PR #43 by a four-lens adversarial workflow confirmed gaps in the new guarantees; each is closed with a pinning test: - an all-zeros file parsed as an empty legacy record and was adopted (truncated + appended to); isLegacyLog now requires a non-empty first record, matching the kernel's own write invariant - the record length field was outside the checksum, so mid-log damage to a length masqueraded as a torn tail and silently truncated acknowledged commits; v1 record headers now checksum their own length field (refuse instead of truncate), and legacy files keep the legacy framing on later appends - a torn-header prefix check now compares against the full 8 expected header bytes, so a short foreign file sharing only the LRDB magic is refused instead of truncated - stale-lock reclaim and forceUnlock were check-then-delete; both now claim the lock atomically by renaming it aside, so two racers can never both acquire and force can never delete a fresh writer's lock - release() verifies ownership on the claimed bytes and restores a lock it does not own - a refused open no longer leaks the WAL file descriptor; close() releases the lock even when the underlying file close throws - getRange snapshots entry references and copies per yield, so an early-exited scan no longer pays for the whole range - transact() rejects async callbacks at compile time as well as runtime - doc()'s relational-name guard runs lazily inside each operation's transaction, so handles can be built inside transact() again - readonlyFileSystem and nodeFileSystem are exported: the supported way to inspect a database a live writer holds locked - node-smoke exercises the CLI entry as the workflow comment claims - docs: downgrade warning (v1 file opened by <=0.1.3 is truncated), legacy torn-first-record refusal, Docker nonroot bind-mount guidance --- .changeset/pre-announcement-hardening.md | 10 +- docs/DOCKER.md | 14 +- docs/RELIABILITY.md | 12 +- scripts/node-smoke.mjs | 9 +- src/adapter/node-fs.ts | 90 ++++++++-- src/core.hardening.test.ts | 91 +++++++++- src/core.recovery.test.ts | 20 ++- src/core.ts | 204 ++++++++++++++++------- src/index.ts | 7 + src/lens/catalog.ts | 14 +- src/lens/document.ts | 38 ++++- src/lens/hardening.test.ts | 24 ++- src/sim/dst.test.ts | 18 +- 13 files changed, 424 insertions(+), 127 deletions(-) diff --git a/.changeset/pre-announcement-hardening.md b/.changeset/pre-announcement-hardening.md index 31dfdfe..7cd4469 100644 --- a/.changeset/pre-announcement-hardening.md +++ b/.changeset/pre-announcement-hardening.md @@ -4,7 +4,9 @@ Durability, safety, and API-contract hardening across the kernel, adapters, lenses, and CLI (the pre-announcement audit wave). -On-disk format: new databases now begin with an 8-byte `LRDB` magic/version header. Files written by earlier releases (headerless) keep opening through a legacy read path; files written by this release are not readable by older releases. The header is what lets `open()` refuse a file that is not a LibreDB database with a clear error instead of destroying it. +On-disk format: new databases now begin with an 8-byte `LRDB` magic/version header, and each record header carries a checksum of its own length field. Files written by earlier releases (headerless) keep opening through a legacy read path, and keep their legacy record framing on later appends. The header is what lets `open()` refuse a file that is not a LibreDB database with a clear error instead of destroying it; the record-header checksum is what lets recovery refuse a damaged length field instead of mistaking it for a torn tail. + +DOWNGRADE WARNING: a file written by this release must never be opened by 0.1.3 or older — the old recovery cannot parse the header, classifies the whole file as a torn tail, and silently truncates it to zero bytes. Back up before any downgrade. Two smaller legacy-behavior changes: a headerless file whose only record is torn/incomplete now refuses to open as `NOT_A_DATABASE` (0.1.3 recovered it to an empty database; refusing is the safe reading, since such a file is indistinguishable from a foreign one), and a legacy length-field corruption still reads as a torn tail (the legacy format has no header checksum — the v1 format exists to close exactly that gap). Kernel: @@ -34,6 +36,8 @@ Lenses: CLI: - Write commands rely on the kernel's exclusive lock; `--force` removes a lock only when its holder is not verifiably alive, and never deletes a file that is not a libredb lock. -- `get`/`scan` escape control characters by default so untrusted values cannot inject terminal escape sequences; `--raw` prints verbatim. +- `get`/`scan` escape control characters (including tab and newline) by default so untrusted values cannot inject terminal escape sequences — scripts that consumed values verbatim should pass `--raw`. + +New exports: `LibreDbError`, `ErrorCode`, `RecoveryInfo`, `nodeFileSystem`, and `readonlyFileSystem` (open a database for inspection with no lock and no writes — the supported way to read a file a live writer holds). -Docker image now runs as a non-root user (distroless `:nonroot`, uid 65532). +Docker image now runs as a non-root user (distroless `:nonroot`, uid 65532): bind-mounted directories must be writable by that uid, or pass `--user "$(id -u):$(id -g)"`. diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 30b2011..77c9976 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -108,6 +108,14 @@ docker buildx build --platform linux/amd64,linux/arm64 -t libredb . any two writers — one wins, the other is refused. LibreDB is single-process. - **Same files everywhere.** A `.libredb` file is byte-identical across the library, the `npx` CLI, the [standalone binary](./BINARY.md), and this image. -- **Permissions:** files the container writes are owned by the container's user; - if that's a problem on Linux, add `--user "$(id -u):$(id -g)"` to the - `docker run`. +- **Permissions:** the image runs as a non-root user (uid 65532, distroless + `:nonroot`) so a container escape holds no root and written files are never + root-owned. The flip side: **writes into a bind-mounted directory fail unless + that directory is writable by uid 65532** — either make it so, or (usually + better) run with your own identity so created files are yours: + + ```sh + docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/data" ghcr.io/libredb/libredb set /data/app.libredb k v + ``` + + Read commands need only read access and work either way. diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md index 8ea0477..659cbc9 100644 --- a/docs/RELIABILITY.md +++ b/docs/RELIABILITY.md @@ -29,9 +29,15 @@ The failure modes *outside* the clean-crash model are handled explicitly rather - **Mid-log corruption refuses to open.** A record that fails its checksum with intact records *after* it cannot be a crash artifact (only the final append can tear) — it is damage to once-durable bytes (bit rot, a partial copy). Recovery throws `code: "CORRUPT_WAL"` and truncates nothing, preserving - the evidence and the committed records behind it. One honest limitation: corruption that hits a - record's *length field* on the final record is byte-for-byte indistinguishable from a torn tail in - format v1 and truncates like one. + the evidence and the committed records behind it. The v1 record header additionally checksums its + own *length field*, so a damaged length is also refused as corruption instead of being mistaken for + a torn tail (a tear cuts bytes off the end; it never rewrites bytes already written). Headerless + v0.1.x files have no header checksum, so a damaged legacy length field still reads as a torn tail — + a documented legacy limitation the v1 format closes. +- **Downgrade warning.** A v1 file opened by LibreDB 0.1.3 or older is silently truncated to zero (the + old recovery cannot recognize the header). Never downgrade past this version with live data; back up + first. Also: a headerless legacy file whose only record is torn now refuses to open (it is + indistinguishable from a foreign file); 0.1.3 opened it as empty. - **A short read is an IO fault, not missing data.** If the filesystem returns fewer bytes than the file holds, recovery throws (`code: "INCOMPLETE_READ"`) instead of mistaking the cut for a torn tail and truncating committed transactions. diff --git a/scripts/node-smoke.mjs b/scripts/node-smoke.mjs index 68c92e1..b655d8b 100644 --- a/scripts/node-smoke.mjs +++ b/scripts/node-smoke.mjs @@ -5,6 +5,7 @@ // lock, reopen, and read everything back. Run with `node scripts/node-smoke.mjs` // after `bun run build`. import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -43,7 +44,13 @@ try { }).get("p1"), { id: "p1", name: "Ada" }); reopened.close(); - console.log("node smoke: ok (open, lenses, lock, reopen all behaved under Node)"); + // One pass of the CLI entry under Node, against the same database. + const out = execFileSync(process.execPath, ["dist/cli/main.js", "get", path, "greeting"], { + encoding: "utf8", + }); + assert.equal(out.trim(), "hello from node"); + + console.log("node smoke: ok (open, lenses, lock, reopen, and the CLI all behaved under Node)"); } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/src/adapter/node-fs.ts b/src/adapter/node-fs.ts index b511e0e..311ecda 100644 --- a/src/adapter/node-fs.ts +++ b/src/adapter/node-fs.ts @@ -28,6 +28,7 @@ import { openSync, readFileSync, readSync, + renameSync, rmSync, writeSync, } from "node:fs"; @@ -114,23 +115,58 @@ function tryCreateLock(lockPath: string, contents: string): boolean { } /** - * Remove a lock file with `--force` semantics: a LibreDb lock is removed + * Atomically claim `lockPath` for removal by renaming it aside, then judge the + * CLAIMED file's contents with `verdict`. Rename is the atomicity primitive: + * of N racers, exactly one wins the rename (the rest see ENOENT and report + * "gone") — so check-then-delete can never remove a lock a NEW writer created + * between the check and the delete. If `verdict` refuses, the file is renamed + * back so the refused lock keeps protecting its holder. + * + * Returns "removed" | "gone" (nothing to claim) | never (verdict threw). + */ +function claimAndRemoveLock(lockPath: string, verdict: (contents: string) => void): "removed" | "gone" { + const claimed = `${lockPath}.claim-${process.pid}-${randomBytes(4).toString("hex")}`; + try { + renameSync(lockPath, claimed); + } catch (error) { + if ((error as { code?: string }).code === "ENOENT") return "gone"; // another racer won + throw error; + } + let contents: string; + try { + contents = readFileSync(claimed, "utf8"); + verdict(contents); + } catch (error) { + // Refused (or unreadable): put the lock back where it was so its holder + // stays protected, then surface the refusal. + renameSync(claimed, lockPath); + throw error; + } + rmSync(claimed, { force: true }); + return "removed"; +} + +/** + * Remove a lock file with `--force` semantics: a LibreDB lock is removed * unless its holder is VERIFIABLY alive (same host, pid exists); a foreign * file is always refused. An unverifiable holder (another host) is removed — * that is exactly the case force exists for — with the risk on the caller. - * Exported for the CLI, which offers this as its `--force` flag. + * The claim is atomic (rename-aside), so force can never delete a lock a new + * writer acquired after the stale one was observed. Exported for the CLI, + * which offers this as its `--force` flag. */ export function forceUnlock(path: string): void { const lockPath = `${path}.lock`; if (!existsSync(lockPath)) return; // nothing to remove - const owner = parseLock(readFileSync(lockPath, "utf8")); - if (owner === null) { - throw new LibreDbError("LOCKED", `refusing to remove ${lockPath}: not a libredb lock file`); - } - if (owner !== undefined && livenessOf(owner) === "alive") { - throw new LibreDbError("LOCKED", `refusing to remove ${lockPath}: holder (pid ${owner.pid}) is alive`); - } - rmSync(lockPath, { force: true }); + claimAndRemoveLock(lockPath, (contents) => { + const owner = parseLock(contents); + if (owner === null) { + throw new LibreDbError("LOCKED", `refusing to remove ${lockPath}: not a libredb lock file`); + } + if (owner !== undefined && livenessOf(owner) === "alive") { + throw new LibreDbError("LOCKED", `refusing to remove ${lockPath}: holder (pid ${owner.pid}) is alive`); + } + }); } /** @@ -203,18 +239,40 @@ export function nodeFileSystem(): FileSystem { for (let attempt = 0; attempt < 2; attempt++) { if (tryCreateLock(lockPath, contents)) { return () => { - // Release only OUR lock: if someone force-removed it and locked - // again, deleting theirs would let a third writer in. + // Release only OUR lock — atomically. If someone force-removed it + // and locked again, the claimed contents carry THEIR nonce; the + // verdict throws, the rename puts their lock back, and we leave it + // alone (a plain check-then-delete would race a fresh acquire). try { - if (parseLock(readFileSync(lockPath, "utf8"))?.nonce !== nonce) return; + claimAndRemoveLock(lockPath, (c) => { + if (parseLock(c)?.nonce !== nonce) throw new Error("not ours"); + }); } catch { - return; // already gone + // Not ours or already gone: either way there is nothing to release. } - rmSync(lockPath, { force: true }); }; } if (!isStaleLock(lockPath)) break; - rmSync(lockPath, { force: true }); // reclaim the stale lock, then retry + // Reclaim the stale lock ATOMICALLY: rename-aside means that of N + // processes racing this reclaim, exactly one removes the stale file — + // the losers see it gone and retry the exclusive create, where again + // exactly one wins. Two concurrent writers can never both acquire. + // The verdict re-checks staleness on the claimed bytes: if a NEW + // writer's lock slid in between the check and the claim, it is put + // back untouched. + try { + if ( + claimAndRemoveLock(lockPath, (c) => { + const owner = parseLock(c); + if (owner === null) throw new Error("foreign file"); + if (owner !== undefined && livenessOf(owner) === "alive") throw new Error("holder alive"); + }) === "gone" + ) { + continue; // another racer reclaimed it; retry the exclusive create + } + } catch { + break; // a live or foreign lock appeared: locked + } } throw new LibreDbError( "LOCKED", diff --git a/src/core.hardening.test.ts b/src/core.hardening.test.ts index 342e02c..d2ea15a 100644 --- a/src/core.hardening.test.ts +++ b/src/core.hardening.test.ts @@ -24,7 +24,7 @@ import { join } from "node:path"; import { afterEach, expect, test } from "bun:test"; import { LibreDbError, open, type FileSystem, type WalFile } from "./core.ts"; -import { open as openNode } from "./index.ts"; +import { doc, open as openNode, readonlyFileSystem, table } from "./index.ts"; const bytes = (...b: number[]): Uint8Array => new Uint8Array(b); @@ -149,13 +149,15 @@ test("an async transact() callback throws ASYNC_TRANSACTION and commits nothing" const db = openNode({ path }); db.transact((tx) => tx.set(bytes(1), bytes(10))); - const error = errorFrom(() => - db.transact(async (tx) => { - tx.set(bytes(2), bytes(20)); - await Promise.resolve(); - tx.set(bytes(3), bytes(30)); - }), - ); + // The public signature rejects an async callback at COMPILE time (the + // PromiseLike -> never intersection), so reaching the runtime guard — + // what an untyped JS caller would hit — requires casting past the types. + const asyncBody = async (tx: Parameters[0]>[0]): Promise => { + tx.set(bytes(2), bytes(20)); + await Promise.resolve(); + tx.set(bytes(3), bytes(30)); + }; + const error = errorFrom(() => db.transact(asyncBody as unknown as () => void)); expect(error.code).toBe("ASYNC_TRANSACTION"); // Nothing from the async body was committed — not even the pre-await write — @@ -213,6 +215,50 @@ test("open() on a non-LibreDB file throws NOT_A_DATABASE and leaves every byte i expect(existsSync(`${path}.lock`)).toBe(false); }); +test("an all-zeros file is refused untouched (it must not parse as an empty legacy record)", () => { + // A zero-filled file — a preallocated image, a zeroed disk region — decodes + // as a size-0 record whose crc32("") is also 0. This kernel never writes an + // empty record (every commit journals at least one op), so zeros are foreign + // bytes; adopting them would truncate and then write into a foreign file. + for (const length of [8, 1001, 1024]) { + const path = tempPath(`zeros-${length}`); + writeFileSync(path, new Uint8Array(length)); // all zeros + expect(errorFrom(() => openNode({ path })).code).toBe("NOT_A_DATABASE"); + const disk = new Uint8Array(readFileSync(path)); + expect(disk.length).toBe(length); + expect(disk.every((byte) => byte === 0)).toBe(true); + } +}); + +test("a short foreign file that shares only the LRDB magic is refused, not truncated", () => { + // 'LRDB' followed by NON-header bytes cannot be a torn first append (a torn + // header is always a prefix of the exact 8 bytes this kernel writes), so it + // is a foreign file and must be left alone. + const path = tempPath("lrdb-text"); + writeFileSync(path, "LRDB\n"); // magic + newline: foreign + expect(errorFrom(() => openNode({ path })).code).toBe("NOT_A_DATABASE"); + expect(readFileSync(path, "utf8")).toBe("LRDB\n"); +}); + +test("a corrupted record length field is corruption (refused), never a torn tail", () => { + const path = tempPath("length-rot"); + const db = openNode({ path }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.transact((tx) => tx.set(bytes(2), bytes(20))); + db.close(); + + // Flip a bit in the FIRST record's length field (offset 8 = right after the + // file header). The v1 record header checksums its own length field, so this + // reads as damage — not as a torn tail that would silently truncate the + // acknowledged second commit. + const disk = new Uint8Array(readFileSync(path)); + disk[8] = (disk[8] as number) ^ 0x40; // flip one bit of the length field + writeFileSync(path, disk); + + expect(errorFrom(() => openNode({ path })).code).toBe("CORRUPT_WAL"); + expect(new Uint8Array(readFileSync(path)).length).toBe(disk.length); // untouched +}); + test("a new database writes the LRDB file header with its first commit", () => { const path = tempPath("fresh"); const db = openNode({ path }); @@ -487,3 +533,32 @@ test("a stale lock from a dead process is reclaimed automatically", () => { db.close(); expect(existsSync(`${path}.lock`)).toBe(false); }); + +test("constructing doc() and table() handles inside a transaction works (no nested transact)", () => { + // Handle construction must not need a transaction of its own: the doc() + // relational-kind guard runs lazily inside each operation's transaction. + const path = tempPath("handles-in-tx"); + const db = openNode({ path }); + const built = db.transact(() => { + return typeof doc(db, "logs").put === "function"; + }); + expect(built).toBe(true); + // The lazy guard still fires on the first OPERATION. + table(db, "accounts", { primaryKey: "id", columns: { id: "string" } }).insert({ id: "a1" }); + const handle = doc(db, "accounts"); // construction alone is fine... + expect(errorFrom(() => handle.get("a1")).code).toBe("INVALID_ARGUMENT"); // ...operations refuse + expect(errorFrom(() => handle.put("a2", {})).code).toBe("INVALID_ARGUMENT"); + db.close(); +}); + +test("readonlyFileSystem opens a database a live writer holds locked", () => { + const path = tempPath("inspect-live"); + const writer = openNode({ path }); + writer.transact((tx) => tx.set(bytes(1), bytes(10))); + // A concurrent inspector: no lock, no writes — reads the committed state. + const inspector = openNode({ path, fs: readonlyFileSystem() }); + expect(inspector.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); + inspector.close(); + writer.transact((tx) => tx.set(bytes(2), bytes(20))); // writer unaffected + writer.close(); +}); diff --git a/src/core.recovery.test.ts b/src/core.recovery.test.ts index 61f4872..5899113 100644 --- a/src/core.recovery.test.ts +++ b/src/core.recovery.test.ts @@ -30,6 +30,17 @@ const bytes = (...b: number[]): Uint8Array => new Uint8Array(b); * crash-injection tests below can hand-craft on-disk frames. */ const u32 = (n: number): Buffer => Buffer.from([(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff]); +/** CRC-32 (IEEE), duplicated here so the crash-injection tests can hand-craft + * frames without borrowing the kernel implementation they judge. */ +const crc32 = (data: Uint8Array): number => { + let crc = 0xffffffff; + for (let i = 0; i < data.length; i++) { + crc ^= data[i] as number; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return (crc ^ 0xffffffff) >>> 0; +}; + /** Temp directories created during the run, cleaned up after each test. */ const dirs: string[] = []; const tempPath = (name: string): string => { @@ -167,10 +178,13 @@ test("a tail record whose checksum fails is discarded on recovery", () => { db.close(); const goodSize = statSync(path).size; - // Append a fully-sized record whose checksum does not match its payload, as a - // half-flushed disk block would produce. payload = [set, keyLen=1, key=0]. + // Append a fully-sized record whose PAYLOAD checksum does not match, as a + // half-flushed disk block would produce. The record header itself must be + // well-formed (its length field carries its own checksum in format v1), so + // the frame is: [len][crc32(len bytes)][bad payload crc][payload]. const payload = Buffer.from([1, 0, 0, 0, 1, 0]); - appendFileSync(path, Buffer.concat([u32(payload.length), u32(0), payload])); + const lenField = u32(payload.length); + appendFileSync(path, Buffer.concat([lenField, u32(crc32(lenField)), u32(0), payload])); const reopened = open({ path }); expect(dump(reopened)).toEqual([[5, 50]]); diff --git a/src/core.ts b/src/core.ts index 527654c..44264f2 100644 --- a/src/core.ts +++ b/src/core.ts @@ -145,8 +145,10 @@ export interface Database { * * `run` MUST be synchronous. An async callback returns a pending Promise — * the kernel cannot see writes made after an `await`, so committing at that - * point would silently lose them. Any thenable return value therefore - * aborts the transaction with {@link ErrorCode} ASYNC_TRANSACTION. + * point would silently lose them. The signature rejects a Promise-returning + * callback at compile time (the `T extends PromiseLike ? never : ...` + * intersection), and any thenable that slips past the types aborts the + * transaction at runtime with {@link ErrorCode} ASYNC_TRANSACTION. * * If a commit fails to reach the disk (the append or fsync throws), the * database LATCHES into a failed state: every later transact() throws with @@ -155,7 +157,7 @@ export interface Database { * acknowledged commits — refusing further writes is what keeps "a returned * transact() is durable" true. */ - transact(run: (tx: Transaction) => T): T; + transact(run: (tx: Transaction) => T & (T extends PromiseLike ? never : unknown)): T; /** Flush pending state and release resources. Safe to call once. Throws if * called from inside a transaction body. */ close(): void; @@ -373,20 +375,25 @@ function makeTransaction(working: StoredEntry[], journal: Op[]): Transaction { journal.push({ kind: "delete", key: ownedKey }); }, *getRange(start, end) { - // Snapshot the matching entries when iteration starts (a generator body - // runs at the first next()). Walking the live array by index instead - // would let a delete-during-scan shift entries under the cursor and - // silently skip them — the classic delete-while-scanning bug. + // Snapshot REFERENCES to the matching entries when iteration starts (a + // generator body runs at the first next()). Walking the live array by + // index instead would let a delete-during-scan shift entries under the + // cursor and silently skip them — the classic delete-while-scanning bug. + // Entries are immutable once stored, so holding references is safe; the + // byte copies happen per-yield, so an early-exited scan pays only for + // the entries it consumed. // locate() returns the first index whose key is >= start (the insertion // point), so the scan is naturally inclusive of start. It stops at the // first key that is not < end, making the range half-open [start, end). - const snapshot: Entry[] = []; + const snapshot: StoredEntry[] = []; for (let i = locate(working, start).index; i < working.length; i++) { const entry = working[i] as StoredEntry; if (compareKeys(entry.key, end) >= 0) break; - snapshot.push({ key: entry.key.slice(), value: entry.value.slice() }); + snapshot.push(entry); + } + for (const entry of snapshot) { + yield { key: entry.key.slice(), value: entry.value.slice() }; } - yield* snapshot; }, }; } @@ -403,11 +410,17 @@ function makeTransaction(working: StoredEntry[], journal: Op[]): Transaction { // // On-disk layout = an 8-byte file header followed by records: // header: [4-byte magic "LRDB"][u16 formatVersion][u16 reserved] -// record: [u32 payloadLength][u32 crc32(payload)][payload] +// record: [u32 payloadLength][u32 crc32(payloadLength bytes)][u32 crc32(payload)][payload] // The magic is what lets open() refuse a file that is NOT a LibreDB database // instead of misparsing arbitrary bytes (and destroying them); the version is -// what lets the format evolve without ambushing older readers. Files written -// by v0.1.x predate the header; recovery still reads them (see recover()). +// what lets the format evolve without ambushing older readers. The record +// header carries a checksum OF ITSELF (the length field): a torn append can +// only ever cut bytes off the end — it cannot rewrite the header it already +// wrote — so a header that is present but fails its own checksum is damage, +// not a crash, and recovery can refuse it instead of trusting a corrupted +// length that would misclassify the damage as a torn tail. Files written by +// v0.1.x predate the file header and use 8-byte record headers with no header +// checksum; recovery still reads them (see recover()). // // The payload is the transaction's mutations back to back. Each op is: // set: [u8 1][u32 keyLength][key][u32 valueLength][value] @@ -421,8 +434,13 @@ function makeTransaction(working: StoredEntry[], journal: Op[]): Transaction { const OP_SET = 1; const OP_DELETE = 0; -/** Bytes in a record header: the payload length and its checksum, both u32. */ -const RECORD_HEADER = 8; +/** Bytes in a v1 record header: payload length, header checksum, payload + * checksum — three u32s. */ +const RECORD_HEADER = 12; +/** Bytes in a headerless v0.1.x record header: payload length and payload + * checksum only (no header checksum — the legacy format cannot distinguish a + * corrupted length field from a torn tail; v1 exists to fix that). */ +const LEGACY_RECORD_HEADER = 8; /** The file magic: "LRDB" in ASCII. A file that does not start with it (and * does not parse as a headerless v0.1.x log) is refused, untouched. */ const MAGIC = Uint8Array.of(0x4c, 0x52, 0x44, 0x42); @@ -478,8 +496,11 @@ function encodeFileHeader(): Uint8Array { } /** Encode one committed transaction's ops as a length-framed, checksummed - * record ready to append to the log. */ -function encodeRecord(ops: readonly Op[]): Uint8Array { + * record ready to append to the log. `legacy` selects the framing: a database + * that opened as a headerless v0.1.x file keeps appending v0.1.x records (a + * format cannot change mid-file), while headered v1 files get the v1 record + * header with its self-checksum. */ +function encodeRecord(ops: readonly Op[], legacy: boolean): Uint8Array { let size = 0; for (const op of ops) { size += 1 + 4 + op.key.length; @@ -502,10 +523,19 @@ function encodeRecord(ops: readonly Op[]): Uint8Array { } } - const record = new Uint8Array(RECORD_HEADER + size); + const header = legacy ? LEGACY_RECORD_HEADER : RECORD_HEADER; + const record = new Uint8Array(header + size); writeU32(record, 0, size); - writeU32(record, 4, crc32(payload)); - record.set(payload, RECORD_HEADER); + if (legacy) { + writeU32(record, 4, crc32(payload)); + } else { + // The header checksum covers the length field, so recovery can tell a + // trustworthy length (torn tail: truncate) from a damaged one (corruption: + // refuse) — see replayLog. + writeU32(record, 4, crc32(record.subarray(0, 4))); + writeU32(record, 8, crc32(payload)); + } + record.set(payload, header); return record; } @@ -547,16 +577,19 @@ function replayPayload(entries: StoredEntry[], payload: Uint8Array): void { /** What {@link recover} learned about the log. */ interface Recovery { entries: StoredEntry[]; - /** True for an empty file: the first append must write the file header. A - * non-empty headerless v0.1.x file keeps appending headerless records — a - * header cannot be inserted mid-file. */ + /** True for an empty file: the first append must write the file header. */ needsHeader: boolean; + /** True for a headerless v0.1.x file: every future append must keep the + * v0.1.x record framing — a file's format cannot change mid-file. */ + legacy: boolean; /** Torn-tail bytes discarded, reported via {@link OpenOptions.onRecovery}. */ truncatedBytes: number; } /** - * Replay every record of `log` from `base` onto a fresh entry array. + * Replay every record of `log` from `base` onto a fresh entry array. `legacy` + * selects the record framing: v1 (12-byte header with a header checksum) or + * headerless v0.1.x (8-byte header without one). * * Failure classification is the heart of recovery: * @@ -564,26 +597,41 @@ interface Recovery { * shorter than a record header, is a TORN TAIL: the append a crash * interrupted. Only the tail can tear (the log is append-only and every * earlier record was fsync'd), so it is truncated away. - * - A record that fails its checksum but ends exactly at the file's end is - * a DAMAGED TAIL — a half-flushed final block — and truncates the same way. - * - A record that fails its checksum with more data AFTER it cannot be a - * crash artifact: bytes after it mean a later append succeeded, which means - * this record was once durable and has since been damaged (bit rot, a - * partial copy, a second writer). That is corruption; recovery THROWS and - * leaves the file untouched rather than destroy the committed records - * behind the damage. + * - A v1 header that fails its OWN checksum is damage, never a tear: a torn + * append cuts bytes off the end, it does not rewrite bytes it already + * wrote. Trusting the corrupted length would misclassify the damage as a + * torn tail and silently truncate every committed record after it, so + * recovery THROWS. (The legacy format has no header checksum; a damaged + * legacy length field still reads as a torn tail — a documented v0.1.x + * limitation the v1 format exists to close.) + * - A record whose payload fails its checksum but ends exactly at the file's + * end is a DAMAGED TAIL — a half-flushed final block — and truncates. + * - A payload-checksum failure with more data AFTER it cannot be a crash + * artifact: bytes after it mean a later append succeeded, which means this + * record was once durable and has since been damaged (bit rot, a partial + * copy, a second writer). That is corruption; recovery THROWS and leaves + * the file untouched rather than destroy the committed records behind the + * damage. */ -function replayLog(log: Uint8Array, base: number): { entries: StoredEntry[]; tail: number; replayed: number } { +function replayLog( + log: Uint8Array, + base: number, + legacy: boolean, +): { entries: StoredEntry[]; tail: number; replayed: number } { + const header = legacy ? LEGACY_RECORD_HEADER : RECORD_HEADER; const entries: StoredEntry[] = []; let offset = base; let replayed = 0; - while (offset + RECORD_HEADER <= log.length) { + while (offset + header <= log.length) { const size = readU32(log, offset); - const start = offset + RECORD_HEADER; + if (!legacy && crc32(log.subarray(offset, offset + 4)) !== readU32(log, offset + 4)) { + throw new LibreDbError("CORRUPT_WAL", `corrupt WAL record header at offset ${offset}; refusing to open`); + } + const start = offset + header; const end = start + size; if (end > log.length) break; // torn tail: fewer bytes than the header promised const payload = log.subarray(start, end); - if (crc32(payload) !== readU32(log, offset + 4)) { + if (crc32(payload) !== readU32(log, offset + (legacy ? 4 : 8))) { if (end < log.length) { throw new LibreDbError( "CORRUPT_WAL", @@ -622,19 +670,25 @@ function recover(file: WalFile): Recovery { // so it is an error, never a recovery. throw new LibreDbError("INCOMPLETE_READ", `WAL read returned ${log.length} of ${size} bytes`); } - if (size === 0) return { entries: [], needsHeader: true, truncatedBytes: 0 }; - - const magicPrefix = Math.min(log.length, MAGIC.length); - const hasMagicPrefix = log.subarray(0, magicPrefix).every((byte, i) => byte === MAGIC[i]); - if (hasMagicPrefix) { - if (log.length < FILE_HEADER) { - // A torn header: the very first commit (header + record in one append) - // was interrupted before the header finished. Nothing was ever - // acknowledged, so start the database over from empty. - file.truncate(0); - file.fsync(); - return { entries: [], needsHeader: true, truncatedBytes: log.length }; - } + if (size === 0) return { entries: [], needsHeader: true, legacy: false, truncatedBytes: 0 }; + + // A torn first append can only leave a PREFIX of the exact 8 header bytes + // this kernel writes (magic + version + reserved), so the recognition test + // compares against all of them — "LRDB" followed by anything else is a + // foreign file, not a torn header. + const expectedHeader = encodeFileHeader(); + const headerPrefix = Math.min(log.length, FILE_HEADER); + const hasHeaderPrefix = log.subarray(0, headerPrefix).every((byte, i) => byte === expectedHeader[i]); + const hasMagic = log.length >= MAGIC.length && MAGIC.every((byte, i) => byte === log[i]); + if (hasHeaderPrefix && log.length < FILE_HEADER) { + // A torn header: the very first commit (header + record in one append) + // was interrupted before the header finished. Nothing was ever + // acknowledged, so start the database over from empty. + file.truncate(0); + file.fsync(); + return { entries: [], needsHeader: true, legacy: false, truncatedBytes: log.length }; + } + if (hasMagic && log.length >= FILE_HEADER) { const fileVersion = ((log[4] as number) << 8) | (log[5] as number); if (fileVersion !== FORMAT_VERSION) { throw new LibreDbError( @@ -642,7 +696,7 @@ function recover(file: WalFile): Recovery { `database format version ${fileVersion} is newer than this library supports (${FORMAT_VERSION})`, ); } - return { ...finishReplay(file, log, FILE_HEADER), needsHeader: false }; + return { ...finishReplay(file, log, FILE_HEADER, false), needsHeader: false, legacy: false }; } // No magic: either a headerless v0.1.x database or a foreign file. Probe the @@ -653,18 +707,24 @@ function recover(file: WalFile): Recovery { if (!isLegacyLog(log)) { throw new LibreDbError("NOT_A_DATABASE", "file is not a libredb database; refusing to touch it"); } - return { ...finishReplay(file, log, 0), needsHeader: false }; + return { ...finishReplay(file, log, 0, true), needsHeader: false, legacy: true }; } -/** Does `log` begin with one complete, checksummed, well-formed v0.1.x record? - * That is the recognition test for a headerless legacy database: real bytes - * from this kernel always start with one, foreign bytes essentially never do. */ +/** Does `log` begin with one complete, checksummed, well-formed, NON-EMPTY + * v0.1.x record? That is the recognition test for a headerless legacy + * database: real bytes from this kernel always start with one (every commit + * journals at least one op — an empty transaction is never appended), foreign + * bytes essentially never do. The non-empty requirement is load-bearing: an + * all-zeros file would otherwise parse as an "empty record" (size 0 and + * crc32("") is 0), and a zero-filled foreign file — a preallocated image, a + * zeroed-out disk region — would be adopted and written into. */ function isLegacyLog(log: Uint8Array): boolean { - if (log.length < RECORD_HEADER) return false; + if (log.length < LEGACY_RECORD_HEADER) return false; const size = readU32(log, 0); - const end = RECORD_HEADER + size; + if (size === 0) return false; // this kernel never writes an empty record + const end = LEGACY_RECORD_HEADER + size; if (end > log.length) return false; - const payload = log.subarray(RECORD_HEADER, end); + const payload = log.subarray(LEGACY_RECORD_HEADER, end); if (crc32(payload) !== readU32(log, 4)) return false; try { replayPayload([], payload); // throwaway replay: structural validation only @@ -680,8 +740,9 @@ function finishReplay( file: WalFile, log: Uint8Array, base: number, + legacy: boolean, ): { entries: StoredEntry[]; truncatedBytes: number } { - const { entries, tail } = replayLog(log, base); + const { entries, tail } = replayLog(log, base, legacy); const truncatedBytes = log.length - tail; if (truncatedBytes > 0) { file.truncate(tail); @@ -708,14 +769,26 @@ function openLog( onRecovery: ((info: RecoveryInfo) => void) | undefined, ): { entries: StoredEntry[]; log: Log } { const file = fs.open(path); - const recovery = recover(file); + let recovery: Recovery; + try { + recovery = recover(file); + } catch (error) { + // A refused open (foreign file, corruption, unsupported version, short + // read) must not leak the file handle; the refusal error stays primary. + try { + file.close(); + } catch { + // The close failed after recovery already failed; surface the original. + } + throw error; + } if (recovery.truncatedBytes > 0) onRecovery?.({ truncatedBytes: recovery.truncatedBytes }); let needsHeader = recovery.needsHeader; return { entries: recovery.entries, log: { append(ops) { - const record = encodeRecord(ops); + const record = encodeRecord(ops, recovery.legacy); if (needsHeader) { // First commit of a new database: header and record go down in ONE // append, so a crash can only ever leave a recognizable prefix @@ -850,9 +923,14 @@ export const open: Open = (options) => { throw new LibreDbError("CLOSE_IN_TRANSACTION", "cannot close the database inside a transaction"); } closed = true; - if (log !== null) log.close(); - releaseLock?.(); - committed = []; + try { + if (log !== null) log.close(); + } finally { + // The lock must not outlive the instance even if the underlying file + // close throws — a leaked lock would wedge every future open. + releaseLock?.(); + committed = []; + } }, }; }; diff --git a/src/index.ts b/src/index.ts index 6fca142..15d4163 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,6 +29,13 @@ export const open: Open = (options) => ? openKernel({ ...options, fs: nodeFileSystem() }) : openKernel(options); +// The adapters, exported so a tool can compose its own open: nodeFileSystem is +// the default durable adapter; readonlyFileSystem opens a database for +// INSPECTION — it never writes (recovery drops a torn tail in memory only) and +// takes no lock, so it can read a file a live writer holds open. +export { nodeFileSystem } from "./adapter/node-fs.ts"; +export { readonlyFileSystem } from "./cli/readonly-fs.ts"; + export { kv } from "./lens/kv.ts"; export type { Kv, KvEntry } from "./lens/kv.ts"; diff --git a/src/lens/catalog.ts b/src/lens/catalog.ts index b5fe977..bcc5d3f 100644 --- a/src/lens/catalog.ts +++ b/src/lens/catalog.ts @@ -99,12 +99,18 @@ export function assertUserName(name: string): void { assertWellFormedText(name, "namespace name"); } -/** The cataloged kind of `name`, or undefined when it is not cataloged. The +/** The cataloged kind of `name` as seen through `read` (a point-read inside + * the caller's own transaction), or undefined when it is not cataloged. The * lens entry points use this to route a name to the right lens: `doc()` refuses * a relational table (its rows are schema-validated), `table()` refuses a - * document collection (its documents never were). */ -export function catalogKindOf(store: Store, name: string): CatalogEntry["kind"] | undefined { - const bytes = store.transact((tx) => tx.get(catalogKey(name))); + * document collection (its documents never were). Taking a read function + * instead of a Store keeps the check inside whatever transaction the caller is + * already running — the kernel forbids nesting a fresh one. */ +export function catalogKindAt( + read: (key: Uint8Array) => Uint8Array | undefined, + name: string, +): CatalogEntry["kind"] | undefined { + const bytes = read(catalogKey(name)); return bytes === undefined ? undefined : (JSON.parse(fromUtf8.decode(bytes)) as CatalogEntry).kind; } diff --git a/src/lens/document.ts b/src/lens/document.ts index c12cb40..691cd78 100644 --- a/src/lens/document.ts +++ b/src/lens/document.ts @@ -13,7 +13,7 @@ */ import { result, type Result, type WriteResult } from "./types.ts"; import { prefixRange } from "../query/range.ts"; -import { assertUserName, assertWellFormedText, catalogKindOf, recordDocument } from "./catalog.ts"; +import { assertUserName, assertWellFormedText, catalogKindAt, recordDocument } from "./catalog.ts"; import type { Store } from "../adapter/store.ts"; import { LibreDbError } from "../core.ts"; @@ -187,21 +187,37 @@ export function doc(store: Store, collection: string): DocCollection { // A collection name may not intrude on the reserved catalog namespace and // must be isolatable in the key layout — reject it before any key is derived. assertUserName(collection); - if (catalogKindOf(store, collection) === "relational") { - throw new LibreDbError( - "INVALID_ARGUMENT", - `${JSON.stringify(collection)} is a relational table; use table() instead of doc()`, - ); - } - return collectionHandle(store, collection); + // The relational-kind guard runs INSIDE each operation's own transaction + // (lazily, memoized after the first pass) rather than here: a construction- + // time check would need a transaction of its own, which would break the + // established pattern of building a handle inside a transact() body. + let checked = false; + const ensure = (read: (key: Uint8Array) => Uint8Array | undefined): void => { + if (checked) return; + if (catalogKindAt(read, collection) === "relational") { + throw new LibreDbError( + "INVALID_ARGUMENT", + `${JSON.stringify(collection)} is a relational table; use table() instead of doc()`, + ); + } + checked = true; + }; + return collectionHandle(store, collection, ensure); } /** * The unguarded collection builder behind {@link doc}. The relational lens uses * it directly: a table IS this handle plus schema validation, so the "is this * name relational?" guard that protects doc() callers must not apply there. + * `ensure` (when given) runs at the start of every operation's transaction — + * doc() uses it to refuse a relational table's name without needing its own + * transaction at construction time. */ -export function collectionHandle(store: Store, collection: string): DocCollection { +export function collectionHandle( + store: Store, + collection: string, + ensure?: (read: (key: Uint8Array) => Uint8Array | undefined) => void, +): DocCollection { // The byte range covering every `:` key. prefixRange computes the // [start, end) bound on raw bytes so it agrees with the kernel's order, which // is what makes the colon a sound collection boundary (a sibling like "users2" @@ -218,6 +234,7 @@ export function collectionHandle(store: Store, collection: string): DocCollectio const scan = (keep: (document: Doc) => boolean): Result => result(() => store.transact((tx) => { + ensure?.((key) => tx.get(key)); const rows: DocEntry[] = []; for (const entry of tx.getRange(start, end)) { const document = decodeDoc(entry.value); @@ -238,6 +255,7 @@ export function collectionHandle(store: Store, collection: string): DocCollectio // encoding — two distinct malformed ids would silently share one key. assertWellFormedText(id, "document id"); store.transact((tx) => { + ensure?.((key) => tx.get(key)); // Register this collection in the catalog on its first write (DESIGN.md // section 6.3). Idempotent and inside the write's own transaction, so the // registration and the document are durable together. A table's inserts @@ -250,6 +268,7 @@ export function collectionHandle(store: Store, collection: string): DocCollectio }, get(id) { return store.transact((tx) => { + ensure?.((key) => tx.get(key)); const bytes = tx.get(keyOf(collection, id)); return bytes === undefined ? undefined : decodeDoc(bytes); }); @@ -258,6 +277,7 @@ export function collectionHandle(store: Store, collection: string): DocCollectio // Read-before-delete in one transaction: the kernel's delete is a silent // no-op on a missing key, so this is how the lens tells 1 from 0 changes. const changed = store.transact((tx) => { + ensure?.((key) => tx.get(key)); const k = keyOf(collection, id); const existed = tx.get(k) !== undefined; tx.delete(k); diff --git a/src/lens/hardening.test.ts b/src/lens/hardening.test.ts index d5da3fb..0920594 100644 --- a/src/lens/hardening.test.ts +++ b/src/lens/hardening.test.ts @@ -93,14 +93,28 @@ test("NaN and the infinities are rejected by number column validation", () => { // --- issue #30: a name belongs to one lens --- -test("doc() refuses a name cataloged as a relational table", () => { +test("doc() operations refuse a name cataloged as a relational table", () => { const db = open(); table(db, "accounts", SCHEMA).insert({ id: "a1", n: 1 }); - const error = errorFrom(() => doc(db, "accounts")); - expect(error.code).toBe("INVALID_ARGUMENT"); - expect(error.message).toMatch(/use table\(\)/); - // The catalog's faithful view survived: still relational, schema intact. + // Construction succeeds (the guard is lazy so handles can be built inside a + // transaction); every OPERATION refuses, so schema validation cannot be + // bypassed through a doc() handle. + const handle = doc(db, "accounts"); + for (const op of [ + () => handle.put("a2", { rogue: true }), + () => handle.get("a1"), + () => handle.delete("a1"), + () => handle.all().toArray(), + () => handle.find({}).toArray(), + ]) { + const error = errorFrom(op); + expect(error.code).toBe("INVALID_ARGUMENT"); + expect(error.message).toMatch(/use table\(\)/); + } + // The catalog's faithful view survived: still relational, schema intact, + // and no rogue row landed. expect(catalog(db).get("accounts")).toEqual({ kind: "relational", schema: SCHEMA }); + expect(table(db, "accounts", SCHEMA).get("a2")).toBeUndefined(); db.close(); }); diff --git a/src/sim/dst.test.ts b/src/sim/dst.test.ts index 4401ac9..e6fa491 100644 --- a/src/sim/dst.test.ts +++ b/src/sim/dst.test.ts @@ -102,22 +102,22 @@ test("a torn in-flight append is discarded; committed state survives the crash", /** The byte offset where records start: past the 8-byte file header. */ const RECORDS_BASE = 8; -test("a corrupted length field on the final record is indistinguishable from a torn tail and truncates", () => { +test("a corrupted length field refuses to open instead of masquerading as a torn tail", () => { const fs = new SimFS(3); const steps: WorkloadStep[] = [{ ops: [{ kind: "set", key: "k0", value: "a" }], abort: false }]; const db = open({ path: WAL, fs }); runWorkload(db, steps); - // Flip a byte in the LAST record's length field. The promised end then lies - // beyond the file, which is byte-for-byte what a torn tail looks like — - // format v1 has no per-record header to tell them apart (see DESIGN.md), so - // recovery truncates from that record on. Every record before it survives; - // here there are none, so the store recovers empty. + // Flip a byte in a record's length field. In format v1 the record header + // carries a checksum of its own length field, precisely so this damage is + // NOT mistaken for a torn tail (a tear cuts bytes off the end; it cannot + // rewrite header bytes that were already written). Recovery must refuse the + // open and leave every byte in place. + const before = fs.durableBytes(WAL); fs.corrupt(WAL, RECORDS_BASE); - const recovered = dump(open({ path: WAL, fs })); - expect(recovered.size).toBe(0); - expect(isCommittedPrefix(recovered, committedPrefixStates(steps))).toBe(true); + expect(() => open({ path: WAL, fs })).toThrow(/corrupt WAL record header/i); + expect(fs.durableBytes(WAL).length).toBe(before.length); }); test("mid-log payload corruption refuses to open instead of truncating committed data", () => { From f333ff32f84583a8c7cf3db438de618c810ebf50 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 07:41:09 +0300 Subject: [PATCH 05/16] fix(kernel,adapter,lens): resolve Copilot review findings on PR #43 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - files shorter than the 8-byte header are refused untouched, even when they share magic-prefix bytes; the torn-first-header auto-repair is removed (destroying an ambiguous file is never the answer; nothing in a sub-header file was ever acknowledged) - commit IO failures are typed: transact() throws LibreDbError FAILED with the adapter error as cause instead of the raw ENOSPC/EIO - lock sentinel is an exact first-line match, so a file merely starting with the sentinel text is foreign - automatic stale-lock reclaim now requires a VERIFIABLY dead holder; anonymous locks (empty, or the sentinel-only 0.1.x format) carry no liveness info — they may even be a concurrent lock() between its exclusive create and its sentinel write — and need --force - isStaleLock treats only ENOENT as vanished; other read errors are real problems, not staleness - tryCreateLock loops writeSync so a partial write cannot leave a malformed lock that reads as an anonymous stray - well-formedness validation no longer depends on String.prototype.isWellFormed (absent on older browser engines the browser entry may reach); a hand-rolled surrogate scan replaces it - recover() comments clarified: the version bytes gate the v1 path, and a foreign file matching the entire 8-byte header is byte-for-byte indistinguishable from a real empty database - size budget 5 kB -> 6 kB for the public entry (lock protocol growth) --- .changeset/pre-announcement-hardening.md | 4 +- .size-limit.json | 2 +- src/adapter/node-fs.test.ts | 40 +++++++++++++++---- src/adapter/node-fs.ts | 50 +++++++++++++++--------- src/cli/run.test.ts | 18 +++++++-- src/core.hardening.test.ts | 28 +++++++------ src/core.ts | 43 ++++++++++++-------- src/lens/catalog.ts | 27 ++++++++++--- 8 files changed, 146 insertions(+), 66 deletions(-) diff --git a/.changeset/pre-announcement-hardening.md b/.changeset/pre-announcement-hardening.md index 7cd4469..f355845 100644 --- a/.changeset/pre-announcement-hardening.md +++ b/.changeset/pre-announcement-hardening.md @@ -6,7 +6,7 @@ Durability, safety, and API-contract hardening across the kernel, adapters, lens On-disk format: new databases now begin with an 8-byte `LRDB` magic/version header, and each record header carries a checksum of its own length field. Files written by earlier releases (headerless) keep opening through a legacy read path, and keep their legacy record framing on later appends. The header is what lets `open()` refuse a file that is not a LibreDB database with a clear error instead of destroying it; the record-header checksum is what lets recovery refuse a damaged length field instead of mistaking it for a torn tail. -DOWNGRADE WARNING: a file written by this release must never be opened by 0.1.3 or older — the old recovery cannot parse the header, classifies the whole file as a torn tail, and silently truncates it to zero bytes. Back up before any downgrade. Two smaller legacy-behavior changes: a headerless file whose only record is torn/incomplete now refuses to open as `NOT_A_DATABASE` (0.1.3 recovered it to an empty database; refusing is the safe reading, since such a file is indistinguishable from a foreign one), and a legacy length-field corruption still reads as a torn tail (the legacy format has no header checksum — the v1 format exists to close exactly that gap). +DOWNGRADE WARNING: a file written by this release must never be opened by 0.1.3 or older — the old recovery cannot parse the header, classifies the whole file as a torn tail, and silently truncates it to zero bytes. Back up before any downgrade. Three smaller legacy-behavior changes: a headerless file whose only record is torn/incomplete now refuses to open as `NOT_A_DATABASE` (0.1.3 recovered it to an empty database; refusing is the safe reading, since such a file is indistinguishable from a foreign one); any file shorter than the 8-byte header is likewise refused untouched (a crash inside the first bytes of a brand-new database's first-ever commit therefore needs a manual delete — nothing in it was acknowledged); and a legacy length-field corruption still reads as a torn tail (the legacy format has no header checksum — the v1 format exists to close exactly that gap). Kernel: @@ -35,7 +35,7 @@ Lenses: CLI: -- Write commands rely on the kernel's exclusive lock; `--force` removes a lock only when its holder is not verifiably alive, and never deletes a file that is not a libredb lock. +- Write commands rely on the kernel's exclusive lock; `--force` removes a lock only when its holder is not verifiably alive, and never deletes a file that is not a libredb lock. Automatic reclaim is stricter still: only a lock whose holder is VERIFIABLY dead (same host, pid gone) is reclaimed without `--force` — anonymous locks (empty, or the sentinel-only 0.1.x format) carry no liveness information and now require `--force`. - `get`/`scan` escape control characters (including tab and newline) by default so untrusted values cannot inject terminal escape sequences — scripts that consumed values verbatim should pass `--raw`. New exports: `LibreDbError`, `ErrorCode`, `RecoveryInfo`, `nodeFileSystem`, and `readonlyFileSystem` (open a database for inspection with no lock and no writes — the supported way to read a file a live writer holds). diff --git a/.size-limit.json b/.size-limit.json index f940e8a..2643d04 100644 --- a/.size-limit.json +++ b/.size-limit.json @@ -8,7 +8,7 @@ "node:os", "node:path" ], - "limit": "5 kB" + "limit": "6 kB" }, { "name": "browser entry (min+brotli)", diff --git a/src/adapter/node-fs.test.ts b/src/adapter/node-fs.test.ts index b410bbf..2447af2 100644 --- a/src/adapter/node-fs.test.ts +++ b/src/adapter/node-fs.test.ts @@ -114,14 +114,36 @@ test("a stale lock (dead pid) is reclaimed and locking proceeds", () => { release(); }); -test("a legacy sentinel-only lock (no owner recorded) is reclaimed", () => { +test("a legacy sentinel-only lock (no owner recorded) is NOT auto-reclaimed", () => { + // It may belong to a live 0.1.x CLI (that format recorded no pid), or be a + // concurrent lock() caught between create and write. No liveness info means + // no automatic stealing; forceUnlock is the explicit escape hatch. const path = tempPath("db"); writeFileSync(`${path}.lock`, `${LOCK_SENTINEL}\n`); // v0.1.x CLI format - const release = nodeFileSystem().lock?.(path) as () => void; - release(); + let caught: unknown; + try { + nodeFileSystem().lock?.(path); + } catch (error) { + caught = error; + } + expect((caught as LibreDbError).code).toBe("LOCKED"); + forceUnlock(path); expect(existsSync(`${path}.lock`)).toBe(false); }); +test("a file that merely starts with the sentinel text is foreign, not a lock", () => { + const path = tempPath("db"); + writeFileSync(`${path}.lock`, `${LOCK_SENTINEL}smith\ndata\n`); // "libredb-locksmith..." + let caught: unknown; + try { + forceUnlock(path); // must refuse: exact first-line match required + } catch (error) { + caught = error; + } + expect((caught as LibreDbError).code).toBe("LOCKED"); + expect(readFileSync(`${path}.lock`, "utf8")).toBe(`${LOCK_SENTINEL}smith\ndata\n`); +}); + test("a lock held on another host is not reclaimed (liveness unverifiable)", () => { const path = tempPath("db"); writeFileSync(`${path}.lock`, otherHostLock()); @@ -184,15 +206,17 @@ test("isStaleLock: a vanished lock file counts as stale (retryable)", () => { expect(isStaleLock(join(tmpdir(), "libredb-vanished-xyz.lock"))).toBe(true); }); -test("isStaleLock: empty stray, dead holder, and legacy sentinel are stale; live and foreign are not", () => { +test("isStaleLock: ONLY a verifiably dead holder is stale", () => { const path = tempPath("db"); const lockPath = `${path}.lock`; - writeFileSync(lockPath, ""); - expect(isStaleLock(lockPath)).toBe(true); writeFileSync(lockPath, deadLock()); - expect(isStaleLock(lockPath)).toBe(true); + expect(isStaleLock(lockPath)).toBe(true); // same host, pid verifiably gone + // Everything without verified death is non-stale: anonymity carries no + // liveness info (and could be a concurrent create-then-write in flight). + writeFileSync(lockPath, ""); + expect(isStaleLock(lockPath)).toBe(false); writeFileSync(lockPath, `${LOCK_SENTINEL}\n`); - expect(isStaleLock(lockPath)).toBe(true); + expect(isStaleLock(lockPath)).toBe(false); writeFileSync(lockPath, liveLock()); expect(isStaleLock(lockPath)).toBe(false); writeFileSync(lockPath, otherHostLock()); diff --git a/src/adapter/node-fs.ts b/src/adapter/node-fs.ts index 311ecda..736bb49 100644 --- a/src/adapter/node-fs.ts +++ b/src/adapter/node-fs.ts @@ -50,12 +50,14 @@ interface LockOwner { } /** Parse a lock file's contents. Returns undefined for an empty file or a - * legacy sentinel-only lock (both are LibreDB strays with no liveness info) and - * throws nothing — a FOREIGN file (no sentinel) returns null. */ + * legacy sentinel-only lock (both are LibreDB artifacts with no liveness info) + * and throws nothing — a FOREIGN file returns null. The sentinel test is an + * EXACT first-line match: a file that merely begins with the sentinel text + * ("libredb-locksmith...") is foreign, not ours. */ function parseLock(contents: string): LockOwner | undefined | null { - if (contents === "") return undefined; // a crashed create left an empty file - if (!contents.startsWith(LOCK_SENTINEL)) return null; // not ours - const [, pid, host, nonce] = contents.split("\n"); + if (contents === "") return undefined; // a create interrupted before the write + const [first, pid, host, nonce] = contents.split("\n"); + if (first !== LOCK_SENTINEL) return null; // not ours if (pid === undefined || host === undefined || nonce === undefined || nonce === "") { return undefined; // sentinel-only legacy lock: ours, but anonymous } @@ -77,22 +79,27 @@ function livenessOf(owner: LockOwner): "alive" | "dead" | "unverifiable" { } /** - * Is the lock at `lockPath` stale — held by a LibreDB process that verifiably - * no longer exists? Foreign files are never stale (they are not locks to - * steal), and a holder that cannot be probed (another host) counts as live: - * auto-reclaim must never race a writer that might still be running. `--force` - * (see {@link forceUnlock}) is the explicit escape hatch for that case. + * Is the lock at `lockPath` stale — held by a LibreDB process that VERIFIABLY + * no longer exists? Everything short of verified-dead is non-stale on purpose: + * a foreign file is not a lock to steal; a holder on another host cannot be + * probed and might be running; an anonymous lock (empty, or the sentinel-only + * v0.1.x format) carries no liveness info — it may even be a concurrent + * lock() between its exclusive create and its sentinel write, so auto- + * reclaiming it could admit two live writers. A read failure other than + * ENOENT (permissions, IO) is a real problem to surface, not staleness. + * `--force` (see {@link forceUnlock}) is the explicit escape hatch for the + * unverifiable cases, with the risk on the human who invoked it. */ export function isStaleLock(lockPath: string): boolean { let contents: string; try { contents = readFileSync(lockPath, "utf8"); - } catch { - return true; // vanished between the failed create and this read: retry + } catch (error) { + // ENOENT: it vanished between the failed create and this read — retry. + return (error as { code?: string }).code === "ENOENT"; } const owner = parseLock(contents); - if (owner === null) return false; // foreign file: refuse to touch it - if (owner === undefined) return true; // anonymous LibreDB stray: reclaim it + if (owner === null || owner === undefined) return false; return livenessOf(owner) === "dead"; } @@ -107,7 +114,12 @@ function tryCreateLock(lockPath: string, contents: string): boolean { return false; } try { - writeSync(fd, contents); + // Loop: writeSync may legally write fewer bytes than asked, and a partial + // sentinel would read as an anonymous stray instead of this holder's lock. + const bytes = Buffer.from(contents, "utf8"); + for (let written = 0; written < bytes.length; ) { + written += writeSync(fd, bytes, written); + } } finally { closeSync(fd); } @@ -264,14 +276,16 @@ export function nodeFileSystem(): FileSystem { if ( claimAndRemoveLock(lockPath, (c) => { const owner = parseLock(c); - if (owner === null) throw new Error("foreign file"); - if (owner !== undefined && livenessOf(owner) === "alive") throw new Error("holder alive"); + // Only a VERIFIED-dead holder is auto-reclaimed; anything else — + // foreign bytes, an anonymous lock, an unverifiable host — stays. + if (owner === null || owner === undefined) throw new Error("not verifiably stale"); + if (livenessOf(owner) !== "dead") throw new Error("holder not verifiably dead"); }) === "gone" ) { continue; // another racer reclaimed it; retry the exclusive create } } catch { - break; // a live or foreign lock appeared: locked + break; // the lock is not verifiably stale after all: locked } } throw new LibreDbError( diff --git a/src/cli/run.test.ts b/src/cli/run.test.ts index 8e294ca..a5b266e 100644 --- a/src/cli/run.test.ts +++ b/src/cli/run.test.ts @@ -253,16 +253,28 @@ test("a write refuses when a live writer holds the lock", () => { expect(r.err.join("\n")).toMatch(/locked/i); }); -test("a stale lock (dead holder) is reclaimed automatically, no --force needed", () => { +test("a stale lock (verifiably dead holder) is reclaimed automatically, no --force needed", () => { const path = fixture(); - // A crashed writer's leftover: an empty lock file carries no live holder. - writeFileSync(`${path}.lock`, ""); + // A crashed writer's leftover, naming a pid above every default pid_max. + writeFileSync(`${path}.lock`, `${LOCK_SENTINEL}\n4194304\n${hostname()}\nnonce\n`); const r = cli("set", path, "k", "v"); expect(r.code).toBe(0); expect(cli("get", path, "k").out).toEqual(["v"]); expect(existsSync(`${path}.lock`)).toBe(false); }); +test("an anonymous (empty) lock is NOT auto-reclaimed; --force removes it", () => { + const path = fixture(); + // An empty lock carries no liveness info — it may even be a concurrent + // writer between its exclusive create and its sentinel write, so stealing + // it automatically could admit two live writers. + writeFileSync(`${path}.lock`, ""); + expect(cli("set", path, "k", "v").code).toBe(1); // locked + const forced = cli("set", path, "k", "v", "--force"); + expect(forced.code).toBe(0); + expect(cli("get", path, "k").out).toEqual(["v"]); +}); + test("--force refuses to remove a live holder's lock", () => { const path = fixture(); writeFileSync(`${path}.lock`, liveLock()); diff --git a/src/core.hardening.test.ts b/src/core.hardening.test.ts index d2ea15a..fdaefd0 100644 --- a/src/core.hardening.test.ts +++ b/src/core.hardening.test.ts @@ -293,17 +293,18 @@ test("a file written by a NEWER format version is refused with UNSUPPORTED_VERSI expect(errorFrom(() => openNode({ path })).code).toBe("UNSUPPORTED_VERSION"); }); -test("a header torn mid-write (first commit interrupted) restarts the database from empty", () => { - const path = tempPath("torn-header"); - writeFileSync(path, Uint8Array.from([0x4c, 0x52])); // "LR": a magic prefix, cut short - const truncations: number[] = []; - const db = openNode({ path, onRecovery: (info) => truncations.push(info.truncatedBytes) }); - expect(truncations).toEqual([2]); // the torn header was reported, not silent - db.transact((tx) => tx.set(bytes(1), bytes(10))); - db.close(); - const reopened = openNode({ path }); - expect(reopened.transact((tx) => tx.get(bytes(1)))).toEqual(bytes(10)); - reopened.close(); +test("a file shorter than the header is refused untouched, even when it shares magic bytes", () => { + // "LR" might be the prefix a torn first-ever append left, or a 2-byte + // foreign file. Identity is ambiguous, so recovery refuses instead of + // adopting (and truncating) the file — the cost is that a crash inside the + // first 8 bytes of a brand-new database's first commit needs a manual + // delete, and nothing in that file was ever acknowledged. + for (const contents of [[0x4c], [0x4c, 0x52], [0x4c, 0x52, 0x44, 0x42], [0x61, 0x62]]) { + const path = tempPath(`short-${contents.length}-${contents[0]}`); + writeFileSync(path, Uint8Array.from(contents)); + expect(errorFrom(() => openNode({ path })).code).toBe("NOT_A_DATABASE"); + expect([...new Uint8Array(readFileSync(path))]).toEqual(contents); // untouched + } }); // --- issue #22: corruption classification --- @@ -383,7 +384,10 @@ test("a partially-written append latches the database; the torn tail cannot pois db.transact((tx) => tx.set(bytes(1), bytes(10))); // commit A: durable file.failAppendAfter = 5; // commit B tears after 5 bytes, then ENOSPC - expect(() => db.transact((tx) => tx.set(bytes(2), bytes(20)))).toThrow(/ENOSPC/); + const commitError = errorFrom(() => db.transact((tx) => tx.set(bytes(2), bytes(20)))); + expect(commitError.code).toBe("FAILED"); // typed, with the adapter error as cause + expect(commitError.message).toMatch(/ENOSPC/); + expect((commitError.cause as Error).message).toMatch(/ENOSPC/); // The database is latched: it refuses commit C outright instead of appending // it after the torn bytes (where the next recovery would destroy it). diff --git a/src/core.ts b/src/core.ts index 44264f2..5d7f3c7 100644 --- a/src/core.ts +++ b/src/core.ts @@ -672,23 +672,24 @@ function recover(file: WalFile): Recovery { } if (size === 0) return { entries: [], needsHeader: true, legacy: false, truncatedBytes: 0 }; - // A torn first append can only leave a PREFIX of the exact 8 header bytes - // this kernel writes (magic + version + reserved), so the recognition test - // compares against all of them — "LRDB" followed by anything else is a - // foreign file, not a torn header. - const expectedHeader = encodeFileHeader(); - const headerPrefix = Math.min(log.length, FILE_HEADER); - const hasHeaderPrefix = log.subarray(0, headerPrefix).every((byte, i) => byte === expectedHeader[i]); - const hasMagic = log.length >= MAGIC.length && MAGIC.every((byte, i) => byte === log[i]); - if (hasHeaderPrefix && log.length < FILE_HEADER) { - // A torn header: the very first commit (header + record in one append) - // was interrupted before the header finished. Nothing was ever - // acknowledged, so start the database over from empty. - file.truncate(0); - file.fsync(); - return { entries: [], needsHeader: true, legacy: false, truncatedBytes: log.length }; + // A file shorter than the 8-byte header cannot be identified: it might be + // the prefix a torn first-ever append left ("LR..."), or it might be a tiny + // foreign file that happens to share those bytes. When identity is + // ambiguous, destroying is never the answer — refuse, untouched. (The cost + // is that a crash inside the first 8 bytes of a database's first-ever + // commit needs the user to delete the file by hand; nothing in it was ever + // acknowledged.) + if (log.length < FILE_HEADER) { + throw new LibreDbError("NOT_A_DATABASE", "file is too short to be a libredb database; refusing to touch it"); } - if (hasMagic && log.length >= FILE_HEADER) { + // The 4-byte magic selects the v1 path. The version bytes then gate it + // further: a foreign file that begins with "LRDB" but carries junk where + // the version belongs is refused as UNSUPPORTED_VERSION — also untouched. + // (A foreign file matching the ENTIRE 8-byte header is byte-for-byte + // indistinguishable from a real empty database; no recognizer can separate + // identical bytes.) + const hasMagic = MAGIC.every((byte, i) => byte === log[i]); + if (hasMagic) { const fileVersion = ((log[4] as number) << 8) | (log[5] as number); if (fileVersion !== FORMAT_VERSION) { throw new LibreDbError( @@ -904,7 +905,15 @@ export const open: Open = (options) => { log.append(journal); } catch (error) { failed = true; - throw error; + // Typed like every other kernel failure — callers branch on the + // stable FAILED code; the adapter's error rides along as `cause` + // (and in the message, for logs that only capture text). + throw new LibreDbError( + "FAILED", + `commit failed to reach the disk (${error instanceof Error ? error.message : String(error)}); ` + + `close and reopen to recover`, + { cause: error }, + ); } } committed = working; diff --git a/src/lens/catalog.ts b/src/lens/catalog.ts index bcc5d3f..1caba57 100644 --- a/src/lens/catalog.ts +++ b/src/lens/catalog.ts @@ -53,14 +53,31 @@ export const CATALOG_PREFIX: string = `${RESERVED_MARKER}libredb:catalog:`; * true. Shared by the kv lens (keys) and the document lens (ids and names). */ export function assertWellFormedText(text: string, what: string): void { - if (!text.isWellFormed()) { - throw new LibreDbError( - "INVALID_ARGUMENT", - `${what} ${JSON.stringify(text)} contains a lone surrogate and cannot round-trip through UTF-8`, - ); + // A hand-rolled scan instead of String.prototype.isWellFormed(): this file + // ships in the browser entry, and isWellFormed is too new to assume on every + // engine the bundle may reach. The scan is the same O(n) as the UTF-8 encode + // that follows it. + for (let i = 0; i < text.length; i++) { + const unit = text.charCodeAt(i); + if (unit >= 0xd800 && unit <= 0xdbff) { + const next = text.charCodeAt(i + 1); // NaN at end-of-string: fails the test below + if (!(next >= 0xdc00 && next <= 0xdfff)) { + throw wellFormednessError(text, what); // high surrogate with no low mate + } + i++; // a valid pair: skip its low half + } else if (unit >= 0xdc00 && unit <= 0xdfff) { + throw wellFormednessError(text, what); // low surrogate with no high mate + } } } +function wellFormednessError(text: string, what: string): LibreDbError { + return new LibreDbError( + "INVALID_ARGUMENT", + `${what} ${JSON.stringify(text)} contains a lone surrogate and cannot round-trip through UTF-8`, + ); +} + /** * Reject a user namespace name the key layout cannot isolate — a loud error, * the same class of correctness rule as the prefix-soundness checks the lenses From 6c21f526b720b296f4437597c062e125069f06a5 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 07:50:53 +0300 Subject: [PATCH 06/16] fix(lens,adapter,ci): resolve second-round Copilot review findings - doc()'s relational-kind guard memoizes only the settled state (name cataloged as document); an uncataloged name keeps re-checking, so a handle built before table() cataloged the name can no longer write around schema validation - parseLock validates the pid field: a sentineled-but-mangled lock downgrades to anonymous (never auto-stale) instead of probing pid=NaN as a dead holder and stealing a live writer's lock - the CI job summary reads the bundle budget from .size-limit.json instead of hardcoding a stale number --- .github/workflows/ci.yml | 3 ++- src/adapter/node-fs.test.ts | 22 ++++++++++++++++++++++ src/adapter/node-fs.ts | 8 +++++++- src/lens/document.ts | 11 +++++++++-- src/lens/hardening.test.ts | 11 +++++++++++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f3ed9a..3ac3c2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: run: | BYTES=$(bun run size --json 2>/dev/null | jq -r '.[0].size // empty' || true) if [ -n "$BYTES" ]; then KB=$(awk "BEGIN{printf \"%.2f\", $BYTES/1024}"); else KB="n/a"; fi + BUDGET=$(jq -r '.[0].limit' .size-limit.json) { echo "## LibreDB CI" echo "" @@ -60,7 +61,7 @@ jobs: echo "| License tripwire (runtime deps) | ${{ steps.license.outcome }} |" echo "" echo "- Coverage: 100% line/function/statement (enforced by bunfig.toml)" - echo "- Bundle: ${KB} kB (min+brotli) / 5 kB budget" + echo "- Bundle: ${KB} kB (min+brotli) / ${BUDGET} budget" } >> "$GITHUB_STEP_SUMMARY" node-smoke: diff --git a/src/adapter/node-fs.test.ts b/src/adapter/node-fs.test.ts index 2447af2..549911d 100644 --- a/src/adapter/node-fs.test.ts +++ b/src/adapter/node-fs.test.ts @@ -131,6 +131,28 @@ test("a legacy sentinel-only lock (no owner recorded) is NOT auto-reclaimed", () expect(existsSync(`${path}.lock`)).toBe(false); }); +test("a sentineled lock with a mangled pid is anonymous, never auto-stale", () => { + // Corruption that keeps the sentinel line but garbles the owner must not + // read as a dead holder: pid=NaN would probe as "dead" and let a LIVE + // holder's lock be auto-reclaimed. Unparseable owner info means no + // liveness info — locked until --force. + const path = tempPath("db"); + const lockPath = `${path}.lock`; + for (const mangled of ["garbage", "-5", "3.14", ""]) { + writeFileSync(lockPath, `${LOCK_SENTINEL}\n${mangled}\n${hostname()}\nnonce\n`); + expect(isStaleLock(lockPath)).toBe(false); + } + let caught: unknown; + try { + nodeFileSystem().lock?.(path); + } catch (error) { + caught = error; + } + expect((caught as LibreDbError).code).toBe("LOCKED"); + forceUnlock(path); // the explicit escape hatch still works + expect(existsSync(lockPath)).toBe(false); +}); + test("a file that merely starts with the sentinel text is foreign, not a lock", () => { const path = tempPath("db"); writeFileSync(`${path}.lock`, `${LOCK_SENTINEL}smith\ndata\n`); // "libredb-locksmith..." diff --git a/src/adapter/node-fs.ts b/src/adapter/node-fs.ts index 736bb49..415d1dc 100644 --- a/src/adapter/node-fs.ts +++ b/src/adapter/node-fs.ts @@ -61,7 +61,13 @@ function parseLock(contents: string): LockOwner | undefined | null { if (pid === undefined || host === undefined || nonce === undefined || nonce === "") { return undefined; // sentinel-only legacy lock: ours, but anonymous } - return { pid: Number(pid), host, nonce }; + // The pid must be a real process id. A sentineled-but-mangled lock (partial + // overwrite, corruption) would otherwise carry pid=NaN, which the liveness + // probe reads as "dead" — and a LIVE holder's lock would be auto-reclaimed. + // Unparseable owner info downgrades to anonymous: never auto-stale. + const parsedPid = Number(pid); + if (!Number.isInteger(parsedPid) || parsedPid <= 0) return undefined; + return { pid: parsedPid, host, nonce }; } /** Can the process behind `owner` be probed on THIS host, and is it alive? diff --git a/src/lens/document.ts b/src/lens/document.ts index 691cd78..525a6f3 100644 --- a/src/lens/document.ts +++ b/src/lens/document.ts @@ -194,13 +194,20 @@ export function doc(store: Store, collection: string): DocCollection { let checked = false; const ensure = (read: (key: Uint8Array) => Uint8Array | undefined): void => { if (checked) return; - if (catalogKindAt(read, collection) === "relational") { + const kind = catalogKindAt(read, collection); + if (kind === "relational") { throw new LibreDbError( "INVALID_ARGUMENT", `${JSON.stringify(collection)} is a relational table; use table() instead of doc()`, ); } - checked = true; + // Memoize ONLY the settled state. Once cataloged as a document collection + // the name can never become relational (recordRelational refuses a name of + // another kind), so the check is done for good. An UNCATALOGED name must + // keep re-checking: a later table() could catalog it as relational, and a + // handle whose guard went quiet on a stale "uncataloged" answer would + // write around that table's schema validation. + if (kind === "document") checked = true; }; return collectionHandle(store, collection, ensure); } diff --git a/src/lens/hardening.test.ts b/src/lens/hardening.test.ts index 0920594..97a03e1 100644 --- a/src/lens/hardening.test.ts +++ b/src/lens/hardening.test.ts @@ -118,6 +118,17 @@ test("doc() operations refuse a name cataloged as a relational table", () => { db.close(); }); +test("a doc() handle built BEFORE the name became relational still refuses (no stale memoization)", () => { + const db = open(); + const handle = doc(db, "ledger"); + expect(handle.get("x")).toBeUndefined(); // used while uncataloged: guard must stay live + table(db, "ledger", SCHEMA).insert({ id: "l1", n: 1 }); // now cataloged relational + expect(errorFrom(() => handle.put("l2", { rogue: true })).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => handle.get("l1")).code).toBe("INVALID_ARGUMENT"); + expect(table(db, "ledger", SCHEMA).get("l2")).toBeUndefined(); // nothing slipped past the schema + db.close(); +}); + test("table() refuses a name cataloged as a document collection", () => { const db = open(); doc(db, "notes").put("n1", { text: "hi" }); From 544b2577736ad9ddd8f180f9b9b99aae24a049ac Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 07:57:32 +0300 Subject: [PATCH 07/16] test(dst): scale the seeded-loop timeout with LIBREDB_DST_SEEDS A 10000-seed soak takes ~6s and tripped bun's default 5s per-test timeout, reporting an invariant failure that was actually the clock. The invariant itself holds across all 10000 seeds (verified standalone, both assertions). The timeout now scales with the seed count so soaks fail only on a broken invariant. --- src/sim/dst.test.ts | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/sim/dst.test.ts b/src/sim/dst.test.ts index e6fa491..3600ca2 100644 --- a/src/sim/dst.test.ts +++ b/src/sim/dst.test.ts @@ -45,16 +45,23 @@ const WAL = "wal"; const SEED_BASE = Number(process.env.LIBREDB_DST_BASE ?? 0) || 0; const SEED_COUNT = Number(process.env.LIBREDB_DST_SEEDS ?? 50) || 50; -test(`the crash/recovery invariant holds across ${SEED_COUNT} seeds`, () => { - for (let i = 0; i < SEED_COUNT; i++) { - const result = runSeed(SEED_BASE + i); - // A clean crash loses only the un-fsync'd torn tail, so the recovered state - // must equal the FULL committed model, and must be a valid committed prefix. - expect(result.passed, describeFailure(result)).toBe(true); - const states = committedPrefixStates(generateWorkload(result.seed)); - expect(isCommittedPrefix(result.recovered, states)).toBe(true); - } -}); +test( + `the crash/recovery invariant holds across ${SEED_COUNT} seeds`, + () => { + for (let i = 0; i < SEED_COUNT; i++) { + const result = runSeed(SEED_BASE + i); + // A clean crash loses only the un-fsync'd torn tail, so the recovered state + // must equal the FULL committed model, and must be a valid committed prefix. + expect(result.passed, describeFailure(result)).toBe(true); + const states = committedPrefixStates(generateWorkload(result.seed)); + expect(isCommittedPrefix(result.recovered, states)).toBe(true); + } + }, + // Scale the per-test timeout with the seed count: the default 50 seeds run in + // milliseconds, but a LIBREDB_DST_SEEDS=10000 soak takes seconds — it must + // fail on a broken invariant, never on bun's default 5s clock. + Math.max(5_000, SEED_COUNT * 5), +); test("runSeed is deterministic: the same seed yields the same recovery", () => { const a = runSeed(424242); From 67d89deb68f4c258cccae631573a5786d4c3b7dd Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:08:35 +0300 Subject: [PATCH 08/16] fix(lens,kernel,cli): close Codex review findings on read-path validation and cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - doc get/delete (and the relational paths through them) validate the id like put already did: a lone-surrogate id encodes to the replacement character's bytes and would silently read — or delete — a document legitimately stored under that id - a throwing onRecovery callback no longer leaks the WAL file handle: the callback runs inside the same guard that closes the file on a refused open - CLI inspect escapes control characters in namespace names (the lens validator rejects ':' and surrogates but control bytes are legal name characters); --raw opts out, matching get/scan --- src/cli/run.test.ts | 17 +++++++++++++++++ src/cli/run.ts | 8 ++++++-- src/core.hardening.test.ts | 36 ++++++++++++++++++++++++++++++++++++ src/core.ts | 9 ++++++--- src/lens/document.ts | 6 ++++++ src/lens/hardening.test.ts | 23 +++++++++++++++++++++++ 6 files changed, 94 insertions(+), 5 deletions(-) diff --git a/src/cli/run.test.ts b/src/cli/run.test.ts index a5b266e..337be91 100644 --- a/src/cli/run.test.ts +++ b/src/cli/run.test.ts @@ -331,6 +331,23 @@ test("get and scan escape control characters so stored data cannot drive the ter expect(scanned.out).toEqual(["evil=\\x1b[2Jcleared\\x07bell"]); }); +test("inspect escapes control characters in namespace names", () => { + const dir = mkdtempSync(join(tmpdir(), "libredb-cli-")); + dirs.push(dir); + const path = join(dir, "evil.libredb"); + const db = open({ path }); + // The lens validator rejects ":" and surrogates, but control characters are + // legal name bytes — so inspect must escape them on the way to a terminal. + doc(db, "evil\u001b[2Jns").put("d1", {}); + db.close(); + const r = cli("inspect", path); + expect(r.code).toBe(0); + expect(r.out.join("\n")).toContain("evil\\x1b[2Jns"); + expect(r.out.join("\n")).not.toContain("\u001b"); + // --raw opts out, matching get/scan. + expect(cli("inspect", path, "--raw").out.join("\n")).toContain("evil\u001b[2Jns"); +}); + test("--raw prints the stored bytes verbatim for callers that want them", () => { const path = fixture(); cli("set", path, "evil", "\u001b[31mred"); diff --git a/src/cli/run.ts b/src/cli/run.ts index ee97c1d..5e95c95 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -104,7 +104,7 @@ const withWriteDb = (path: string, force: boolean, fn: (db: Database) => T): } }; -function inspect({ path, io }: Ctx): number { +function inspect({ path, io, raw }: Ctx): number { return withReadDb(path, (db) => { const registry = catalog(db); io.out(`${path} ${statSync(path).size} bytes`); @@ -113,8 +113,12 @@ function inspect({ path, io }: Ctx): number { return 0; } for (const [name, entry] of registry) { + // Namespace names are user data too: the lens validator rejects ":" and + // surrogates but not control characters, so a name could otherwise carry + // terminal escapes into whoever inspects the file. Schemas are safe as + // JSON.stringify output (it escapes control characters itself). const schema = entry.schema === undefined ? "" : ` ${JSON.stringify(entry.schema)}`; - io.out(` ${name} ${entry.kind}${schema}`); + io.out(` ${sanitize(name, raw)} ${entry.kind}${schema}`); } return 0; }); diff --git a/src/core.hardening.test.ts b/src/core.hardening.test.ts index fdaefd0..a11750c 100644 --- a/src/core.hardening.test.ts +++ b/src/core.hardening.test.ts @@ -344,6 +344,42 @@ test("a CRC-valid record with a malformed payload is corruption, not ops", () => expect(errorFrom(() => openNode({ path: path2 })).code).toBe("CORRUPT_WAL"); }); +test("a throwing onRecovery callback does not leak the WAL file handle", () => { + const { fs, file } = faultFs(); + let closed = 0; + const countingFs: FileSystem = { + open(path) { + const inner = fs.open(path); + return { + ...inner, + close() { + closed++; + inner.close(); + }, + }; + }, + }; + // A database with a torn tail, so reopening will fire onRecovery. + const db = open({ path: "wal", fs: countingFs }); + db.transact((tx) => tx.set(bytes(1), bytes(10))); + db.close(); + file.data.push(0xff, 0xff, 0xff); // torn fragment shorter than a record header + expect(closed).toBe(1); + + expect(() => + open({ + path: "wal", + fs: countingFs, + onRecovery: () => { + throw new Error("user callback exploded"); + }, + }), + ).toThrow(/user callback exploded/); + // The refused open still closed its file handle (and the truncation it + // performed before the callback fired remains applied — it was fsync'd). + expect(closed).toBe(2); +}); + test("recovery reports a torn tail through onRecovery instead of dropping it silently", () => { const path = tempPath("reported"); const db = openNode({ path }); diff --git a/src/core.ts b/src/core.ts index 5d7f3c7..4bd4aa2 100644 --- a/src/core.ts +++ b/src/core.ts @@ -773,17 +773,20 @@ function openLog( let recovery: Recovery; try { recovery = recover(file); + // Inside the same guard as recover(): the callback is user code, and a + // throw from it must not leak the file handle either. + if (recovery.truncatedBytes > 0) onRecovery?.({ truncatedBytes: recovery.truncatedBytes }); } catch (error) { // A refused open (foreign file, corruption, unsupported version, short - // read) must not leak the file handle; the refusal error stays primary. + // read, a throwing onRecovery) must not leak the file handle; the + // original error stays primary. try { file.close(); } catch { - // The close failed after recovery already failed; surface the original. + // The close failed after the open already failed; surface the original. } throw error; } - if (recovery.truncatedBytes > 0) onRecovery?.({ truncatedBytes: recovery.truncatedBytes }); let needsHeader = recovery.needsHeader; return { entries: recovery.entries, diff --git a/src/lens/document.ts b/src/lens/document.ts index 525a6f3..3f11588 100644 --- a/src/lens/document.ts +++ b/src/lens/document.ts @@ -274,6 +274,11 @@ export function collectionHandle( return { changed: 1 }; }, get(id) { + // Validated like put(): a lone-surrogate id encodes to the replacement + // character's bytes, which would silently ALIAS a document legitimately + // stored under "\ufffd" — reading (and below, deleting) someone else's + // document instead of failing loudly. + assertWellFormedText(id, "document id"); return store.transact((tx) => { ensure?.((key) => tx.get(key)); const bytes = tx.get(keyOf(collection, id)); @@ -283,6 +288,7 @@ export function collectionHandle( delete(id) { // Read-before-delete in one transaction: the kernel's delete is a silent // no-op on a missing key, so this is how the lens tells 1 from 0 changes. + assertWellFormedText(id, "document id"); // same aliasing hazard as get() const changed = store.transact((tx) => { ensure?.((key) => tx.get(key)); const k = keyOf(collection, id); diff --git a/src/lens/hardening.test.ts b/src/lens/hardening.test.ts index 97a03e1..2f58833 100644 --- a/src/lens/hardening.test.ts +++ b/src/lens/hardening.test.ts @@ -177,6 +177,29 @@ test("document ids and namespace names with a lone surrogate are rejected", () = db.close(); }); +test("get and delete reject a lone-surrogate id too — a malformed id must not alias a real document", () => { + const db = open(); + const users = doc(db, "users"); + // A document stored under the REPLACEMENT CHARACTER id is perfectly legal. + // A malformed id encodes to those same bytes, so without validation on the + // read/delete paths it would silently read — or destroy — this document. + users.put("id-�", { owner: "legitimate" }); + expect(errorFrom(() => users.get("id-\ud800")).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => users.delete("id-\ud800")).code).toBe("INVALID_ARGUMENT"); + expect(users.get("id-�")).toEqual({ owner: "legitimate" }); // unharmed + db.close(); +}); + +test("relational get and delete reject a lone-surrogate primary key the same way", () => { + const db = open(); + const t = table(db, "rows", SCHEMA); + t.insert({ id: "pk-�", n: 1 }); + expect(errorFrom(() => t.get("pk-\ud800")).code).toBe("INVALID_ARGUMENT"); + expect(errorFrom(() => t.delete("pk-\ud800")).code).toBe("INVALID_ARGUMENT"); + expect(t.get("pk-�")).toEqual({ id: "pk-�", n: 1 }); // unharmed + db.close(); +}); + test("well-formed non-ASCII keys, ids, and values round-trip exactly", () => { const db = open(); const store = kv(db); From c6b72c3d85c70315dc986eef687f822f56b2b624 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:17:17 +0300 Subject: [PATCH 09/16] chore(cli,test,ci): apply Kimi review nits and add a binary smoke to CI - the control-regex suppression in sanitize() targets oxlint by name (only oxlint's rule fires there; the eslint-disable form was an unused directive for eslint while still being consumed by oxlint) - types.test.ts builds comparison arrays with Array.from().concat() instead of map-spread, clearing the last standing lint warning - the node-smoke CI job also compiles the standalone binary and runs a set/get round-trip: previously a compile regression would first surface during publish, after npm publish had already succeeded --- .github/workflows/ci.yml | 10 ++++++++++ src/cli/run.ts | 5 ++++- src/lens/types.test.ts | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ac3c2a..6abd495 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,3 +84,13 @@ jobs: - run: bun run build - name: Smoke-test the built package under Node run: node scripts/node-smoke.mjs + # The standalone binary is otherwise only compiled during publish (and + # only after npm publish succeeded), so a compile regression would first + # surface at release time. One cheap compile + round-trip here keeps it + # honest on every PR. + - name: Smoke-test the compiled standalone binary + run: | + bun build --compile src/cli/main.ts --outfile /tmp/libredb-smoke + /tmp/libredb-smoke set /tmp/smoke.libredb greeting hello + test "$(/tmp/libredb-smoke get /tmp/smoke.libredb greeting)" = "hello" + /tmp/libredb-smoke stats /tmp/smoke.libredb diff --git a/src/cli/run.ts b/src/cli/run.ts index 5e95c95..12f05ce 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -49,7 +49,10 @@ interface Ctx { */ function sanitize(text: string, raw: boolean): string { if (raw) return text; - // eslint-disable-next-line no-control-regex + // Control characters in this regex are the entire point (they are what gets + // escaped). Only oxlint's no-control-regex fires here, so the suppression + // targets it specifically — an eslint-disable would be an unused directive. + // oxlint-disable-next-line no-control-regex return text.replace(/[\u0000-\u001f\u007f-\u009f]/g, (c) => `\\x${c.charCodeAt(0).toString(16).padStart(2, "0")}`); } diff --git a/src/lens/types.test.ts b/src/lens/types.test.ts index 48ac016..66ee6df 100644 --- a/src/lens/types.test.ts +++ b/src/lens/types.test.ts @@ -52,7 +52,7 @@ describe("Result", () => { }); db.close(); - expect(rows.map((e) => [...e.key, ...e.value])).toEqual([ + expect(rows.map((e) => Array.from(e.key).concat(Array.from(e.value)))).toEqual([ [1, 10], [2, 20], ]); From 567240c25c8e64063925c9013517979a55907a6e Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:20:00 +0300 Subject: [PATCH 10/16] docs(kernel): state getRange snapshot cost precisely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment claimed an early-exited scan pays only for consumed entries; in fact the reference walk over the matching range is eager at first next() — only the per-yield byte copies are lazy. Say so. --- src/core.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/core.ts b/src/core.ts index 4bd4aa2..ba04a50 100644 --- a/src/core.ts +++ b/src/core.ts @@ -379,9 +379,11 @@ function makeTransaction(working: StoredEntry[], journal: Op[]): Transaction { // generator body runs at the first next()). Walking the live array by // index instead would let a delete-during-scan shift entries under the // cursor and silently skip them — the classic delete-while-scanning bug. - // Entries are immutable once stored, so holding references is safe; the - // byte copies happen per-yield, so an early-exited scan pays only for - // the entries it consumed. + // Entries are immutable once stored, so holding references is safe. + // Cost, precisely: the reference walk is O(matching range) and runs + // up-front at the first next(); only the BYTE COPIES are per-yield, so + // an early-exited scan skips the copies (usually the dominant cost) but + // not the walk. // locate() returns the first index whose key is >= start (the insertion // point), so the scan is naturally inclusive of start. It stops at the // first key that is not < end, making the range half-open [start, end). From 24177ac2717670e1e705d9c69d0a409af40e81a8 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:29:16 +0300 Subject: [PATCH 11/16] fix(cli,browser,docs): close GPT review findings on import validation and stale claims - CLI import enforces the kv lens's well-formed string invariant: a JSON file whose \uD800-style escapes decode to lone surrogates was previously written through the kernel directly, where two distinct malformed keys collide on the same UTF-8 bytes - BrowserOpenOptions carries onRecovery, so the browser type surface matches the runtime (RecoveryInfo was exported but the option was not expressible); pinned by a compile-level test - README line-count claim updated honestly: the kernel is one file of under a thousand lines, roughly half explanatory prose (measured: 449 code / 456 comment); DESIGN.md gains a dated addendum with the same numbers so the decision log tracks reality - the redundant reclaim continue-branch in lock() is folded away (removed and gone both retry the exclusive create), restoring true 100% line coverage across all files (node-fs was at 99.31%) --- README.md | 3 ++- docs/DESIGN.md | 7 +++++++ src/adapter/node-fs.ts | 21 ++++++++++----------- src/browser.test.ts | 31 +++++++++++++++++++++++++++++++ src/browser.ts | 6 +++--- src/cli/run.test.ts | 18 ++++++++++++++++++ src/cli/run.ts | 14 +++++++++++++- 7 files changed, 84 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d87287f..f7fa243 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ learn how a database actually works, and serious enough to grow into more. key-value engine (FoundationDB-style), not three engines bolted together. - **Multi-model** — raw strings, JSON documents, and schema-validated typed tables in the same database, even the same file. -- **Readable by design** — the kernel is under 600 lines; open the source and learn how a database +- **Readable by design** — the kernel is one file of under a thousand lines, roughly half of it + explanatory prose; open the source and learn how a database actually works. - **Embeddable, zero dependencies** — `bun add @libredb/libredb` and go; nothing else to install or run. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 8e9648d..73d816a 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -169,6 +169,13 @@ The real reliability bar (DESIGN principle 4). The first lens shipped on standar change the DST work sanctioned; it is still far under the ~1,000-line starting target. The DST harness (`src/sim/`, 675 lines) is test-only and never ships, so it does not count against the budget. The budget held with headroom. + **Update (2026-07-03, the pre-announcement hardening wave):** `core.ts` now measures 950 physical + lines — but 456 of them are explanatory comments and 45 are blank; the CODE is 449 lines, still + under the original ~500-line shape. The growth is the audit-driven hardening (the `LRDB` file + header and self-checksummed record headers, recovery corruption classification, the IO-failure + latch, the exclusive-lock seam, typed errors) plus the prose that keeps each mechanism teachable. + Comprehension time remains the governing metric; the ~1,000-physical-line starting target still + holds, with the code half comfortably inside the original budget. - **Reliability tooling. (RESOLVED 2026-06-24 — DST built and green.)** The open question "how is deterministic simulation testing actually implemented in a TS project" is now answered by shipped, green code, exactly per the §6.4 locked design: an injectable FS seam in `core.ts` (S1), a seeded diff --git a/src/adapter/node-fs.ts b/src/adapter/node-fs.ts index 415d1dc..f6f00c1 100644 --- a/src/adapter/node-fs.ts +++ b/src/adapter/node-fs.ts @@ -279,17 +279,16 @@ export function nodeFileSystem(): FileSystem { // writer's lock slid in between the check and the claim, it is put // back untouched. try { - if ( - claimAndRemoveLock(lockPath, (c) => { - const owner = parseLock(c); - // Only a VERIFIED-dead holder is auto-reclaimed; anything else — - // foreign bytes, an anonymous lock, an unverifiable host — stays. - if (owner === null || owner === undefined) throw new Error("not verifiably stale"); - if (livenessOf(owner) !== "dead") throw new Error("holder not verifiably dead"); - }) === "gone" - ) { - continue; // another racer reclaimed it; retry the exclusive create - } + // "removed" (we reclaimed it) and "gone" (another racer did) call + // for the same next step: retry the exclusive create, where exactly + // one contender wins. Only a refusal breaks out as locked. + claimAndRemoveLock(lockPath, (c) => { + const owner = parseLock(c); + // Only a VERIFIED-dead holder is auto-reclaimed; anything else — + // foreign bytes, an anonymous lock, an unverifiable host — stays. + if (owner === null || owner === undefined) throw new Error("not verifiably stale"); + if (livenessOf(owner) !== "dead") throw new Error("holder not verifiably dead"); + }); } catch { break; // the lock is not verifiably stale after all: locked } diff --git a/src/browser.test.ts b/src/browser.test.ts index 6ed9070..cd5beac 100644 --- a/src/browser.test.ts +++ b/src/browser.test.ts @@ -131,3 +131,34 @@ test("the node entry's import graph DOES pull in node:fs (the walker discriminat const specifiers = transitiveBareSpecifiers(resolve(import.meta.dir, "index.ts")); expect(specifiers.has("node:fs")).toBe(true); }); + +test("the browser open type accepts an onRecovery callback alongside path+fs", () => { + // The kernel supports onRecovery and the browser entry exports RecoveryInfo; + // the option must therefore be expressible in BrowserOpenOptions — this test + // is primarily a COMPILE-TIME assertion (it would fail typecheck if the + // option were missing from the type), with a runtime pass over a fresh store. + const store: number[] = []; + const memFs = { + open: () => ({ + size: () => store.length, + read: (offset: number, length: number) => Uint8Array.from(store.slice(offset, offset + length)), + append: (b: Uint8Array) => { + for (const byte of b) store.push(byte); + }, + fsync: () => {}, + truncate: (length: number) => { + store.length = length; + }, + close: () => {}, + }), + }; + const reports: number[] = []; + const db = open({ + path: "recovery-typed", + fs: memFs, + onRecovery: (info) => reports.push(info.truncatedBytes), + }); + db.transact((tx) => tx.set(Uint8Array.of(1), Uint8Array.of(10))); + db.close(); + expect(reports).toEqual([]); // a clean open has nothing to report +}); diff --git a/src/browser.ts b/src/browser.ts index 3703e80..a771cb3 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -11,7 +11,7 @@ * this entry is the import graph: it reaches nothing in `node:`, so a bundler * can ship it to a browser. The node:fs adapter lives behind Node only. */ -import { open as openKernel, type Database, type FileSystem } from "./core.ts"; +import { open as openKernel, type Database, type FileSystem, type RecoveryInfo } from "./core.ts"; export { version, LibreDbError } from "./core.ts"; export type { ErrorCode, RecoveryInfo } from "./core.ts"; @@ -30,8 +30,8 @@ export type { Database, FileSystem, WalFile } from "./core.ts"; * needs no filesystem. */ export type BrowserOpenOptions = - | { readonly path: string; readonly fs: FileSystem } - | { readonly path?: never; readonly fs?: FileSystem }; + | { readonly path: string; readonly fs: FileSystem; readonly onRecovery?: (info: RecoveryInfo) => void } + | { readonly path?: never; readonly fs?: FileSystem; readonly onRecovery?: never }; /** * Open a database in the browser. The same runtime as the kernel's `open`, typed diff --git a/src/cli/run.test.ts b/src/cli/run.test.ts index 337be91..b214613 100644 --- a/src/cli/run.test.ts +++ b/src/cli/run.test.ts @@ -227,6 +227,24 @@ test("import rejects a non-string value", () => { expect(r.err.join("\n")).toMatch(/object of string values/i); }); +test("import rejects lone-surrogate keys and values (the kv lens invariant holds for bulk loads)", () => { + const path = fixture(); + // JSON.parse happily produces lone surrogates from \uD800 escapes; without + // validation the import would write them through the kernel directly, where + // two distinct malformed keys collide on the same UTF-8 bytes. + const file = `${path}.surrogate.json`; + writeFileSync(file, String.raw`{"bad-\ud800-key": "v"}`); + const badKey = cli("import", path, file); + expect(badKey.code).toBe(2); + expect(badKey.err.join("\n")).toMatch(/lone surrogate/i); + + writeFileSync(file, String.raw`{"ok": "bad-\udfff-value"}`); + const badValue = cli("import", path, file); + expect(badValue.code).toBe(2); + expect(badValue.err.join("\n")).toMatch(/lone surrogate/i); + expect(cli("get", path, "ok").code).toBe(1); // nothing was written +}); + test("import with no file is a usage error", () => { const r = cli("import", fixture()); expect(r.code).toBe(2); diff --git a/src/cli/run.ts b/src/cli/run.ts index 12f05ce..2f9ec2a 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -19,7 +19,7 @@ import { parseArgs } from "node:util"; import { forceUnlock } from "../adapter/node-fs.ts"; import { LibreDbError, type Database } from "../core.ts"; import { open } from "../index.ts"; -import { catalog, isReservedKey } from "../lens/catalog.ts"; +import { assertWellFormedText, catalog, isReservedKey } from "../lens/catalog.ts"; import { kv } from "../lens/kv.ts"; import { readonlyFileSystem } from "./readonly-fs.ts"; @@ -233,6 +233,18 @@ function importKeys({ path, args, io, force }: Ctx): number { io.err(`import: refusing to write a reserved key: ${key}`); return 2; } + try { + // The same invariant the kv lens enforces: a lone-surrogate string + // cannot round-trip through UTF-8, so two distinct malformed keys would + // silently collide on one stored key (and a malformed value would read + // back altered). Import writes through the kernel directly (one atomic + // transaction), so it must hold the line itself. + assertWellFormedText(key, "import key"); + assertWellFormedText(value, "import value"); + } catch (error) { + io.err(error instanceof Error ? error.message : String(error)); + return 2; + } pairs.push([key, value]); } return withWriteDb(path, force, (db) => { From aa5f37f1ee148d120e07eed7ef0a5c9e1b793b15 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:31:34 +0300 Subject: [PATCH 12/16] test(adapter): cover the reclaim race window deterministically The one uncovered line in the repo was lock()'s refusal branch for the reclaim race: the holder looks dead at the isStaleLock pre-check but alive when the claimed bytes are re-judged. Desyncing the two liveness probes (spyOn process.kill) simulates the race exactly, pinning both the LOCKED refusal and the rename-back. All files are back at true 100% line/function/statement coverage. --- src/adapter/node-fs.test.ts | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/adapter/node-fs.test.ts b/src/adapter/node-fs.test.ts index 549911d..421aede 100644 --- a/src/adapter/node-fs.test.ts +++ b/src/adapter/node-fs.test.ts @@ -12,7 +12,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, expect, test } from "bun:test"; +import { afterEach, expect, spyOn, test } from "bun:test"; import { LibreDbError } from "../core.ts"; import { forceUnlock, fsyncDirectoryOf, isStaleLock, LOCK_SENTINEL, nodeFileSystem } from "./node-fs.ts"; @@ -204,6 +204,41 @@ test("a non-EEXIST failure creating the lock surfaces unchanged (not LOCKED)", ( expect((caught as { code?: string }).code).toBe("ENOENT"); }); +test("a lock that turns out live at claim time (the reclaim race) is put back and refused", () => { + // The race the double-check exists for: the holder looks dead at the + // isStaleLock pre-check but is alive when the claimed bytes are re-judged + // (in reality: a new writer's lock slid in between the two). Simulated + // deterministically by desyncing the two liveness probes. + const path = tempPath("db"); + const lockPath = `${path}.lock`; + const contents = deadLock(); + writeFileSync(lockPath, contents); + let probes = 0; + const killSpy = spyOn(process, "kill").mockImplementation(() => { + probes++; + if (probes === 1) { + const dead = new Error("ESRCH") as Error & { code: string }; + dead.code = "ESRCH"; + throw dead; // pre-check: verifiably dead + } + return true; // claim re-check: alive after all + }); + try { + let caught: unknown; + try { + nodeFileSystem().lock?.(path); + } catch (error) { + caught = error; + } + expect((caught as LibreDbError).code).toBe("LOCKED"); + // The refused lock was renamed back exactly as it was. + expect(readFileSync(lockPath, "utf8")).toBe(contents); + expect(probes).toBe(2); + } finally { + killSpy.mockRestore(); + } +}); + test("release() does not delete a lock someone else re-acquired after a force", () => { const path = tempPath("db"); const fs = nodeFileSystem(); From 81fa9413b280117465d406ea2c5903d5aaeef91c Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:38:52 +0300 Subject: [PATCH 13/16] test(kernel,dst): aim the mid-log corruption tests at actual payload bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both corruption tests still used pre-v1 offset math (8-byte record headers). One landed in the second record's header-checksum field, the other in the first record's stored payload-checksum — each still drove recovery into a CORRUPT_WAL refusal, but neither exercised the exact payload-CRC-mismatch-with-intact-data-after path its description claimed. The offsets now target real payload bytes (v1 framing: 8-byte file header, 12-byte record headers), and the DST assertion matches the payload-path error message specifically, so it can no longer pass via the header-checksum path. --- src/core.hardening.test.ts | 10 ++++++---- src/sim/dst.test.ts | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/core.hardening.test.ts b/src/core.hardening.test.ts index a11750c..19837e4 100644 --- a/src/core.hardening.test.ts +++ b/src/core.hardening.test.ts @@ -316,10 +316,12 @@ test("mid-log corruption refuses to open with CORRUPT_WAL and truncates nothing" db.transact((tx) => tx.set(bytes(2), bytes(20))); db.close(); - // Flip a payload byte of the FIRST record (offset: 8 header + 8 record - // header = 16). Intact record 2 sits after it, so this is damage to - // once-durable bytes, not a crash artifact. - file.data[16] = (file.data[16] as number) ^ 0xff; + // Flip a byte inside the FIRST record's PAYLOAD: 8 file-header bytes plus + // the 12-byte v1 record header (length, header checksum, payload checksum) + // put the payload at offset 20. Its checksum then fails while intact + // record 2 sits after it — damage to once-durable bytes, not a crash + // artifact. + file.data[20] = (file.data[20] as number) ^ 0xff; const sizeBefore = file.data.length; expect(errorFrom(() => open({ path: "wal", fs })).code).toBe("CORRUPT_WAL"); diff --git a/src/sim/dst.test.ts b/src/sim/dst.test.ts index 3600ca2..db6620d 100644 --- a/src/sim/dst.test.ts +++ b/src/sim/dst.test.ts @@ -137,19 +137,20 @@ test("mid-log payload corruption refuses to open instead of truncating committed const db = open({ path: WAL, fs }); runWorkload(db, steps); - // Corrupt a byte inside the SECOND record's payload: record 0 spans - // [RECORDS_BASE, RECORDS_BASE+8+len0); record 1's payload starts 8 bytes - // after that. Its CRC then fails while INTACT data (record 2) sits after it - // — that is not a crash artifact (only the final append can tear), it is - // damage to once-durable bytes. Truncating would destroy record 2's - // committed transaction, so recovery must refuse the open and leave every - // byte in place. + // Corrupt a byte inside the SECOND record's PAYLOAD. v1 records carry a + // 12-byte header (length, header checksum, payload checksum), so record 0 + // spans [RECORDS_BASE, RECORDS_BASE+12+len0) and record 1's payload begins + // 12 header bytes after that. The payload CRC then fails while INTACT data + // (record 2) sits after it — that is not a crash artifact (only the final + // append can tear), it is damage to once-durable bytes. Truncating would + // destroy record 2's committed transaction, so recovery must refuse the + // open and leave every byte in place. const durable = fs.durableBytes(WAL); const len0 = readU32(durable, RECORDS_BASE); const before = fs.durableBytes(WAL); - fs.corrupt(WAL, RECORDS_BASE + 8 + len0 + 8); + fs.corrupt(WAL, RECORDS_BASE + 12 + len0 + 12); - expect(() => open({ path: WAL, fs })).toThrow(/corrupt/i); + expect(() => open({ path: WAL, fs })).toThrow(/corrupt WAL record at offset/i); // Refuse means refuse: the file was not truncated or rewritten (only the // one deliberately-flipped byte differs). const after = fs.durableBytes(WAL); From 243f3177b2a0da50a145e7aeda3d65ae9c8bccc0 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:40:20 +0300 Subject: [PATCH 14/16] docs: remove the distribution channels design document --- ...2026-06-29-distribution-channels-design.md | 227 ------------------ 1 file changed, 227 deletions(-) delete mode 100644 docs/superpowers/specs/2026-06-29-distribution-channels-design.md diff --git a/docs/superpowers/specs/2026-06-29-distribution-channels-design.md b/docs/superpowers/specs/2026-06-29-distribution-channels-design.md deleted file mode 100644 index bf489f2..0000000 --- a/docs/superpowers/specs/2026-06-29-distribution-channels-design.md +++ /dev/null @@ -1,227 +0,0 @@ -# Distribution Channels — Research and Design - -Design and research document for [issue #6](https://github.com/libredb/libredb-database/issues/6): -*Explore additional packaging/distribution channels (JSR, CDN/browser, CLI, standalone binary, Docker).* - -Status: design approved (2026-06-29). Implementation deferred to per-phase specs. -This document is the research that the issue mandated before development begins. - -## 1. Scope and intent - -LibreDB ships today through a single channel: the npm package `@libredb/libredb` -(ESM-only, Node 22+/Bun, zero runtime dependencies, public surface is the lens API). -This document evaluates five additional channels and lays out a phased roadmap. - -It is one research/design document covering all five channels — not five -implementation specs. Each channel is decomposed into its own phase; when a phase -is greenlit it gets its own spec, plan, and implementation cycle. - -**Non-goal (carried from the issue and the manifesto):** no networked -client/server engine. LibreDB is an embedded, in-process library (SQLite-style), -not a server/daemon (Postgres-style). Every channel below preserves that posture — -the CLI, binary, and Docker image are all *embedded tooling shells*, never a -listening service. - -## 2. Current state (verified against code) - -These facts shape every decision and were verified against the source, not memory. - -- **Public API.** `open({ path?, fs? }) -> Database`; lenses `kv`, `doc`, `table`, - and the `catalog` registry sit on top (`src/index.ts`). Synchronous throughout - (`transact`, `fsyncSync`). Sync core is a ratified DESIGN decision; an async - face may be layered later as an adapter over the sync core. -- **node:fs coupling (pre-Phase-0 baseline; resolved by this work).** At the start - of this work, `core.ts` statically imported `node:fs` at module top - (`import { ... } from "node:fs"`). The `FileSystem` seam already existed and the - default `nodeFileSystem()` was only used when a `path` was given — but the static - import meant importing the package at all dragged `node:fs` into the import graph, - breaking browser use even for in-memory (`path`-less) databases. This was the - single linchpin blocking the browser channel, and Phase 0 below removes it. -- **File format.** A `.libredb` file IS the write-ahead log; there is no separate - data file. Record framing: `[u32 payloadLength][u32 crc32(payload)][payload]`, - where payload is a sequence of ops (`[1 byte kind][u32 keyLen][key]`, and for a - set `+[u32 valueLen][value]`). Recovery replays records, stops at the first - incomplete/bad-CRC record, and truncates the torn tail. The whole file is read - into memory on open (`readFileSync`). -- **Catalog.** `catalog(store)` returns a registry keyed by namespace with each - namespace's kind (`kv`/`document`/`relational`) and, for a table, its schema. - Reserved prefix `\x00libredb:catalog:`. This is what lets a tool render faithful - per-kind views from a cold file. -- **Build / JSR readiness.** Source uses honest `.ts` import specifiers - (`./core.ts`); the build rewrites them to `.js` via - `rewriteRelativeImportExtensions`. `isolatedDeclarations: true` is on. This is - ideal for JSR, which prefers explicit `.ts` specifiers and rewards explicit - types (its "slow types" check passes cleanly). -- **Constraints.** 100% line/function/statement coverage held by the gate; - changesets for user-facing changes; conventional commits; English-only; no emoji; - zero runtime dependencies. - -## 3. Central architectural decision: decouple node:fs - -The browser channel and a genuinely runtime-agnostic core both depend on one -refactor, so it is decided up front and pulled to Phase 0. - -**Chosen approach (A): split the adapter out of the core and add conditional -exports.** - -- Move `nodeFileSystem()` from `core.ts` to `src/adapter/node-fs.ts`. After this, - `core.ts` has zero `node:` imports — a genuinely runtime-agnostic kernel. -- `core.ts`'s `open` no longer imports a default filesystem. If a `path` is given - without an `fs`, it throws a clear error ("no filesystem provided for path"). -- Two entry points: - - `src/index.ts` (default, Node): re-exports an `open` pre-bound with - `nodeFileSystem()` as the default `fs`, so the Node experience is unchanged - and backward compatible. - - `src/browser.ts` (new): exposes the core `open` as-is. In-memory works - out of the box; a `path` requires an injected `fs` (e.g. a future OPFS - adapter). -- `package.json` `exports` gains `browser`/`import` conditions and a `./browser` - subpath. `node:fs` leaves the browser import graph entirely. - -Rejected alternatives: **(B) dynamic `await import("node:fs")`** breaks the -ratified synchronous contract; **(C) rely on bundler aliasing** is fragile, -depends on the consumer's build, and is not true browser support. - -This change touches the public API (additive: a new subpath, an unchanged Node -default), so it carries a **minor** changeset. The honesty discipline holds: the -core gets smaller and purer, not more complex. - -## 4. Per-channel design - -### Channel 1 — JSR (`jsr.io`) - -- **Effort:** low. **Risk:** low. **Value:** high. -- Add `jsr.json` (`name: "@libredb/libredb"`, `exports` mapping `.` to - `./src/index.ts` and `./browser` to `./src/browser.ts`). JSR publishes from - source and runs its own transpile/type generation. -- Version stays single-sourced from `package.json`: extend `scripts/sync-version.ts` - to also write `jsr.json`, backstopped by a test as today. -- CI: add `npx jsr publish` to the publish workflow (OIDC, token-less), running in - parallel with the npm publish on the same tag. -- Watch item: `node:` built-in imports can be flagged by JSR — already mitigated - because, after Phase 0, only `adapter/node-fs.ts` imports `node:`. - -### Channel 2 — CDN / browser - -- **Effort:** low (docs) + the Phase 0 entry. **Risk:** low. **Value:** high. -- esm.sh / jsdelivr / unpkg already serve the npm package. Document a pinned-version - browser import example (`import { open, kv } from "https://esm.sh/@libredb/libredb"`). -- True browser support is the `src/browser.ts` entry from Phase 0: in-memory fully - works; `path` errors clearly. -- CI proof: a test asserting the browser entry's import graph contains no `node:` - specifier, so the boundary cannot silently regress. -- **Persistence (future, Phase 5 — not in this round):** wire an OPFS adapter into - the existing `FileSystem` seam. OPFS sync access handles (available inside a Web - Worker) preserve the core's synchronous contract and are the recommended path. - IndexedDB is async and would require the DESIGN-anticipated async-face adapter. - The doc recommends OPFS-in-Worker; the final call is deferred to the Phase 5 spec. - -### Channel 3 — CLI (`npx libredb`) — read + write - -- **Effort:** high. **Risk:** medium-high. **Value:** high. -- New `src/cli/` entry, a thin wrapper over the public API. Zero dependencies via - `node:util` `parseArgs`. `package.json` `bin: { "libredb": "./dist/cli/main.js" }`. -- Commands (as shipped): - - Read: `inspect` (catalog summary grouped by kind), `stats` (file size + - namespace count + counts by kind), `get `, `scan `. - - Write: `set`, `delete`, `import`. - - `repl` was deferred (the issue marked it optional; it is interactive and adds - disproportionate test surface). -- **Data-safety design (DBA-critical):** - - LibreDB is single-process with no file locking. Two concurrent writers corrupt - the file. Write commands acquire an advisory lock (`.lock`, carrying a - sentinel so `--force` refuses to delete a non-lock file); if held, they refuse - unless `--force`. - - Writes reject reserved keys (`isReservedKey`) so the CLI cannot overwrite the - catalog/lens keyspace. - - `open()` truncates a torn tail during recovery, so even a read-intent command - can mutate the file. Read commands therefore use a read-only `FileSystem` - adapter that opens `O_RDONLY` and turns `truncate` into a no-op plus a warning. - This is the default for `inspect`/`get`/`scan`/`stats`. - - `import` and bulk writes commit in a single atomic transaction; a crash mid-way - is rolled back by recovery. -- Size/packaging: the CLI bin is separate from the 4 kB library budget, but - `dist/cli` must be included in knip/publint checks and given its own size guard. - -### Channel 4 — standalone binary - -- **Effort:** medium. **Risk:** low-medium. **Value:** medium. -- `bun build --compile` packages the CLI into a single self-contained executable - (embeds the Bun runtime, ~50-90 MB). Cross-compile targets: `linux-x64`, - `linux-arm64`, `darwin-x64`, `darwin-arm64`, `windows-x64`. -- DevOps: a CI matrix builds per target on tag and attaches artifacts to the - GitHub Release with a `SHA256SUMS` file. SLSA provenance / cosign signing is a - later enhancement. Not published to npm or JSR. -- The binary embeds Bun, not Node; the `node:fs` adapter runs unchanged on Bun. - -### Channel 5 — Docker - -- **Effort:** low. **Risk:** low. **Value:** medium. -- A minimal image wrapping the static binary (distroless or `scratch` + a static - linux binary). Volume-mount `.libredb` files. It is a CLI shell, not a server — - honoring the non-goal. -- DevOps: `docker buildx` multi-arch (amd64/arm64), published to GHCR - (`ghcr.io/libredb/libredb`), tagged with the version and `latest`. -- Usage: `docker run -v $PWD:/data ghcr.io/libredb/libredb inspect /data/app.libredb`. - -## 5. Dependency order and phased roadmap - -The real dependency chain differs from the issue's suggested order: Phase 0 (the -node:fs refactor) is the linchpin, and the CLI is a prerequisite for the binary -and Docker. - -``` -Phase 0: node:fs decoupling (browser.ts + exports) <- foundation of everything - | - +- Phase 1: JSR + CDN docs (depends on 0; fastest value) - | - +- Phase 2: CLI (read+write) (depends on 0; independent of browser) - | - +- Phase 3: standalone binary (depends on CLI) - | | - | +- Phase 4: Docker (depends on binary) - | - +- Phase 5: browser persistence (OPFS) (depends on 0+1; optional, last) -``` - -| Phase | Work | Why here | Changeset | -|-------|------|----------|-----------| -| 0 | Decouple `node:fs` -> `adapter/node-fs.ts`; add `src/browser.ts` + `exports` conditions | Linchpin: browser, JSR-cleanliness, and a testable pure core all depend on it | Yes (minor — additive subpath, backward-compatible) | -| 1 | JSR publish + CDN/browser docs | Lowest effort, fastest reach; free once Phase 0 lands | Browser entry shipping: yes; docs-only: no | -| 2 | CLI (read-only adapter for reads + advisory lock for writes) | Prerequisite for CI automation and the binary | Yes (ships a `bin`) | -| 3 | `bun build --compile` cross-compile + GitHub Release matrix | Meaningless without the CLI | No (not in the npm package) | -| 4 | Multi-arch Docker -> GHCR | Wraps the binary | No | -| 5 | OPFS persistence adapter (browser) | Highest architectural risk; after value is proven | Yes (minor) | - -## 6. Risk / effort / value matrix - -| Phase | Effort | Risk | Value | Primary risk item | -|-------|--------|------|-------|-------------------| -| 0 node:fs decouple | Medium | Medium | High | Backward compatibility — Node `open({path})` must behave identically; 100% coverage across two entry points | -| 1 JSR + CDN | Low | Low | High | JSR slow-types / built-in import warnings (likely none); CDN version-pinning hygiene | -| 2 CLI | High | Medium-High | High | Advisory-lock races; read-only recovery suppression; `parseArgs` UX; 100% coverage | -| 3 binary | Medium | Low-Medium | Medium | Cross-compile matrix fragility; ~50-90 MB size expectations; signing/provenance | -| 4 Docker | Low | Low | Medium | Multi-arch buildx; GHCR permissions; preserving "not a server" positioning | -| 5 OPFS persistence | High | High | Medium | Sync access handles are Worker-only; sync contract vs IndexedDB async; browser matrix | - -**Three points needing the most care:** - -1. **Phase 2 — CLI data safety.** The single-writer rule, advisory lock, and - read-only recovery suppression. Done wrong, the CLI can corrupt a live - `.libredb`. The highest "do no harm" risk in the project. -2. **Phase 0 — 100% coverage across two entry points.** The gate holds coverage at - 100%; the node-free browser path and the "path but no fs" error branch must be - fully tested. -3. **Phase 5 — sync/async impedance mismatch.** OPFS-in-Worker vs an async-face - adapter. The doc recommends OPFS-in-Worker and leaves the final decision to the - Phase 5 spec. - -## 7. Execution model (autonomous loop) - -Implementation will run as a six-phase autonomous development loop: - -1. Implement one phase's first step. -2. Wait for GitHub Actions; resolve every problem until all checks are green. -3. Only then advance to the next step/phase. -4. As a software architect, make decisions on ambiguous points and proceed. -5. When everything is complete, do not merge the PR — stop for final human review. From 13544d3f4f03c538027cf03e3b9fdaece244befc Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 08:59:50 +0300 Subject: [PATCH 15/16] docs: sync every document with the hardened code A 40-agent docs-vs-code audit (one verifier per document, every factual claim checked against the source; fixes applied per file) found and corrected ~100 stale claims across 15 files. Highlights: - ARCHITECTURE and UNDERSTANDING-THE-KERNEL now describe the v1 on-disk format (8-byte LRDB file header, 12-byte self-checksummed record headers) with a regenerated, byte-verified hex example, the recovery classification (torn tail vs corruption vs short read), the failure latch, and the exclusive lock; all core.ts line references re-anchored - README: early-beta status badge and prose, 6 kB bundle claim, honest --force semantics (auto-reclaim only for verifiably dead holders), correct issue links (libredb-database repo), correct binary matrix - CODE-METRICS: full table regenerated against wc -l (core.ts 950/449) - TOOLCHAIN: gate step list, size budgets, node-smoke and binary-smoke jobs, Dependabot, provenance - DESIGN: stale present-tense line counts turned into dated narrative - CLI/BINARY/BROWSER/STUDIO/guides/CONTRIBUTING/PR template: lock semantics, --raw flag, error codes, export surface, OPFS caveat Every replacement was written with code evidence (file:line or a reproduced byte dump); relative links verified unbroken. --- .github/PULL_REQUEST_TEMPLATE.md | 3 +- ARCHITECTURE.md | 72 +++++++++---- CONTRIBUTING.md | 7 +- README.md | 26 +++-- docs/BINARY.md | 10 +- docs/BROWSER.md | 7 +- docs/CLI.md | 6 +- docs/CODE-METRICS.md | 56 +++++----- docs/DESIGN.md | 25 ++--- docs/STUDIO.md | 5 +- docs/TOOLCHAIN.md | 80 ++++++++------ docs/UNDERSTANDING-THE-KERNEL.md | 177 +++++++++++++++++-------------- docs/guides/README.md | 2 +- docs/guides/catalog.md | 8 +- docs/guides/relational.md | 6 +- 15 files changed, 289 insertions(+), 201 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 59d7955..7829c36 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,7 @@ ## What and why diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3503fcb..c4f7d09 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -169,7 +169,11 @@ interface Database { close(): void; } -const open: (options?: { path?: string; fs?: FileSystem }) => Database; +const open: (options?: { + path?: string; + fs?: FileSystem; + onRecovery?: (info: RecoveryInfo) => void; // reports a truncated torn tail +}) => Database; ``` With a `path`, the database is file-backed and durable. Without one, it is purely @@ -340,8 +344,17 @@ time comes, without the three-box machinery becoming mandatory. Each committed transaction is one length-framed, checksummed record: ``` - record = [ u32 payloadLength ] [ u32 crc32(payload) ] [ payload ] - 4 bytes 4 bytes + file = [ 4-byte magic "LRDB" ] [ u16 formatVersion ] [ u16 reserved ] 8-byte file header + ...followed by records, back to back + + record = [ u32 payloadLength ] [ u32 crc32(length bytes) ] [ u32 crc32(payload) ] [ payload ] + 4 bytes 4 bytes 4 bytes + + The record header carries a checksum of itself (the length field), so + recovery can tell a trustworthy length from a damaged one. Files written by + v0.1.x predate the file header and use 8-byte record headers + [ u32 payloadLength ][ u32 crc32(payload) ]; they still open, and keep that + legacy framing on appends (a file's format cannot change mid-file). payload = one or more ops, back to back: set = [ u8 1 ] [ u32 keyLen ] [ key ] [ u32 valLen ] [ value ] @@ -354,8 +367,10 @@ A real record from a live file, decoded (the first write of a relational table - its schema, before any row): ``` + 4c 52 44 42 00 01 00 00 file header: magic "LRDB", format version 1 00 00 00 9d payloadLength = 157 - ea 44 78 48 crc32 + af fa 30 e5 crc32 of the header's own length bytes + ea 44 78 48 crc32 of the payload 01 op = SET 00 00 00 16 keyLen = 22 00 6c 69 62 72 65 64 62 3a ... key = "\x00libredb:catalog:users" @@ -375,9 +390,13 @@ The fsync happens *before* the in-memory commit becomes visible: on disk, survives a crash now visible in memory ``` -So when `transact()` returns, the change is on disk. If the append fails, the code -throws with memory and disk still agreeing on the *prior* state. A read-only -transaction writes nothing to the log. +So when `transact()` returns, the change is on disk. If the append or fsync +fails, `transact()` throws a typed error (code FAILED) with memory keeping the +prior state -- and the database latches: every later `transact()` throws FAILED +until it is closed and reopened. Appending past a possibly-torn tail could let +the next recovery silently discard later, acknowledged commits; refusing +further writes is what keeps "a returned transact() is durable" true. A +read-only transaction writes nothing to the log. ### Recovery and torn writes @@ -387,10 +406,13 @@ Because the log is append-only and fsynced, a crash can only ever damage the ``` recover(file): replay each intact record in order, rebuilding the sorted array - stop at the first record that is: - - torn (header promises more bytes than exist), or - - corrupt (crc32 of payload does not match) - truncate the file at that point # next append starts from a clean boundary + a TORN TAIL -- a header promising more bytes than exist, or the FINAL + record's payload failing its crc32 -- is truncated away and fsynced, + and reported through the open option onRecovery({ truncatedBytes }) + a payload crc32 failure with intact data AFTER it, or a v1 record + header failing its own checksum, is CORRUPTION: recovery throws + CORRUPT_WAL and leaves the file untouched -- it never truncates + committed records to get past damage ``` ``` @@ -610,7 +632,10 @@ The kernel never calls `node:fs` directly. Every byte to disk goes through one small interface: ```ts -interface FileSystem { open(path: string): WalFile; } +interface FileSystem { + open(path: string): WalFile; + lock?(path: string): () => void; // optional exclusive lock; a second open throws LOCKED +} interface WalFile { size(): number; @@ -662,7 +687,7 @@ what reopening each decision would entail, not a committed roadmap. | Working set | the whole store lives in memory | bounded by RAM, not disk | | Log growth | append-only, no compaction | the file grows with write *history* | | Multi-key atomicity | lenses auto-commit per operation | for atomic multi-writes use `transact` | -| Durability edge | no directory fsync on first create | see 10.3 -- a known hardening gap | +| Durability edge | browser OPFS `flush()` is weaker than POSIX fsync | power-loss durability in the browser is engine-dependent | None of these are hidden. The cost is concentrated where it is cheapest to reason about, and the throughline is that **every one of them could be addressed above the @@ -717,16 +742,17 @@ cleanly into two kinds of work. These close real correctness gaps on the existing design. They are tracked as known limitations, not new directions: -- **Directory fsync on first file creation.** Creating a file durably requires - fsyncing the *directory*, not just the file -- otherwise a power loss can lose - the directory entry for a freshly created database. Currently not done. +- **Directory fsync on first file creation** -- done. The `node:fs` adapter + fsyncs the parent directory when it creates the database file, so a power + loss can no longer lose the directory entry of a freshly created database. - **WAL compaction / checkpointing.** Today the log only grows (section 5). A checkpoint -- fold the committed state into a compact snapshot, then trim the log -- bounds file size and speeds recovery. This is the single most important hardening item for any long-lived database. -- **Short-read recovery robustness.** A note carried out of the crash-recovery - work: make sure a partial read at the tail is always treated as a torn record, - never as data. +- **Short-read recovery robustness** -- done, and stricter than first sketched: + a read that returns fewer bytes than the file holds throws a typed + INCOMPLETE_READ error instead of being treated as data or as a torn tail, so + a transient IO fault can never cause recovery to truncate committed records. **Scaling features (each reopens a locked decision).** These are not on the v1 path and would each force a deliberate decision to be @@ -769,7 +795,11 @@ by staying small and correct, not by absorbing every feature. | `lens/document.ts` | lens | JSON documents, by-id CRUD, scan and find | | `lens/relational.ts` | lens | schema-validated tables, where/select/join | | `lens/catalog.ts` | edge | reserved namespace, registry, validate-on-reopen | -| `index.ts` | public | the npm export surface | +| `adapter/node-fs.ts` | edge | the real `node:fs` WAL adapter (fd reads, directory fsync, lock file) | +| `adapter/opfs.ts` | edge | the browser OPFS WAL adapter | +| `cli/` | tooling | the libredb CLI (inspect, stats, get, scan, set, delete, import) and the read-only filesystem | +| `index.ts` | public | the Node npm export surface | +| `browser.ts` | public | the browser export surface (no Node built-ins) | | `sim/` | test harness | simulated filesystem and crash-recovery oracle (DST) | --- @@ -793,7 +823,7 @@ Putting it together -- what happens when you insert a row into a file-backed tab append + fsync # durable here committed = working # visible here - on disk (demo.libredb), now two records: + on disk (demo.libredb), now an 8-byte file header followed by two records: [ \x00libredb:catalog:users -> {relational, schema} ] [ users:1 -> {"id":"1","name":"Ada","age":36,"active":true} ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3b3b7c..e4243ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,8 +48,8 @@ typecheck -> format -> lint -> knip -> build -> size -> test - **lint** — Oxlint plus a narrow type-aware typescript-eslint pass. - **knip** — no unused files, exports, or dependencies. - **build** — `tsc` emits `dist/` with isolated declarations. -- **size** — a byte budget on the shipped entry (`.size-limit.json`); a heavy or non-tree-shakeable - import fails it. +- **size** — byte budgets on the shipped entries, public and browser (`.size-limit.json`); a heavy + or non-tree-shakeable import fails it. - **test** — `bun test --coverage`. Coverage is held at **100%** line/function/statement (`bunfig.toml`); a change that drops coverage fails the gate. @@ -70,7 +70,8 @@ typecheck -> format -> lint -> knip -> build -> size -> test `docs: clarify the recovery invariant`. - **English only**, and **no emoji** anywhere — code, comments, commits, docs. - PRs are **squash-merged**: the individual commit messages are discarded and the PR title becomes the - single commit (and the changelog entry), so keep the **PR title** conventional and descriptive. + single commit, so keep the **PR title** conventional and descriptive. (The changelog comes from + changesets, not commit messages — see below.) - If your change is user-facing, add a changeset: `bun run changeset`. This is what generates the changelog and version bump at release time. - The CI gate mirrors `bun run gate` and runs on every PR, including forks. diff --git a/README.md b/README.md index f7fa243..db916b1 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ [![types: included](https://img.shields.io/badge/types-included-blue.svg)](https://www.typescriptlang.org/) [![dependencies: 0](https://img.shields.io/badge/dependencies-0-brightgreen.svg)](./package.json) [![bundle size](https://img.shields.io/bundlephobia/minzip/@libredb/libredb)](https://bundlephobia.com/package/@libredb/libredb) -[![status: pre-alpha](https://img.shields.io/badge/status-pre--alpha-orange.svg)](#project-status--roadmap) +[![status: early beta](https://img.shields.io/badge/status-early%20beta-orange.svg)](#project-status--roadmap) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/libredb/libredb-database) LibreDB is a small, readable, embeddable, multi-model database written in TypeScript. It is built on @@ -25,8 +25,8 @@ one idea: a database can be powerful and still be understood by opening its sour key-value core handles durability and transactions; key-value, document, and relational APIs are thin *lenses* over that one core — not three separate engines. It runs in-memory for tests or file-backed for durability, ships **zero runtime dependencies**, and proves its crash recovery with deterministic -simulation testing. Today it is pre-alpha, aimed at test and development environments — small enough to -learn how a database actually works, and serious enough to grow into more. +simulation testing. Today it is an early beta, aimed at test and development environments — small +enough to learn how a database actually works, and serious enough to grow into more. ## Highlights @@ -41,7 +41,7 @@ learn how a database actually works, and serious enough to grow into more. run. - **In-memory or durable** — `open()` for tests, `open({ path })` for a crash-safe, WAL-backed, fsync-on-commit file. -- **TypeScript-native** — full types shipped, ESM-only, tree-shakeable, under 4 kB min+brotli. +- **TypeScript-native** — full types shipped, ESM-only, tree-shakeable, under 6 kB min+brotli. - **Crash recovery you can trust** — 100% line coverage on the core, plus deterministic simulation testing that tortures the write-ahead log under a simulated crashing filesystem. - **Nothing hidden** — queries are plain in-engine scans, errors surface, and costs are obvious (O(n) @@ -153,12 +153,15 @@ npx libredb import data.libredb seed.json # bulk-set from a JSON object, atomica ``` Read commands open the file read-only, so inspection never mutates it. Write commands take an -advisory `.lock` to refuse a second concurrent writer. Use `--force` only to clear a stale -lock left by a crashed writer: the lock is advisory and LibreDB is single-process, so two writers -that force at the same time can still race and corrupt the file. +advisory `.lock` to refuse a second concurrent writer. A lock left by a writer that crashed on +the same host is reclaimed automatically — no flag needed. Use `--force` only for a lock whose +holder cannot be verified (an anonymous lock or one written on another host); it refuses a holder +that is verifiably alive, so it cannot knowingly admit two live writers. The remaining risk is the +unverifiable case: a live writer on another machine sharing the file can still be forced past, which +can corrupt the file. Prefer a standalone binary with no Node or Bun installed? Each release attaches self-contained -executables (Linux, macOS, Windows; x64 and arm64) with `.sha256` checksums on its +executables (Linux and macOS on x64 and arm64; Windows on x64) with `.sha256` checksums on its [GitHub Release](https://github.com/libredb/libredb-database/releases). Or build one locally with `bun run compile`. @@ -295,7 +298,7 @@ Honesty about scale (comprehension is the budget in v1, not throughput): (or use `libredb import`, which already does): one copy, one fsync, one record for the whole batch. - **No secondary indexes**: a `find`/`where` is an O(n) scan by design in v1. - **The log grows without bound** until compaction lands (tracked in - [#12](https://github.com/libredb/libredb/issues/12)); reopening replays the whole log. + [#12](https://github.com/libredb/libredb-database/issues/12)); reopening replays the whole log. ## Documentation @@ -324,8 +327,9 @@ the recommended home is still test/dev data. codes; the DST harness with IO-fault injection and binary fuzz; 100% line/function/statement coverage. - **Next:** secondary indexes and a richer query surface; more query operators; additional lenses; - WAL compaction/checkpointing ([#12](https://github.com/libredb/libredb/issues/12)); real-browser - OPFS verification ([#10](https://github.com/libredb/libredb/issues/10)). + WAL compaction/checkpointing + ([#12](https://github.com/libredb/libredb-database/issues/12)); real-browser OPFS verification + ([#10](https://github.com/libredb/libredb-database/issues/10)). ## The LibreDB family diff --git a/docs/BINARY.md b/docs/BINARY.md index 428ac5f..9f6f937 100644 --- a/docs/BINARY.md +++ b/docs/BINARY.md @@ -1,7 +1,8 @@ # Standalone binaries -Every LibreDB release attaches **self-contained executables** of the `libredb` -CLI to its [GitHub Release](https://github.com/libredb/libredb-database/releases). +Every stable LibreDB release attaches **self-contained executables** of the `libredb` +CLI to its [GitHub Release](https://github.com/libredb/libredb-database/releases) +(pre-releases skip the binaries). They embed the Bun runtime, so they run with **no Node, no Bun, and no `npm install`** — just download one file and run it. @@ -91,8 +92,9 @@ builds.) ## Notes -- **Size:** each binary is ~80–90 MB because it bundles the Bun runtime. That is - the cost of "no install / no dependencies." +- **Size:** each binary is roughly 60–100 MB depending on the platform (macOS + builds are the smallest, Windows the largest) because it bundles the Bun + runtime. That is the cost of "no install / no dependencies." - **Not on npm/JSR:** binaries are a GitHub Releases artifact only; the package registries ship the importable library + the `libredb` bin instead. - **Same data files everywhere:** a `.libredb` file written by the library, the diff --git a/docs/BROWSER.md b/docs/BROWSER.md index 6bb0367..c270502 100644 --- a/docs/BROWSER.md +++ b/docs/BROWSER.md @@ -283,8 +283,11 @@ setup from §4.1. on the file — only one handle per file at a time. So one Worker owns the database; a second tab/Worker cannot open the same file concurrently. For multi-tab apps, route all access through a single owner (e.g. a `SharedWorker`, or elect one tab - as writer). This matches LibreDB's "single-process, no internal file locking" - model — it is the foundation, not a server. + as writer). This matches LibreDB's single-writer model — on Node the kernel + enforces it with an exclusive `.lock` file (a second `open` throws + `LOCKED`); in the browser the sync access handle's own exclusivity provides + the same guarantee, so the OPFS adapter needs no lock file. It is the + foundation, not a server. - **OPFS needs a Worker and a secure context.** Sync access handles exist only in dedicated Web Workers, over HTTPS or `localhost`. In-memory `open()` has neither requirement. diff --git a/docs/CLI.md b/docs/CLI.md index 4309d43..229f2bd 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -61,7 +61,7 @@ schema for relational tables. ```sh $ libredb inspect app.libredb app.libredb 412 bytes - logs document + logs document people relational {"primaryKey":"id","columns":{"id":"string","name":"string"}} ``` @@ -190,8 +190,8 @@ file copy — with one rule. | Code | Meaning | Examples | | --- | --- | --- | | `0` | Success | a read/write completed | -| `1` | Runtime error | file not found, `get` on a missing key | -| `2` | Usage error | unknown command/option, missing argument, malformed import JSON, reserved key, lock held | +| `1` | Runtime error | file not found, `get` on a missing key, lock held by another writer | +| `2` | Usage error | unknown command/option, missing argument, malformed import JSON, reserved key | This makes the CLI scriptable — e.g. in CI: diff --git a/docs/CODE-METRICS.md b/docs/CODE-METRICS.md index ed6cc7f..43bf7a6 100644 --- a/docs/CODE-METRICS.md +++ b/docs/CODE-METRICS.md @@ -13,21 +13,27 @@ For each file: ## Shipped engine -The code that ships in the published package: the durability core plus the thin -model lenses on top of it. +The code that ships in the published package: the durability core, the thin +model lenses on top of it, the file-system adapters, the browser entry, and the CLI. | File | Responsibility | Total | Code | |---|---|---:|---:| -| `src/core.ts` | Durability core (ordered KV, transactions, WAL/recovery) | 548 | 273 | -| `src/lens/relational.ts` | Relational lens (CRUD, query, joins) | 361 | 159 | -| `src/lens/document.ts` | Document lens | 235 | 99 | -| `src/lens/catalog.ts` | Catalog / reserved-namespace contract | 210 | 70 | -| `src/lens/kv.ts` | KV lens (the proof layer) | 124 | 61 | +| `src/core.ts` | Durability core (ordered KV, transactions, WAL/recovery, on-disk format) | 950 | 449 | +| `src/lens/relational.ts` | Relational lens (CRUD, query, joins) | 369 | 160 | +| `src/cli/run.ts` | CLI command implementations | 317 | 245 | +| `src/lens/document.ts` | Document lens | 311 | 137 | +| `src/adapter/node-fs.ts` | Node/Bun file-system adapter (fd reads, fsync, lock) | 302 | 184 | +| `src/lens/catalog.ts` | Catalog / reserved-namespace contract | 286 | 115 | +| `src/lens/kv.ts` | KV lens (the proof layer) | 133 | 65 | +| `src/adapter/opfs.ts` | Browser OPFS adapter | 96 | 45 | | `src/query/range.ts` | Range-query helpers | 70 | 19 | | `src/lens/types.ts` | Shared types | 69 | 16 | +| `src/browser.ts` | Browser entry / export surface | 61 | 19 | +| `src/index.ts` | Public entry / export surface | 51 | 19 | +| `src/cli/readonly-fs.ts` | Read-only file system for CLI reads | 47 | 27 | | `src/adapter/store.ts` | FS/store adapter interface | 32 | 4 | -| `src/index.ts` | Public entry / export surface | 28 | 11 | -| **Subtotal** | | **1677** | **712** | +| `src/cli/main.ts` | CLI entry point | 16 | 6 | +| **Subtotal** | | **3110** | **1510** | ## Simulation / test-running harness (DST) @@ -36,19 +42,19 @@ ship in the product. Kept separate from the shipped engine. | File | Responsibility | Total | Code | |---|---|---:|---:| +| `src/sim/simfs.ts` | Simulated file system (fault injection, crash, corruption) | 174 | 97 | | `src/sim/workload.ts` | Workload generator | 159 | 88 | -| `src/sim/dst.ts` | Crash/recovery oracle runner | 156 | 69 | -| `src/sim/simfs.ts` | Simulated file system | 142 | 79 | +| `src/sim/dst.ts` | Crash/recovery oracle runner | 157 | 69 | | `src/sim/prng.ts` | Deterministic PRNG | 24 | 9 | -| **Subtotal** | | **481** | **245** | +| **Subtotal** | | **514** | **263** | ## Grand total | Category | Total | Code | |---|---:|---:| -| Shipped engine | 1677 | 712 | -| Simulation harness | 481 | 245 | -| **All production source** | **2158** | **957** | +| Shipped engine | 3110 | 1510 | +| Simulation harness | 514 | 263 | +| **All production source** | **3624** | **1773** | ## Shipped size @@ -57,20 +63,22 @@ embed*. Same proof, different axis. (Measured at the version in `package.json`.) | What | Size | Meaning | |---|---:|---| -| Public entry, bundled | **2.83 kB** | What a consumer's app pays after their bundler tree-shakes, minifies, and brotli-compresses `import ... from "@libredb/libredb"`. Node built-ins (`node:fs`) are runtime-provided, not counted. | -| npm tarball | ~29 kB | The download (`bun pm pack`): 21 files including `.js`, `.d.ts` types, README, and LICENSE. | -| Unpacked `dist/` | ~136 kB | On disk after install — readable (unminified) JS plus full type declarations. | +| Public entry, bundled | **5.08 kB** | What a consumer's app pays after their bundler tree-shakes, minifies, and brotli-compresses `import ... from "@libredb/libredb"`. Node built-ins (`node:fs`) are runtime-provided, not counted. | +| npm tarball | ~53 kB | The download (`bun pm pack`): 33 files including `.js`, `.d.ts` types, README, and LICENSE. | +| Unpacked `dist/` | ~157 kB | On disk after install — readable (unminified) JS plus full type declarations. | The bundled figure is **machine-enforced**: `size-limit` holds the public entry -under a **4 kB** budget as part of `bun run gate`, so an accidental heavy -dependency or a non-tree-shakeable import fails the build. Raising the budget has -to be a conscious edit — the byte-level analog of the core line-count discipline. +under a **6 kB** budget (and the browser entry under **5 kB**) as part of +`bun run gate`, so an accidental heavy dependency or a non-tree-shakeable +import fails the build. Raising the budget has to be a conscious edit — the +byte-level analog of the core line-count discipline. ## Notes -- The entire durability core lives in a single file (`src/core.ts`, 273 lines of - code), with everything else being thin lenses layered on top — consistent with - the FoundationDB-style architecture described in `DESIGN.md`. +- The entire durability core lives in a single file (`src/core.ts`, 449 lines of + code), with everything else being thin lenses, adapters, and tooling layered + on top — consistent with the FoundationDB-style architecture described in + `DESIGN.md`. - **Code heuristic (reproducible).** The Code column is produced by: `grep -cvE '^\s*($|//|/\*|\*/?\s*$|\*\s)' ` — it strips blank lines, `//` line comments, `/*` and `*/` block delimiters, and ` * ` JSDoc continuation diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 73d816a..75c32b7 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -62,14 +62,14 @@ The empty space we claim: **small + readable + embedded + multi-model**, where t The two-tier trust model is expressed physically. A single file cannot encode "these lines are guarded, those are flexible"; the file boundary makes the trust model visible and CI-enforceable. ``` -@libredb/core +@libredb/libredb |-- core.ts KERNEL: storage + transactions + recovery. Starts ~1k lines. | Strictest line budget. Heavy tests. CODEOWNERS-guarded. | Reading this file teaches "how a database works." |-- lens/ | |-- kv.ts First lens (natural fit: the core is an ordered key-value store) -| |-- document.ts Later -| |-- relational.ts Later +| |-- document.ts Second lens (shipped; section 6.1) +| |-- relational.ts Third lens (shipped; section 6.2) |-- adapter/ , query/ Edges: flexible, fast, open contribution ``` @@ -121,11 +121,11 @@ The relational lens is the reach lens. It is the identity-risk lens: a SQL engin - **Honest deferrals (later, not v1):** SQL, secondary indexes, outer / non-equi joins, foreign keys, unique constraints, nullable/optional columns, aggregation / group-by, and order-by beyond primary-key order. - The kernel is **unchanged**; the lens is `lens/relational.ts`, built on top of the document lens and reusing `lens/types.ts`, `adapter/store.ts`, and `query/`. -## 6.3 Catalog — design for a future milestone (NOT built yet) +## 6.3 Catalog — v1 locked design (built as `lens/catalog.ts`; see section 7) **Motivation.** A LibreDB file is raw ordered key-value bytes. Which lens a key belongs to (kv / document / relational) and a relational table's schema live in *application code* (`table(db, name, schema)`), NOT on disk. So a tool that opens a file cold — e.g. the LibreDB Studio provider — can only show the raw KV store grouped by key prefix; it cannot know that `orders:` is a relational table with schema X. The catalog persists that interpretation so tools can offer richer, faithful views. (Studio's raw-KV browser, "Option A", ships without this; the richer "Option B" views depend on it.) -**Locked-by-default design (ratify before building):** +**Locked design (as built):** - **A lens-level convention, NOT a kernel feature.** The kernel stays pure ordered-KV and unchanged. The catalog is just additional KV entries written by the lenses under a reserved key prefix. Honesty discipline: do not grow `core.ts` for this. - **Reserved prefix.** Catalog entries live under a reserved namespace (e.g. a low-byte prefix like `\x00libredb:catalog:`, sorting before user data). User collection/table names starting with the reserved marker are rejected — a correctness rule, like the prefix-soundness rule already enforced. @@ -163,12 +163,12 @@ The real reliability bar (DESIGN principle 4). The first lens shipped on standar (ordered-KV kernel + transactions + WAL crash recovery), comfortably under the ~1,000-line starting target and far under the ~10,000-line ceiling. Comprehension time, not line count, remains the governing metric. The "lenses land on top of (not inside) the core" bet is VERIFIED, not predicted: the - document (229 lines) and relational (339 lines) lenses, and the entire catalog (`lens/catalog.ts`, - 194 lines), all landed ON TOP of the kernel. `core.ts` grew only once — the S1 injectable-FS seam for + document (311 lines) and relational (369 lines) lenses, and the entire catalog (`lens/catalog.ts`, + 286 lines), all landed ON TOP of the kernel. `core.ts` grew only once — the S1 injectable-FS seam for DST (see §6.4) took it from 481 to 548 lines (+67, against an estimated ~520), the one guarded-core change the DST work sanctioned; it is still far under the ~1,000-line starting target. The DST harness - (`src/sim/`, 675 lines) is test-only and never ships, so it does not count against the budget. The - budget held with headroom. + (`src/sim/`, roughly 1,200 lines including its tests) is test-only and never ships, so it does not count + against the budget. The budget held with headroom. **Update (2026-07-03, the pre-announcement hardening wave):** `core.ts` now measures 950 physical lines — but 456 of them are explanatory comments and 45 are blank; the CODE is 449 lines, still under the original ~500-line shape. The growth is the audit-driven hardening (the `LRDB` file @@ -187,9 +187,10 @@ The real reliability bar (DESIGN principle 4). The first lens shipped on standar DST layer §6.4 and DESIGN principle 4 promised, no longer hand-waved. - **Production arc.** Still open. Define the concrete bar that moves LibreDB from "test/dev" to "earns production." The deterministic-simulation-testing dependency is now DISCHARGED (resolved above); what - remains is a hardening checklist (e.g. directory fsync on first file creation, WAL - compaction/checkpointing — both known limitations, and DST's own short-read - recovery note from S4), all of which are tracked-later tasks. + remains is a hardening checklist. Directory fsync on first file creation shipped in the 2026-07 hardening + wave (`src/adapter/node-fs.ts` fsyncs the parent directory when the file is created), and a short + read during recovery now fails loudly with INCOMPLETE_READ instead of being a deferred note; WAL + compaction/checkpointing remains a known limitation and a tracked-later task. ## 8. Lineage of the idea diff --git a/docs/STUDIO.md b/docs/STUDIO.md index 82369f2..25c08ca 100644 --- a/docs/STUDIO.md +++ b/docs/STUDIO.md @@ -22,8 +22,9 @@ A LibreDB connection is just a path to a `.libredb` file on the Studio server's ``` The `database` field carries the file path (reused exactly like SQLite — there is no separate `path` -field). The file is opened with `open({ path })`; a path that does not exist yet is created on the -first write. Connecting without a path is rejected — there is no in-memory mode for a connection. +field). The file is opened with `open({ path })`; a path that does not exist yet is created empty at +open time (together with its `.lock` file), and the on-disk format header lands with the first write. +Connecting without a path is rejected — there is no in-memory mode for a connection. ## How editor commands map to the lens API diff --git a/docs/TOOLCHAIN.md b/docs/TOOLCHAIN.md index 7aca650..b7596b1 100644 --- a/docs/TOOLCHAIN.md +++ b/docs/TOOLCHAIN.md @@ -1,7 +1,7 @@ # LibreDB Toolchain - 2026 Decision Record -> Status: implemented (local phases done and committed; the committed CI workflows activate automatically -> once the repo is on GitHub). Captures the per-tool decisions from a researched-then-adversarially-verified evaluation of +> Status: implemented and live (local phases done and committed; the repo is on GitHub and the CI, +> SonarCloud, and publish workflows are active). Captures the per-tool decisions from a researched-then-adversarially-verified evaluation of > 2026 gold-standard OSS-TypeScript tooling, judged against LibreDB's manifesto and `DESIGN.md`. Every > adopted tool plugs into `bun run gate` or a documented CI phase. @@ -41,10 +41,10 @@ errors in the first-pass research; the corrections are baked into the configs be | Release | ADOPT | `@changesets/cli` | Human-curated changelog; local now, CI later | | License hygiene | ADOPT | `license-checker-rseidelsohn` | Dev-only allowlist; fails on non-permissive deps | | Security (local) | ADOPT | `bun audit` + `secretlint` | Dependency audit + secret scan at the edge (npm-native) | -| Security (CI) | DEFER-TO-CI | CodeQL, Scorecard, provenance, osv-scanner, dependency-review | Need GitHub Actions | -| Dependency updates | DEFER-TO-CI | Renovate (or Dependabot) | Need GitHub; `bun outdated` is the manual local stand-in | -| Code quality (CI) | DEFER-TO-CI (prepared) | SonarCloud | Cloud SAST + coverage; workflow committed, inert until SONAR_TOKEN | -| CI gate | DEFER-TO-CI (prepared) | GitHub Actions | Mirrors `bun run gate`; workflow committed, activates on push | +| Security (CI) | PARTLY ADOPTED | npm provenance (live in publish.yml); CodeQL, Scorecard, osv-scanner, dependency-review still deferred | Provenance ships with every release | +| Dependency updates | ADOPT (CI) | Dependabot | `.github/dependabot.yml`: weekly actions/docker/npm update PRs, devDependencies grouped | +| Code quality (CI) | ADOPT (live) | SonarCloud | Cloud SAST + coverage; analysis runs on every push/PR, quality-gate and coverage badges in the README | +| CI gate | ADOPT (live) | GitHub Actions | Mirrors `bun run gate` on every push/PR, plus a Node 22 smoke job | ## Cross-cutting integration realities (verifier-surfaced) @@ -144,7 +144,7 @@ export default tseslint.config( Scripts: `"format": "biome format src eslint.config.js"`, `"format:fix": "biome format --write src eslint.config.js"`, `"lint": "oxlint && eslint ."`. A `biome.json` enables the formatter only (`linter` and `assist` disabled). Remove `@eslint/js` from devDependencies. -Packages: `oxlint@^1.71`, `@biomejs/biome@^2.5` (formatter only), `typescript-eslint@^8.62`, `eslint@^10.5`. +Packages: `oxlint@^1.71`, `@biomejs/biome@^2.5` (formatter only), `typescript-eslint@^8.0`, `eslint@^10.0`. ### Packaging correctness: attw + publint @@ -174,10 +174,11 @@ import-time side effects, so this lets consumer bundlers tree-shake unused expor The runtime analog of the line-count discipline: a checked-in byte ceiling on the shipped public entry. size-limit bundles + treeshakes + minifies + brotli-compresses exactly as a consumer's bundler would and -exits non-zero on regression. As implemented, the public entry measures **2.83 kB** (min+brotli, all -deps); the budget is set to **4 kB** - meaningful headroom for active development while still catching a -real regression (an accidental heavy dep or a non-treeshakeable import). Raising the limit must be a -conscious edit, the same discipline as the core LOC budget. +exits non-zero on regression. As implemented, the public entry measures **5.08 kB** (min+brotli, all +deps) against a **6 kB** budget, and the browser entry measures **4.24 kB** against a **5 kB** budget - +meaningful headroom for active development while still catching a real regression (an accidental heavy +dep or a non-treeshakeable import). Raising the limit must be a conscious edit, the same discipline as +the core LOC budget. ```jsonc // .size-limit.json @@ -186,10 +187,15 @@ conscious edit, the same discipline as the core LOC budget. "name": "public entry (min+brotli)", "path": "dist/index.js", // node builtins are runtime-provided, not shipped; esbuild must not try to - // bundle them. Only node:fs is in the shipped graph today; os/path are - // listed defensively. - "ignore": ["node:fs", "node:os", "node:path"], - "limit": "4 kB" + // bundle them. The shipped graph now reaches node:crypto, node:fs, node:os, + // and node:path (fd-based I/O plus the lock file's pid/host/nonce). + "ignore": ["node:crypto", "node:fs", "node:os", "node:path"], + "limit": "6 kB" + }, + { + "name": "browser entry (min+brotli)", + "path": "dist/browser.js", + "limit": "5 kB" } ] ``` @@ -317,7 +323,8 @@ lefthook is the documented upgrade path if parallelism / staged-file scoping is `@changesets/cli@^2.31`, run locally. `changeset init` config (`.changeset/config.json`), adjusted to `access: public` (the package is public; init defaults to `restricted`) and the built-in -`@changesets/cli/changelog` (no GitHub-changelog dependency while the repo is private). `$schema` pins +`@changesets/cli/changelog` (no GitHub-changelog dependency; chosen while the repo was still private and +kept now that it is public, for the minimal dependency surface). `$schema` pins `@changesets/config@3.1.4`. Scripts: `"changeset"` (write an intent file) and `"changeset:version"` (`changeset version` + `bun run sync-version` - bump package.json + write CHANGELOG.md, then sync the exported `version`). There is deliberately NO `changeset publish` script: publishing is done by @@ -363,44 +370,51 @@ coverage path exists. ## Defer to CI (document now, apply when the repo is on GitHub) -These need GitHub Actions. The workflow files are now PREPARED but inert until the repo is on GitHub -(and, for SonarCloud, until a secret is set). What is committed now: +These need GitHub Actions. The repo is on GitHub and all three workflows are live: `ci.yml` and +`sonarcloud.yml` run on every push and PR, and `publish.yml` runs on published GitHub Releases. What is +committed: -Both workflows are hardened the way CodeQL's actions queries and OpenSSF Scorecard expect: every action is +All three workflows are hardened the way CodeQL's actions queries and OpenSSF Scorecard expect: every action is pinned to a full commit SHA (with a `# vX.Y.Z` comment), not a mutable tag, and each workflow declares a least-privilege `permissions: contents: read`. Bump the pins by resolving the new release tag to its SHA (`gh api repos///commits/ --jq .sha`). -- **CI gate** (`.github/workflows/ci.yml`): runs `bun run gate` (+ secrets, audit, license) on push/PR. +- **CI gate** (`.github/workflows/ci.yml`): runs `bun run gate` (+ secrets, audit, license) on push/PR, + plus a `node 22 smoke` job that builds `dist/`, smoke-tests the built package under Node 22 + (`scripts/node-smoke.mjs`), and compiles + round-trips the standalone CLI binary. Needs no secrets, so it runs on FORK PRs too - this is the check that gates external contributions. For green to actually block a merge, enable branch protection on `main` (require the `gate` status check + a PR review); the workflow only reports, it does not block by itself. - **SonarCloud** (`.github/workflows/sonarcloud.yml` + `sonar-project.properties`): CI-based analysis with LCOV coverage (lcov reporter set in `bunfig.toml`; `bun test --coverage` -> `coverage/lcov.info`). Project - keys captured: `projectKey=libredb_libredb-database`, `organization=libredb`. Activation - (post-push fine-tuning): bind the repo to the SonarCloud `libredb` org, add a `SONAR_TOKEN` repo - secret, DISABLE automatic analysis (CI-based is required for coverage), and confirm the latest - `sonarqube-scan-action` major. The DST harness `src/sim/` is marked test code, not production source. + keys captured: `projectKey=libredb_libredb-database`, `organization=libredb`. Activation is complete: the + repo is bound to the SonarCloud `libredb` org, `SONAR_TOKEN` is set, automatic analysis is disabled, and + the quality-gate and coverage badges are live in the README. `src/cli/main.ts` is additionally excluded + from Sonar coverage (`sonar.coverage.exclusions`), mirroring `coveragePathIgnorePatterns` in + `bunfig.toml`. The DST harness `src/sim/` is marked test code, not production source. **Fork PRs skip this job** (`if: push || head.repo == repo`): GitHub does not pass secrets to fork-triggered runs, so the scan would fail through no fault of the contributor. Fork contributions are analyzed after they merge to main; the `gate` (ci.yml) still runs on their PR. - **Publish** (`.github/workflows/publish.yml`): triggers ONLY on `release: [published]` (never on push/PR), runs the full gate, then `npm publish` authenticated via `setup-node` + the `NPMJS_TOKEN` secret. npm runs `prepublishOnly` (build + attw + publint) automatically. Releasing = create a tag + GitHub Release. - Provenance is omitted while private; add `--provenance` + `id-token: write` once public. + Publishes with `--provenance` (`id-token: write`) so npm carries a signed attestation verifiable via + `npm audit signatures`; a guard step verifies the release tag matches package.json, and follow-up jobs + publish to JSR (pinned `jsr@0.14.3`), attach standalone binaries to the GitHub Release, and push + multi-arch images to GHCR and Docker Hub. Deliberately NOT added now (minimalism - the layers above already cover this ground; each can be enabled later if the project wants a stronger posture): -- **Dependency updates:** none for now; `bun outdated` is the manual stand-in. When PR volume justifies - automation, prefer **Dependabot** (GitHub-native, just `.github/dependabot.yml`, no external app, - Bun/`bun.lock` supported) over Renovate (stronger grouping/automerge but needs the external Renovate - GitHub App - a service dependency the minimalism rule avoids until it earns its place). +- **Dependency updates:** Dependabot is configured (`.github/dependabot.yml`): weekly update PRs for + GitHub Actions SHA pins, Docker base digests, and npm devDependencies (grouped into a single PR). + Renovate stays rejected: stronger grouping/automerge, but it needs the external Renovate GitHub App - + a service dependency the minimalism rule avoids. - **SAST:** CodeQL via GitHub's **default setup** (Settings -> Code security -> enable) - no committed workflow file; it auto-detects the language. An advanced `.github/workflows/codeql.yml` (SHA-pinned `github/codeql-action/{init,analyze}`) is only for custom queries/paths - not needed here. -- **npm provenance:** a one-line change to `publish.yml` (`--provenance` + `id-token: write`) once the - repo is public; the placeholder comment is already in the workflow. +- **npm provenance:** implemented - `publish.yml` publishes with `--provenance` + `id-token: write`, + attaching a signed attestation verifiable via `npm audit signatures`. - **Optional security workflows - evaluated and skipped:** OpenSSF Scorecard, GitHub dependency-review, and osv-scanner each add a workflow file plus maintenance, and the existing stack (CodeQL default setup, SonarCloud, secretlint, `bun audit`, the license tripwire) already covers the ground. Add any @@ -428,10 +442,10 @@ Each phase ended green through the gate, committed individually. 1. **Lint + format (done):** Oxlint + Biome (formatter only), ESLint reduced to type-aware, `@eslint/js` removed. 2. **Build (done):** `isolatedDeclarations` in `tsconfig.build.json`, `catalog.ts` annotation added. 3. **Packaging (done):** attw + publint + `prepublishOnly` + `sideEffects: false` (no knip entries needed - scripts suffice). -4. **Size budget (done):** size-limit, measured budget 4 kB (2.83 kB actual), gate reordered (build before size). +4. **Size budget (done):** size-limit, measured budget 6 kB public entry (5.08 kB actual) and 5 kB browser entry (4.24 kB actual), gate reordered (build before size). 5. **Environment (done):** `.editorconfig`, `.bun-version`, `bunfig.toml [install] exact`, CI reads `.bun-version` (preinstall guard dropped; `.npmrc` token remediation advised). 6. **Security + hooks (done):** `bun audit` + secretlint + `.githooks` + `core.hooksPath` (secrets + audit also in CI). 7. **Commit quality (done):** commitlint + config-conventional + `commit-msg` hook. 8. **License (done):** runtime-only tripwire via `scripts/license-check.sh` (bunx, no devDependency). 9. **Release (done):** changesets init + config + first changeset (Node 22 / ES2024 / sideEffects, patch). -10. **CI (done):** `ci.yml` + `sonarcloud.yml` + `publish.yml` committed, SHA-pinned, inert until pushed; dependency bot and optional security workflows deliberately deferred (see "Deliberately NOT added now"). +10. **CI (done):** `ci.yml` + `sonarcloud.yml` + `publish.yml` committed, SHA-pinned, and now live on GitHub; Dependabot was added afterwards (`.github/dependabot.yml`), optional security workflows remain deferred (see "Deliberately NOT added now"). diff --git a/docs/UNDERSTANDING-THE-KERNEL.md b/docs/UNDERSTANDING-THE-KERNEL.md index 823f9f8..cdf8a54 100644 --- a/docs/UNDERSTANDING-THE-KERNEL.md +++ b/docs/UNDERSTANDING-THE-KERNEL.md @@ -51,7 +51,7 @@ A database does not work this way, and understanding why is the first real step. The LibreDB file is not the current state -- it is an **append-only log of every change that ever happened**. You never seek back into the file to edit a spot; you only ever **append to the end**. This structure is called a **write-ahead log -(WAL)** ([`core.ts`](../src/core.ts) L275-294). +(WAL)** ([`core.ts`](../src/core.ts) L403-435). ``` JSON / snapshot model LibreDB / WAL model @@ -74,7 +74,7 @@ recognizable once named: - **`app.log`.** You append log lines; you never rewrite the file to change line 400. Even a delete is an append: it writes a *tombstone* record that says "this key is -gone now" (`OP_DELETE`, [`core.ts`](../src/core.ts) L239), not a physical erasure. +gone now" (`OP_DELETE`, [`core.ts`](../src/core.ts) L438), not a physical erasure. The file only ever grows. **Why append-only?** Because appending is atomic and safe. Rewriting the middle @@ -122,7 +122,7 @@ always showed you the top costume ("l"). `xxd` just takes the costume off. - A **bit** is one yes/no: high voltage (1) or not (0). - A **byte** is 8 bits, giving 2^8 = 256 combinations, so it holds a number from - **0 to 255**. `Key = Uint8Array` in the kernel ([`core.ts`](../src/core.ts) L35) + **0 to 255**. `Key = Uint8Array` in the kernel ([`core.ts`](../src/core.ts) L81) means "unsigned 8-bit integers" -- a sequence of bytes. - **Hex** is just a compact way to *write* a byte. Because 4 bits map exactly to one hex digit (2^4 = 16), one byte is always exactly two hex digits. `0x6c` @@ -133,7 +133,7 @@ The deepest point: **a byte is a universal medium.** A string, a number, an image, a video -- on disk they are all the same thing, a sequence of bytes. The only difference is *interpretation*: the byte `01100001` is "97" through one lens and "a" through another. This is exactly why the kernel keeps values as opaque -bytes and never interprets them ([`core.ts`](../src/core.ts) L37-39) -- that +bytes and never interprets them ([`core.ts`](../src/core.ts) L83-85) -- that decision is a lens concern, not a kernel one, and it is what lets one storage substrate carry three data models. @@ -150,27 +150,31 @@ sample `libredb-studio/data/demo.libredb` begins: 00000020: 6572 7300 0000 7e7b 226b 696e 6422 3a22 ers...~{"kind":" ``` -The record format ([`core.ts`](../src/core.ts) L285-291) is: +This sample predates the v1 on-disk format, so it uses the legacy v0.1.x framing: no file header, and an 8-byte record header. The formats ([`core.ts`](../src/core.ts) L413-434) are: ``` -record = [u32 payloadLength][u32 crc32(payload)][payload] +v1 file = [4-byte magic "LRDB"][u16 formatVersion][u16 reserved], then records +v1 record = [u32 payloadLength][u32 crc32(the 4 length bytes)][u32 crc32(payload)][payload] +legacy record = [u32 payloadLength][u32 crc32(payload)][payload] <- decoded below payload = one or more ops, back to back set: [u8 1][u32 keyLength][key][u32 valueLength][value] delete: [u8 0][u32 keyLength][key] integers are big-endian (most significant byte first) ``` +The kernel still opens legacy files (recognized by their first record replaying cleanly) and keeps appending legacy-framed records to them; a new database starts with the 8-byte file header and uses the 12-byte record header, whose extra checksum covers the length field itself. + Decoding the first record byte by byte: | Bytes | Value | Meaning | Code | | --- | --- | --- | --- | -| `00 00 00 9d` | 157 | payloadLength | `encodeRecord` ([`core.ts`](../src/core.ts) L363) | -| `ea 44 78 48` | 0xea447848 | crc32 of the payload | L364 | -| `01` | 1 | OP_SET | L349 | -| `00 00 00 16` | 22 | keyLength | L351 | -| `00 6c 69 ... 73` | `\x00libredb:catalog:users` | the key (22 bytes) | L353 | -| `00 00 00 7e` | 126 | valueLength | L357 | -| `7b 22 6b ...` | `{"kind":"relational",...}` | the value (126 bytes) | L359 | +| `00 00 00 9d` | 157 | payloadLength | `encodeRecord` ([`core.ts`](../src/core.ts) L530) | +| `ea 44 78 48` | 0xea447848 | crc32 of the payload (legacy framing) | L532 | +| `01` | 1 | OP_SET | L515 | +| `00 00 00 16` | 22 | keyLength | L516 | +| `00 6c 69 ... 73` | `\x00libredb:catalog:users` | the key (22 bytes) | L518 | +| `00 00 00 7e` | 126 | valueLength | L521 | +| `7b 22 6b ...` | `{"kind":"relational",...}` | the value (126 bytes) | L523 | Note the key begins with a `0x00` byte. That is the catalog's reserved marker ([`lens/catalog.ts`](../src/lens/catalog.ts), `RESERVED_MARKER`): `0x00` is the @@ -188,19 +192,19 @@ text? Four reasons, each of which you can now see in the bytes: scanning for it. Fixed-width fields are a `struct`; JSON is free text you must parse. 2. **Length-prefixing kills delimiter-hunting.** "Read 4 bytes for the length, - then read exactly that many bytes" ([`core.ts`](../src/core.ts) L377-383). + then read exactly that many bytes" ([`core.ts`](../src/core.ts) L559-571). No scanning for a closing quote, no escaping. Crucially, the value can then contain *any* byte -- even `0x00`, even `"` -- so the store is byte-safe and can hold arbitrary blobs. Text formats are not byte-safe. 3. **Space and speed.** Binary integers are smaller than their text spellings and - need no parser; `readU32` ([`core.ts`](../src/core.ts) L311) turns 4 bytes into + need no parser; `readU32` ([`core.ts`](../src/core.ts) L464) turns 4 bytes into a number with one shift-and-add. 4. **Sortability.** Ordering over raw bytes is exact and stable, and big-endian makes byte-lexicographic order match numeric order. String order is not stable across encodings (`"10" < "2"` in UTF-16). See `compareKeys` - ([`core.ts`](../src/core.ts) L189). + ([`core.ts`](../src/core.ts) L285). -The **CRC-32** ([`core.ts`](../src/core.ts) L326, written without a lookup table +The **CRC-32** ([`core.ts`](../src/core.ts) L479, written without a lookup table so the mechanism stays visible) is the record's "is this complete?" checksum. It is what lets recovery tell a fully-written record from one a crash left half-flushed. @@ -212,14 +216,14 @@ half-flushed. A common early misconception: "we opened the file in append mode, so writing to the file must keep memory in sync automatically." It does not. There is **no magic** and **no automatic bridge** between the file and the in-memory data. -Opening in append mode (`openSync(path, "a")`, +Opening in append mode (`openSync(path, "a+")` -- read plus append-only writes, [`adapter/node-fs.ts`](../src/adapter/node-fs.ts)) only means "writes go to the end of the file." It says nothing about memory. Memory and disk are two separate worlds -- a JavaScript array on the heap, and a byte stream on disk -- and *the kernel's code explicitly bridges them*. Writing `employees.insert(...)` runs through `transact` -([`core.ts`](../src/core.ts) L493): +([`core.ts`](../src/core.ts) L882): ```ts transact(run) { @@ -227,11 +231,11 @@ transact(run) { const journal = []; const result = run(makeTransaction(working, journal)); // inside, each tx.set does two things: - // applySet(working, key, value) // (A) update the in-memory COPY L259 - // journal.push({kind:"set",...}) // records the op in a list L260 + // applySet(working, key, value) // (A) update the in-memory COPY L369 + // journal.push({kind:"set",...}) // records the op in a list L370 if (log !== null && journal.length > 0) - log.append(journal); // (B) write to DISK (append + fsync) L508 - committed = working; // (C) make it official (atomic swap) L509 + log.append(journal); // (B) write to DISK (append + fsync) L910 + committed = working; // (C) make it official (atomic swap) L924 } ``` @@ -245,17 +249,19 @@ Two design choices fall out of this being explicit: - **Copy-on-write commit.** The transaction works on a *copy* (`working`), and a successful commit is a single atomic reference assignment `committed = working` - ([`core.ts`](../src/core.ts) L509). If `run` throws, that line is never reached + ([`core.ts`](../src/core.ts) L924). If `run` throws, that line is never reached and `committed` keeps its old value -- so an abort applies nothing, with no undo logic. Atomicity is "the reference either changed or it did not." - **The order matters: disk before memory.** (B) runs before (C). If the disk - append fails, we throw *before* the swap, leaving memory and disk agreeing on - the prior state. This is only possible *because* they are separate operations - you can order. + append fails, we throw a typed error (code `FAILED`) *before* the swap, so memory + keeps the prior state -- and the database latches: every later `transact()` throws + `FAILED` until close and reopen, because the file tail may hold a torn record that + only recovery can repair. This is only possible *because* they are separate + operations you can order. A subtle bonus: writes within one transaction accumulate in `journal` (a list in RAM) and are flushed as **one** appended record ([`core.ts`](../src/core.ts) -L508). Five `set`s in a transaction produce one disk append, not five -- atomicity +L910). Five `set`s in a transaction produce one disk append, not five -- atomicity and throughput at once. > Aside: memory-mapped files (mmap) *do* let the OS couple a memory region to a @@ -276,7 +282,7 @@ the WAL, `fsync`, CRC, and recovery. Here is the danger. `append` hands bytes to the OS, but the OS may keep them in its **page cache** (RAM) and write to the physical device "later." If the power fails in that window, a commit you already reported as successful is lost. -**`fsync`** ([`core.ts`](../src/core.ts) L127, called at L437; `fsyncSync` in the +**`fsync`** ([`core.ts`](../src/core.ts) L209, called at L810; `fsyncSync` in the Node adapter) is the command that closes this window: "OS, flush this file to durable storage now, and do not return until it is there." It is the single line that gives the word "durability" its weight. @@ -292,16 +298,16 @@ writeback) is unsafe. Delegating is not the same as forgetting. - Committed (fsync'd) data survives; recovery brings it back. - An in-flight transaction not yet fsync'd is lost -- which is *correct* (atomicity: it never happened). -- A torn last record is truncated away on recovery - ([`core.ts`](../src/core.ts) L413). +- A torn last record is truncated away on recovery, fsync'd, and reported through + the `onRecovery` open option ([`core.ts`](../src/core.ts) L742-755). **The trust boundary.** Good systems do not "trust the language" broadly; they shrink and make explicit exactly what they must trust. The kernel imports nothing from `node:`. Every byte to disk goes through one tiny interface, the -`FileSystem` seam ([`core.ts`](../src/core.ts) L109-134): +`FileSystem` seam ([`core.ts`](../src/core.ts) L178-214): ```ts -interface FileSystem { open(path): WalFile } +interface FileSystem { open(path): WalFile; lock?(path): release } // lock is optional interface WalFile { size; read; append; fsync; truncate; close } // that is all ``` @@ -333,26 +339,33 @@ write format was designed for the reader.** Every record is length-framed, so th reader never guesses where a record ends -- it reads the header, jumps exactly that many bytes, and lands on the next record. The format is *self-describing*. -**Cold read -- `recover`** ([`core.ts`](../src/core.ts) L397) reads the whole file +**Cold read -- `recover`** ([`core.ts`](../src/core.ts) L666) reads the whole file and replays every record: ``` -offset = 0 -loop: - read 8-byte header -> payloadLength, crc L402-403 - end = offset + 8 + payloadLength - if end > file length -> STOP: record is torn L406 - if crc32(payload) != crc -> STOP: record corrupt L408 - replayPayload(payload) -> apply set/delete L409 - offset = end jump to the next record -if leftover bytes remain -> truncate them L413 +identify the file first (`recover`, L666): + empty file -> new database (header written with the first commit) + starts with "LRDB" magic -> v1; a newer version -> throw UNSUPPORTED_VERSION + no magic, but the first record replays cleanly -> legacy v0.1.x + anything else -> throw NOT_A_DATABASE; the file is left untouched +then replay records (`replayLog`, L618): + read the record header (12 bytes v1, 8 legacy) -> payloadLength, checksums + v1: header fails its own checksum -> throw CORRUPT_WAL L629 + end = header end + payloadLength + if end > file length -> STOP: torn tail L634 + if crc32(payload) != stored crc: + with data after it -> throw CORRUPT_WAL, never truncate L638 + at the file's end -> STOP: damaged tail L643 + replayPayload(payload) -> apply set/delete L645 + offset = end +leftover tail bytes -> truncate + fsync, reported via onRecovery L742-755 ``` Three things happen as the flat log becomes the in-memory state -- this is the whole of "reading": 1. **Collapse.** Superseded records and tombstones are replayed and dropped via - `applySet` / `applyDelete` ([`core.ts`](../src/core.ts) L219, L227) -- the same + `applySet` / `applyDelete` ([`core.ts`](../src/core.ts) L315, L323) -- the same functions live writes use, so "what a set means" has exactly one definition. The last write of a key wins. In `demo.libredb`, `project:1` is written twice on disk but collapses to one live key in memory. @@ -377,25 +390,27 @@ whole of "reading": Sorting is what makes queries cheap (below); the cold read pays for it once. 3. **Restructure.** A flat, framed byte stream becomes a JavaScript array of - `{key, value}` objects (`StoredEntry[]`, [`core.ts`](../src/core.ts) L175). The + `{key, value}` objects (`StoredEntry[]`, [`core.ts`](../src/core.ts) L271). The framing (lengths, CRC) is gone -- the JS array holds structure natively. Torn-tail handling is the payoff of append-only: because a crash can only damage -the last record, recovery trusts every record up to the first one that is -incomplete or fails its checksum, and truncates the rest so the next append starts -on a clean boundary. +the last record, recovery trusts every record up to a genuinely torn or half-flushed +tail, truncates that tail away (fsync'd, and reported through the `onRecovery` open +option so it is never silent), and refuses to open with a `CORRUPT_WAL` error when a +record fails its checksum with intact data after it -- that is corruption, not a crash +artifact, and truncating there would silently destroy committed records. **Hot read -- queries.** Once in memory, reads never touch disk: - `get(key)` is a **binary search** over the sorted array (`locate`, - [`core.ts`](../src/core.ts) L203; `get` at L250) -- O(log n), like finding a + [`core.ts`](../src/core.ts) L299; `get` at L360) -- O(log n), like finding a word in a dictionary by halving. - `getRange(start, end)` is "find the start, walk until the end" - ([`core.ts`](../src/core.ts) L262-270). Because the array is sorted, all keys in + ([`core.ts`](../src/core.ts) L377-399). Because the array is sorted, all keys in a range are contiguous -- which is why `prefix("users:")` (scanning a table) is cheap even though `users:1` and `users:2` were far apart on disk. - **Read-your-writes**: a transaction reads from its `working` copy, so it sees - its own not-yet-committed writes ([`core.ts`](../src/core.ts) L242-247). + its own not-yet-committed writes ([`core.ts`](../src/core.ts) L358-363). The resolution to the "reading is hard" worry: all the hard work is concentrated into a *single moment* (open), which produces a clean sorted structure that makes @@ -459,20 +474,22 @@ db.close(); // free memory, close the file descri - **`open`** ([`index.ts`](../src/index.ts) wires the `node:fs` adapter when a path is given without one) calls `openLog` -> `recover` - ([`core.ts`](../src/core.ts) L429, L397). It opens an append-mode descriptor, - reads the **whole file** into memory (`readFileSync` in the Node adapter), and - replays it into the sorted `committed` array. This is the cold boot. + ([`core.ts`](../src/core.ts) L769, L666). It opens an append-mode descriptor, + reads the **whole file** into memory (a positional-read loop over the file + descriptor in the Node adapter), and replays it into the sorted `committed` + array. This is the cold boot. - **Use.** Reads come from memory; writes append + fsync through the descriptor that stays open for the session. -- **`close`** ([`core.ts`](../src/core.ts) L517) sets `closed`, closes the file - descriptor (`log.close()`), and drops the array (`committed = []`) for GC. It is - idempotent. Importantly, **durability does not depend on `close`** -- every - commit was already fsync'd, so committed data survives even a crash with no - `close`. `close` only releases resources. +- **`close`** ([`core.ts`](../src/core.ts) L932) sets `closed`, closes the file + descriptor (`log.close()`), releases the exclusive open lock, and drops the + array (`committed = []`) for GC. It is idempotent. Importantly, **durability + does not depend on `close`** -- every commit was already fsync'd, so + committed data survives even a crash with no `close`. `close` only releases + resources. **The memory model, and its one big consequence.** Two memory moments matter: -- *Transient peak* at open: `readFileSync` loads the entire file (say 500 MB) into +- *Transient peak* at open: recovery reads the entire file (say 500 MB) into a buffer while replaying. - *Retained*: after replay, that buffer is GC'd; what stays is the live data only (say 50 MB), because recovery collapsed the dead records. @@ -531,7 +548,7 @@ machinery (MVCC + conflict detection). In LibreDB it is *free*: the API is synchronous and single-threaded, so each transaction body runs to completion before the next begins -- the schedule is serial by construction, and the kernel forbids nested transactions to keep it that way -([`core.ts`](../src/core.ts) L490-497). Cheap serializability is a gift of the +([`core.ts`](../src/core.ts) L887-889). Cheap serializability is a gift of the constraint, not a feature that was built. --- @@ -549,9 +566,10 @@ constraint, not a feature that was built. - **A byte-honest multi-model foundation.** The kernel stores opaque bytes in a single ordered keyspace; three data models (kv, document, relational) are lenses over it with no duplicated storage. -- **A small, explicit trust boundary.** The `FileSystem` seam is six operations. - The same kernel runs on Node, in the browser (OPFS), and against a fault- - injecting fake, with zero `node:` imports in the core. +- **A small, explicit trust boundary.** The `FileSystem` seam is `open` (plus an + optional advisory `lock`) returning a six-operation file handle. The same + kernel runs on Node, in the browser (OPFS), and against a fault-injecting + fake, with zero `node:` imports in the core. - **Free serializability.** The serial execution model gives the strongest isolation level at no cost. - **Reliability discipline.** 100% line/function/statement coverage, plus a @@ -582,13 +600,11 @@ kernel small; see [`ARCHITECTURE.md` section 10.2/10.3](../ARCHITECTURE.md) and project explicitly refuses (see [`MANIFESTO.md`](../MANIFESTO.md)). - **mmap and OS-coupled memory.** Explicit hand-coded sync is chosen over memory-mapped files for clarity and control over the durability point. -- **Group-commit / fsync-batching, versioned WAL headers, expanded fault - profiles.** Exploratory durability-hardening directions tracked in issue - [#9](https://github.com/libredb/libredb-database/issues/9); intentionally not in - the current core. -- **Directory fsync on first file creation.** A known durability gap - ([`ARCHITECTURE.md` 10.3](../ARCHITECTURE.md)): creating a file durably needs a - directory fsync, currently not done. +- **Group-commit / fsync-batching.** An exploratory durability-hardening direction + tracked in issue [#9](https://github.com/libredb/libredb-database/issues/9); + intentionally not in the current core. (Two other directions from #9 have since + landed: the on-disk format now carries a versioned file header, and the DST + harness has fault-injection profiles.) The unifying rule: **durability hardening lands inside the guarded core under heavy review; scaling features are pushed above the trust boundary wherever the @@ -633,14 +649,17 @@ correct, not by absorbing every feature. **Source (all line numbers are [`src/core.ts`](../src/core.ts) unless noted):** -- Types: `Key`/`Value` (L35-39), `Transaction` (L62-75), `FileSystem`/`WalFile` - (L109-134), `OpenOptions` (L143-155). -- Ordered store: `compareKeys` (L189), `locate` (L203), `applySet` (L219), - `applyDelete` (L227), `makeTransaction` (L248). -- Record codec: format comment (L275-294), `writeU32` (L302), `readU32` (L311), - `crc32` (L326), `encodeRecord` (L339), `replayPayload` (L371). -- Durability + recovery: `recover` (L397), `openLog` (L429, fsync at L437), - `open` (L453), `transact` (L493, disk-then-memory at L508-509), `close` (L517). +- Types: `LibreDbError`/`ErrorCode` (L44-70), `Key`/`Value` (L81-85), `Transaction` + (L113-129), `FileSystem`/`WalFile` (L178-214), `RecoveryInfo` (L218-222), + `OpenOptions` (L231-250). +- Ordered store: `compareKeys` (L285), `locate` (L299), `applySet` (L315), + `applyDelete` (L323), `makeTransaction` (L358). +- Record codec: format comment (L403-435), `writeU32` (L455), `readU32` (L464), + `crc32` (L479), `encodeFileHeader` (L491), `encodeRecord` (L505), + `replayPayload` (L551). +- Durability + recovery: `replayLog` (L618), `recover` (L666), `isLegacyLog` (L724), + `finishReplay` (L742), `openLog` (L769, fsync at L810), `open` (L826), + `transact` (L882, disk-then-memory at L910-924), `close` (L932). - Adapters and lenses: [`src/adapter/node-fs.ts`](../src/adapter/node-fs.ts), [`src/adapter/opfs.ts`](../src/adapter/opfs.ts), [`src/adapter/store.ts`](../src/adapter/store.ts), diff --git a/docs/guides/README.md b/docs/guides/README.md index 80ec28c..4ca505f 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -9,6 +9,6 @@ move up to documents and typed tables. All three are thin lenses over the same o reach lens. - [Catalog](./catalog.md) — the self-describing registry of what each namespace holds. -See also: the [README](../../README.md) for the quick start, [`../ARCHITECTURE.md`](../ARCHITECTURE.md) +See also: the [README](../../README.md) for the quick start, [`../../ARCHITECTURE.md`](../../ARCHITECTURE.md) for how it all works under the hood, and [`../RELIABILITY.md`](../RELIABILITY.md) for the durability and crash-recovery story. diff --git a/docs/guides/catalog.md b/docs/guides/catalog.md index 232561b..44e2e4c 100644 --- a/docs/guides/catalog.md +++ b/docs/guides/catalog.md @@ -27,7 +27,8 @@ registry.get("logs"); ``` The catalog lives under a reserved key prefix that sorts below all user data, so its entries never -appear in a `kv`, `doc`, or `table` scan — and no user row leaks into the registry. `kv` namespaces +appear in a `doc` or `table` scan, and no `kv` scan over user keys crosses into them — and no user +row leaks into the registry. `kv` namespaces are deliberately not cataloged: `kv` is the raw layer with full keyspace access. A tool that renders the raw `kv` layer (so it sees everything, including the catalog) should hide @@ -42,8 +43,9 @@ const db = open(); // ... user writes through doc/table, which also write catalog entries ... // range is half-open [start, end). "" encodes to the lowest bytes, and -// "\u{10FFFF}" (the highest Unicode code point) encodes above any UTF-8 text key -// the lenses produce, so this interval covers the whole keyspace. (kv.prefix +// "\u{10FFFF}" (the highest Unicode code point) encodes above any key that starts +// with a lower code point, so this interval covers every practical key. (A key +// beginning with U+10FFFF itself falls outside the half-open interval.) (kv.prefix // cannot scan everything — it rejects an empty prefix.) This is the same // full-keyspace pattern LibreDB Studio's provider uses. const visible = kv(db) diff --git a/docs/guides/relational.md b/docs/guides/relational.md index 51753a2..d1d4109 100644 --- a/docs/guides/relational.md +++ b/docs/guides/relational.md @@ -31,8 +31,10 @@ users.get("missing"); // undefined db.close(); ``` -A column type is one of `"string"`, `"number"`, `"boolean"`, or `"object"` (a plain JSON object). The -`primaryKey` must name a declared `"string"` column — it becomes the kernel key. +A column type is one of `"string"`, `"number"`, `"boolean"`, or `"object"` (a plain JSON object). A +`"number"` column accepts only finite numbers — `NaN` and `Infinity` are rejected at insert (JSON +cannot represent them, so they would round-trip as a schema-violating `null`). The `primaryKey` must +name a declared `"string"` column — it becomes the kernel key. ## Validation is strict at insert From a0978c33d8d1abc44058e3de0cb39e75b380c146 Mon Sep 17 00:00:00 2001 From: cevheri Date: Fri, 3 Jul 2026 09:46:42 +0300 Subject: [PATCH 16/16] fix(adapter,dst): narrow directory-fsync error handling; document the empty append prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fsyncDirectoryOf no longer swallows every error: codes that mean the platform cannot fsync a directory are tolerated, while a real disk failure (EIO) propagates — a database that cannot make its own existence durable must say so (the fsyncgate lesson). Syscalls are injectable so the classification is pinned by tests; changeset notes the behavior change - SimFS armAppendError comments now state that the kept prefix may be empty (a failure before any byte reached disk is a real ENOSPC outcome); the deterministic non-empty poisoned-tail case stays pinned in core.hardening.test.ts --- .changeset/pre-announcement-hardening.md | 2 +- src/adapter/node-fs.test.ts | 29 +++++++++++++++++++ src/adapter/node-fs.ts | 37 +++++++++++++++++++----- src/sim/simfs.ts | 18 ++++++++---- 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/.changeset/pre-announcement-hardening.md b/.changeset/pre-announcement-hardening.md index f355845..5060108 100644 --- a/.changeset/pre-announcement-hardening.md +++ b/.changeset/pre-announcement-hardening.md @@ -22,7 +22,7 @@ Kernel: Adapters: -- node-fs: creating a database fsyncs the parent directory (a fresh database can no longer vanish wholesale on power loss); recovery truncation is fsync'd; reads are positional on the WAL's own file descriptor instead of re-reading the whole file per call. +- node-fs: creating a database fsyncs the parent directory (a fresh database can no longer vanish wholesale on power loss); a directory-fsync failure that is not a platform limitation (e.g. EIO) now surfaces as an error instead of being silently ignored; recovery truncation is fsync'd; reads are positional on the WAL's own file descriptor instead of re-reading the whole file per call. - OPFS: reads loop until filled, so a legal short read can no longer masquerade as a torn tail; recovery treats an incomplete read as an IO fault (`INCOMPLETE_READ`), never as license to truncate. Lenses: diff --git a/src/adapter/node-fs.test.ts b/src/adapter/node-fs.test.ts index 421aede..a3eb29c 100644 --- a/src/adapter/node-fs.test.ts +++ b/src/adapter/node-fs.test.ts @@ -71,6 +71,35 @@ test("fsyncDirectoryOf tolerates a directory that cannot be opened", () => { expect(() => fsyncDirectoryOf(join(tmpdir(), "libredb-no-such-dir-xyz", "file"))).not.toThrow(); }); +test("fsyncDirectoryOf surfaces real IO failures and absorbs only unsupported-platform codes", () => { + const fail = (code: string): never => { + const error = new Error(code) as Error & { code: string }; + error.code = code; + throw error; + }; + const io = (fsyncCode: string) => ({ + openSync: () => 42, + fsyncSync: () => fail(fsyncCode), + closeSync: () => {}, + }); + // "This platform cannot fsync a directory" codes are tolerated... + for (const code of ["EINVAL", "ENOTSUP", "EPERM", "EACCES", "EBADF", "EISDIR"]) { + expect(() => fsyncDirectoryOf("/any/file", io(code))).not.toThrow(); + } + // ...a real disk failure is not: the new directory entry may not be durable, + // and silence here would be exactly the fsyncgate mistake. + expect(() => fsyncDirectoryOf("/any/file", io("EIO"))).toThrow(/EIO/); + // A codeless throw is treated as unsupported (UNKNOWN), not as a disk fault. + const codeless = { + openSync: () => 42, + fsyncSync: () => { + throw new Error("no code"); + }, + closeSync: () => {}, + }; + expect(() => fsyncDirectoryOf("/any/file", codeless)).not.toThrow(); +}); + test("fsyncDirectoryOf fsyncs an existing parent directory without error", () => { const path = tempPath("synced"); writeFileSync(path, "x"); diff --git a/src/adapter/node-fs.ts b/src/adapter/node-fs.ts index f6f00c1..8f2aa55 100644 --- a/src/adapter/node-fs.ts +++ b/src/adapter/node-fs.ts @@ -187,23 +187,44 @@ export function forceUnlock(path: string): void { }); } +/** Error codes that mean "this platform cannot fsync a directory" (Windows + * refuses to open one; some filesystems refuse the fsync) — the only failures + * a directory fsync may silently absorb. ENOENT is included for the caller + * that probes a path whose directory is already gone. A code outside this set + * (EIO above all) is a REAL failure: the new directory entry may not be + * durable, and pretending otherwise would be the exact silence the fsyncgate + * lesson warns about. */ +const DIR_FSYNC_UNSUPPORTED = new Set(["EACCES", "EBADF", "EINVAL", "EISDIR", "ENOENT", "ENOTSUP", "EPERM", "UNKNOWN"]); + +/** The syscalls {@link fsyncDirectoryOf} performs, injectable so a test can + * exercise the failure classification without a faulty real disk. */ +interface DirSyncIo { + openSync(path: string, flags: string): number; + fsyncSync(fd: number): void; + closeSync(fd: number): void; +} + /** * Fsync the directory containing `path`, making a just-created file's directory * entry durable. POSIX leaves a new entry volatile until the directory itself * is fsync'd — without this, a freshly created database (and every commit in - * it) can vanish wholesale on power loss. Exported for the tests that pin it. + * it) can vanish wholesale on power loss. Platforms that cannot fsync a + * directory are tolerated (see {@link DIR_FSYNC_UNSUPPORTED}); any other + * failure — an EIO from the disk — propagates, because a database that cannot + * make its own existence durable must say so rather than carry on. Exported + * for the tests that pin it. */ -export function fsyncDirectoryOf(path: string): void { +export function fsyncDirectoryOf(path: string, io: DirSyncIo = { openSync, fsyncSync, closeSync }): void { try { - const dirFd = openSync(dirname(path), "r"); + const dirFd = io.openSync(dirname(path), "r"); try { - fsyncSync(dirFd); + io.fsyncSync(dirFd); } finally { - closeSync(dirFd); + io.closeSync(dirFd); } - } catch { - // Platforms without directory fsync (Windows) throw on the open or the - // fsync; directory-entry durability is the OS's best effort there. + } catch (error) { + const code = (error as { code?: string }).code ?? "UNKNOWN"; + if (!DIR_FSYNC_UNSUPPORTED.has(code)) throw error; } } diff --git a/src/sim/simfs.ts b/src/sim/simfs.ts index ee259a7..690326c 100644 --- a/src/sim/simfs.ts +++ b/src/sim/simfs.ts @@ -43,7 +43,9 @@ export class SimFS implements FileSystem { /** When set, the NEXT read returns a seeded-short prefix, then disarms. */ private shortReadArmed = false; /** When set, the NEXT append persists only a seeded STRICT prefix of its - * bytes, then throws — a partial write cut short by ENOSPC/EIO. */ + * bytes — possibly EMPTY (the write failed before anything reached disk), + * never the full record — then throws, modelling ENOSPC/EIO cutting a write + * short at an arbitrary point including before it started. */ private appendErrorArmed = false; /** When set, the NEXT fsync throws — the bytes stay pending (not durable), * modelling a durability point that failed after the write. */ @@ -66,8 +68,11 @@ export class SimFS implements FileSystem { append: (b) => { if (this.appendErrorArmed) { this.appendErrorArmed = false; - // A STRICT prefix (never the full record): the fault is "the write - // was cut short", so the record on disk must be torn. + // A STRICT prefix, never the full record. kept may be ZERO — a + // failure before any byte reached the disk is as real an ENOSPC + // outcome as a mid-record tear, and the seeded range covers both. + // (The deterministic poisoned-tail case, which needs a non-empty + // tear, is pinned separately in core.hardening.test.ts.) const kept = Math.floor(this.random() * b.length); for (const byte of b.subarray(0, kept)) f.pending.push(byte); throw new Error("simfs: injected append fault (ENOSPC)"); @@ -90,9 +95,10 @@ export class SimFS implements FileSystem { } /** Arm a one-shot append fault: the next {@link WalFile.append} persists a - * seeded strict prefix of its bytes and throws. The torn record this leaves - * is exactly the poisoned-tail scenario the kernel's failure latch exists - * for (audit finding B3 / fsyncgate). */ + * seeded strict prefix of its bytes (possibly none of them) and throws. + * A non-empty prefix is exactly the poisoned-tail scenario the kernel's + * failure latch exists for (audit finding B3 / fsyncgate); an empty one is + * the clean-failure variant the latch must also survive. */ armAppendError(): void { this.appendErrorArmed = true; }