From b907f870f6008121fbc479e0f1ace230b5c8109c Mon Sep 17 00:00:00 2001 From: User Joe <5625795+jtrotsky@users.noreply.github.com> Date: Sat, 13 Jun 2026 21:33:38 +1200 Subject: [PATCH] feat: node26 image using built-in node:sqlite (drop native toolchain) Moves the FreeBSD Papra image to node26 (matching upstream's .nvmrc) and replaces the better-sqlite3 driver swap with node26's built-in node:sqlite, driven through drizzle-orm/sqlite-proxy. This removes the entire C toolchain from the builder (no FreeBSD-clang/lld/toolchain/clibs-dev, no node-gyp, python, gmake) -- there is no native addon to compile -- and decouples the DB layer from node ABI churn (the better-sqlite3 11->12 break on node26 was exactly this class of problem). node26 specifics: - node26 ships only in FreeBSD's `latest` pkg branch (the base image points at `quarterly`, which tops out at node25), so switch the branch before install in both stages. - papra uses the TC39 Temporal global. Official node26 ships it built-in, but FreeBSD's node26 package is compiled WITHOUT it (and --harmony-temporal is a no-op here), so preload a spec-compliant polyfill via `node --import temporal-polyfill/global`. database.ts maps node:sqlite to the libsql semantics papra expects: - sqlite-proxy callback uses StatementSync.setReturnArrays(true) for the positional rows drizzle's query builder wants. - attachBatch: custom db.batch (sqlite-proxy's native batch can't handle papra's eagerly-executed db.run() results). - attachLibsqlRaw: raw db.run/all/get return libsql's row OBJECTS (migrations 0010-0019 read db.run(...).rows; the health check reads db.all(...)). The other three FreeBSD patches (health-check, tasks-driver, migration 0006) are unchanged. Remote libSQL/Turso URLs remain unsupported (local file DB only) -- same as before. Co-Authored-By: Claude Opus 4.8 --- Containerfile | 70 ++++---- Containerfile.j2 | 70 ++++---- patches/database.ts | 249 ++++++++++++++++----------- root/etc/services.d/papra-server/run | 11 +- 4 files changed, 211 insertions(+), 189 deletions(-) diff --git a/Containerfile b/Containerfile index ac35bf1..bde16b2 100644 --- a/Containerfile +++ b/Containerfile @@ -20,24 +20,17 @@ FROM ghcr.io/daemonless/base:${BASE_VERSION} AS builder ARG UPSTREAM_URL # Build dependencies. -# - node24 / npm-node24 / corepack: JS toolchain (Papra targets node26; node24 is -# the newest LTS in FreeBSD pkg — node26 is not yet packaged — and runs Papra fine). +# - node26 / npm-node26 / corepack: JS toolchain. This image uses node26's +# BUILT-IN `node:sqlite` for the database (see patches/database.ts), so there +# is NO native addon to compile — hence NO C toolchain / node-gyp / python here. # - git-lite jq: clone the release tag resolved from the GitHub API. -# - pkgconf python3 gmake: node-gyp's build driver dependencies. -# The daemonless base jail ships NO C toolchain and NO /usr/include, so the -# better-sqlite3 node-gyp build needs a full FreeBSD-native toolchain. We pull the -# pkgbase packages so everything lands in /usr/bin under the default names node-gyp -# expects (cc, c++, ld, ar, ranlib) — no CC/CXX overrides needed: -# - FreeBSD-clang → /usr/bin/cc, clang, c++, clang++ -# - FreeBSD-lld → /usr/bin/ld (LLVM linker) -# - FreeBSD-toolchain → /usr/bin/ar, nm, ranlib (the archiver; `ar: not found` fix) -# - FreeBSD-clibs-dev → /usr/include libc + libc++ headers and crt objects -# All builder-stage only — discarded, does not affect the final runtime image size. -RUN pkg update && pkg install -y \ - node24 npm-node24 corepack \ +# +# node26 ships only in the FreeBSD `latest` pkg branch (the stock base points at +# `quarterly`, which tops out at node25), so switch the branch before installing. +RUN sed -i '' -e 's,/quarterly,/latest,' /etc/pkg/FreeBSD.conf && \ + pkg update && pkg install -y \ + node26 npm-node26 corepack \ git-lite jq \ - pkgconf python3 gmake \ - FreeBSD-clang FreeBSD-lld FreeBSD-toolchain FreeBSD-clibs-dev \ && pkg clean -ay # Enable pnpm via corepack (Papra pins pnpm 11.1.1). @@ -85,12 +78,13 @@ RUN for f in \ # FreeBSD DB driver swap: upstream's @libsql/client file: driver loads the native # `libsql` package which has no FreeBSD prebuilt and no buildable source. Replace -# the local-file path with better-sqlite3 (builds from bundled SQLite via node-gyp) -# and add a libsql-compatible db.batch() polyfill. See patches/database.ts. +# the local-file path with node26's built-in `node:sqlite` (driven via +# drizzle-orm/sqlite-proxy) and add libsql-compatible db.batch()/db.run() polyfills. +# See patches/database.ts. COPY patches/database.ts ./apps/papra-server/src/modules/app/database/database.ts # Companion fix: the DB health check reads a libsql-specific result shape, which -# better-sqlite3 doesn't produce. Use a driver-agnostic query so /api/health is +# node:sqlite doesn't produce. Use a driver-agnostic query so /api/health is # accurate. See patches/health-check.repository.ts. COPY patches/health-check.repository.ts ./apps/papra-server/src/modules/app/health-check/health-check.repository.ts @@ -101,16 +95,14 @@ COPY patches/health-check.repository.ts ./apps/papra-server/src/modules/app/heal COPY patches/tasks-driver.registry.ts ./apps/papra-server/src/modules/tasks/drivers/tasks-driver.registry.ts # Companion fix: migration 0006 uses libSQL's non-standard `ALTER COLUMN ... TO` -# syntax, which standard SQLite (better-sqlite3) rejects. Replace it with the +# syntax, which standard SQLite (node:sqlite) rejects. Replace it with the # SQLite-supported table-rebuild recipe (same resulting schema). # See patches/0006-organizations-invitations-improvement.migration.ts. COPY patches/0006-organizations-invitations-improvement.migration.ts ./apps/papra-server/src/migrations/list/0006-organizations-invitations-improvement.migration.ts -# Add better-sqlite3 to the server workspace so the patched database.ts can -# resolve it. We edit package.json directly (rather than `pnpm add`) so the -# install stays a single deterministic resolve. better-sqlite3 ships a SQLite -# amalgamation it compiles from source via node-gyp — no system SQLite needed. -RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p.dependencies['better-sqlite3']='^11.10.0';require('fs').writeFileSync(f,JSON.stringify(p,null,2)+'\n')" +# No database dependency to inject: the patched database.ts uses node26's built-in +# `node:sqlite` driven through `drizzle-orm/sqlite-proxy` (drizzle-orm is already a +# papra dependency). Nothing to compile, no system SQLite, no node-gyp. # sharp 0.34.x ships no FreeBSD native prebuilt (its optionalDependencies cover only # darwin/linux/linuxmusl/win32/wasm32). sharp is statically imported by @papra/lecture @@ -123,24 +115,18 @@ RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p. # self-contained — no system libvips and no libvips dependency chain in the image.) RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p.dependencies['@img/sharp-wasm32']='0.34.5';require('fs').writeFileSync(f,JSON.stringify(p,null,2)+'\n')" -# pnpm v11 refuses to run a dependency's build scripts unless it is allow-listed, -# and treats an ignored build as a hard error. Allow better-sqlite3 to compile its -# native addon by appending it to the workspace allowBuilds map. -RUN node -e "const f='pnpm-workspace.yaml';let s=require('fs').readFileSync(f,'utf8');if(!/allowBuilds:/.test(s)){s+='\nallowBuilds:\n';}s=s.replace(/allowBuilds:\n/, 'allowBuilds:\n better-sqlite3: true\n');require('fs').writeFileSync(f,s)" +# Papra uses the TC39 Temporal global, which official node26 ships built-in but +# FreeBSD's node26 package is compiled WITHOUT. Add a spec-compliant polyfill as a +# prod dep; the s6 run script preloads it via `node --import temporal-polyfill/global` +# so Temporal exists before any app code runs. (Pure JS — nothing to compile.) +RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p.dependencies['temporal-polyfill']='0.3.2';require('fs').writeFileSync(f,JSON.stringify(p,null,2)+'\n')" -# Install all needed workspaces. better-sqlite3 compiles here (it is allow-listed). +# Install all needed workspaces. No native addon compiles here — node:sqlite is +# built into node26, and sharp uses its WebAssembly runtime (added above). RUN pnpm install --frozen-lockfile=false \ --filter "@papra/app-client..." \ --filter "@papra/app-server..." -# Fail fast if the better-sqlite3 native addon didn't compile during install. -# (pnpm's hoisted linker may place it at the workspace root rather than the -# server package, so check both locations.) The deploy step below recompiles it -# for the shipped /app/production tree, so no explicit rebuild is needed here. -RUN ( test -f apps/papra-server/node_modules/better-sqlite3/build/Release/better_sqlite3.node || \ - test -f node_modules/better-sqlite3/build/Release/better_sqlite3.node ) && \ - echo "better-sqlite3 native addon present." - # Build frontend (Vite) then server (esbuild bundle). RUN pnpm --filter "@papra/app-client..." run build RUN pnpm --filter "@papra/app-server..." run build @@ -190,7 +176,7 @@ RUN if [ ! -f /app/production/node_modules/.bin/tsx ]; then \ FROM ghcr.io/daemonless/base:${BASE_VERSION} ARG FREEBSD_ARCH=amd64 -ARG PACKAGES="node24 nginx tesseract tesseract-data" +ARG PACKAGES="node26 nginx tesseract tesseract-data" ARG UPSTREAM_URL="https://api.github.com/repos/papra-hq/papra/releases" ARG UPSTREAM_JQ="[.[].tag_name | select(startswith(\"@papra/app@\"))][0] | ltrimstr(\"@papra/app@\")" ARG HEALTHCHECK_ENDPOINT="http://localhost:1221" @@ -214,8 +200,10 @@ LABEL org.opencontainers.image.title="Papra" \ io.daemonless.healthcheck-url="${HEALTHCHECK_ENDPOINT}" \ io.daemonless.packages="${PACKAGES}" -# Install runtime dependencies. -RUN pkg update && \ +# Install runtime dependencies. node26 lives in the `latest` pkg branch (see +# builder stage) — switch before install. +RUN sed -i '' -e 's,/quarterly,/latest,' /etc/pkg/FreeBSD.conf && \ + pkg update && \ pkg install -y ${PACKAGES} && \ mkdir -p /app /app/public /app-data/db /app-data/documents /ingestion \ /var/log/nginx /var/run /var/tmp/nginx && \ diff --git a/Containerfile.j2 b/Containerfile.j2 index b5ae6f5..01bfeba 100644 --- a/Containerfile.j2 +++ b/Containerfile.j2 @@ -14,24 +14,17 @@ FROM ghcr.io/daemonless/base:${BASE_VERSION} AS builder ARG UPSTREAM_URL # Build dependencies. -# - node24 / npm-node24 / corepack: JS toolchain (Papra targets node26; node24 is -# the newest LTS in FreeBSD pkg — node26 is not yet packaged — and runs Papra fine). +# - node26 / npm-node26 / corepack: JS toolchain. This image uses node26's +# BUILT-IN `node:sqlite` for the database (see patches/database.ts), so there +# is NO native addon to compile — hence NO C toolchain / node-gyp / python here. # - git-lite jq: clone the release tag resolved from the GitHub API. -# - pkgconf python3 gmake: node-gyp's build driver dependencies. -# The daemonless base jail ships NO C toolchain and NO /usr/include, so the -# better-sqlite3 node-gyp build needs a full FreeBSD-native toolchain. We pull the -# pkgbase packages so everything lands in /usr/bin under the default names node-gyp -# expects (cc, c++, ld, ar, ranlib) — no CC/CXX overrides needed: -# - FreeBSD-clang → /usr/bin/cc, clang, c++, clang++ -# - FreeBSD-lld → /usr/bin/ld (LLVM linker) -# - FreeBSD-toolchain → /usr/bin/ar, nm, ranlib (the archiver; `ar: not found` fix) -# - FreeBSD-clibs-dev → /usr/include libc + libc++ headers and crt objects -# All builder-stage only — discarded, does not affect the final runtime image size. -RUN pkg update && pkg install -y \ - node24 npm-node24 corepack \ +# +# node26 ships only in the FreeBSD `latest` pkg branch (the stock base points at +# `quarterly`, which tops out at node25), so switch the branch before installing. +RUN sed -i '' -e 's,/quarterly,/latest,' /etc/pkg/FreeBSD.conf && \ + pkg update && pkg install -y \ + node26 npm-node26 corepack \ git-lite jq \ - pkgconf python3 gmake \ - FreeBSD-clang FreeBSD-lld FreeBSD-toolchain FreeBSD-clibs-dev \ && pkg clean -ay # Enable pnpm via corepack (Papra pins pnpm 11.1.1). @@ -79,12 +72,13 @@ RUN for f in \ # FreeBSD DB driver swap: upstream's @libsql/client file: driver loads the native # `libsql` package which has no FreeBSD prebuilt and no buildable source. Replace -# the local-file path with better-sqlite3 (builds from bundled SQLite via node-gyp) -# and add a libsql-compatible db.batch() polyfill. See patches/database.ts. +# the local-file path with node26's built-in `node:sqlite` (driven via +# drizzle-orm/sqlite-proxy) and add libsql-compatible db.batch()/db.run() polyfills. +# See patches/database.ts. COPY patches/database.ts ./apps/papra-server/src/modules/app/database/database.ts # Companion fix: the DB health check reads a libsql-specific result shape, which -# better-sqlite3 doesn't produce. Use a driver-agnostic query so /api/health is +# node:sqlite doesn't produce. Use a driver-agnostic query so /api/health is # accurate. See patches/health-check.repository.ts. COPY patches/health-check.repository.ts ./apps/papra-server/src/modules/app/health-check/health-check.repository.ts @@ -95,16 +89,14 @@ COPY patches/health-check.repository.ts ./apps/papra-server/src/modules/app/heal COPY patches/tasks-driver.registry.ts ./apps/papra-server/src/modules/tasks/drivers/tasks-driver.registry.ts # Companion fix: migration 0006 uses libSQL's non-standard `ALTER COLUMN ... TO` -# syntax, which standard SQLite (better-sqlite3) rejects. Replace it with the +# syntax, which standard SQLite (node:sqlite) rejects. Replace it with the # SQLite-supported table-rebuild recipe (same resulting schema). # See patches/0006-organizations-invitations-improvement.migration.ts. COPY patches/0006-organizations-invitations-improvement.migration.ts ./apps/papra-server/src/migrations/list/0006-organizations-invitations-improvement.migration.ts -# Add better-sqlite3 to the server workspace so the patched database.ts can -# resolve it. We edit package.json directly (rather than `pnpm add`) so the -# install stays a single deterministic resolve. better-sqlite3 ships a SQLite -# amalgamation it compiles from source via node-gyp — no system SQLite needed. -RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p.dependencies['better-sqlite3']='^11.10.0';require('fs').writeFileSync(f,JSON.stringify(p,null,2)+'\n')" +# No database dependency to inject: the patched database.ts uses node26's built-in +# `node:sqlite` driven through `drizzle-orm/sqlite-proxy` (drizzle-orm is already a +# papra dependency). Nothing to compile, no system SQLite, no node-gyp. # sharp 0.34.x ships no FreeBSD native prebuilt (its optionalDependencies cover only # darwin/linux/linuxmusl/win32/wasm32). sharp is statically imported by @papra/lecture @@ -117,24 +109,18 @@ RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p. # self-contained — no system libvips and no libvips dependency chain in the image.) RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p.dependencies['@img/sharp-wasm32']='0.34.5';require('fs').writeFileSync(f,JSON.stringify(p,null,2)+'\n')" -# pnpm v11 refuses to run a dependency's build scripts unless it is allow-listed, -# and treats an ignored build as a hard error. Allow better-sqlite3 to compile its -# native addon by appending it to the workspace allowBuilds map. -RUN node -e "const f='pnpm-workspace.yaml';let s=require('fs').readFileSync(f,'utf8');if(!/allowBuilds:/.test(s)){s+='\nallowBuilds:\n';}s=s.replace(/allowBuilds:\n/, 'allowBuilds:\n better-sqlite3: true\n');require('fs').writeFileSync(f,s)" +# Papra uses the TC39 Temporal global, which official node26 ships built-in but +# FreeBSD's node26 package is compiled WITHOUT. Add a spec-compliant polyfill as a +# prod dep; the s6 run script preloads it via `node --import temporal-polyfill/global` +# so Temporal exists before any app code runs. (Pure JS — nothing to compile.) +RUN node -e "const f='apps/papra-server/package.json';const p=require('./'+f);p.dependencies['temporal-polyfill']='0.3.2';require('fs').writeFileSync(f,JSON.stringify(p,null,2)+'\n')" -# Install all needed workspaces. better-sqlite3 compiles here (it is allow-listed). +# Install all needed workspaces. No native addon compiles here — node:sqlite is +# built into node26, and sharp uses its WebAssembly runtime (added above). RUN pnpm install --frozen-lockfile=false \ --filter "@papra/app-client..." \ --filter "@papra/app-server..." -# Fail fast if the better-sqlite3 native addon didn't compile during install. -# (pnpm's hoisted linker may place it at the workspace root rather than the -# server package, so check both locations.) The deploy step below recompiles it -# for the shipped /app/production tree, so no explicit rebuild is needed here. -RUN ( test -f apps/papra-server/node_modules/better-sqlite3/build/Release/better_sqlite3.node || \ - test -f node_modules/better-sqlite3/build/Release/better_sqlite3.node ) && \ - echo "better-sqlite3 native addon present." - # Build frontend (Vite) then server (esbuild bundle). RUN pnpm --filter "@papra/app-client..." run build RUN pnpm --filter "@papra/app-server..." run build @@ -184,7 +170,7 @@ RUN if [ ! -f /app/production/node_modules/.bin/tsx ]; then \ FROM ghcr.io/daemonless/base:${BASE_VERSION} ARG FREEBSD_ARCH=amd64 -ARG PACKAGES="node24 nginx tesseract tesseract-data" +ARG PACKAGES="node26 nginx tesseract tesseract-data" ARG UPSTREAM_URL="https://api.github.com/repos/papra-hq/papra/releases" ARG UPSTREAM_JQ="[.[].tag_name | select(startswith(\"@papra/app@\"))][0] | ltrimstr(\"@papra/app@\")" ARG HEALTHCHECK_ENDPOINT="http://localhost:1221" @@ -208,8 +194,10 @@ LABEL org.opencontainers.image.title="Papra" \ io.daemonless.healthcheck-url="${HEALTHCHECK_ENDPOINT}" \ io.daemonless.packages="${PACKAGES}" -# Install runtime dependencies. -RUN pkg update && \ +# Install runtime dependencies. node26 lives in the `latest` pkg branch (see +# builder stage) — switch before install. +RUN sed -i '' -e 's,/quarterly,/latest,' /etc/pkg/FreeBSD.conf && \ + pkg update && \ pkg install -y ${PACKAGES} && \ mkdir -p /app /app/public /app-data/db /app-data/documents /ingestion \ /var/log/nginx /var/run /var/tmp/nginx && \ diff --git a/patches/database.ts b/patches/database.ts index 2bc5eaf..0af85e8 100644 --- a/patches/database.ts +++ b/patches/database.ts @@ -1,51 +1,43 @@ -// FreeBSD daemonless patch for Papra's database setup. +// FreeBSD daemonless patch for Papra's database setup — node:sqlite variant. // // WHY: Upstream uses `@libsql/client` whose `file:` driver loads the native -// `libsql` npm package. That package only ships prebuilt binaries for -// linux/darwin/win32 (see pnpm-lock.yaml: @libsql/{linux,darwin,win32}-* only, -// no freebsd), and the Rust source does not build cleanly under FreeBSD pkg. -// Critically, even *importing* `@libsql/client` eagerly loads the native module -// and crashes on FreeBSD with `Cannot find module '@libsql/freebsd-x64'`. +// `libsql` npm package, which ships NO FreeBSD prebuilt and no source-build +// fallback (even *importing* @libsql/client crashes on FreeBSD with +// `Cannot find module '@libsql/freebsd-x64'`). // -// FIX: This image only ever uses a local `file:` SQLite database, so we replace -// the driver entirely with `better-sqlite3` (which compiles its bundled SQLite -// amalgamation from source via node-gyp on FreeBSD — no system SQLite, no libsql). -// `@libsql/client` is NOT imported at all, so nothing pulls the native libsql -// module. `drizzle-orm/better-sqlite3` speaks the same SQLite dialect, so all -// queries and migrations work unchanged. +// This variant replaces the driver with node26's BUILT-IN `node:sqlite` +// (DatabaseSync) instead of better-sqlite3. The win: no native addon, so the +// builder needs NO C toolchain / node-gyp / python — node:sqlite ships inside +// the node binary. drizzle has no first-class node:sqlite adapter, so we drive +// it through `drizzle-orm/sqlite-proxy` (a generic callback driver) and map +// node:sqlite's results into the positional `string[][]` shape the proxy wants. // -// Two gaps remain between the two drivers, both polyfilled below: +// node:sqlite specifics relied on (node>=26, StatementSync): +// - `.setReturnArrays(true)` → rows come back as positional value arrays, +// exactly the sqlite-proxy contract (no object-key collisions on joins). +// - `.columns()` → non-empty for readers (SELECT/PRAGMA); used to reproduce +// libsql's `db.run().rows` shape (see attachLibsqlRun). // -// 1. `.batch()` — drizzle's better-sqlite3 driver has no `.batch()` method (that -// is a libsql-only API). Papra uses `db.batch([...])` in migrations and a -// couple of runtime repositories. We polyfill it by running the queued -// statements inside a single synchronous transaction, matching libsql's batch -// semantics (all-or-nothing, results returned in order). +// Two libsql-only behaviours papra depends on, polyfilled below (same intent as +// the better-sqlite3 patch): +// 1. `db.batch([...])` — custom override (see attachBatch); sqlite-proxy's native +// batch can't handle our eagerly-executed db.run() results. +// 2. `db.run/all/get(sql).rows`/objects — libsql returns row OBJECTS for raw SQL; +// the bare sqlite-proxy callback returns positional arrays. Upstream reads by +// column name (migrations 0010-0019 use `db.run(...).rows`; the health check +// uses `db.all(...)`), so we override the raw helpers (see attachLibsqlRaw). // -// 2. `db.run()` result SHAPE — libsql's `db.run(sql)` returns a ResultSet with -// `.rows`/`.columns`/`.rowsAffected` for ANY statement (it always reads back -// rows). better-sqlite3's drizzle `.run()` returns only `{changes, -// lastInsertRowid}` with NO `.rows`. Papra reads `db.run(sql`PRAGMA -// table_info(...)`).rows` in SEVEN migrations (0010/0011/0012/0014/0016/0017/ -// 0019) and in the health-check repo. Rather than patch each call site, we -// override `db.run` to return the libsql shape: for statements that return -// data (PRAGMA/SELECT — `stmt.reader === true`) we `.all()` and populate -// `.rows`; for DDL/DML we `.run()` and populate `.rowsAffected`. This makes the -// libsql-shaped reads work everywhere unchanged. -// -// NOTE: remote libSQL/Turso URLs (DATABASE_URL=libsql://… or http(s)://…) are NOT -// supported in this FreeBSD image because the native libsql client can't load -// here. The container defaults to a local file DB, which is the standard self-host -// setup; pointing at a remote libSQL server would need a different base. +// NOTE: remote libSQL/Turso URLs are NOT supported here (no native client on +// FreeBSD). The container always uses a local `file:` DB, the standard self-host +// setup. node:sqlite is a Node release-candidate API (stability 1.2). import type { ShutdownServices } from '../graceful-shutdown/graceful-shutdown.services'; -import Database from 'better-sqlite3'; -import { drizzle as drizzleBetterSqlite } from 'drizzle-orm/better-sqlite3'; +import { DatabaseSync } from 'node:sqlite'; +import { drizzle as drizzleProxy } from 'drizzle-orm/sqlite-proxy'; export { setupDatabase }; -// Convert a `file:./app-data/db/db.sqlite` style URL into a filesystem path that -// better-sqlite3 understands. +// Convert a `file:./app-data/db/db.sqlite` style URL into a filesystem path. function fileUrlToPath(url: string): string { if (url === ':memory:' || url.startsWith(':memory:')) { return ':memory:'; @@ -53,7 +45,7 @@ function fileUrlToPath(url: string): string { let path = url.startsWith('file:') ? url.slice('file:'.length) : url; - // Strip any query string (e.g. ?mode=rwc) — better-sqlite3 takes options separately. + // Strip any query string (e.g. ?mode=rwc) — node:sqlite takes the bare path. const queryIndex = path.indexOf('?'); if (queryIndex !== -1) { path = path.slice(0, queryIndex); @@ -62,95 +54,143 @@ function fileUrlToPath(url: string): string { return path; } -// Polyfill libsql's `db.batch(queries)` for the better-sqlite3 driver. +// drizzle passes only primitives (string/number/bigint/null/Uint8Array) for `?` +// markers, so a straight spread into node:sqlite's positional binding is safe +// (no plain object that would be misread as named parameters). +function bind(params: unknown[] | undefined): unknown[] { + return params ?? []; +} + +// sqlite-proxy main callback: execute one statement and return rows as positional +// value arrays. `method` is 'run' | 'all' | 'values' | 'get'. +function makeProxyCallback(sqlite: DatabaseSync) { + return (sql: string, params: unknown[], method: string) => { + const stmt = sqlite.prepare(sql); + + if (method === 'run') { + stmt.run(...bind(params)); + return { rows: [] as unknown[] }; + } + + stmt.setReturnArrays(true); + + if (method === 'get') { + // sqlite-proxy wants a single flat array for 'get'. + const row = stmt.get(...bind(params)) as unknown[] | undefined; + return { rows: row ?? [] }; + } + + // 'all' | 'values' → array of value-arrays. + return { rows: stmt.all(...bind(params)) as unknown[][] }; + }; +} + +// Override `db.batch([...])` (libsql-only API papra uses in migrations/repos). // -// Subtlety: drizzle's better-sqlite3 driver is SYNCHRONOUS, so query objects -// behave differently depending on how they were created (Papra mixes both): -// - `db.run(sql`...`)` → executes EAGERLY during argument evaluation and -// returns a plain RunResult ({changes,lastInsertRowid}) with no .run()/.all(). -// By the time batch() is called these statements have ALREADY run. -// - `db.update(...)…` (a query builder, e.g. in the FTS repositories) → is LAZY; -// it exposes .run()/.all() and has not executed yet. +// We can't use sqlite-proxy's NATIVE batch: it calls `query._prepare()` on each +// item, but papra passes `db.run(sql`...`)` results — and our db.run (see +// attachLibsqlRun) executes EAGERLY and returns a plain libsql-shaped object with +// no `_prepare`. So we provide our own batch that runs any still-lazy drizzle +// builders and passes already-executed eager results through, wrapped in a +// transaction (skipped if we're already inside one — SQLite has no nested BEGIN). // -// So we run any builder that still has a .run()/.all() method, and pass through -// already-executed results untouched. We wrap the whole thing in the RAW -// better-sqlite3 connection's `.transaction()` (which forwards its arguments to -// the wrapped fn, unlike drizzle's `db.transaction()` whose callback gets a tx -// object) so the lazy builders commit atomically. (The eagerly-run statements -// already auto-committed before we got here — atomicity across the eager ones is -// the one semantic we can't preserve, which is fine for Papra's idempotent, -// IF-NOT-EXISTS migrations and its result-discarding runtime write batches.) -function attachBatch(db: any, sqlite: any) { - if (typeof db.batch === 'function') { - return db; - } - - const runInTransaction = sqlite.transaction((queries: any[]) => { - const results: unknown[] = []; - for (const query of queries) { - // .run() works for any statement type (DDL/INSERT/UPDATE/DELETE and even - // SELECT — it returns {changes,lastInsertRowid} without rows), whereas - // .all() THROWS on non-SELECT. Papra's batched statements are all writes - // whose return values are discarded, so prefer .run(). - if (typeof query?.run === 'function') { - results.push(query.run()); - } else if (typeof query?.all === 'function') { - results.push(query.all()); - } else { - // Already-executed RunResult from an eager db.run(sql`...`); pass through. - results.push(query); +// Note: statements created via `db.run(sql`...`)` have already auto-committed by +// the time batch() runs (they execute during array construction), so atomicity +// across those is not preserved — fine for papra's idempotent IF-NOT-EXISTS +// migrations and result-discarding write batches, same as the better-sqlite3 patch. +function attachBatch(db: any, sqlite: DatabaseSync) { + db.batch = async (queries: any[]) => { + const ownTx = !sqlite.isTransaction; + if (ownTx) { + sqlite.exec('BEGIN'); + } + try { + const results: unknown[] = []; + for (const query of queries) { + if (query && typeof query.execute === 'function') { + results.push(await query.execute()); + } else if (query && typeof query.run === 'function') { + results.push(await query.run()); + } else if (query && typeof query.all === 'function') { + results.push(await query.all()); + } else { + // Already-executed eager db.run(sql`...`) result — pass through. + results.push(query); + } } + if (ownTx) { + sqlite.exec('COMMIT'); + } + return results; + } catch (error) { + if (ownTx && sqlite.isTransaction) { + sqlite.exec('ROLLBACK'); + } + throw error; } - return results; - }); - - db.batch = async (queries: any[]) => runInTransaction(queries); + }; return db; } -// Override `db.run` to return libsql's ResultSet shape (see gap #2 above). -// drizzle's better-sqlite3 `.run()` drops rows for read statements, so libsql-era -// call sites that do `db.run(...).rows` break. We re-implement `run` against the -// raw better-sqlite3 connection: build the SQL string + params via drizzle's own -// dialect, then choose `.all()` (readers: PRAGMA/SELECT → populate `.rows`) vs -// `.run()` (writers/DDL → populate `.rowsAffected`). Returns synchronously; the -// `await db.run(...)` call sites resolve a plain value, which is fine. -function attachLibsqlRun(db: any, sqlite: any) { +// Override the top-level raw-SQL helpers `db.run` / `db.all` / `db.get` to return +// libsql's shapes (rows as OBJECTS, like libsql), executed directly against +// node:sqlite. drizzle's QUERY BUILDER (db.select().from(), etc.) does NOT go +// through these — it uses the session/proxy callback (positional arrays, which +// drizzle maps back to objects via the schema) — so overriding the convenience +// methods is safe and only affects raw `db.run/all/get(sql`...`)` call sites: +// - db.run(sql`PRAGMA table_info(...)`).rows → 7 upstream migrations (0010-0019) +// - db.all(sql`SELECT 1 AS ok`) → the health-check repository +// libsql returns row OBJECTS for these; the bare sqlite-proxy callback returns +// positional arrays, so without this override those by-name reads break. +function attachLibsqlRaw(db: any, sqlite: DatabaseSync) { const dialect = db.dialect; - - db.run = (query: any) => { - const { sql: queryString, params } = typeof query === 'string' - ? { sql: query, params: [] as unknown[] } + const render = (query: any): { sql: string; params: unknown[] } => + typeof query === 'string' + ? { sql: query, params: [] } : dialect.sqlToQuery(query.getSQL()); + db.run = (query: any) => { + const { sql: queryString, params } = render(query); const stmt = sqlite.prepare(queryString); - if (stmt.reader) { + const columns = stmt.columns(); + + if (columns.length > 0) { + // Reader (PRAGMA/SELECT) — return objects in libsql's `.rows` shape. return { - rows: stmt.all(...params), - columns: stmt.columns().map((c: any) => c.name), + rows: stmt.all(...bind(params)), + columns: columns.map((c: any) => c.name), rowsAffected: 0, - changes: 0, lastInsertRowid: 0, }; } - const info = stmt.run(...params); + const info = stmt.run(...bind(params)); return { rows: [], columns: [], - rowsAffected: info.changes, - changes: info.changes, + rowsAffected: Number(info.changes), lastInsertRowid: info.lastInsertRowid, }; }; + // libsql's db.all/db.get return row OBJECTS (keyed by column name). + db.all = (query: any) => { + const { sql: queryString, params } = render(query); + return sqlite.prepare(queryString).all(...bind(params)); + }; + + db.get = (query: any) => { + const { sql: queryString, params } = render(query); + return sqlite.prepare(queryString).get(...bind(params)); + }; + return db; } function setupDatabase({ url, - // authToken/encryptionKey are libsql-only; ignored in the better-sqlite3 path. + // authToken/encryptionKey are libsql-only; ignored on the node:sqlite path. authToken: _authToken, encryptionKey: _encryptionKey, shutdownServices, @@ -161,14 +201,17 @@ function setupDatabase({ shutdownServices?: ShutdownServices; }) { const path = fileUrlToPath(url); - const sqlite = new Database(path); + const sqlite = new DatabaseSync(path); // Pragmas matching libsql's sensible defaults for a local single-writer file DB. - sqlite.pragma('journal_mode = WAL'); - sqlite.pragma('foreign_keys = ON'); - sqlite.pragma('busy_timeout = 5000'); - - const db = attachBatch(attachLibsqlRun(drizzleBetterSqlite(sqlite), sqlite), sqlite); + sqlite.exec('PRAGMA journal_mode = WAL'); + sqlite.exec('PRAGMA foreign_keys = ON'); + sqlite.exec('PRAGMA busy_timeout = 5000'); + + const db = attachBatch( + attachLibsqlRaw(drizzleProxy(makeProxyCallback(sqlite)), sqlite), + sqlite, + ); shutdownServices?.registerShutdownHandler({ id: 'database-client-close', diff --git a/root/etc/services.d/papra-server/run b/root/etc/services.d/papra-server/run index 5895c1d..e253e8d 100755 --- a/root/etc/services.d/papra-server/run +++ b/root/etc/services.d/papra-server/run @@ -8,11 +8,14 @@ # first makes it fail with "s6-supervise not running". The node commands cd into # /app inside their own subshell instead. -# Papra targets node 26, which has the TC39 Temporal API as a global. FreeBSD's -# node 24 only exposes it behind V8's --harmony-temporal flag. That flag is NOT -# permitted in NODE_OPTIONS, so it must be passed directly on each node command. +# Papra uses the TC39 Temporal global (rate-limit config). Official node26 ships +# Temporal built-in, but FreeBSD's node26 package is compiled WITHOUT it and even +# `--harmony-temporal` is a no-op here. So preload a spec-compliant polyfill via +# node's --import: `temporal-polyfill/global` installs the Temporal global before +# any app code runs. Both node commands below inherit $NODE_FLAGS. (Resolved from +# /app/node_modules — temporal-polyfill is added as a prod dep in the Containerfile.) NODE=/usr/local/bin/node -NODE_FLAGS="--harmony-temporal" +NODE_FLAGS="--import temporal-polyfill/global" # better-auth refuses to start in production (NODE_ENV=production) with the default # auth secret. If the operator supplied AUTH_SECRET (compose/secrets.env), use it.