diff --git a/.changeset/transform-cache-relocation.md b/.changeset/transform-cache-relocation.md new file mode 100644 index 0000000..b64fbb5 --- /dev/null +++ b/.changeset/transform-cache-relocation.md @@ -0,0 +1,9 @@ +--- +"vitest-native": patch +--- + +Native-engine transform cache rework: project-local, content-keyed, lazy Babel. + +- **The transform disk cache moves from `os.tmpdir()` to the project's `node_modules/.cache/vitest-native/`** (tmpdir fallback when node_modules is absent or unwritable). tmpdir is ephemeral on CI runners — every job paid a full cold Babel transform of React Native's ~250-file boot graph — and macOS purges it periodically. The new location persists across runs and is restorable by standard CI dependency-cache actions. The V8 compile cache is colocated. +- **Disk entries are keyed by content hash (platform + path + source), not mtime + size.** Content keys survive fresh installs, Docker mtime normalization, and CI cache restores — and eliminate the stale-hit class where a same-size, same-mtime file with different content served wrong executable code. The path stays in the key because Babel's output embeds the filename (`_jsxFileName` in transformed JSX), so identical sources at different paths must not share an entry; restores are valid wherever the checkout path is stable, which CI workspaces are. The cache directory name now also carries the `@babel/core` version alongside the preset version, so a Babel upgrade invalidates cleanly. +- **`@babel/core` loads lazily, only on a cache miss.** Loading Babel costs ~35ms vs ~0.5ms for the resolve-only version check, and under the default engine every isolated worker paid it even when every file came from the disk cache. Measured on the package's own native suite (warm cache): aggregate worker setup down ~30%, wall clock ~11% — the effect scales with test-file count. diff --git a/packages/vitest-native/src/native/compile-cache.mjs b/packages/vitest-native/src/native/compile-cache.mjs index 3cb73b5..b34d82d 100644 --- a/packages/vitest-native/src/native/compile-cache.mjs +++ b/packages/vitest-native/src/native/compile-cache.mjs @@ -14,22 +14,27 @@ import nodeModule from "node:module"; import os from "node:os"; import path from "node:path"; +import { cacheRootFor } from "./transform.mjs"; let _enabled = false; /** * Enable Node's persistent V8 compile cache for this worker/thread. Idempotent and - * guarded per realm. Colocated under the `vitest-native-cache` tmp prefix so it - * shares the transform cache's lifecycle (cleared with it on a cold run, reused on - * warm). Must be called before RN's modules are first compiled. + * guarded per realm. Colocated with the transform disk cache (the project's + * node_modules/.cache/vitest-native when available — persistent and CI-restorable — + * else tmpdir) so both caches share a lifecycle. Must be called before RN's modules + * are first compiled. */ -export function enableV8CompileCache() { +export function enableV8CompileCache(projectRoot) { if (_enabled) return; _enabled = true; const enable = nodeModule.enableCompileCache; if (typeof enable !== "function") return; // Node < 22.8: feature absent try { - enable.call(nodeModule, path.join(os.tmpdir(), "vitest-native-cache-v8")); + const root = projectRoot + ? cacheRootFor(projectRoot) + : path.join(os.tmpdir(), "vitest-native-cache"); + enable.call(nodeModule, path.join(root, "v8")); // No atomic-write handling needed (unlike the transform cache, which writes // executable source): Node CRC32-validates each compile-cache entry on read, // so a torn or raced write from concurrent workers is a cache miss and a diff --git a/packages/vitest-native/src/native/setup.mjs b/packages/vitest-native/src/native/setup.mjs index cd9e5bd..ce57787 100644 --- a/packages/vitest-native/src/native/setup.mjs +++ b/packages/vitest-native/src/native/setup.mjs @@ -99,7 +99,7 @@ for (const name of presetNames) { // Enable the V8 compile cache before RN is compiled (it loads when the test file // imports react-native, after this setup runs) so its bytecode is cached to disk // and reused on the next file/worker/run. Covers the stock (non-hot) path. -enableV8CompileCache(); +enableV8CompileCache(projectRoot); installGlobals(); // RNTL 13+ auto-registers matchers through the global expect when imported. // Expose Vitest's expect only when the consumer has not enabled globals. diff --git a/packages/vitest-native/src/native/transform.mjs b/packages/vitest-native/src/native/transform.mjs index fcb0353..c3e358d 100644 --- a/packages/vitest-native/src/native/transform.mjs +++ b/packages/vitest-native/src/native/transform.mjs @@ -2,27 +2,66 @@ // (the only transformer that lowers RN's `component` syntax). Used by BOTH the loader // hook (import) and the require hook, which run in separate threads: each gets its own // instance of this module, so the in-memory `mem` cache below is per-thread (no shared -// mutable state to race on). The disk cache — keyed by path + mtime + size + preset -// version — is the layer shared across them. +// mutable state to race on). The disk cache — keyed by CONTENT hash + platform, in a +// directory versioned by preset + @babel/core versions — is the layer shared across +// threads, workers, runs, and (because content-keyed entries survive fresh installs +// and mtime normalization) CI cache restores. The path is part of the key too — +// Babel output embeds the filename — so restores are valid wherever the checkout +// path is stable (CI runners use a fixed workspace path). +// +// @babel/core itself is loaded lazily, only on a cache MISS: on a warm cache the +// default engine pays this module's init in every isolated worker, and requiring +// Babel costs ~35ms vs ~0.5ms for resolving versions — pure waste when every file +// is served from disk. import { createRequire } from "node:module"; import path from "node:path"; import fs from "node:fs"; import os from "node:os"; import crypto from "node:crypto"; +let _req; let _babel; let _preset; let _cacheDir; let _writeSeq = 0; const mem = new Map(); -const TRANSFORM_CACHE_VERSION = 2; +const TRANSFORM_CACHE_VERSION = 3; + +/** + * Root directory for vitest-native's on-disk caches. Prefers the project's + * node_modules/.cache — persistent across runs, per-project, and restorable by + * standard CI dependency-cache actions (unlike os.tmpdir(), which is ephemeral + * on CI runners and periodically purged on macOS). Falls back to tmpdir when + * node_modules is absent or unwritable. + */ +export function cacheRootFor(projectRoot) { + const nm = path.join(projectRoot, "node_modules"); + if (fs.existsSync(nm)) { + try { + const dir = path.join(nm, ".cache", "vitest-native"); + fs.mkdirSync(dir, { recursive: true }); + return dir; + } catch { + // Read-only node_modules (some CI sandboxes) — fall through to tmpdir. + } + } + const dir = path.join(os.tmpdir(), "vitest-native-cache"); + fs.mkdirSync(dir, { recursive: true }); + return dir; +} function init(projectRoot) { - if (_babel) return; - const req = createRequire(path.join(projectRoot, "package.json")); + if (_cacheDir) return; + _req = createRequire(path.join(projectRoot, "package.json")); + let presetVersion; + let babelVersion; try { - _babel = req("@babel/core"); - _preset = req.resolve("@react-native/babel-preset"); + // Resolve-only here (cheap); the actual require of @babel/core happens + // lazily on the first cache miss. + _preset = _req.resolve("@react-native/babel-preset"); + _req.resolve("@babel/core"); + presetVersion = _req("@react-native/babel-preset/package.json").version; + babelVersion = _req("@babel/core/package.json").version; } catch { throw new Error( "[vitest-native] engine 'native' requires '@react-native/babel-preset' and " + @@ -30,15 +69,24 @@ function init(projectRoot) { "(they ship with React Native projects by default).", ); } - const presetVersion = req("@react-native/babel-preset/package.json").version; + // Both transformer versions key the directory — a preset or Babel upgrade + // must never serve output produced by the previous version — and so does the + // Babel environment: the preset's dev-mode JSX transform produces different + // output (e.g. _jsxFileName injection) under NODE_ENV=development than under + // test/production. + const babelEnv = process.env.BABEL_ENV || process.env.NODE_ENV || "none"; _cacheDir = path.join( - os.tmpdir(), - "vitest-native-cache", - `${presetVersion}-v${TRANSFORM_CACHE_VERSION}`, + cacheRootFor(projectRoot), + `transform-${presetVersion}-b${babelVersion}-${babelEnv}-v${TRANSFORM_CACHE_VERSION}`, ); fs.mkdirSync(_cacheDir, { recursive: true }); } +/** The active transform disk-cache directory (resolved on first use). Test hook. */ +export function transformCacheDir() { + return _cacheDir ?? null; +} + /** Returns true if the source contains RN Flow syntax that must be transformed. */ export function isFlow(src) { return /@flow|import typeof|\bcomponent\s+\w/.test(src); @@ -47,6 +95,13 @@ export function isFlow(src) { /** Transform an RN source file to runnable CJS. Cached in-memory + on disk. */ export function transformRN(file, src, projectRoot, platform = "ios") { init(projectRoot); + // The in-memory key uses mtime+size (one statSync) so the hot path skips + // hashing; the DISK key hashes the actual content, so entries stay valid + // across fresh installs, Docker mtime normalization, and CI cache restores — + // and a same-path file with different content can never produce a wrong hit. + // The FILENAME is part of the key because Babel's output depends on it: the + // preset embeds the absolute path (_jsxFileName) in transformed JSX, so two + // identical sources at different paths must not share an entry. const st = fs.statSync(file); const memKey = `${platform}\0${file}\0${st.mtimeMs}\0${st.size}`; const memHit = mem.get(memKey); @@ -54,7 +109,11 @@ export function transformRN(file, src, projectRoot, platform = "ios") { const key = crypto .createHash("sha1") - .update(platform + ":" + file + ":" + st.mtimeMs + ":" + st.size) + .update(platform) + .update("\0") + .update(file) + .update("\0") + .update(src) .digest("hex"); const cachePath = path.join(_cacheDir, key + ".js"); try { @@ -63,6 +122,7 @@ export function transformRN(file, src, projectRoot, platform = "ios") { return cached; } catch {} + if (!_babel) _babel = _req("@babel/core"); const out = _babel.transformSync(src, { filename: file, presets: [[_preset, { disableStaticViewConfigsCodegen: true }]], diff --git a/packages/vitest-native/src/native/worker.mjs b/packages/vitest-native/src/native/worker.mjs index e88fcf8..e3b9c5c 100644 --- a/packages/vitest-native/src/native/worker.mjs +++ b/packages/vitest-native/src/native/worker.mjs @@ -55,7 +55,7 @@ if (diagnostics) { // per-file resets rather than wrongly torn down with test pollution. // Enable the V8 compile cache before any RN module compiles, so the resident // graph's bytecode is cached to disk for the next worker/run. -enableV8CompileCache(); +enableV8CompileCache(projectRoot); installGlobals(); installRequireHooks(projectRoot, transformPkgs, platform, reactNativeVersion, assetExts); try { diff --git a/packages/vitest-native/tests/compile-cache.test.ts b/packages/vitest-native/tests/compile-cache.test.ts index 4b51d31..159477a 100644 --- a/packages/vitest-native/tests/compile-cache.test.ts +++ b/packages/vitest-native/tests/compile-cache.test.ts @@ -1,7 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import Module from "node:module"; +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; // `enableV8CompileCache` reads `nodeModule.enableCompileCache` at call time, so we // drive its three branches by swapping the real (singleton) `Module.enableCompileCache` @@ -9,7 +11,21 @@ import path from "node:path"; // sticky, so `vi.resetModules()` is what lets each case start from a clean slate. // Swapping also prevents these tests from actually enabling the cache on the test // process. -const EXPECTED_DIR = path.join(os.tmpdir(), "vitest-native-cache-v8"); +const HERE = path.dirname(fileURLToPath(import.meta.url)); +function findUp(rel: string, start: string): string { + let dir = start; + for (;;) { + const candidate = path.join(dir, rel); + if (fs.existsSync(candidate)) return candidate; + const parent = path.dirname(dir); + if (parent === dir) throw new Error(`${rel} not found from ${start}`); + dir = parent; + } +} +const projectRoot = path.dirname(findUp("package.json", HERE)); +// Colocated with the transform cache under the project's node_modules/.cache. +const EXPECTED_DIR = path.join(projectRoot, "node_modules", ".cache", "vitest-native", "v8"); +const FALLBACK_DIR = path.join(os.tmpdir(), "vitest-native-cache", "v8"); describe("enableV8CompileCache", () => { let original: unknown; @@ -28,7 +44,7 @@ describe("enableV8CompileCache", () => { return (await import("../src/native/compile-cache.mjs")).enableV8CompileCache; } - it("enables the cache once, under the vitest-native-cache-v8 tmp dir", async () => { + it("enables the cache once, colocated with the transform cache", async () => { const dirs: string[] = []; (Module as unknown as Record).enableCompileCache = (dir: string) => { dirs.push(dir); @@ -36,12 +52,25 @@ describe("enableV8CompileCache", () => { }; const enableV8CompileCache = await load(); - enableV8CompileCache(); - enableV8CompileCache(); // idempotent: the per-realm guard suppresses the second call + enableV8CompileCache(projectRoot); + enableV8CompileCache(projectRoot); // idempotent: the per-realm guard suppresses the second call expect(dirs).toEqual([EXPECTED_DIR]); }); + it("falls back to the tmpdir root when no project root is supplied", async () => { + const dirs: string[] = []; + (Module as unknown as Record).enableCompileCache = (dir: string) => { + dirs.push(dir); + return { status: 1, directory: dir }; + }; + const enableV8CompileCache = await load(); + + enableV8CompileCache(); + + expect(dirs).toEqual([FALLBACK_DIR]); + }); + it("is a no-op when the API is absent (Node < 22.8)", async () => { (Module as unknown as Record).enableCompileCache = undefined; const enableV8CompileCache = await load(); diff --git a/packages/vitest-native/tests/transform-cache.test.ts b/packages/vitest-native/tests/transform-cache.test.ts new file mode 100644 index 0000000..4ea4cfd --- /dev/null +++ b/packages/vitest-native/tests/transform-cache.test.ts @@ -0,0 +1,148 @@ +/** + * Disk-cache behavior of the native-engine transform: project-local cache + * location, content-hash keying, and lazy @babel/core loading on warm caches. + */ +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import crypto from "node:crypto"; +import path from "node:path"; +import fs from "node:fs"; +import os from "node:os"; +import { fileURLToPath, pathToFileURL } from "node:url"; +// @ts-expect-error — runtime .mjs, no types +import { transformRN, transformCacheDir, cacheRootFor } from "../src/native/transform.mjs"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +function findUp(rel: string, start: string): string { + let dir = start; + for (;;) { + const candidate = path.join(dir, rel); + if (fs.existsSync(candidate)) return candidate; + const parent = path.dirname(dir); + if (parent === dir) throw new Error(`${rel} not found from ${start}`); + dir = parent; + } +} +const projectRoot = path.dirname(findUp("package.json", HERE)); + +describe("cacheRootFor", () => { + it("prefers the project's node_modules/.cache when node_modules exists", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "vn-cacheroot-")); + try { + fs.mkdirSync(path.join(tmp, "node_modules")); + const root = cacheRootFor(tmp); + expect(root).toBe(path.join(tmp, "node_modules", ".cache", "vitest-native")); + expect(fs.existsSync(root)).toBe(true); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("falls back to tmpdir when node_modules is absent", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "vn-cacheroot-")); + try { + const root = cacheRootFor(tmp); + expect(root).toBe(path.join(os.tmpdir(), "vitest-native-cache")); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); + +describe("transform disk cache", () => { + // Exact-key assertions, not directory-entry counts: other test files + // transform into the same shared project cache dir from parallel workers, + // so counting entries races. The expected key mirrors the implementation's + // sha1(platform, path, content) scheme. + const diskKeyFor = (platform: string, file: string, src: string) => + crypto + .createHash("sha1") + .update(platform) + .update("\0") + .update(file) + .update("\0") + .update(src) + .digest("hex"); + + it("keys entries by path + content, not mtime — CI-restore friendly", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "vn-contentkey-")); + try { + const src = `// @flow\n// ${crypto.randomUUID()}\ntype T = string;\nmodule.exports = (x: T) => x;`; + const fileA = path.join(dir, "a.js"); + fs.writeFileSync(fileA, src); + transformRN(fileA, src, projectRoot); + const cacheDir = transformCacheDir(); + expect(cacheDir).toBeTruthy(); + const entryA = path.join(cacheDir, diskKeyFor("ios", fileA, src) + ".js"); + expect(fs.existsSync(entryA)).toBe(true); + const writtenAt = fs.statSync(entryA).mtimeMs; + + // Same path + same content with a DIFFERENT source mtime (fresh install, + // Docker mtime normalization, CI cache restore) hits the SAME disk + // entry: it is read, not rewritten. + const later = new Date(Date.now() + 5000); + fs.utimesSync(fileA, later, later); + transformRN(fileA, src, projectRoot); + expect(fs.statSync(entryA).mtimeMs).toBe(writtenAt); + + // Identical content at a DIFFERENT path gets its OWN entry: Babel output + // can embed the filename (the preset's dev JSX transform writes + // _jsxFileName when NODE_ENV isn't test/production), so sharing an entry + // across paths could serve file A's identity as file B's. + const fileB = path.join(dir, "b.js"); + fs.writeFileSync(fileB, src); + transformRN(fileB, src, projectRoot); + const entryB = path.join(cacheDir, diskKeyFor("ios", fileB, src) + ".js"); + expect(entryB).not.toBe(entryA); + expect(fs.existsSync(entryB)).toBe(true); + + // The cache directory name carries preset + babel versions + Babel env. + expect(path.basename(cacheDir)).toMatch(/^transform-.+-b.+-v\d+$/); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("does not load @babel/core when every file is served from the disk cache", () => { + // Two subprocesses sharing the disk cache: the first (cold) must load + // Babel; the second (warm) must serve from disk without ever requiring it. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "vn-lazybabel-")); + try { + const file = path.join(dir, "mod.js"); + // Unique content per test invocation: the disk cache is content-keyed and + // shared across runs, so a constant source would make the "cold" run warm. + fs.writeFileSync( + file, + `// @flow\n// ${crypto.randomUUID()}\ntype Q = number;\nmodule.exports = (q: Q) => q + 1;`, + ); + const script = ` + import { transformRN } from ${JSON.stringify( + // file:// URL, not a raw path: Windows absolute paths ("D:\\…") are + // rejected by Node's ESM loader as an unsupported URL scheme. + pathToFileURL(path.join(projectRoot, "src", "native", "transform.mjs")).href, + )}; + import fs from "node:fs"; + const file = ${JSON.stringify(file)}; + transformRN(file, fs.readFileSync(file, "utf8"), ${JSON.stringify(projectRoot)}); + const { default: Module } = await import("node:module"); + // The version read (@babel/core/package.json) is cheap and expected on + // both runs; "loaded" means Babel's actual entry module was required. + const loadedBabel = Object.keys(Module._cache ?? {}).some( + (p) => p.includes(${JSON.stringify(path.join("@babel", "core") + path.sep)}) && + !p.endsWith("package.json"), + ); + console.log("BABEL_LOADED=" + loadedBabel); + `; + const run = () => + execFileSync(process.execPath, ["--input-type=module", "-e", script], { + encoding: "utf8", + }); + const cold = run(); + expect(cold).toContain("BABEL_LOADED=true"); + const warm = run(); + expect(warm).toContain("BABEL_LOADED=false"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +});