From aab6245965f6351ec9c85cfb221cba9f16ff5f0e Mon Sep 17 00:00:00 2001 From: Dario Piotrowicz Date: Mon, 13 Jul 2026 11:56:29 +0100 Subject: [PATCH] Cache package dependency collection results to avoid redundant filesystem walks --- .changeset/cache-package-dependencies.md | 7 + packages/deploy-helpers/package.json | 1 + packages/deploy-helpers/src/deploy/deploy.ts | 4 +- .../deploy/helpers/package-dependencies.ts | 209 ++++++++++++- .../src/deploy/versions-upload.ts | 4 +- packages/deploy-helpers/src/shared/types.ts | 2 + .../tests/package-dependencies.test.ts | 276 ++++++++++++++++++ .../deployment-bundle/merge-config-args.ts | 3 + pnpm-lock.yaml | 3 + 9 files changed, 502 insertions(+), 7 deletions(-) create mode 100644 .changeset/cache-package-dependencies.md diff --git a/.changeset/cache-package-dependencies.md b/.changeset/cache-package-dependencies.md new file mode 100644 index 0000000000..de72fffc3a --- /dev/null +++ b/.changeset/cache-package-dependencies.md @@ -0,0 +1,7 @@ +--- +"@cloudflare/deploy-helpers": patch +--- + +Cache package dependency collection results to avoid redundant filesystem walks + +`collectPackageDependencies` now accepts an optional `cacheDir` parameter. When provided, collected results are persisted to disk and reused on subsequent calls as long as neither `package.json` nor the project's lockfile have been modified. This speeds up repeated deploys when dependencies haven't changed. diff --git a/packages/deploy-helpers/package.json b/packages/deploy-helpers/package.json index 6027898c83..7b8117528c 100644 --- a/packages/deploy-helpers/package.json +++ b/packages/deploy-helpers/package.json @@ -59,6 +59,7 @@ "@types/node": "catalog:default", "concurrently": "^8.2.2", "devtools-protocol": "^0.0.1182435", + "empathic": "^2.0.0", "esbuild": "catalog:default", "json-diff": "^1.0.6", "ts-dedent": "^2.2.0", diff --git a/packages/deploy-helpers/src/deploy/deploy.ts b/packages/deploy-helpers/src/deploy/deploy.ts index c4a4851fbc..d09110b5d7 100644 --- a/packages/deploy-helpers/src/deploy/deploy.ts +++ b/packages/deploy-helpers/src/deploy/deploy.ts @@ -328,7 +328,9 @@ export default async function deploy( cache: config.cache, package_dependencies: config.dependencies_instrumentation?.enabled !== false && projectRoot - ? await collectPackageDependencies(projectRoot) + ? await collectPackageDependencies(projectRoot, { + cacheDir: props.cacheDir, + }) : undefined, }; diff --git a/packages/deploy-helpers/src/deploy/helpers/package-dependencies.ts b/packages/deploy-helpers/src/deploy/helpers/package-dependencies.ts index c15cc73acd..55c4b9adb6 100644 --- a/packages/deploy-helpers/src/deploy/helpers/package-dependencies.ts +++ b/packages/deploy-helpers/src/deploy/helpers/package-dependencies.ts @@ -1,9 +1,11 @@ -import { access, readFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { getInstalledPackageVersion, parsePackageJSON, } from "@cloudflare/workers-utils"; +import * as find from "empathic/find"; import { logger } from "../../shared/context"; /** @@ -26,6 +28,156 @@ export type PackageDependency = { */ const MAX_PACKAGE_DEPENDENCIES = 200; +/** Filename used for the cached dependency results. */ +const CACHE_FILENAME = "package-dependencies.json"; + +/** + * Lockfile names to look for when computing the cache key. + * Checked in order; the first match wins. + */ +const KNOWN_LOCKFILES = [ + "pnpm-lock.yaml", + "package-lock.json", + "yarn.lock", + "bun.lock", + "bun.lockb", +] as const; + +/** + * Shape of the on-disk cache file written to the caller-supplied cache directory. + */ +type PackageDependenciesCache = { + /** SHA-256 hex digest of the project's package.json content. */ + packageJsonHash: string; + /** SHA-256 hex digest of the lockfile content, or `null` if none was found. */ + lockfileHash: string | null; + /** Absolute path of the lockfile that was hashed, or `null` if none. */ + lockfilePath: string | null; + /** The collected dependency entries. */ + dependencies: PackageDependency[]; +}; + +/** + * Computes the SHA-256 hex digest of a buffer or string. + * + * @param content - The content to hash + * @returns The hex-encoded SHA-256 digest + */ +function sha256(content: Buffer | string): string { + return createHash("sha256").update(content).digest("hex"); +} + +/** + * Searches for the first known lockfile starting at `projectPath` and walking + * up parent directories toward the filesystem root. This ensures lockfiles at + * a monorepo/workspace root are detected even when the Worker lives in a + * sub-package. + * + * @param projectPath - The project directory to start searching from + * @returns An object with the lockfile's absolute `filePath` and its content `hash`, + * or `null` if no lockfile is found + */ +async function getLockfileInfo( + projectPath: string +): Promise<{ filePath: string; hash: string } | null> { + for (const name of KNOWN_LOCKFILES) { + const filePath = find.file(name, { cwd: projectPath }); + if (filePath) { + const content = await readFile(filePath); + return { filePath, hash: sha256(content) }; + } + } + return null; +} + +/** + * Validates that a parsed JSON value conforms to the {@link PackageDependenciesCache} shape. + * Returns `false` if any field is missing or has the wrong type, causing the + * entire cache to be discarded. + * + * @param value - The parsed JSON value to validate + * @returns `true` if the value matches the expected cache structure, `false` otherwise + */ +function isValidDependenciesCache( + value: unknown +): value is PackageDependenciesCache { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + + const obj = value as Record; + + if (typeof obj.packageJsonHash !== "string") { + return false; + } + if (obj.lockfileHash !== null && typeof obj.lockfileHash !== "string") { + return false; + } + if (obj.lockfilePath !== null && typeof obj.lockfilePath !== "string") { + return false; + } + if (!Array.isArray(obj.dependencies)) { + return false; + } + + for (const dep of obj.dependencies) { + if (typeof dep !== "object" || dep === null || Array.isArray(dep)) { + return false; + } + const d = dep as Record; + if ( + typeof d.name !== "string" || + typeof d.packageJsonVersion !== "string" || + typeof d.installedVersion !== "string" + ) { + return false; + } + } + + return true; +} + +/** + * Attempts to read a previously-written cache file. + * + * @param cachePath - Full path to the cache JSON file + * @returns The parsed cache contents, or `null` if the file is missing or malformed + */ +async function readDependenciesCache( + cachePath: string +): Promise { + try { + const raw = await readFile(cachePath, "utf-8"); + const parsed: unknown = JSON.parse(raw); + + if (!isValidDependenciesCache(parsed)) { + return null; + } + + return parsed; + } catch { + return null; + } +} + +/** + * Writes the dependency cache to disk (best-effort; failures are silently ignored). + * + * @param cachePath - Full path to the cache JSON file + * @param data - The cache payload to persist + */ +async function writeDependenciesCache( + cachePath: string, + data: PackageDependenciesCache +): Promise { + try { + await mkdir(path.dirname(cachePath), { recursive: true }); + await writeFile(cachePath, JSON.stringify(data, null, 2)); + } catch { + // best-effort — a failure to write cache should never block a deploy + } +} + /** * Collects npm package dependency metadata from the project's package.json. * @@ -38,12 +190,22 @@ const MAX_PACKAGE_DEPENDENCIES = 200; * * The result is capped at {@link MAX_PACKAGE_DEPENDENCIES} entries. * + * When `opts.cacheDir` is provided the results are cached to disk and reused + * on subsequent calls as long as neither `package.json` nor the project's + * lockfile have been modified (compared by content hash). + * * @param projectPath - Path to the project directory (where package.json is located) + * @param opts - Optional settings + * @param opts.cacheDir - Directory to store the cache file. If omitted, caching is disabled. * @returns An array of package dependency entries, or `undefined` if package.json * cannot be read or no valid dependencies are found */ export async function collectPackageDependencies( - projectPath: string + projectPath: string, + opts?: { + /** Directory to store the cache file. If omitted, caching is disabled. */ + cacheDir?: string; + } ): Promise { const packageJsonPath = path.join(projectPath, "package.json"); @@ -54,8 +216,35 @@ export async function collectPackageDependencies( } try { - const content = await readFile(packageJsonPath, "utf-8"); - const packageJson = parsePackageJSON(content, packageJsonPath); + const packageJsonContent = await readFile(packageJsonPath, "utf-8"); + + const cachePath = opts?.cacheDir + ? path.join(opts.cacheDir, CACHE_FILENAME) + : null; + + // Only compute hashes and resolve lockfile info when caching is enabled, + // to avoid unnecessary filesystem I/O (directory walks and large file + // reads) when cacheDir is not provided. + const packageJsonHash = cachePath ? sha256(packageJsonContent) : undefined; + const lockfileInfo = cachePath + ? await getLockfileInfo(projectPath) + : undefined; + + if (cachePath && packageJsonHash !== undefined) { + // Attempt to return cached results when available + const cached = await readDependenciesCache(cachePath); + if ( + cached && + cached.packageJsonHash === packageJsonHash && + cached.lockfileHash === (lockfileInfo?.hash ?? null) && + cached.lockfilePath === (lockfileInfo?.filePath ?? null) + ) { + logger?.debug("Using cached package dependency results"); + return cached.dependencies.length > 0 ? cached.dependencies : undefined; + } + } + + const packageJson = parsePackageJSON(packageJsonContent, packageJsonPath); const allDependencies = { ...packageJson.dependencies, @@ -96,9 +285,19 @@ export async function collectPackageDependencies( }); } + // Persist the cache for future calls + if (cachePath && packageJsonHash !== undefined) { + await writeDependenciesCache(cachePath, { + packageJsonHash, + lockfileHash: lockfileInfo?.hash ?? null, + lockfilePath: lockfileInfo?.filePath ?? null, + dependencies: result, + }); + } + return result.length > 0 ? result : undefined; } catch (error) { - logger.debug( + logger?.debug( `Failed to collect package dependencies: ${error instanceof Error ? error.message : error}` ); return undefined; diff --git a/packages/deploy-helpers/src/deploy/versions-upload.ts b/packages/deploy-helpers/src/deploy/versions-upload.ts index 70c78bedd1..d2f226e846 100644 --- a/packages/deploy-helpers/src/deploy/versions-upload.ts +++ b/packages/deploy-helpers/src/deploy/versions-upload.ts @@ -192,7 +192,9 @@ export default async function versionsUpload( cache: config.cache, // cache is a versioned setting package_dependencies: config.dependencies_instrumentation?.enabled !== false && projectRoot - ? await collectPackageDependencies(projectRoot) + ? await collectPackageDependencies(projectRoot, { + cacheDir: props.cacheDir, + }) : undefined, }; diff --git a/packages/deploy-helpers/src/shared/types.ts b/packages/deploy-helpers/src/shared/types.ts index 5477f2a149..1d0a722171 100644 --- a/packages/deploy-helpers/src/shared/types.ts +++ b/packages/deploy-helpers/src/shared/types.ts @@ -99,6 +99,8 @@ export type SharedDeployVersionsProps = { accountId: string | undefined; /** Resolved from getMetricsUsageHeaders() / config.send_metrics. Controls whether usage metrics headers are sent with upload requests. */ sendMetrics: boolean; + /** Optional directory for caching intermediate results (e.g. package dependency collection). Callers set this to a project-local cache directory. */ + cacheDir?: string; /** Resolved from getFlag("RESOURCES_PROVISION"). Controls whether bindings are auto-provisioned before upload. */ resourcesProvision: boolean; /** Controls whether provisioned resource IDs are written back to the config file. */ diff --git a/packages/deploy-helpers/tests/package-dependencies.test.ts b/packages/deploy-helpers/tests/package-dependencies.test.ts index 0d8e953a3a..282cb2abed 100644 --- a/packages/deploy-helpers/tests/package-dependencies.test.ts +++ b/packages/deploy-helpers/tests/package-dependencies.test.ts @@ -340,4 +340,280 @@ describe("collectPackageDependencies", () => { }, ]); }); + + describe("caching", () => { + /** + * Sets up a minimal project with a single installed npm package in node_modules. + * + * @param cwd - The directory to set up the project in + * @param opts - Options for the package name, version constraint, and installed version + */ + function seedProjectWithPackage( + cwd: string, + opts: { + packageName?: string; + packageJsonVersion?: string; + installedVersion?: string; + } = {} + ) { + const { + packageName = "test-pkg", + packageJsonVersion = "^1.0.0", + installedVersion = "1.0.0", + } = opts; + const nodeModulesPath = path.join(cwd, "node_modules"); + const pkgPath = path.join(nodeModulesPath, packageName); + fs.mkdirSync(pkgPath, { recursive: true }); + fs.writeFileSync(path.join(pkgPath, "index.js"), "module.exports = {}"); + fs.writeFileSync( + path.join(pkgPath, "package.json"), + JSON.stringify( + { name: packageName, version: installedVersion }, + null, + 2 + ) + ); + fs.writeFileSync( + path.join(cwd, "package.json"), + JSON.stringify( + { + name: "test-project", + dependencies: { [packageName]: packageJsonVersion }, + }, + null, + 2 + ) + ); + } + + it("should write a cache file and return cached results on second call", async ({ + expect, + }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + const first = await collectPackageDependencies(cwd, { cacheDir }); + + expect(first).toEqual([ + { + name: "test-pkg", + packageJsonVersion: "^1.0.0", + installedVersion: "1.0.0", + }, + ]); + + // Cache file should exist on disk + const cachePath = path.join(cacheDir, "package-dependencies.json"); + expect(fs.existsSync(cachePath)).toBe(true); + + const second = await collectPackageDependencies(cwd, { cacheDir }); + expect(second).toEqual(first); + }); + + it("should invalidate cache when package.json changes", async ({ + expect, + }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + const first = await collectPackageDependencies(cwd, { cacheDir }); + expect(first).toHaveLength(1); + + // Add a second dependency + const newPkgPath = path.join(cwd, "node_modules", "new-pkg"); + fs.mkdirSync(newPkgPath, { recursive: true }); + fs.writeFileSync( + path.join(newPkgPath, "index.js"), + "module.exports = {}" + ); + fs.writeFileSync( + path.join(newPkgPath, "package.json"), + JSON.stringify({ name: "new-pkg", version: "2.0.0" }, null, 2) + ); + fs.writeFileSync( + path.join(cwd, "package.json"), + JSON.stringify( + { + name: "test-project", + dependencies: { + "test-pkg": "^1.0.0", + "new-pkg": "^2.0.0", + }, + }, + null, + 2 + ) + ); + + const second = await collectPackageDependencies(cwd, { cacheDir }); + expect(second).toHaveLength(2); + expect(second).toEqual( + expect.arrayContaining([expect.objectContaining({ name: "new-pkg" })]) + ); + }); + + it("should invalidate cache when lockfile content changes", async ({ + expect, + }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + // Create a lockfile + fs.writeFileSync( + path.join(cwd, "package-lock.json"), + JSON.stringify({ lockfileVersion: 1 }) + ); + + const first = await collectPackageDependencies(cwd, { cacheDir }); + expect(first).toHaveLength(1); + + // Modify the lockfile content (simulates a new `npm install`) + fs.writeFileSync( + path.join(cwd, "package-lock.json"), + JSON.stringify({ lockfileVersion: 2 }) + ); + + // Update the installed version to verify the cache was really invalidated + fs.writeFileSync( + path.join(cwd, "node_modules", "test-pkg", "package.json"), + JSON.stringify({ name: "test-pkg", version: "1.0.1" }, null, 2) + ); + + const second = await collectPackageDependencies(cwd, { cacheDir }); + expect(second).toEqual([ + { + name: "test-pkg", + packageJsonVersion: "^1.0.0", + installedVersion: "1.0.1", + }, + ]); + }); + + it("should work when no lockfile exists", async ({ expect }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + // No lockfile present — cache should still work based on package.json hash + const first = await collectPackageDependencies(cwd, { cacheDir }); + expect(first).toHaveLength(1); + + const second = await collectPackageDependencies(cwd, { cacheDir }); + expect(second).toEqual(first); + }); + + it("should detect lockfile in a parent directory (monorepo)", async ({ + expect, + }) => { + const cwd = process.cwd(); + + // Simulate a monorepo: lockfile at workspace root, sub-package below + const subPkg = path.join(cwd, "packages", "my-worker"); + fs.mkdirSync(subPkg, { recursive: true }); + + // Lockfile lives at the workspace root (cwd), not in the sub-package + fs.writeFileSync( + path.join(cwd, "pnpm-lock.yaml"), + "lockfileVersion: 9.0" + ); + + // Set up the sub-package with a dependency + seedProjectWithPackage(subPkg); + + const cacheDir = path.join(subPkg, ".cache"); + const first = await collectPackageDependencies(subPkg, { cacheDir }); + expect(first).toHaveLength(1); + + // Second call should hit cache + const second = await collectPackageDependencies(subPkg, { cacheDir }); + expect(second).toEqual(first); + + // Changing the root lockfile content should invalidate the cache + fs.writeFileSync( + path.join(cwd, "pnpm-lock.yaml"), + "lockfileVersion: 9.1" + ); + + // Update installed version to prove cache was invalidated + fs.writeFileSync( + path.join(subPkg, "node_modules", "test-pkg", "package.json"), + JSON.stringify({ name: "test-pkg", version: "1.0.1" }, null, 2) + ); + + const third = await collectPackageDependencies(subPkg, { cacheDir }); + expect(third).toEqual([ + { + name: "test-pkg", + packageJsonVersion: "^1.0.0", + installedVersion: "1.0.1", + }, + ]); + }); + + it("should handle corrupted cache file gracefully", async ({ expect }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + // Write invalid JSON to the cache file + fs.mkdirSync(cacheDir, { recursive: true }); + fs.writeFileSync( + path.join(cacheDir, "package-dependencies.json"), + "NOT VALID JSON{{{{" + ); + + const result = await collectPackageDependencies(cwd, { cacheDir }); + + // Should fall through to fresh collection despite the bad cache + expect(result).toEqual([ + { + name: "test-pkg", + packageJsonVersion: "^1.0.0", + installedVersion: "1.0.0", + }, + ]); + }); + + it("should discard cache with structurally invalid content", async ({ + expect, + }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + // Write valid JSON that doesn't match the expected cache shape + fs.mkdirSync(cacheDir, { recursive: true }); + fs.writeFileSync( + path.join(cacheDir, "package-dependencies.json"), + JSON.stringify({ unexpected: "shape", numbers: [1, 2, 3] }) + ); + + const result = await collectPackageDependencies(cwd, { cacheDir }); + + // Should fall through to fresh collection despite the bad cache shape + expect(result).toEqual([ + { + name: "test-pkg", + packageJsonVersion: "^1.0.0", + installedVersion: "1.0.0", + }, + ]); + }); + + it("should not cache when cacheDir is not provided", async ({ expect }) => { + const cwd = process.cwd(); + const cacheDir = path.join(cwd, ".cache"); + seedProjectWithPackage(cwd); + + // Call without cacheDir + const result = await collectPackageDependencies(cwd); + expect(result).toHaveLength(1); + + // No cache file should exist + expect(fs.existsSync(cacheDir)).toBe(false); + }); + }); }); diff --git a/packages/wrangler/src/deployment-bundle/merge-config-args.ts b/packages/wrangler/src/deployment-bundle/merge-config-args.ts index 08aad89406..5959672c49 100644 --- a/packages/wrangler/src/deployment-bundle/merge-config-args.ts +++ b/packages/wrangler/src/deployment-bundle/merge-config-args.ts @@ -1,8 +1,10 @@ +import path from "node:path"; import { generatePreviewAlias } from "@cloudflare/deploy-helpers"; import { getCIGeneratePreviewAlias, getCIOverrideName, getTodaysCompatDate, + getWranglerHiddenDirPath, getWranglerTmpDir, UserError, } from "@cloudflare/workers-utils"; @@ -92,6 +94,7 @@ async function mergeSharedConfigArgs( experimentalAutoCreate: args.experimentalAutoCreate, accountId, sendMetrics, + cacheDir: path.join(getWranglerHiddenDirPath(entry.projectRoot), "cache"), resourcesProvision: getFlag("RESOURCES_PROVISION") ?? false, skipProvisioningConfigWriteback: false, strict: args.strict ?? false, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4658ab1598..7a174b3e22 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2016,6 +2016,9 @@ importers: devtools-protocol: specifier: ^0.0.1182435 version: 0.0.1182435 + empathic: + specifier: ^2.0.0 + version: 2.0.1 esbuild: specifier: catalog:default version: 0.28.1