-
Notifications
You must be signed in to change notification settings - Fork 1.4k
[benchmark] lock file resolution #14764
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # Lockfile resolution benchmark | ||
|
|
||
| This fixture compares dry-run deploy performance between the workspace Wrangler | ||
| and the prerelease from PR | ||
| [#14703](https://github.com/cloudflare/workers-sdk/pull/14703). | ||
|
|
||
| Its `test:ci` script warms both CLIs, runs five alternating trials, and prints a | ||
| single timing summary. The fixture test suite runs automatically on Linux, | ||
| macOS, and Windows in the main CI workflow. | ||
|
|
||
| Run it locally from the repository root with: | ||
|
|
||
| ```sh | ||
| pnpm test:ci --filter @fixture/lockfile-resolution-benchmark | ||
| ``` | ||
| Original file line number | Diff line number | Diff line change | ||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,93 @@ | ||||||||||||
| import { spawn } from "node:child_process"; | ||||||||||||
| import path from "node:path"; | ||||||||||||
| import { performance } from "node:perf_hooks"; | ||||||||||||
| import { fileURLToPath } from "node:url"; | ||||||||||||
|
|
||||||||||||
| const projectPath = path.dirname(fileURLToPath(import.meta.url)); | ||||||||||||
| const trials = 5; | ||||||||||||
| const binaries = { | ||||||||||||
| stable: path.join(projectPath, "node_modules/wrangler/bin/wrangler.js"), | ||||||||||||
| pr: path.join(projectPath, "node_modules/wrangler-pr/bin/wrangler.js"), | ||||||||||||
| }; | ||||||||||||
|
|
||||||||||||
| await runWrangler("stable"); | ||||||||||||
| await runWrangler("pr"); | ||||||||||||
|
|
||||||||||||
| const results = { stable: [], pr: [] }; | ||||||||||||
| for (let trial = 0; trial < trials; trial++) { | ||||||||||||
| const order = trial % 2 === 0 ? ["stable", "pr"] : ["pr", "stable"]; | ||||||||||||
| for (const variant of order) { | ||||||||||||
| results[variant].push(await runWrangler(variant)); | ||||||||||||
| } | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| const stable = summarize(results.stable); | ||||||||||||
| const pr = summarize(results.pr); | ||||||||||||
| const pairedDeltas = results.pr.map( | ||||||||||||
| (duration, index) => duration - results.stable[index] | ||||||||||||
| ); | ||||||||||||
| const delta = summarize(pairedDeltas); | ||||||||||||
|
|
||||||||||||
| console.log(` | ||||||||||||
| Lockfile resolution benchmark (${trials} warmed dry-run deploys) | ||||||||||||
|
|
||||||||||||
| median mean min max | ||||||||||||
| Wrangler stable ${format(stable.median)} ${format(stable.mean)} ${format(stable.min)} ${format(stable.max)} | ||||||||||||
| PR #14703 ${format(pr.median)} ${format(pr.mean)} ${format(pr.min)} ${format(pr.max)} | ||||||||||||
|
|
||||||||||||
| Median paired overhead: ${format(delta.median)} | ||||||||||||
| Median slowdown: ${((pr.median / stable.median - 1) * 100).toFixed(1)}% | ||||||||||||
| `); | ||||||||||||
|
|
||||||||||||
| async function runWrangler(variant) { | ||||||||||||
| const startedAt = performance.now(); | ||||||||||||
| await new Promise((resolve, reject) => { | ||||||||||||
| const child = spawn( | ||||||||||||
| process.execPath, | ||||||||||||
| [binaries[variant], "deploy", "--dry-run"], | ||||||||||||
| { | ||||||||||||
| cwd: projectPath, | ||||||||||||
| env: { | ||||||||||||
| ...process.env, | ||||||||||||
| WRANGLER_SEND_METRICS: "false", | ||||||||||||
| WRANGLER_WRITE_LOGS: "false", | ||||||||||||
| }, | ||||||||||||
| stdio: ["ignore", "pipe", "pipe"], | ||||||||||||
| } | ||||||||||||
| ); | ||||||||||||
|
|
||||||||||||
| let stdout = ""; | ||||||||||||
| let stderr = ""; | ||||||||||||
| child.stdout.setEncoding("utf8"); | ||||||||||||
| child.stderr.setEncoding("utf8"); | ||||||||||||
| child.stdout.on("data", (chunk) => (stdout += chunk)); | ||||||||||||
| child.stderr.on("data", (chunk) => (stderr += chunk)); | ||||||||||||
| child.on("error", reject); | ||||||||||||
| child.on("exit", (code, signal) => { | ||||||||||||
| if (code === 0) { | ||||||||||||
| resolve(); | ||||||||||||
| return; | ||||||||||||
| } | ||||||||||||
| reject( | ||||||||||||
| new Error( | ||||||||||||
| `${variant} Wrangler failed (${signal ?? code})\n${stdout}\n${stderr}` | ||||||||||||
| ) | ||||||||||||
| ); | ||||||||||||
| }); | ||||||||||||
| }); | ||||||||||||
| return performance.now() - startedAt; | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| function summarize(values) { | ||||||||||||
| const sorted = values.toSorted((a, b) => a - b); | ||||||||||||
| return { | ||||||||||||
| min: sorted[0], | ||||||||||||
| median: sorted[Math.floor(sorted.length / 2)], | ||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||||
| mean: values.reduce((sum, value) => sum + value, 0) / values.length, | ||||||||||||
| max: sorted[sorted.length - 1], | ||||||||||||
| }; | ||||||||||||
| } | ||||||||||||
|
|
||||||||||||
| function format(milliseconds) { | ||||||||||||
| return `${milliseconds.toFixed(1).padStart(7)} ms`; | ||||||||||||
| } | ||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| { | ||
| "name": "@fixture/lockfile-resolution-benchmark", | ||
| "private": true, | ||
| "scripts": { | ||
| "test:ci": "node benchmark.mjs" | ||
| }, | ||
| "dependencies": { | ||
| "jsonc-parser": "3.2.0" | ||
| }, | ||
| "devDependencies": { | ||
| "wrangler": "workspace:*", | ||
| "wrangler-pr": "catalog:lockfile-benchmark" | ||
| }, | ||
| "volta": { | ||
| "extends": "../../package.json" | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| import { parse } from "jsonc-parser"; | ||
|
|
||
| export default { | ||
| fetch() { | ||
| return new Response(parse('"Hello World!"')); | ||
| }, | ||
| }; |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| { | ||
| "$schema": "http://turbo.build/schema.json", | ||
| "extends": ["//"], | ||
| "tasks": { | ||
| "test:ci": { | ||
| "dependsOn": ["wrangler#build"] | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| { | ||
| "$schema": "node_modules/wrangler/config-schema.json", | ||
| "name": "lockfile-resolution-benchmark", | ||
| "main": "src/index.js", | ||
| "compatibility_date": "2026-07-20", | ||
| } |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This documents that the benchmark runs automatically in the main CI workflow on all three OSes. Because
test:ciis the standard CI task, every CI run will now fetch thepkg.pr.newprerelease (wrangler-pr) and spawn wrangler ~12 times on Linux/macOS/Windows. That's a real recurring cost and an external network dependency (pkg.pr.new) baked intopnpm-lock.yaml/the catalog. Given the[benchmark]title and the exploratory PR body, this looks like a one-off measurement that shouldn't be merged intomainpermanently — consider dropping it before merge, or gating the script so it's opt-in (e.g. behind an env flag) rather than part of the default CI graph.