Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions fixtures/lockfile-resolution-benchmark/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Lockfile resolution benchmark

This fixture compares dry-run deploy performance between the workspace Wrangler
and the prerelease from PR
[#14703](https://github.com/cloudflare/workers-sdk/pull/14703).

Its `test:ci` script warms both CLIs, runs five alternating trials, and prints a
single timing summary. The fixture test suite runs automatically on Linux,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This documents that the benchmark runs automatically in the main CI workflow on all three OSes. Because test:ci is the standard CI task, every CI run will now fetch the pkg.pr.new prerelease (wrangler-pr) and spawn wrangler ~12 times on Linux/macOS/Windows. That's a real recurring cost and an external network dependency (pkg.pr.new) baked into pnpm-lock.yaml/the catalog. Given the [benchmark] title and the exploratory PR body, this looks like a one-off measurement that shouldn't be merged into main permanently — consider dropping it before merge, or gating the script so it's opt-in (e.g. behind an env flag) rather than part of the default CI graph.

macOS, and Windows in the main CI workflow.

Run it locally from the repository root with:

```sh
pnpm test:ci --filter @fixture/lockfile-resolution-benchmark
```
93 changes: 93 additions & 0 deletions fixtures/lockfile-resolution-benchmark/benchmark.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import { spawn } from "node:child_process";
import path from "node:path";
import { performance } from "node:perf_hooks";
import { fileURLToPath } from "node:url";

const projectPath = path.dirname(fileURLToPath(import.meta.url));
const trials = 5;
const binaries = {
stable: path.join(projectPath, "node_modules/wrangler/bin/wrangler.js"),
pr: path.join(projectPath, "node_modules/wrangler-pr/bin/wrangler.js"),
};

await runWrangler("stable");
await runWrangler("pr");

const results = { stable: [], pr: [] };
for (let trial = 0; trial < trials; trial++) {
const order = trial % 2 === 0 ? ["stable", "pr"] : ["pr", "stable"];
for (const variant of order) {
results[variant].push(await runWrangler(variant));
}
}

const stable = summarize(results.stable);
const pr = summarize(results.pr);
const pairedDeltas = results.pr.map(
(duration, index) => duration - results.stable[index]
);
const delta = summarize(pairedDeltas);

console.log(`
Lockfile resolution benchmark (${trials} warmed dry-run deploys)

median mean min max
Wrangler stable ${format(stable.median)} ${format(stable.mean)} ${format(stable.min)} ${format(stable.max)}
PR #14703 ${format(pr.median)} ${format(pr.mean)} ${format(pr.min)} ${format(pr.max)}

Median paired overhead: ${format(delta.median)}
Median slowdown: ${((pr.median / stable.median - 1) * 100).toFixed(1)}%
`);

async function runWrangler(variant) {
const startedAt = performance.now();
await new Promise((resolve, reject) => {
const child = spawn(
process.execPath,
[binaries[variant], "deploy", "--dry-run"],
{
cwd: projectPath,
env: {
...process.env,
WRANGLER_SEND_METRICS: "false",
WRANGLER_WRITE_LOGS: "false",
},
stdio: ["ignore", "pipe", "pipe"],
}
);

let stdout = "";
let stderr = "";
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk) => (stdout += chunk));
child.stderr.on("data", (chunk) => (stderr += chunk));
child.on("error", reject);
child.on("exit", (code, signal) => {
if (code === 0) {
resolve();
return;
}
reject(
new Error(
`${variant} Wrangler failed (${signal ?? code})\n${stdout}\n${stderr}`
)
);
});
});
return performance.now() - startedAt;
}

function summarize(values) {
const sorted = values.toSorted((a, b) => a - b);
return {
min: sorted[0],
median: sorted[Math.floor(sorted.length / 2)],

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

median only picks a single element, so for an even number of trials it returns the upper-middle value rather than the average of the two middle values. Harmless while trials = 5 (odd), but the whole point of this fixture is accurate stats, so it's worth making it robust in case trials changes:

Suggested change
median: sorted[Math.floor(sorted.length / 2)],
median:
sorted.length % 2 === 0
? (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2
: sorted[Math.floor(sorted.length / 2)],

mean: values.reduce((sum, value) => sum + value, 0) / values.length,
max: sorted[sorted.length - 1],
};
}

function format(milliseconds) {
return `${milliseconds.toFixed(1).padStart(7)} ms`;
}
17 changes: 17 additions & 0 deletions fixtures/lockfile-resolution-benchmark/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"name": "@fixture/lockfile-resolution-benchmark",
"private": true,
"scripts": {
"test:ci": "node benchmark.mjs"
},
"dependencies": {
"jsonc-parser": "3.2.0"
},
"devDependencies": {
"wrangler": "workspace:*",
"wrangler-pr": "catalog:lockfile-benchmark"
},
"volta": {
"extends": "../../package.json"
}
}
7 changes: 7 additions & 0 deletions fixtures/lockfile-resolution-benchmark/src/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { parse } from "jsonc-parser";

export default {
fetch() {
return new Response(parse('"Hello World!"'));
},
};
9 changes: 9 additions & 0 deletions fixtures/lockfile-resolution-benchmark/turbo.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"$schema": "http://turbo.build/schema.json",
"extends": ["//"],
"tasks": {
"test:ci": {
"dependsOn": ["wrangler#build"]
}
}
}
6 changes: 6 additions & 0 deletions fixtures/lockfile-resolution-benchmark/wrangler.jsonc
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"$schema": "node_modules/wrangler/config-schema.json",
"name": "lockfile-resolution-benchmark",
"main": "src/index.js",
"compatibility_date": "2026-07-20",
}
86 changes: 86 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pnpm-workspace.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ patchedDependencies:
# Catalog
# ──────────────────────────────────────────────────────────────────────────────

catalogs:
lockfile-benchmark:
wrangler-pr: "https://pkg.pr.new/wrangler@14703"

catalog:
chalk: "5.3.0"
command-exists: "1.2.9"
Expand Down
Loading