Skip to content

Commit ef85b59

Browse files
oetroetr
committed
bench(engine): add backend smoke harness
1 parent a634660 commit ef85b59

9 files changed

Lines changed: 497 additions & 0 deletions

File tree

‎benchmarks/engine_smoke/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
node_modules/
2+
package-lock.json
3+
work/

‎benchmarks/engine_smoke/anomaly.js‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
* Copyright 2026 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
const { spawnSync } = require("child_process");
18+
const fs = require("fs");
19+
const path = require("path");
20+
21+
const benchmarkDirectory = __dirname;
22+
const workDirectory = path.join(benchmarkDirectory, "work", "anomalies");
23+
const engineTarget = path.join(
24+
benchmarkDirectory,
25+
"..",
26+
"..",
27+
"tests",
28+
"engine",
29+
"fuzz.js",
30+
);
31+
const asyncTarget = path.join(benchmarkDirectory, "anomaly_fuzz.js");
32+
33+
function removeIfExists(targetPath) {
34+
fs.rmSync(targetPath, { force: true, recursive: true });
35+
}
36+
37+
function ensureDirectory(targetPath) {
38+
fs.mkdirSync(targetPath, { recursive: true });
39+
}
40+
41+
function runCommand(label, args, cwd, outputDirectory, expectedStatus = 0) {
42+
console.log(`\n[anomaly] ${label}`);
43+
console.log(`[anomaly] command: npx ${args.join(" ")}`);
44+
ensureDirectory(outputDirectory);
45+
const sanitizedLabel = label.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
46+
const stdoutPath = path.join(outputDirectory, `${sanitizedLabel}.stdout.log`);
47+
const stderrPath = path.join(outputDirectory, `${sanitizedLabel}.stderr.log`);
48+
const stdoutFd = fs.openSync(stdoutPath, "w");
49+
const stderrFd = fs.openSync(stderrPath, "w");
50+
const startedAt = Date.now();
51+
const proc = spawnSync("npx", args, {
52+
cwd,
53+
env: { ...process.env },
54+
shell: true,
55+
stdio: ["ignore", stdoutFd, stderrFd],
56+
windowsHide: true,
57+
});
58+
const elapsedMs = Date.now() - startedAt;
59+
fs.closeSync(stdoutFd);
60+
fs.closeSync(stderrFd);
61+
62+
if (proc.status !== expectedStatus) {
63+
throw new Error(
64+
`${label} failed with exit code ${proc.status}\nSTDOUT (${stdoutPath}):\n${fs.readFileSync(stdoutPath, "utf8")}\nSTDERR (${stderrPath}):\n${fs.readFileSync(stderrPath, "utf8")}`,
65+
);
66+
}
67+
68+
return {
69+
elapsedMs,
70+
stderrPath,
71+
stdoutPath,
72+
};
73+
}
74+
75+
function parseExecsPerSecond(stderrPath) {
76+
const stderr = fs.readFileSync(stderrPath, "utf8");
77+
const match = stderr.match(/speed:\s+([0-9.]+) exec\/s/);
78+
if (!match) {
79+
throw new Error(`No LibAFL done line found in ${stderrPath}`);
80+
}
81+
return Number.parseFloat(match[1]);
82+
}
83+
84+
function runGuidedNumericSmoke() {
85+
const outputDirectory = path.join(workDirectory, "guided-numeric");
86+
const corpusDirectory = path.join(outputDirectory, "corpus");
87+
removeIfExists(outputDirectory);
88+
ensureDirectory(corpusDirectory);
89+
fs.writeFileSync(path.join(corpusDirectory, "seed"), Buffer.alloc(4));
90+
91+
const result = runCommand(
92+
"guided numeric solve",
93+
[
94+
"jazzer",
95+
engineTarget,
96+
"-f",
97+
"guided_numeric",
98+
"--engine=afl",
99+
"--sync",
100+
"--disable_bug_detectors=.*",
101+
"--runs=4000",
102+
"--seed=1337",
103+
"--maxLen=16",
104+
`--artifactPrefix=${outputDirectory}${path.sep}`,
105+
corpusDirectory,
106+
],
107+
benchmarkDirectory,
108+
outputDirectory,
109+
77,
110+
);
111+
112+
const output =
113+
fs.readFileSync(result.stdoutPath, "utf8") +
114+
fs.readFileSync(result.stderrPath, "utf8");
115+
if (!output.includes("AFL numeric guidance finding")) {
116+
throw new Error("Guided numeric smoke did not report the expected finding");
117+
}
118+
119+
return {
120+
name: "guided-numeric",
121+
elapsedMs: result.elapsedMs,
122+
};
123+
}
124+
125+
function runAsyncSmoke() {
126+
const outputDirectory = path.join(workDirectory, "async-smoke");
127+
const corpusDirectory = path.join(outputDirectory, "corpus");
128+
removeIfExists(outputDirectory);
129+
ensureDirectory(corpusDirectory);
130+
fs.writeFileSync(path.join(corpusDirectory, "seed"), "async-seed");
131+
132+
const result = runCommand(
133+
"async throughput smoke",
134+
[
135+
"jazzer",
136+
asyncTarget,
137+
"-f",
138+
"async_smoke",
139+
"--engine=afl",
140+
"--disable_bug_detectors=.*",
141+
"--runs=2000",
142+
"--seed=9001",
143+
"--maxLen=128",
144+
`--artifactPrefix=${outputDirectory}${path.sep}`,
145+
corpusDirectory,
146+
],
147+
benchmarkDirectory,
148+
outputDirectory,
149+
);
150+
151+
const execsPerSecond = parseExecsPerSecond(result.stderrPath);
152+
if (execsPerSecond <= 0) {
153+
throw new Error("Async smoke reported a non-positive exec/sec rate");
154+
}
155+
if (result.elapsedMs > 30000) {
156+
throw new Error(
157+
`Async smoke took unexpectedly long: ${result.elapsedMs} ms`,
158+
);
159+
}
160+
161+
return {
162+
name: "async-smoke",
163+
elapsedMs: result.elapsedMs,
164+
execsPerSecond,
165+
};
166+
}
167+
168+
function main() {
169+
ensureDirectory(workDirectory);
170+
const results = [runGuidedNumericSmoke(), runAsyncSmoke()];
171+
for (const result of results) {
172+
const stats = [`elapsed_ms=${result.elapsedMs}`];
173+
if (result.execsPerSecond !== undefined) {
174+
stats.push(`execs_per_second=${result.execsPerSecond}`);
175+
}
176+
console.log(`[anomaly] ${result.name}: ${stats.join(" ")}`);
177+
}
178+
fs.writeFileSync(
179+
path.join(workDirectory, "results.json"),
180+
JSON.stringify(results, null, 2),
181+
);
182+
console.log(
183+
`\n[anomaly] wrote machine-readable results to ${path.join(workDirectory, "results.json")}`,
184+
);
185+
}
186+
187+
main();

‎benchmarks/engine_smoke/anomaly_fuzz.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright 2026 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
module.exports.async_smoke = function (data) {
18+
let checksum = 0;
19+
for (const byte of data) {
20+
checksum = ((checksum * 33) ^ byte) & 0xffff;
21+
}
22+
23+
return new Promise((resolve) => {
24+
setImmediate(() => {
25+
if (checksum === 0x1337) {
26+
// Exercise an extra branch without turning this into a finding target.
27+
checksum ^= data.length;
28+
}
29+
resolve(checksum);
30+
});
31+
});
32+
};

‎benchmarks/engine_smoke/fuzz.js‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright 2026 Code Intelligence GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
const qs = require("qs");
18+
19+
const { FuzzedDataProvider } = require("@jazzer.js/core");
20+
21+
module.exports.fuzz = function (data) {
22+
const provider = new FuzzedDataProvider(data);
23+
const input = provider.consumeRemainingAsString();
24+
25+
const parseOptions = {
26+
allowDots: provider.consumeBoolean(),
27+
allowEmptyArrays: provider.consumeBoolean(),
28+
allowPrototypes: provider.consumeBoolean(),
29+
arrayLimit: provider.consumeIntegralInRange(0, 32),
30+
charset: provider.pickValue(["utf-8", "iso-8859-1"]),
31+
charsetSentinel: provider.consumeBoolean(),
32+
comma: provider.consumeBoolean(),
33+
decodeDotInKeys: provider.consumeBoolean(),
34+
depth: provider.consumeIntegralInRange(0, 16),
35+
duplicates: provider.pickValue(["combine", "first", "last"]),
36+
ignoreQueryPrefix: provider.consumeBoolean(),
37+
interpretNumericEntities: provider.consumeBoolean(),
38+
parameterLimit: provider.consumeIntegralInRange(1, 256),
39+
parseArrays: provider.consumeBoolean(),
40+
plainObjects: provider.consumeBoolean(),
41+
strictDepth: provider.consumeBoolean(),
42+
strictNullHandling: provider.consumeBoolean(),
43+
};
44+
45+
let parsed;
46+
try {
47+
parsed = qs.parse(input, parseOptions);
48+
} catch {
49+
return;
50+
}
51+
52+
try {
53+
qs.stringify(parsed, {
54+
addQueryPrefix: provider.consumeBoolean(),
55+
allowDots: provider.consumeBoolean(),
56+
allowEmptyArrays: provider.consumeBoolean(),
57+
arrayFormat: provider.pickValue([
58+
"indices",
59+
"brackets",
60+
"repeat",
61+
"comma",
62+
]),
63+
charset: provider.pickValue(["utf-8", "iso-8859-1"]),
64+
charsetSentinel: provider.consumeBoolean(),
65+
commaRoundTrip: provider.consumeBoolean(),
66+
delimiter: provider.pickValue(["&", ";"]),
67+
encode: provider.consumeBoolean(),
68+
encodeDotInKeys: provider.consumeBoolean(),
69+
indices: provider.consumeBoolean(),
70+
skipNulls: provider.consumeBoolean(),
71+
strictNullHandling: provider.consumeBoolean(),
72+
});
73+
} catch {
74+
// Smoke target: ignore library-level parse/stringify failures.
75+
}
76+
};

‎benchmarks/engine_smoke/package.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"name": "jazzerjs-engine-smoke",
3+
"version": "1.0.0",
4+
"private": true,
5+
"description": "Manual 30-second smoke benchmark for libFuzzer vs LibAFL.",
6+
"scripts": {
7+
"smoke": "node run.js",
8+
"smoke:anomalies": "node anomaly.js"
9+
},
10+
"devDependencies": {
11+
"@jazzer.js/core": "file:../../packages/core",
12+
"istanbul-lib-coverage": "^3.2.2",
13+
"qs": "^6.14.0"
14+
}
15+
}

0 commit comments

Comments
 (0)