From a44c27b9e478d0b8f1e6d976120c24ab252f3945 Mon Sep 17 00:00:00 2001 From: Larry Gritz Date: Fri, 24 Jul 2026 16:55:19 -0700 Subject: [PATCH] test(fuzz): changes to fuzz corpus seeding for better coverage The old rule copied only the 5 smallest files (<=100 KB) per format into the run corpus. Sorting by size and taking the first few systematically selected degenerate inputs (decompression bombs, truncated headers) and discarded the real, structurally rich images -- the worst bias for a coverage-guided mutation fuzzer, which needs valid deep-decode states to mutate around to find subtle bugs. - populate_corpora.py: drop the MAX_FILES count cap; gather every example from each format's FORMAT_SOURCES (testsuite fixtures + companion repos). Raise the per-file size cap from 100 KB to 5 MB (a throughput guard only; covers essentially all our test images). - ci-fuzztest.bash: gather seeds into a scratch seed_dir, then run `oiio_fuzz_image -merge=1` to fold only the coverage-increasing subset into the cached run corpus before the timed session. A crash during merge (a committed seed or testsuite fixture that itself trips the decoder) fails the job. - docs/dev/fuzzing.md: document the gather-all + merge model and the rule that regression fixtures live in testsuite/ (wired into FORMAT_SOURCES), not duplicated into src/fuzz/corpora/. Assisted-by: Claude Code / Claude Opus 4.8 Signed-off-by: Larry Gritz --- docs/dev/fuzzing.md | 62 +++++++++++++++--------------- src/build-scripts/ci-fuzztest.bash | 34 ++++++++++++++-- src/fuzz/populate_corpora.py | 34 +++++++++++----- 3 files changed, 88 insertions(+), 42 deletions(-) diff --git a/docs/dev/fuzzing.md b/docs/dev/fuzzing.md index e3ea222c86..b2d93f4199 100644 --- a/docs/dev/fuzzing.md +++ b/docs/dev/fuzzing.md @@ -116,50 +116,52 @@ merging the fix. ## Adding seeds for a new format +Seeds are **not** hand-curated into `src/fuzz/corpora/` per format. At fuzz +time `ci-fuzztest.bash` gathers every example we have for the format from the +directories listed in `FORMAT_SOURCES` (`src/fuzz/populate_corpora.py`) — the +format's own `testsuite/` fixtures (valid images *and* malformed/regression +files) plus the companion image repos — capped only by a per-file size limit +(`MAX_BYTES`, 5 MB), with **no cap on the number of seeds**. It then runs +`oiio_fuzz_image -merge=1` to distill that pile down to the coverage-increasing +subset before the timed run. Feeding in the full, size-diverse set (not just a +few tiny files) is deliberate: mutating structurally rich real images reaches +the deep decode paths where subtle bugs hide. + When a new format plugin is added to OIIO: 1. The format automatically appears in `--list-formats` (no harness changes needed). -2. The CI lint job (`fuzz-corpus-lint` in `.github/workflows/ci.yml`) will +2. The CI lint job (`fuzz-corpus-lint` in `.github/workflows/fuzz.yml`) will **fail** until `src/fuzz/corpora//` exists. This is intentional — - it enforces that every compiled-in format has at least a corpus directory. -3. Create the directory and add 1–5 representative seed files (each ≤ 100 KB): + it enforces that every compiled-in format has at least a corpus directory + (a `.gitkeep` is enough). +3. Wire the format's seed sources into `FORMAT_SOURCES` in + `src/fuzz/populate_corpora.py`: its `testsuite//src` (and any + dedicated malformed-fixture dirs, e.g. `testsuite/-corrupt/src`) + and the relevant companion-repo path. Regression fixtures added to + `testsuite/` are then picked up automatically — do **not** also copy them + into `src/fuzz/corpora/`. +4. Only when a format has **no** testsuite fixture or companion source (e.g. + `hdr`, `iff`, `sgi`, `jpegxl`), commit a small synthetic seed directly: ```bash -mkdir src/fuzz/corpora// -# copy or generate seed files -cp path/to/sample. src/fuzz/corpora// -# Or generate a small synthetic seed: oiiotool --create 64x64 3 --ch R,G,B -o src/fuzz/corpora//seed. ``` -4. Verify the seeds parse cleanly: +5. Verify the gathered seeds parse cleanly (`-runs=0` processes seeds without + mutating): ```bash -OIIO_FUZZ_FORMAT= \ - build/src/fuzz/oiio_fuzz_image \ - src/fuzz/corpora// \ - -runs=0 -``` - -5. Commit both the corpus directory and any seed files. - -To populate corpus seeds from testsuite and companion image repos run: - -```bash -# Populate src/fuzz/corpora// (local dev) -python3 src/fuzz/populate_corpora.py [--format ] - -# Or write directly into a working corpus dir (as CI does): -python3 src/fuzz/populate_corpora.py --format --dest corpus/ +python3 src/fuzz/populate_corpora.py --format --dest /tmp/seeds +OIIO_FUZZ_FORMAT= build/src/fuzz/oiio_fuzz_image /tmp/seeds// -runs=0 ``` -The script auto-detects companion repos at `../oiio-images` (sibling of the -repo root, as checked out by the fuzz CI job) or at `build/testsuite/oiio-images` -(fetched by `oiio_setup_test_data` during a regular test run). Only synthetic -seeds (formats with no existing source files, such as `seed.hdr`) are committed -to `src/fuzz/corpora/`; all other seeds are sourced from `testsuite/` or -companion repos at fuzz time. +`populate_corpora.py` auto-detects companion repos at `../oiio-images` (sibling +of the repo root, as checked out by the fuzz CI job) or at +`build/testsuite/oiio-images` (fetched by `oiio_setup_test_data` during a +regular test run). The committed `src/fuzz/corpora//` dir holds only +synthetic seeds that have no testsuite fixture; everything else is sourced from +`testsuite/` or companion repos at fuzz time. ## How format selection works diff --git a/src/build-scripts/ci-fuzztest.bash b/src/build-scripts/ci-fuzztest.bash index 60443f1f7a..0d7b2597b0 100755 --- a/src/build-scripts/ci-fuzztest.bash +++ b/src/build-scripts/ci-fuzztest.bash @@ -38,10 +38,20 @@ fi # Seed the corpus, run the fuzzer, and write a job summary for one format. # if [[ -n "${OIIO_FUZZ_FORMAT}" ]]; then + # corpus_dir persists across runs via the CI cache and accumulates the + # coverage-increasing inputs libFuzzer discovers. seed_dir is gathered + # fresh each run and then minimized into corpus_dir below. corpus_dir="corpus/${OIIO_FUZZ_FORMAT}" - mkdir -p "$corpus_dir" - python3 src/fuzz/populate_corpora.py --format "$OIIO_FUZZ_FORMAT" --dest corpus - cp -rn "src/fuzz/corpora/${OIIO_FUZZ_FORMAT}/"* "$corpus_dir/" 2>/dev/null || true + seed_dir="seeds/${OIIO_FUZZ_FORMAT}" + mkdir -p "$corpus_dir" "$seed_dir" + + # Gather every available seed for this format (testsuite fixtures + + # companion image repos, each within populate_corpora's per-file size cap) + # plus any committed synthetic seeds. There is intentionally no cap on the + # number of seeds -- the -merge step below keeps only the coverage- + # increasing subset, so feeding in the full, size-diverse set is the point. + python3 src/fuzz/populate_corpora.py --format "$OIIO_FUZZ_FORMAT" --dest seeds + cp -rn "src/fuzz/corpora/${OIIO_FUZZ_FORMAT}/"* "$seed_dir/" 2>/dev/null || true if [[ ! -x "$FUZZ_BIN" ]]; then echo "::error::$FUZZ_BIN not found — was OIIO_BUILD_FUZZ_TARGETS=ON passed to cmake?" @@ -56,6 +66,19 @@ if [[ -n "${OIIO_FUZZ_FORMAT}" ]]; then skipped=1 else set +e + # Corpus minimization: add only coverage-increasing seeds from seed_dir + # into the (possibly cached) run corpus. A crash/hang here means a + # committed seed or testsuite fixture itself trips the decoder -- that + # is a finding, surfaced via merge_status below. + "$FUZZ_BIN" -merge=1 \ + -rss_limit_mb=4096 \ + -malloc_limit_mb=2048 \ + -timeout=60 \ + -detect_leaks=0 \ + -artifact_prefix="crash_${OIIO_FUZZ_FORMAT}_" \ + "$corpus_dir" "$seed_dir" + merge_status=$? + # Timed, coverage-guided fuzzing over the minimized corpus. "$FUZZ_BIN" "$corpus_dir" \ -max_total_time="${OIIO_FUZZ_MAX_TIME:-3600}" \ -max_len=16777216 \ @@ -67,6 +90,11 @@ if [[ -n "${OIIO_FUZZ_FORMAT}" ]]; then -jobs=$(nproc) -workers=$(nproc) fuzz_status=$? set -e + # If minimization tripped a crash but the timed run was clean, still + # fail the job on the merge-time finding. + if [[ "$merge_status" -ne 0 && "$fuzz_status" -eq 0 ]]; then + fuzz_status=$merge_status + fi fi crash_count=$(find . -maxdepth 1 -name "crash_${OIIO_FUZZ_FORMAT}_*" | wc -l | tr -d ' ') diff --git a/src/fuzz/populate_corpora.py b/src/fuzz/populate_corpora.py index 4c85905a7c..837682f332 100644 --- a/src/fuzz/populate_corpora.py +++ b/src/fuzz/populate_corpora.py @@ -3,13 +3,23 @@ # SPDX-License-Identifier: Apache-2.0 # https://github.com/AcademySoftwareFoundation/OpenImageIO """ -Populate src/fuzz/corpora/ with seed files for each image format. +Gather fuzz seed files for each image format from its configured sources. -Run from the repository root. Companion image repos are expected as siblings -of the repo root (e.g. ../oiio-images, ../fits-images, etc.). +For every format, FORMAT_SOURCES lists the directories to draw seeds from: +the format's own testsuite fixture dirs (valid images and malformed/regression +files) and, when available, the companion image repos (../oiio-images, etc.). +Every file up to the per-file size cap (MAX_BYTES) is gathered -- there is no +cap on seed count. Coverage-based minimization happens downstream in +ci-fuzztest.bash via `oiio_fuzz_image -merge=1`, which keeps only the +coverage-increasing subset, so it is fine (and intended) to feed in everything. + +Run from the repository root. In CI this is invoked with --dest to collect +seeds into a scratch dir for minimization; without --dest it writes into +src/fuzz/corpora// (which otherwise holds only synthetic seeds that have +no testsuite fixture, e.g. hdr/iff/sgi/jpegxl). Usage: - python src/fuzz/populate_corpora.py [--format ] [--dry-run] + python src/fuzz/populate_corpora.py [--format ] [--dest DIR] [--dry-run] """ import argparse @@ -21,8 +31,14 @@ REPO_ROOT = Path(__file__).resolve().parents[2] CORPUS_ROOT = REPO_ROOT / "src" / "fuzz" / "corpora" -MAX_FILES = 5 -MAX_BYTES = 100 * 1024 # 100 KB per file +# Per-file size cap only. There is deliberately no cap on the *number* of +# seeds: we gather every example we have (valid + malformed, all sizes up to +# the cap) so the fuzzer can mutate structurally rich inputs, not just tiny +# degenerate ones. Coverage-based minimization (libFuzzer -merge, run by +# ci-fuzztest.bash) then distills them to the covering subset. The size cap is +# purely a throughput guard -- very large seeds slow mutation without adding +# much reach; 5 MB comfortably covers our test images. +MAX_BYTES = 5 * 1024 * 1024 # 5 MB per file # Companion image repos may live: # (a) as siblings of the repo root (local dev or fuzz CI checkout) @@ -186,7 +202,7 @@ def candidate_files(sources: list) -> list[Path]: def populate_format(fmt: str, dry_run: bool, dest: Path | None = None) -> tuple[int, list[str]]: """ - Copy up to MAX_FILES files ≤ MAX_BYTES into dest (default: src/fuzz/corpora//). + Copy every file ≤ MAX_BYTES into dest (default: src/fuzz/corpora//). Returns (files_copied, missing_reasons). """ dest = dest if dest is not None else CORPUS_ROOT / fmt @@ -203,7 +219,7 @@ def populate_format(fmt: str, dry_run: bool, dest: Path | None = None) -> tuple[ if not small: if files: missing.append( - f"found {len(files)} file(s) but all exceed {MAX_BYTES // 1024} KB" + f"found {len(files)} file(s) but all exceed {MAX_BYTES // (1024 * 1024)} MB" ) else: dirs_checked = [resolve(r) for r, _ in sources] @@ -216,7 +232,7 @@ def populate_format(fmt: str, dry_run: bool, dest: Path | None = None) -> tuple[ ) return 0, missing - chosen = small[:MAX_FILES] + chosen = small if not dry_run: dest.mkdir(parents=True, exist_ok=True) copied = 0