Check the example report fiches the way the figures are checked

.github/labeler.yml

@@ -30,6 +30,7 @@ figures:
           - scripts/generate_graphs.py
           - scripts/generate_diagrams.py
           - scripts/check_figures.py
+          - scripts/generated_assets.py
           - src/phonometry/_plot/**
           - src/phonometry/_plotting.py
           - requirements-figures.txt
@@ -41,6 +42,13 @@ reports:
           - .github/reports/**
           - scripts/generate_reports.py
           - scripts/generate_site_reports.py
+          - scripts/check_reports.py
+          - scripts/generated_assets.py
+          - requirements-reports.txt
+          # The fiche job installs both requirement files, and the embedded
+          # plot comes off the matplotlib/NumPy stack pinned in this one, so a
+          # bump here can move the rendered PDF and its preview.
+          - requirements-figures.txt
 
 conformance:
   - changed-files:

.github/workflows/python-app.yml

@@ -242,6 +242,46 @@ jobs:
       # for the whole set) and needs no rendering stack of its own.
       run: python scripts/check_figure_contrast.py
 
+  # The committed example .report() fiches (.github/reports) must match a fresh
+  # `make reports` run. Same drift gate as the figures, one layer further down
+  # the pipeline: the fiches are what the documentation links to as worked
+  # examples of what the library prints, so a stale one publishes a wrong
+  # answer. Nothing checked them until this job existed, and two of them sat a
+  # plot-styling release behind the code for weeks. The rendering stack is
+  # pinned across both requirements files (matplotlib/numpy in -figures,
+  # svglib/reportlab/pypdfium2 in -reports) because every link in the chain
+  # fixes the rendered page the check compares.
+  reports:
+    name: Example report fiches up to date
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        persist-credentials: false
+    - name: Set up Python 3.13
+      uses: actions/setup-python@v7
+      with:
+        python-version: "3.13"
+        cache: 'pip'
+    - name: Install the locked fiche-rendering stack
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        # Pin the render+compute stack last so it wins over looser deps.
+        pip install -r requirements-figures.txt -r requirements-reports.txt
+    - name: Regenerate the example fiches
+      run: make reports
+    - name: Fail if any committed fiche is stale
+      # Tolerance-aware compare instead of a byte diff: GitHub's heterogeneous
+      # runner CPUs shift a few of the embedded plot's path coordinates ~1 ULP,
+      # which rewrites the PDF page stream for no visible reason. The script
+      # compares the extracted text exactly and the rendered page within a
+      # calibrated pixel tolerance, so a real fiche change fails while
+      # cross-CPU coordinate noise passes. See scripts/check_reports.py.
+      run: python scripts/check_reports.py
+
   tests:
     runs-on: ${{ matrix.os }}
     permissions:

CHANGELOG.md

@@ -999,6 +999,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Changed
 
+- CI fails when `.github/reports` no longer matches a fresh `make reports`
+  run, which is the reason the fiches were able to drift for weeks in the
+  first place: the conformance report, the generated API reference, the
+  evidence pages and the documentation figures all had a staleness gate, and
+  the fiches had none. The new `Example report fiches up to date` job
+  regenerates the whole set and compares it through
+  `scripts/check_reports.py`. A byte diff is not an option, for the same
+  reason it was abandoned for the figures: every fiche embeds its plot as
+  vector geometry, and GitHub's heterogeneous runners compute a few of those
+  coordinates a last bit apart, which rewrites the page for no visible reason.
+  So each fiche is compared on what it says and on what it looks like: the
+  extracted text exactly, page by page, and the committed preview within the
+  same two-criteria pixel tolerance the figures use, recalibrated for a
+  document page. The thresholds are 64 pixels and an RMS of 0.5, and the
+  reasoning behind them is written out in the script rather than summarised
+  here, because it is not the clean separation it first looked like: moving
+  every plotted coordinate by one unit in the last decimal, which is an upper
+  bound rather than a forecast, pushes five of the 67 fiches past those
+  thresholds, all of them plots whose bar edges sit on a pixel boundary.
+  Moving a single coordinate in the worst of them changes nothing at all, and
+  the coordinates in question come from exact layout arithmetic, so the bound
+  is not what a real machine does. Raising the RMS far enough to clear it
+  would stop catching the restyled fills, which is the failure the check
+  exists to prevent. The two stale fiches moved 28710 and 27629 pixels and the
+  subtler fills moved the RMS to 0.79 and 1.33, so those are caught. They are
+  tighter than the figures' on purpose: a fiche preview is a flat document
+  page rasterized by a pinned binary rasterizer, a much quieter thing to
+  compare than a plot raster, and the figures' looser bound would have let
+  the restyled fills through. The rendering stack the job runs is pinned in
+  `requirements-reports.txt` alongside the existing figure pins, `make
+  reports` now renders into a scratch directory and swaps it in only once the
+  whole set is written, so a fiche that is no longer produced is actually
+  removed without a generator that dies halfway leaving the working tree
+  stripped of the committed examples, and the generator pins the numerical
+  thread pools the way the figure generator does, so a runner with a
+  different core count cannot reorder a floating-point sum into the rendered
+  page. The staleness helpers the two checks share moved to
+  `scripts/generated_assets.py`. A unit test also asserts every registered
+  fiche has a committed render, which fails in the same second as the commit
+  that forgets one, with no regeneration needed.
+
 - The figures now spell the decimal logarithm the way the pages around them
   do. Every axis label, legend entry, title, annotation box and diagram
   caption the generators write still carried the ISO `lg`, so a guide that
@@ -1550,6 +1591,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Fixed
 
+- The committed example `.report()` fiches were not checked against the code
+  that writes them, and two of them had fallen behind it. `iso1999_nipts` and
+  `iso1996_tone_audibility` still carried the shaded regions as they were
+  drawn before fills started being derived from the page they sit on: the
+  fractile band was the old 50 % composite `#ffddbb` instead of the
+  `#ffe7d2` wash, and the decisive tone's critical band was the hard-coded
+  `#eaf0f8` instead of `#c8deed`. Nothing a reader would misread, since every
+  printed value and every verdict was already right, but the two fiches the
+  documentation offers as examples of what the library prints were showing a
+  paler band than the library prints. `ebu_r128_loudness` and the two
+  `iec61260_filter` fiches carried the same kind of drift a shade smaller.
+  All five are regenerated here, together with the `iec61043_intensity`
+  fiche, which had been registered in the generator and documented in the
+  intensity guide without its PDF and preview ever being committed; the guide
+  now embeds it in both languages, like every other worked example.
+
 - The `<link rel="alternate" type="text/markdown">` in every page's head
   pointed at a file that does not exist on 147 of them. It was written for
   `<page>/index.md` on every page but the two splash pages, while the copies

Makefile

@@ -132,11 +132,31 @@ lighthouse:
 	cd site && pnpm run lighthouse
 
 # Regenerate the committed example .report() fiches under .github/reports/,
-# which the documentation links to as rendered normative-report examples.
-# Not byte-checked in CI (the embedded vector plot differs by ~1 ULP across
-# CPUs); tests/test_generate_reports.py only checks the generator still works.
+# which the documentation links to as rendered normative-report examples. CI
+# fails if this drifts (see the `reports` job in python-app.yml). The compare
+# is tolerance-aware rather than a byte diff, for the same reason the figures'
+# is: the embedded vector plot differs by ~1 ULP across CPUs. See
+# scripts/check_reports.py.
+#
+# Renders into a scratch directory and swaps it in only once the whole set is
+# written. Clearing the output first, which is how this used to work, means a
+# generator that dies halfway leaves the working tree stripped of the committed
+# examples and the maintainer reaching for `git checkout`. The clearing itself
+# has to stay, because the generator only overwrites and never deletes, so a
+# fiche that is no longer produced would survive as a stale orphan and slip
+# past the staleness check; it just belongs after a successful run rather than
+# before an attempted one. The trap covers the interrupted run the same way.
 reports:
-	$(PYTHON) scripts/generate_reports.py
+	set -e; \
+	tmp=$$(mktemp -d .github/reports.tmp.XXXXXX); \
+	old=$$(mktemp -d .github/reports.old.XXXXXX); \
+	trap 'if [ -d "$$old"/current ]; then rm -rf .github/reports; mv "$$old"/current .github/reports; fi; rm -rf "$$tmp" "$$old"' EXIT INT TERM HUP; \
+	$(FIGURE_ENV) $(PYTHON) scripts/generate_reports.py --output-dir "$$tmp"; \
+	[ -n "$$(ls -A "$$tmp")" ] || { echo "no fiche was generated" >&2; exit 1; }; \
+	mv .github/reports "$$old"/current; \
+	mv "$$tmp" .github/reports; \
+	mv "$$old"/current "$$old"/replaced; \
+	rm -rf "$$old"
 
 # Regenerate the committed, versioned numerical conformance report, then bring
 # every count quoted from it into line. The --file-header flag prepends the

docs/intensity.md

@@ -394,6 +394,11 @@ EN/IEC text (see the [errata registry](ERRATA.md)). Clause 8 combines separately
 `instrument_class_from_components(probe_class, processor_class)` returns 1
 only when both are class 1, and 2 for every other pairing.
 
+The example fiche, regenerated with `make reports`, is kept rendered in the
+repository. Click the preview to open the PDF:
+
+[![One-page instrument-class-verification fiche: a metadata header, a per-band table listing the class 1 and class 2 minima, the measured residual index, the margin and the class achieved in each one-third-octave band from 50 Hz to 6.3 kHz, the measured index drawn as a step curve over the two Table 2 masks with the 100 Hz band ringed below the class 1 minimum, the boxed Class 2 - COMPLIES (binding margin +4.20 dB) result, the microphone separation and equivalent phase mismatch, and a FAIL verdict against the required class 1](https://raw.githubusercontent.com/jmrplens/phonometry/main/.github/reports/iec61043_intensity_example.webp)](https://raw.githubusercontent.com/jmrplens/phonometry/main/.github/reports/iec61043_intensity_example.pdf)
+
 ### Reading `δpI0` as a phase error
 
 The requirement is really a phase-matching requirement in disguise. In an

requirements-reports.txt

@@ -0,0 +1,23 @@
+# Locked fiche-rendering stack for the reports CI check (the "Example report
+# fiches up to date" job in python-app.yml). Install it on top of
+# requirements-figures.txt, which pins the matplotlib/numpy half of the chain.
+#
+# A fiche goes matplotlib -> SVG -> svglib -> reportlab -> PDF -> pypdfium2 ->
+# WebP preview, and the staleness check compares the rendered page. Every link
+# after matplotlib therefore fixes what the check sees: svglib and its CSS/XML
+# parsers decide how the SVG becomes vector graphics, reportlab lays out the
+# page and embeds the fonts, and pypdfium2 rasterizes it. Cross-CPU ~1-ULP
+# coordinate drift is absorbed by the tolerance in scripts/check_reports.py, so
+# the stack does not have to byte-match the runner hardware, but a version bump
+# in any of these moves the rendered pixels well past that tolerance.
+#
+# Bump these together with a fresh `make reports` regeneration (generate the
+# fiches with exactly this stack, then commit both). Generated from the
+# maintainer's venv that produced the committed fiches.
+reportlab==5.0.0
+svglib==2.0.2
+pypdfium2==5.12.1
+lxml==6.1.1
+cssselect2==0.9.0
+tinycss2==1.5.1
+webencodings==0.5.1

scripts/check_figures.py

@@ -17,14 +17,10 @@
   must be identical, and every numeric token must agree within an absolute or
   relative tolerance. A moved element, changed label or new path fails; a
   last-bit coordinate wobble passes.
-* **Raster (WebP/PNG)** -- identical dimensions, and *both* (a) at most
-  ``RASTER_MAX_SIG_PIXELS`` pixels whose per-channel difference exceeds
-  ``RASTER_LEVEL_TOL`` and (b) a per-pixel root-mean-square difference below
-  ``RASTER_RMS_TOL``. The pixel count
-  catches a *localised* change (a moved line, a relabelled axis) that a global
-  RMS would dilute in a large image; the RMS catches a *broad* change (a
-  recoloured background) that few-but-everywhere pixels would slip past the
-  count. Cross-CPU sub-ULP coordinate drift changes neither meaningfully.
+* **Raster (WebP/PNG)** -- identical dimensions, at most
+  :data:`RASTER_TOL`\\ ``.max_sig_pixels`` meaningfully changed pixels and a
+  bounded root-mean-square difference (see
+  :class:`~generated_assets.RasterTolerance`).
 * **Anything else** -- exact byte compare.
 
 Added or removed files always fail: a new figure must be committed, and a
@@ -37,14 +33,20 @@
 
 from __future__ import annotations
 
-import io
 import re
-import subprocess
 import sys
 from pathlib import Path
 
-import numpy as np
-from PIL import Image
+_SCRIPTS = Path(__file__).resolve().parent
+if str(_SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS))
+
+from generated_assets import (
+    RasterTolerance,
+    committed_bytes,
+    raster_problem,
+    tracked_files,
+)
 
 IMG_DIR = ".github/images"
 
@@ -55,18 +57,12 @@
 SVG_REL_TOL = 1e-4
 
 # A pixel counts as "meaningfully changed" if any channel differs by more than
-# RASTER_LEVEL_TOL (0..255). Cross-CPU drift perturbs a coordinate by ~1e-6 units,
+# ``level`` (0..255). Cross-CPU drift perturbs a coordinate by ~1e-6 units,
 # i.e. a ~1e-5-pixel geometric shift, whose anti-aliasing effect rounds to at
-# most a level or two on a few edge pixels -- far below this threshold.
-RASTER_LEVEL_TOL = 12
-# Allowed number of meaningfully-changed pixels. A real edit moves a plotted
-# line or glyph, changing hundreds to thousands of edge pixels; noise changes
-# a handful at most.
-RASTER_MAX_SIG_PIXELS = 100
-# Broad-change guard: maximum root-mean-square per-channel difference. Catches
-# a change spread thinly over the whole image (e.g. a recoloured background)
-# that stays individually under RASTER_LEVEL_TOL.
-RASTER_RMS_TOL = 2.0
+# most a level or two on a few edge pixels -- far below this threshold. A real
+# edit moves a plotted line or glyph, changing hundreds to thousands of edge
+# pixels, and a restyled fill moves the whole image past the RMS bound.
+RASTER_TOL = RasterTolerance(level=12, max_sig_pixels=100, rms=2.0)
 
 # Integers and decimals, with optional sign and exponent. ``split``/``findall``
 # with this pattern partition a file into fixed text and numeric values.
@@ -84,27 +80,6 @@
 _IDREF = re.compile(rb'(\bid="|url\(#|xlink:href="#|\bhref="#)([A-Za-z_][\w.:\-]*)')
 
 
-def _committed(path: str) -> bytes | None:
-    """Return the bytes of ``path`` as committed at HEAD, or ``None``."""
-    result = subprocess.run(
-        ["git", "show", f"HEAD:{path}"],
-        capture_output=True,
-        check=False,
-    )
-    return result.stdout if result.returncode == 0 else None
-
-
-def _tracked_files() -> set[str]:
-    """Return the set of ``.github/images`` paths tracked at HEAD."""
-    result = subprocess.run(
-        ["git", "ls-tree", "-r", "--name-only", "HEAD", IMG_DIR],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return {line for line in result.stdout.splitlines() if line}
-
-
 def _canonicalize_ids(data: bytes) -> bytes:
     """Rewrite hash-derived ids/references to first-appearance placeholders."""
     mapping: dict[bytes, bytes] = {}
@@ -136,26 +111,9 @@ def _svg_within_tolerance(old: bytes, new: bytes) -> bool:
     return True
 
 
-def _raster_problem(old: bytes, new: bytes) -> str | None:
-    """Describe how two rasters differ beyond tolerance, or ``None`` if within it."""
-    a = np.asarray(Image.open(io.BytesIO(old)).convert("RGBA"), dtype=np.float64)
-    b = np.asarray(Image.open(io.BytesIO(new)).convert("RGBA"), dtype=np.float64)
-    if a.shape != b.shape:
-        return f"dimensions changed {a.shape} != {b.shape}"
-    diff = np.abs(a - b)
-    sig_pixels = int(np.count_nonzero(diff.max(axis=-1) > RASTER_LEVEL_TOL))
-    if sig_pixels > RASTER_MAX_SIG_PIXELS:
-        return (f"{sig_pixels} pixels changed by >{RASTER_LEVEL_TOL} "
-                f"(> {RASTER_MAX_SIG_PIXELS})")
-    rms = float(np.sqrt(np.mean(diff**2)))
-    if rms > RASTER_RMS_TOL:
-        return f"RMS {rms:.3f} > {RASTER_RMS_TOL}"
-    return None
-
-
 def main() -> int:
     disk = {str(p) for p in Path(IMG_DIR).rglob("*") if p.is_file()}
-    tracked = _tracked_files()
+    tracked = tracked_files(IMG_DIR)
     problems: list[str] = []
 
     for path in sorted(disk - tracked):
@@ -165,7 +123,7 @@ def main() -> int:
 
     for path in sorted(disk & tracked):
         new = Path(path).read_bytes()
-        old = _committed(path)
+        old = committed_bytes(path)
         if old is None:
             problems.append(f"cannot read committed {path}")
             continue
@@ -175,7 +133,7 @@ def main() -> int:
             if not _svg_within_tolerance(old, new):
                 problems.append(f"SVG changed beyond tolerance: {path}")
         elif path.endswith((".webp", ".png")):
-            reason = _raster_problem(old, new)
+            reason = raster_problem(old, new, RASTER_TOL)
             if reason is not None:
                 problems.append(f"raster changed beyond tolerance ({reason}): {path}")
         else: