diff --git a/.husky/pre-commit b/.husky/pre-commit index 7c1a7fad..bd56732a 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -18,9 +18,18 @@ else fi # fallow — block the commit on new findings in the changes you're staging. -# --health-baseline grandfathers the same known-complex functions CI does -# (.github/workflows/fallow.yml), so editing a file that merely CONTAINS a -# pre-existing god-function doesn't false-RED; genuinely new complexity (not in -# the baseline) still blocks. Keeps local === CI. +# Keep both commands aligned with .github/workflows/fallow.yml: audit catches +# dead code/duplication and the dedicated production health pass enforces the +# same complexity threshold that gates pull requests. The staged diff keeps +# unrelated working-tree edits out of the line-level verdict. echo "› fallow" -npx fallow audit --base HEAD --health-baseline .fallow/health-baseline.json +FALLOW_STAGED_DIFF="$(mktemp)" +trap 'rm -f "$FALLOW_STAGED_DIFF"' EXIT HUP INT TERM +git diff --cached --unified=0 -- . > "$FALLOW_STAGED_DIFF" +npx --no-install fallow audit --base HEAD --health-baseline .fallow/health-baseline.json +npx --no-install fallow health \ + --production \ + --baseline .fallow/health-baseline.json \ + --changed-since HEAD \ + --diff-file "$FALLOW_STAGED_DIFF" \ + --fail-on-issues diff --git a/CHANGELOG.md b/CHANGELOG.md index b7d1d149..a81c4943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +### Fixed + +- **Report-only correspondence can no longer erase reviewable certification + evidence and render a false “all surfaces identical” result.** If unique path + pairing collapses every presentation finding while the raw differ still has + reviewable DOM/style/state evidence, `reportConsistency` now fails closed as + `presentation_collapsed_while_raw_reviewable`. The report explains the + contradiction and the Action maps it to `CERTIFICATION_FAILED`, which visual + approval cannot clear. + ## [4.7.5] - 2026-08-06 ### Fixed diff --git a/README.md b/README.md index f7f5d587..7e52f4c8 100644 --- a/README.md +++ b/README.md @@ -1087,7 +1087,7 @@ Non-visual and framework-injected elements (``/``/`<script>`/`<styl - `styleproof-init` — scaffold the gate: the capture spec (inventory guard on; Next.js repos get route discovery + the coverage guard, others a crawl-by-default spec), a dedicated `playwright.styleproof.config.ts` (production-build `webServer`, parallel capture), `.gitignore` cache entries, the restore-first report workflow, the approval workflow, and the restore-or-publish pre-push hook. One command. The CI hot path restores exact-SHA maps and runs no browser; a compatible base hit plus head miss captures only the head; a base miss captures the pair; every fallback is published for reuse. Generated commands follow the repo's lockfile (`bun.lock`/`bun.lockb`, `pnpm-lock.yaml`, `yarn.lock`, or npm by default), respect pnpm/Corepack version pins, and detect Vite/Next production preview commands instead of assuming every repo has `start`. Generated files carrying StyleProof's ownership marker are **machine-owned** thin wrappers over packaged commands: after upgrading styleproof, `styleproof-init --check` reports whether they drifted from the release's templates (exit 1 — wire it into CI), and `styleproof-init --upgrade` refreshes them in place without touching your spec, playwright config, or a repository-owned Husky hook. Use the explicit `--hook` command only when you intend to replace that hook. - `styleproof-map` — capture the current commit's computed-style map through Playwright. By default it writes `.styleproof/maps/current`, keeps screenshots for reports, writes a manifest, and uploads to `styleproof-maps` outside CI when the working tree was clean and a git remote exists. Pass `--crawl-base-url` plus repeated `--crawl-route` to run `styleproof-variants` before capture, `--no-upload`, `--restore --sha <commit>`, `--spec`, `--dir`, `--base-dir`, `--no-screenshots`, or repeated `--dirty-allow <path>` (a tracked file a dev tool rewrites on every run — e.g. `next dev` regenerating a `tsconfig.json` — that must not mark the capture dirty) for custom flows. - `styleproof-diff` — the certify gate. With no args, it restores cached maps for the current commit and inferred base (`GITHUB_BASE_REF`, `branch.<name>.gh-merge-base`, `gh pr view`, then main/master fallbacks); `styleproof-diff main` / `styleproof-diff master` pins the base; `styleproof-diff <beforeDir> <afterDir>` keeps the manual two-directory form for CI fallback captures. Exits `0` certified (identical); `1` on a reviewable diff — computed-style/DOM/state differences, and equally an unacknowledged inventory removal, an unacknowledged failing data endpoint under an armed `dataResidue: 'gate'`, an incomplete coverage registry, or an unproven-determinism capture; `2` on a usage/capture error (including a **manifest-less side** — since **v4**, a two-directory compare where a side ships maps but no `styleproof-manifest.json` is refused loudly, naming the bare side(s), because the same-environment guard can't be enforced without one; re-capture with current StyleProof; **and** a **missing map** — a bundle that claims to exist yet holds zero captures, i.e. a `styleproof-manifest.json` present with no maps, on either side, or a head capture that produced nothing; refused loudly rather than mislabelled as all-new — **and** the no-args case where the cached base map can't be restored at all: no map-store remote, no cached bundle, nothing to compare. A "nothing was compared" outcome always exits `2`, never a soft `0` that would read as certified; the error names the two ways forward — run in CI where the base is restorable, or use the two-directory form); `3` when only new surfaces are present — surfaces captured only on the **head** side (a surface present only on the **base** side is a **removed** surface, a reviewable change: exit `1`) — (no baseline for _those_ surfaces to diff against — new surfaces against an existing baseline, or a base dir with no maps at all (and hence no manifest), meaning no baseline was ever captured: the first-adoption review path; approval policy decides whether to gate). A clean run prints `0 changed surfaces across N captured surface(s)`, and `--json` includes `compared`. The human output **groups the same way the report does**: surfaces that changed identically collapse into one finding (with the per-surface count on its header), longhands fold into shorthands, and size/position-derived longhands fold behind a `(+N derived longhands)` count — so one real change reads as one entry, not dozens of raw lines. A change that rode the shared frame every view draws (a persistent nav/header/footer) is promoted to a "🧱 Global chrome change" callout up top. `--json` stays the complete, unchanged machine contract — every surface and every raw longhand — regardless of the human grouping. -- `styleproof-report` — render the diff to a Markdown report with before/after crops. With no args, it reports cached maps for the current commit against the inferred base; `styleproof-report main` / `styleproof-report master` pins the base; `styleproof-report <beforeDir> <afterDir> --out <dir>` keeps the manual two-directory form. Add `--include-content` for the opt-in, advisory content section (see above). Shares the same comparison truth as `styleproof-diff` (`reviewableCounts` / `reportConsistency` in `report.json`): raw-only derived reflow noise never claims “all surfaces identical” and exits `1` fail-closed instead of green-with-no-crops. +- `styleproof-report` — render the diff to a Markdown report with before/after crops. With no args, it reports cached maps for the current commit against the inferred base; `styleproof-report main` / `styleproof-report master` pins the base; `styleproof-report <beforeDir> <afterDir> --out <dir>` keeps the manual two-directory form. Add `--include-content` for the opt-in, advisory content section (see above). Shares the same comparison truth as `styleproof-diff` (`reviewableCounts` / `reportConsistency` in `report.json`): raw-only derived reflow noise never claims “all surfaces identical,” and neither does report-only path correspondence when it collapses every presentation finding while raw reviewable evidence remains. Both contradictions fail closed instead of producing a green report with no crops. - `styleproof-capture` — one-shot capture of any URL (no spec): `styleproof-capture <url> --key <name> --out <dir>`, with `--widths` (omit to auto-detect `@media` bands), `--wait <selector>`, `--ignore <selector>`, `--no-screenshots`, and the crawler flags (`--crawl`, `--setup <file>`, `--require-full-coverage` → exit 4 on residue, `--until-covered`, `--workers <n>`, `--no-data-states`) described in [Match a design pixel-for-pixel](#match-a-design-pixel-for-pixel). - `styleproof-variants` — crawl a running app for one-step state variants and write `styleproof.variants.generated.json`. Pass `--base-url`, repeat `--route`, and use `--strict` when unresolved skipped/live candidates should fail automation. - `styleproof-prepush` — the canonical pre-push flow, packaged: reads git's refspecs from stdin, captures the pushed commit only when its tip is the checked-out tree, skips docs-only pushes, restores an already-published exact-SHA map or captures and publishes once, then runs the advisory diff. The hook `styleproof-init` writes is a two-line shim that execs the installed local binary directly, so the rules update with each release instead of drifting in a copied hook file and a missing install fails instead of falling through to a package-registry download — refresh an old hook with `styleproof-init --hook`. diff --git a/action.yml b/action.yml index 7b3ab104..e6570282 100644 --- a/action.yml +++ b/action.yml @@ -145,6 +145,34 @@ runs: # Both need human review in approval mode. Every defined verdict publishes a receipt. if [ "$code" -eq 1 ] || [ "$code" -eq 3 ]; then echo "changed=true" >> "$GITHUB_OUTPUT"; else echo "changed=false" >> "$GITHUB_OUTPUT"; fi + # Generate the human report before trust classification. Report-only structural + # correspondence can discover a presentation/certification contradiction that the + # raw differ cannot know about; merge that one coherence verdict back into the + # machine payload so approval can never clear hidden raw review evidence. + - id: report + shell: bash + run: | + set -euo pipefail + rm -rf styleproof-report + set +e + node "$GITHUB_ACTION_PATH/bin/styleproof-report.mjs" "${{ inputs.baseline-dir }}" "${{ inputs.fresh-dir }}" --out styleproof-report + report_exit_code=$? + set -e + if [ "$report_exit_code" -ne 0 ] && [ "$report_exit_code" -ne 1 ]; then + echo "styleproof-report exited with code $report_exit_code — no trustworthy report was generated" >&2 + exit "$report_exit_code" + fi + node <<'NODE' + const fs = require('fs'); + const diff = JSON.parse(fs.readFileSync('styleproof-diff.json', 'utf8')); + const generated = JSON.parse(fs.readFileSync('styleproof-report/report.json', 'utf8')); + if (!generated.reportConsistency || typeof generated.reportConsistency.ok !== 'boolean') { + throw new Error('styleproof-report/report.json has no valid reportConsistency verdict'); + } + diff.reportConsistency = generated.reportConsistency; + fs.writeFileSync('styleproof-diff.json', `${JSON.stringify(diff, null, 2)}\n`); + NODE + # One machine-readable verdict for every defined diff. Consumers should not # re-parse styleproof-diff.json or guess whether approval can clear the run. - id: verdict @@ -227,16 +255,6 @@ runs: GH_TOKEN: ${{ inputs.github-token }} run: | set -euo pipefail - rm -rf styleproof-report - set +e - node "$GITHUB_ACTION_PATH/bin/styleproof-report.mjs" "${{ inputs.baseline-dir }}" "${{ inputs.fresh-dir }}" --out styleproof-report - report_exit_code=$? - set -e - if [ "$report_exit_code" -ne 0 ] && [ "$report_exit_code" -ne 1 ]; then - echo "styleproof-report exited with code $report_exit_code — no trustworthy report was generated" >&2 - exit "$report_exit_code" - fi - PR='${{ steps.context.outputs.pr-number }}' REPORT_PATH="pr-${PR:-${{ github.run_id }}}" REPORT_SHA='${{ steps.context.outputs.head-sha }}' diff --git a/bin/styleproof-report.mjs b/bin/styleproof-report.mjs index 22ad2ce7..f468d4ec 100755 --- a/bin/styleproof-report.mjs +++ b/bin/styleproof-report.mjs @@ -183,16 +183,15 @@ try { } const newNote = result.newSurfaces ? ` (+${result.newSurfaces} new surface(s) with no baseline)` : ''; -if (result.comparison?.rawOnlyNoReviewable) { - console.log( - `⚠ report consistency: raw certification deltas exist but no reviewable crops — not a clean no-change (fail closed)`, - ); +const consistencyFailed = result.reportConsistency?.ok === false; +if (consistencyFailed) { + console.log(`⚠ report consistency: ${result.reportConsistency.reason} — not a clean no-change (fail closed)`); } console.log( result.changedSurfaces === 0 ? result.newSurfaces === 0 - ? result.comparison?.rawOnlyNoReviewable - ? '⚠ no reviewable changes — consistency failure written (raw-only derived longhands)' + ? consistencyFailed + ? '⚠ no presentation changes — report consistency failure written' : '✓ no changes — empty report written' : `ℹ ${result.newSurfaces} new surface(s) with no baseline — report written for review` : `✗ ${result.changedSurfaces} changed surface(s), ${result.totalFindings} finding(s)${newNote}`, @@ -201,8 +200,6 @@ console.log(`report: ${result.reportMdPath}`); if (includeContent && result.contentChanges > 0) { console.log(`📝 ${result.contentChanges} advisory content change(s) — does not affect the exit code`); } -// Exit 1 when there is anything to review OR a raw-only consistency failure (never -// exit 0 for "identical" when the certification differ saw deltas). -process.exit( - result.changedSurfaces === 0 && result.newSurfaces === 0 && !result.comparison?.rawOnlyNoReviewable ? 0 : 1, -); +// Exit 1 when there is anything to review OR any report-consistency failure (never +// exit 0 for "identical" when certification evidence was hidden by presentation). +process.exit(result.changedSurfaces === 0 && result.newSurfaces === 0 && !consistencyFailed ? 0 : 1); diff --git a/package-lock.json b/package-lock.json index 92af1a2b..c961ff97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "@types/node": "^26.0.0", "@types/pngjs": "^6.0.5", "eslint": "^10.5.0", - "fallow": "^3.2.0", + "fallow": "3.14.0", "globals": "^17.6.0", "husky": "^9.1.7", "prettier": "^3.4.2", @@ -512,9 +512,9 @@ } }, "node_modules/@fallow-cli/darwin-arm64": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/darwin-arm64/-/darwin-arm64-3.4.2.tgz", - "integrity": "sha512-6DFe/JzATrNrnZyIKXNcxl1/nfQm5yqg3kVomww6aoEL+v6/Jm955J8pdSBsc69G59EGoAcsvxcz3uj66kp+rw==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/darwin-arm64/-/darwin-arm64-3.14.0.tgz", + "integrity": "sha512-seaix3OqADcq90PkPOOPrN8Jl7QZDGjFziHA85Ikp+lGSHVjJ2IMyGh/QKLhvi9Q6Ha3Y//F0+PFFGNd0Gi4mw==", "cpu": [ "arm64" ], @@ -526,9 +526,9 @@ ] }, "node_modules/@fallow-cli/darwin-x64": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/darwin-x64/-/darwin-x64-3.4.2.tgz", - "integrity": "sha512-6sILz44YD73x+tXl/w664L4T+KjAUNT9gIj19OqfLkyuRAPtpAJon/V4DU9pzMPPPYF13P1CMDy4QsTkHRhEZA==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/darwin-x64/-/darwin-x64-3.14.0.tgz", + "integrity": "sha512-qgYZHHTFobuJCtON7aVe/tXCUjiF/9nKT4tNLVfUPH4Qe9KwieO4rAuknDOnxkjzsWPO4MfHyCbcVdslf4JMnw==", "cpu": [ "x64" ], @@ -540,9 +540,9 @@ ] }, "node_modules/@fallow-cli/linux-arm64-gnu": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/linux-arm64-gnu/-/linux-arm64-gnu-3.4.2.tgz", - "integrity": "sha512-sMIMJcOU/WvtZuF7M3GyTOUmS1tFBy2KGsx5SDowcv8uCXZbbd4YRZr1yVsC+qpimzOmtLt0zTKs3G3R/wR7RA==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/linux-arm64-gnu/-/linux-arm64-gnu-3.14.0.tgz", + "integrity": "sha512-FwmIG391lMViBXh1cAvOcwQxbI2sLjEYkugFauVbmntjuJt6xVxQXAxjybdFXSL/jjfFnbAzPKy9oHlhq3sA6A==", "cpu": [ "arm64" ], @@ -554,9 +554,9 @@ ] }, "node_modules/@fallow-cli/linux-arm64-musl": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/linux-arm64-musl/-/linux-arm64-musl-3.4.2.tgz", - "integrity": "sha512-38GbTbpVbwbplCeFgKLx8uIYMlGtqSSfKk+ZUyXX4E6JoXCLyFzz11AJjNRg+QSP/fobD7jk0Z++gILvljPczw==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/linux-arm64-musl/-/linux-arm64-musl-3.14.0.tgz", + "integrity": "sha512-x4R9aNT2kIoSHRIZ5s2MZ+swyb3uutV6SSYCwWhPSE5UI/oCEYWNWiYS7uKMLM0cTuMPVcsm2qMTd4iYA3ACLw==", "cpu": [ "arm64" ], @@ -568,9 +568,9 @@ ] }, "node_modules/@fallow-cli/linux-x64-gnu": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/linux-x64-gnu/-/linux-x64-gnu-3.4.2.tgz", - "integrity": "sha512-d7IGOZbn9tf2EcICPt+XLQXWaPfzo62gHqlQyAXLimay8PvaVDKXGeNjMJBXJCceKAU+zeBdIdbyea9pFOuSWg==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/linux-x64-gnu/-/linux-x64-gnu-3.14.0.tgz", + "integrity": "sha512-8kBxy9FkwKjiH1C19T0nSkmbtldwVyWLQWajIqvyHE0x4rurHhve7RQrwEDPzNMFu7OgYCMU1nXr/AMIJ6Kg9w==", "cpu": [ "x64" ], @@ -582,9 +582,9 @@ ] }, "node_modules/@fallow-cli/linux-x64-musl": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/linux-x64-musl/-/linux-x64-musl-3.4.2.tgz", - "integrity": "sha512-98TG142inap3rNA0DdsLJ/VPXAeQ+yXZjXQtGQ2R3JTTGP/eIDBpPv0CKgciusKRo54xeweYkHtSfVNHo3Werg==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/linux-x64-musl/-/linux-x64-musl-3.14.0.tgz", + "integrity": "sha512-9TAdXSpYvk9WkKy2GR8T/PHbq7QCTO/SYUmiZ+EAJUMzXBvSfnpj8cCGQpeAD000Nb+o7rL94lIAPJMAqMKO3Q==", "cpu": [ "x64" ], @@ -596,9 +596,9 @@ ] }, "node_modules/@fallow-cli/win32-arm64-msvc": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/win32-arm64-msvc/-/win32-arm64-msvc-3.4.2.tgz", - "integrity": "sha512-5U7fmUdgC2S1Exsv4IbG9VMuzOIS2E9TS594qOeiYfVTx12bqFnqoxC9+gX3ThkVQdekkXaJeGZqJC3StGJ/kQ==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/win32-arm64-msvc/-/win32-arm64-msvc-3.14.0.tgz", + "integrity": "sha512-xc495cracyJL+0UKwcqNhofIz2K5XiJbb+YFhxrbSrrbm+EVDnQKpCOzzOJYbm7RPxF6XOxJpATn7ciBy7Rqrw==", "cpu": [ "arm64" ], @@ -610,9 +610,9 @@ ] }, "node_modules/@fallow-cli/win32-x64-msvc": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/@fallow-cli/win32-x64-msvc/-/win32-x64-msvc-3.4.2.tgz", - "integrity": "sha512-qfUhAmvLIdfPyGSO3F9SIJOWTYRAqboApIe73gpR/CFIFsq3CPT3ovEMlFzw8YosaC7zwymxTOq5Vslx38Gd9A==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/@fallow-cli/win32-x64-msvc/-/win32-x64-msvc-3.14.0.tgz", + "integrity": "sha512-ihyCZ5GMoYGnzlPP6b5pZBBcGZhDHDW2u1YEGkZEuf3pbW6E45U7Hhf213cE2CCheScXOirK6AKWnLM/Qj4c2A==", "cpu": [ "x64" ], @@ -825,6 +825,346 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/acorn": { "version": "8.17.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", @@ -1301,9 +1641,9 @@ } }, "node_modules/fallow": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/fallow/-/fallow-3.4.2.tgz", - "integrity": "sha512-ufxwwtszht6bODr3rTZfD3FV94qIkSGIykKrtogWxayGAcmCnjTN1iH+58NjKsbPKOTKeNR1sFrOnIP8/Odz+g==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/fallow/-/fallow-3.14.0.tgz", + "integrity": "sha512-hse+sChmWkrffLiZDPGdNSyiLybVEhgoiYCIYgqLwdYhfTCO+f2PS1g+gMhGAJdLlYpZ75tFwRoMBFmZjN3Q6g==", "dev": true, "license": "MIT", "dependencies": { @@ -1318,14 +1658,68 @@ "node": ">=22" }, "optionalDependencies": { - "@fallow-cli/darwin-arm64": "3.4.2", - "@fallow-cli/darwin-x64": "3.4.2", - "@fallow-cli/linux-arm64-gnu": "3.4.2", - "@fallow-cli/linux-arm64-musl": "3.4.2", - "@fallow-cli/linux-x64-gnu": "3.4.2", - "@fallow-cli/linux-x64-musl": "3.4.2", - "@fallow-cli/win32-arm64-msvc": "3.4.2", - "@fallow-cli/win32-x64-msvc": "3.4.2" + "@fallow-cli/darwin-arm64": "3.14.0", + "@fallow-cli/darwin-x64": "3.14.0", + "@fallow-cli/linux-arm64-gnu": "3.14.0", + "@fallow-cli/linux-arm64-musl": "3.14.0", + "@fallow-cli/linux-x64-gnu": "3.14.0", + "@fallow-cli/linux-x64-musl": "3.14.0", + "@fallow-cli/win32-arm64-msvc": "3.14.0", + "@fallow-cli/win32-x64-msvc": "3.14.0", + "fallow-type-aware": "3.14.0" + } + }, + "node_modules/fallow-type-aware": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/fallow-type-aware/-/fallow-type-aware-3.14.0.tgz", + "integrity": "sha512-D8+bNELjmoVV2dKpmHplQ1YzuheHltmZKaOVJ//hx7h8vFNyp5H3hcJV8rR26ofBU51z9K5eyA7FpgP3hebjwg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "typescript": "7.0.2" + }, + "bin": { + "fallow-type-aware": "fallow-type-aware.mjs" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fallow-type-aware/node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" } }, "node_modules/fast-deep-equal": { diff --git a/package.json b/package.json index 3c38ac23..eb996488 100644 --- a/package.json +++ b/package.json @@ -93,7 +93,7 @@ "@types/node": "^26.0.0", "@types/pngjs": "^6.0.5", "eslint": "^10.5.0", - "fallow": "^3.2.0", + "fallow": "3.14.0", "globals": "^17.6.0", "husky": "^9.1.7", "prettier": "^3.4.2", diff --git a/src/report.ts b/src/report.ts index 0a071654..c69d6bad 100644 --- a/src/report.ts +++ b/src/report.ts @@ -154,6 +154,8 @@ export type ReportResult = { * computed-style deltas exist — callers must fail closed, never approve. */ comparison: ComparisonTruth; + /** Presentation-vs-certification coherence. Any false value must fail closed. */ + reportConsistency: ReportConsistency; reportMdPath: string; reportJsonPath: string; }; @@ -1377,14 +1379,40 @@ function changedSurfaceSummaryLines( ]; } +function reportConsistencyFailureSummaryLines( + reportConsistency: ReportConsistency, + rawCounts: DiffCounts | undefined, + baselineSurfaceFailures: SurfaceCaptureFailure[], +): string[] | undefined { + if (reportConsistency.ok || !rawCounts) return undefined; + + const explanation = + reportConsistency.reason === 'raw_only_no_reviewable' + ? 'every delta is a derived/reflow longhand the visual report strips — **no reviewable crops or change sections**.' + : 'report-only path correspondence collapsed every presentation finding — **no reviewable crops or change sections remain**.'; + const remediation = + reportConsistency.reason === 'raw_only_no_reviewable' + ? '_This is **not** a clean no-change and **not** a visual-approval gate. Fail closed (`CERTIFICATION_FAILED`): fix the reflow source, or re-run with `--include-layout-noise` to inspect the raw longhands._' + : '_This is **not** a clean no-change and cannot be approved visually. Fail closed (`CERTIFICATION_FAILED`): inspect the raw path churn or tighten the correspondence signal before trusting this comparison._'; + const md = [ + `⚠ **Report consistency failure:** the certification differ found **${rawCounts.dom} DOM**, **${rawCounts.style} computed-style**, and **${rawCounts.state} state** difference(s), but ${explanation}`, + '', + remediation, + ]; + if (baselineSurfaceFailures.length > 0) { + md.push('', ...baselineFailureSummaryLines(baselineSurfaceFailures)); + } + return md; +} + function summaryLines(args: { changeGroups: ChangeGroup[]; missing: PreparedSurface[]; shown: DiffCounts; changedScope: { bases: number; variants: number }; contentCount: number; - /** Raw-only derived noise: must not claim "identical". */ - rawOnlyNoReviewable?: boolean; + /** Any raw-vs-presentation contradiction: must not claim "identical". */ + reportConsistency: ReportConsistency; rawCounts?: DiffCounts; baselineSurfaceFailures: SurfaceCaptureFailure[]; }): string[] { @@ -1394,7 +1422,7 @@ function summaryLines(args: { shown, changedScope, contentCount, - rawOnlyNoReviewable, + reportConsistency, rawCounts, baselineSurfaceFailures, } = args; @@ -1408,17 +1436,8 @@ function summaryLines(args: { surfaceMissingMatchesBaselineFailure(p.sd.surface, baselineSurfaceFailures), ); if (changeGroups.length === 0 && missing.length === 0) { - if (rawOnlyNoReviewable && rawCounts) { - const md = [ - `⚠ **Report consistency failure:** the certification differ found **${rawCounts.dom} DOM**, **${rawCounts.style} computed-style**, and **${rawCounts.state} state** difference(s), but every delta is a derived/reflow longhand the visual report strips — **no reviewable crops or change sections**.`, - '', - '_This is **not** a clean no-change and **not** a visual-approval gate. Fail closed (`CERTIFICATION_FAILED`): fix the reflow source, or re-run with `--include-layout-noise` to inspect the raw longhands._', - ]; - if (baselineSurfaceFailures.length > 0) { - md.push('', ...baselineFailureSummaryLines(baselineSurfaceFailures)); - } - return md; - } + const failureSummary = reportConsistencyFailureSummaryLines(reportConsistency, rawCounts, baselineSurfaceFailures); + if (failureSummary) return failureSummary; if (baselineSurfaceFailures.length === 0) { return [ contentCount > 0 @@ -1446,7 +1465,7 @@ function reportHeadline(args: { volatileCount: number; liveCandidateLabels: string[]; contentCount: number; - rawOnlyNoReviewable?: boolean; + reportConsistency: ReportConsistency; rawCounts?: DiffCounts; baselineSurfaceFailures: SurfaceCaptureFailure[]; }): string[] { @@ -1458,7 +1477,7 @@ function reportHeadline(args: { volatileCount, liveCandidateLabels, contentCount, - rawOnlyNoReviewable, + reportConsistency, rawCounts, baselineSurfaceFailures, } = args; @@ -1468,7 +1487,7 @@ function reportHeadline(args: { shown, changedScope, contentCount, - rawOnlyNoReviewable, + reportConsistency, rawCounts, baselineSurfaceFailures, }); @@ -1866,11 +1885,31 @@ function prepareReportSurfaces( .filter((p) => p.sd.missing || p.findings.length > 0); } +type ReportConsistency = + | { ok: true; reason: 'aligned' } + | { + ok: false; + reason: 'raw_only_no_reviewable' | 'presentation_collapsed_while_raw_reviewable'; + }; + +/** + * The presentation may simplify raw evidence, but it may never erase every + * reviewable finding and then claim the surfaces are identical. + */ +function assessReportConsistency(comparison: ComparisonTruth, hasPresentationEvidence: boolean): ReportConsistency { + if (comparison.rawOnlyNoReviewable) return { ok: false, reason: 'raw_only_no_reviewable' }; + if (comparison.hasReviewableEvidence && !hasPresentationEvidence) { + return { ok: false, reason: 'presentation_collapsed_while_raw_reviewable' }; + } + return { ok: true, reason: 'aligned' }; +} + function writeReportArtifacts( outDir: string, md: string[], shown: DiffCounts, comparison: ComparisonTruth, + reportConsistency: ReportConsistency, surfacesJson: Array<Record<string, unknown>>, ): { reportMdPath: string; reportJsonPath: string } { const reportMdPath = path.join(outDir, 'report.md'); @@ -1883,9 +1922,7 @@ function writeReportArtifacts( counts: shown, rawCounts: comparison.rawCounts, reviewableCounts: comparison.reviewableCounts, - reportConsistency: comparison.rawOnlyNoReviewable - ? { ok: false, reason: 'raw_only_no_reviewable' } - : { ok: true, reason: 'aligned' }, + reportConsistency, surfaces: surfacesJson, }, null, @@ -1950,6 +1987,7 @@ export function generateStyleMapReport(opts: ReportOptions): ReportResult { const changedScope = countChangedSurfaceScope(changeGroups, surfaceKeyOf); const baselineSurfaceFailures = readMapManifest(beforeDir)?.surfaceCaptureFailures ?? []; const comparison = comparisonForReport(rawComparison, includeNoise, prepared.length - missing.length); + const reportConsistency = assessReportConsistency(comparison, changeGroups.length > 0 || missing.length > 0); const md: string[] = []; const json: Array<Record<string, unknown>> = []; @@ -1986,7 +2024,7 @@ export function generateStyleMapReport(opts: ReportOptions): ReportResult { volatileCount, liveCandidateLabels, contentCount: contentSection.count, - rawOnlyNoReviewable: comparison.rawOnlyNoReviewable, + reportConsistency, rawCounts: comparison.rawCounts, baselineSurfaceFailures, }), @@ -2047,13 +2085,14 @@ export function generateStyleMapReport(opts: ReportOptions): ReportResult { } md.push(...contentSection.md); - const { reportMdPath, reportJsonPath } = writeReportArtifacts(outDir, md, shown, comparison, json); + const { reportMdPath, reportJsonPath } = writeReportArtifacts(outDir, md, shown, comparison, reportConsistency, json); return { changedSurfaces: prepared.length - missing.length, newSurfaces: missing.length, totalFindings, contentChanges: contentSection.count, comparison, + reportConsistency, reportMdPath, reportJsonPath, }; diff --git a/test/action.test.mjs b/test/action.test.mjs index b0a856c9..559e8058 100644 --- a/test/action.test.mjs +++ b/test/action.test.mjs @@ -24,17 +24,19 @@ test('composite action builds its checkout before running local bins', () => { }); test('composite action publishes a durable no-change report on a clean first run', () => { + const reportStep = extractActionStep('- id: report', '\\n\\s{4}#|\\n\\s{4}- id:'); const publishStep = extractActionStep('- id: publish', '\\n\\s{4}- name: Upsert PR comment'); const commentStep = extractActionStep('- name: Upsert PR comment', '\\n\\s{4}#|\\n\\s{4}- name:'); + assert.ok(reportStep, 'action.yml should include a local report generation step'); assert.ok(publishStep, 'action.yml should include a report publish step'); assert.ok(commentStep, 'action.yml should include a PR comment step'); - assert.doesNotMatch(publishStep[0], /if: steps\.diff\.outputs/); - assert.match(publishStep[0], /rm -rf styleproof-report/); - assert.match(publishStep[0], /styleproof-report\.mjs/); - assert.doesNotMatch(publishStep[0], /styleproof-report\.mjs[^\n]*\|\| true/); - assert.match(publishStep[0], /report_exit_code=\$\?/); - assert.match(publishStep[0], /"\$report_exit_code" -ne 0.*"\$report_exit_code" -ne 1/); + assert.doesNotMatch(reportStep[0], /if: steps\.diff\.outputs/); + assert.match(reportStep[0], /rm -rf styleproof-report/); + assert.match(reportStep[0], /styleproof-report\.mjs/); + assert.doesNotMatch(reportStep[0], /styleproof-report\.mjs[^\n]*\|\| true/); + assert.match(reportStep[0], /report_exit_code=\$\?/); + assert.match(reportStep[0], /"\$report_exit_code" -ne 0.*"\$report_exit_code" -ne 1/); // The run receipt is embedded by the API publisher before upload. assert.match(publishStep[0], /styleproof-publish-report\.mjs/); assert.match( @@ -250,6 +252,30 @@ test('composite action maps raw-only report inconsistency to CERTIFICATION_FAILE assert.match(commentStep[0], /report\/diff consistency|reflow source/i); }); +test('composite action classifies report-time correspondence collapse before approval or publication', () => { + const report = actionYml.match(/- id: report[\s\S]*?(?=\n\s{4}- id:|\n\s{4}- name:|\n\s{4}#)/); + assert.ok(report, 'action.yml should generate the report before classifying trust'); + assert.match(report[0], /styleproof-report\.mjs/); + assert.match(report[0], /styleproof-report\/report\.json/); + assert.match(report[0], /diff\.reportConsistency\s*=\s*generated\.reportConsistency/); + assert.match(report[0], /writeFileSync\('styleproof-diff\.json'/); + + const reportIndex = actionYml.indexOf('- id: report'); + const verdictIndex = actionYml.indexOf('- id: verdict'); + const gateIndex = actionYml.indexOf('- id: gate'); + const publishIndex = actionYml.indexOf('- id: publish'); + assert.ok( + reportIndex > 0 && verdictIndex > reportIndex, + 'report consistency must exist before verdict classification', + ); + assert.ok(gateIndex > verdictIndex, 'approval lookup must consume the final trust verdict ordering'); + assert.ok(publishIndex > gateIndex, 'network publication follows local report generation and classification'); + + const publish = actionYml.match(/- id: publish[\s\S]*?(?=\n\s{4}- id:|\n\s{4}- name:|\n\s{4}#)/); + assert.ok(publish); + assert.doesNotMatch(publish[0], /styleproof-report\.mjs/, 'publication must not regenerate a second report'); +}); + test('composite action blocks unapproved changes by default (opt out with "blocking": false)', () => { // The policy default flipped in v4: absent/blank config → blocking ON, so the config // step emits 'true' unless the file explicitly sets "blocking": false. diff --git a/test/cli.test.mjs b/test/cli.test.mjs index 6001fcb9..37261f5e 100644 --- a/test/cli.test.mjs +++ b/test/cli.test.mjs @@ -62,6 +62,52 @@ function identicalPair({ bare = false } = {}) { return { root, A, B }; } +function correspondenceCollapsedPair() { + const root = mkTmp(); + const A = path.join(root, 'a'); + const B = path.join(root, 'b'); + const button = { + tag: 'button', + rect: [10, 10, 80, 24], + ownTextLength: 3, + style: { color: 'rgb(0, 0, 0)' }, + }; + const label = { + tag: 'span', + rect: [10, 50, 80, 20], + ownTextLength: 4, + style: { color: 'rgb(0, 0, 0)' }, + }; + const shell = { tag: 'div', rect: [0, 0, 200, 100], ownTextLength: 0, style: {} }; + writeCapture( + A, + 'home@400', + makeMap({ + elements: { + 'body > div:nth-child(1)': shell, + 'body > div:nth-child(1) > button:nth-child(1)': button, + 'body > div:nth-child(1) > span:nth-child(2)': label, + }, + }), + null, + ); + writeCapture( + B, + 'home@400', + makeMap({ + elements: { + 'body > div:nth-child(1)': shell, + 'body > div:nth-child(1) > span:nth-child(1)': label, + 'body > div:nth-child(1) > button:nth-child(2)': button, + }, + }), + null, + ); + writeManifest(A, 'base-sha', 'same-env-key'); + writeManifest(B, 'head-sha', 'same-env-key'); + return { root, A, B }; +} + // ---------------------------------------------------------------- styleproof-map test('styleproof-map runs Playwright with local cache defaults', () => { @@ -836,6 +882,21 @@ test('report CLI exits 1 and writes a report when surfaces changed', () => { rmTmp(root); }); +test('report CLI exits 1 when correspondence collapses every presentation finding', () => { + const { root, A, B } = correspondenceCollapsedPair(); + const out = path.join(root, 'out'); + const r = run(REPORT, [A, B, '--out', out]); + assert.equal(r.status, 1, r.stderr); + assert.match(r.stdout, /presentation_collapsed_while_raw_reviewable/); + assert.doesNotMatch(r.stdout, /✓ no changes/); + const json = JSON.parse(fs.readFileSync(path.join(out, 'report.json'), 'utf8')); + assert.deepEqual(json.reportConsistency, { + ok: false, + reason: 'presentation_collapsed_while_raw_reviewable', + }); + rmTmp(root); +}); + test('report CLI exits 2 on a manifest-less pair (v4 refuses)', () => { const { root, A, B } = differingPair({ bare: true }); const out = path.join(root, 'out'); diff --git a/test/fallow-hook.test.mjs b/test/fallow-hook.test.mjs new file mode 100644 index 00000000..e63ed213 --- /dev/null +++ b/test/fallow-hook.test.mjs @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; + +const hook = readFileSync(new URL('../.husky/pre-commit', import.meta.url), 'utf8'); +const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const lock = JSON.parse(readFileSync(new URL('../package-lock.json', import.meta.url), 'utf8')); + +test('pre-commit runs the same audit and production complexity gates as CI', () => { + const syntax = spawnSync('sh', ['-n', '.husky/pre-commit'], { encoding: 'utf8' }); + assert.equal(syntax.status, 0, syntax.stderr); + + assert.match(hook, /git diff --cached --unified=0/); + assert.match(hook, /npx --no-install fallow audit --base HEAD --health-baseline/); + assert.match(hook, /npx --no-install fallow health/); + assert.match(hook, /--production/); + assert.match(hook, /--baseline \.fallow\/health-baseline\.json/); + assert.match(hook, /--changed-since HEAD/); + assert.match(hook, /--diff-file "\$FALLOW_STAGED_DIFF"/); + assert.match(hook, /--fail-on-issues/); +}); + +test('local Fallow is exactly pinned so pre-commit and the Action resolve the same release', () => { + assert.equal(pkg.devDependencies.fallow, '3.14.0'); + assert.equal(lock.packages[''].devDependencies.fallow, '3.14.0'); + assert.equal(lock.packages['node_modules/fallow'].version, '3.14.0'); +}); diff --git a/test/report.test.mjs b/test/report.test.mjs index 09263e66..2d269b97 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -1021,6 +1021,106 @@ test('correspondence: ambiguous duplicates stay unpaired; raw/reviewable gate co rmTmp(root); }); +test('correspondence: sibling reorder never reports identical while raw reviewable evidence remains', () => { + const button = { + tag: 'button', + cls: 'action', + rect: [10, 10, 80, 24], + ownTextLength: 3, + style: { color: 'rgb(0, 0, 0)' }, + }; + const label = { + tag: 'span', + cls: 'label', + rect: [10, 50, 80, 20], + ownTextLength: 4, + style: { color: 'rgb(0, 0, 0)' }, + }; + const shell = { tag: 'div', cls: 'shell', rect: [0, 0, 200, 100], ownTextLength: 0, style: {} }; + const before = makeMap({ + elements: { + body: { tag: 'body', rect: [0, 0, 400, 200], ownTextLength: 0, style: {} }, + 'body > div:nth-child(1)': shell, + 'body > div:nth-child(1) > button:nth-child(1)': button, + 'body > div:nth-child(1) > span:nth-child(2)': label, + }, + }); + const after = makeMap({ + elements: { + body: { tag: 'body', rect: [0, 0, 400, 200], ownTextLength: 0, style: {} }, + 'body > div:nth-child(1)': shell, + 'body > div:nth-child(1) > span:nth-child(1)': label, + 'body > div:nth-child(1) > button:nth-child(2)': button, + }, + }); + const { beforeDir, afterDir, outDir, root } = pairFixture({ + surface: 's@400', + before, + after, + beforePng: solidPng(400, 200), + afterPng: solidPng(400, 200), + }); + const res = generateStyleMapReport({ beforeDir, afterDir, outDir }); + const md = fs.readFileSync(res.reportMdPath, 'utf8'); + const json = JSON.parse(fs.readFileSync(res.reportJsonPath, 'utf8')); + assert.deepEqual(json.counts, { dom: 0, style: 0, state: 0 }); + assert.equal(res.comparison.hasReviewableEvidence, true); + assert.deepEqual(json.reportConsistency, { + ok: false, + reason: 'presentation_collapsed_while_raw_reviewable', + }); + assert.deepEqual(res.reportConsistency, json.reportConsistency); + assert.match(md, /Report consistency failure/); + assert.match(md, /path correspondence collapsed every presentation finding/); + assert.doesNotMatch(md, /✓ All surfaces identical/); + rmTmp(root); +}); + +test('correspondence: nesting bijection never reports identical while raw reviewable evidence remains', () => { + const outer = { tag: 'div', cls: 'outer', rect: [0, 0, 200, 100], ownTextLength: 0, style: {} }; + const inner = { + tag: 'span', + cls: 'inner', + rect: [10, 10, 100, 20], + ownTextLength: 4, + style: { color: 'rgb(0, 0, 0)' }, + }; + const before = makeMap({ + elements: { + body: { tag: 'body', rect: [0, 0, 400, 200], ownTextLength: 0, style: {} }, + 'body > div:nth-child(1)': outer, + 'body > div:nth-child(1) > span:nth-child(1)': inner, + }, + }); + const after = makeMap({ + elements: { + body: { tag: 'body', rect: [0, 0, 400, 200], ownTextLength: 0, style: {} }, + 'body > span:nth-child(1)': inner, + 'body > span:nth-child(1) > div:nth-child(1)': outer, + }, + }); + const { beforeDir, afterDir, outDir, root } = pairFixture({ + surface: 's@400', + before, + after, + beforePng: solidPng(400, 200), + afterPng: solidPng(400, 200), + }); + const res = generateStyleMapReport({ beforeDir, afterDir, outDir }); + const md = fs.readFileSync(res.reportMdPath, 'utf8'); + const json = JSON.parse(fs.readFileSync(res.reportJsonPath, 'utf8')); + assert.deepEqual(json.counts, { dom: 0, style: 0, state: 0 }); + assert.equal(res.comparison.hasReviewableEvidence, true); + assert.deepEqual(json.reportConsistency, { + ok: false, + reason: 'presentation_collapsed_while_raw_reviewable', + }); + assert.deepEqual(res.reportConsistency, json.reportConsistency); + assert.match(md, /Report consistency failure/); + assert.doesNotMatch(md, /✓ All surfaces identical/); + rmTmp(root); +}); + // Regression, seen in a downstream report: a gradient diff rendered as the same // "representative" rgba in BOTH cells — the real change (a dropped `0px` stop) // was invisible. Long values must excerpt around the differing substring.