From f94808c24fd755d31924abf192a949843919a147 Mon Sep 17 00:00:00 2001 From: Daniel Fry Date: Sun, 5 Jul 2026 14:45:32 +0100 Subject: [PATCH 1/3] feat(trust): publish the per-version fidelity matrix from CI's own reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RN × Vitest matrix has always run the mock-vs-real-RN cross-check per cell and discarded the per-cell reports. Those reports are now uploaded as artifacts (on failure too — a diverging cell's report is the forensic record) and aggregated into a published dashboard: - scripts/fidelity-matrix.mjs renders website/guide/fidelity-matrix.md from a directory of per-cell report.json files: a React Native × Vitest summary table plus a per-probe divergence table, with corpus text escaped against the VitePress hazards (pipes, bare tags, Vue interpolation). With no reports it renders the committed placeholder, so local site builds and dead-link checks never depend on matrix availability. - pages.yml gains a workflow_run trigger (matrix completed on main) and a best-effort pre-build step that downloads the latest successful matrix run's report artifacts and regenerates the page in the deploy workspace — the site refreshes with every matrix run without committing generated data. Failed matrix runs are deliberately not published: the site shows the last green state while CI is red. - crosscheck.mjs records the resolved Vitest version in the ephemeral report (the matrix's second axis), and the fidelity report page links to the matrix. The dashboard turns 'CI runs this range on every commit' from a claim into a page generated by the runs themselves. --- .github/workflows/native-rn-matrix.yml | 13 ++ .github/workflows/pages.yml | 26 ++++ packages/vitest-native/scripts/crosscheck.mjs | 10 ++ .../vitest-native/scripts/fidelity-matrix.mjs | 147 ++++++++++++++++++ .../vitest-native/scripts/fidelity-report.mjs | 1 + .../tests/fidelity-matrix.test.ts | 97 ++++++++++++ website/.vitepress/config.ts | 1 + website/guide/fidelity-matrix.md | 20 +++ website/guide/fidelity.md | 1 + 9 files changed, 316 insertions(+) create mode 100644 packages/vitest-native/scripts/fidelity-matrix.mjs create mode 100644 packages/vitest-native/tests/fidelity-matrix.test.ts create mode 100644 website/guide/fidelity-matrix.md diff --git a/.github/workflows/native-rn-matrix.yml b/.github/workflows/native-rn-matrix.yml index d7d2925..1cd5c13 100644 --- a/.github/workflows/native-rn-matrix.yml +++ b/.github/workflows/native-rn-matrix.yml @@ -103,3 +103,16 @@ jobs: - name: Cross-check (mock ≡ real RN ${{ matrix.rn }}) run: bun run crosscheck + + # The per-cell report feeds the published fidelity matrix (pages.yml pulls + # the latest successful run's artifacts and renders + # website/guide/fidelity-matrix.md at deploy time). Uploaded on failure + # too — a diverging cell's report is the forensic record. + - name: Upload cross-check report + if: always() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: crosscheck-report-rn${{ matrix.rn }}-${{ matrix.vitest }} + path: packages/vitest-native/crosscheck/.out/report.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 37b168b..3b03e6e 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -6,6 +6,12 @@ on: paths: - 'website/**' - '.github/workflows/pages.yml' + # Redeploy when the RN × Vitest matrix completes on main, so the published + # fidelity matrix reflects the latest run's per-cell cross-check reports. + workflow_run: + workflows: ['Native engine — Vitest × RN matrix'] + types: [completed] + branches: [main] workflow_dispatch: permissions: @@ -23,6 +29,8 @@ jobs: permissions: pages: write id-token: write + # Read the matrix workflow's artifacts (per-cell cross-check reports). + actions: read environment: name: github-pages @@ -36,6 +44,24 @@ jobs: - run: bun install --frozen-lockfile + # Render the fidelity matrix from the latest successful matrix run's + # per-cell cross-check reports. Best-effort: with no run or no artifacts + # (expired retention, first deploy), the committed placeholder page ships + # instead — the site build must never depend on matrix availability. + - name: Render fidelity matrix from the latest matrix run + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + run_id=$(gh run list --repo "$GITHUB_REPOSITORY" \ + --workflow native-rn-matrix.yml --branch main --status success \ + --limit 1 --json databaseId --jq '.[0].databaseId' || true) + if [ -n "$run_id" ]; then + gh run download "$run_id" --repo "$GITHUB_REPOSITORY" \ + --pattern 'crosscheck-report-*' --dir /tmp/crosscheck-reports || true + fi + node packages/vitest-native/scripts/fidelity-matrix.mjs --reports /tmp/crosscheck-reports + - name: Build website run: bun run build:website diff --git a/packages/vitest-native/scripts/crosscheck.mjs b/packages/vitest-native/scripts/crosscheck.mjs index f91cdcc..ff35c58 100644 --- a/packages/vitest-native/scripts/crosscheck.mjs +++ b/packages/vitest-native/scripts/crosscheck.mjs @@ -30,6 +30,15 @@ function resolveReactNativeVersion() { } } +function resolveVitestVersion() { + try { + const req = createRequire(path.join(root, "package.json")); + return req("vitest/package.json").version; + } catch { + return null; + } +} + function runEngine(engine) { const out = path.join(outDir, `${engine}.json`); fs.rmSync(out, { force: true }); @@ -76,6 +85,7 @@ console.log(`${matched}/${names.length} probes match between mock and real React const failureByName = new Map(failures.map((f) => [f.name, f])); const report = { reactNativeVersion: resolveReactNativeVersion(), + vitestVersion: resolveVitestVersion(), generatedAt: new Date().toISOString(), summary: { total: names.length, matching: matched }, probes: names.map((name) => { diff --git a/packages/vitest-native/scripts/fidelity-matrix.mjs b/packages/vitest-native/scripts/fidelity-matrix.mjs new file mode 100644 index 0000000..7744ed3 --- /dev/null +++ b/packages/vitest-native/scripts/fidelity-matrix.mjs @@ -0,0 +1,147 @@ +// Fidelity matrix page: aggregate the per-cell cross-check reports produced by +// the CI matrix (one report.json per RN × Vitest cell) into a published +// probes-across-versions dashboard. +// +// CI already proves mock ≡ real-RN parity per RN version on every commit — but +// it used to throw the per-cell reports away, leaving the published fidelity +// page a single-version snapshot. This script turns those reports into the +// dashboard page: +// +// node scripts/fidelity-matrix.mjs --reports # /**/report.json +// node scripts/fidelity-matrix.mjs # placeholder page +// +// The page is rendered at DEPLOY time by pages.yml from the latest successful +// matrix run's artifacts; the committed website/guide/fidelity-matrix.md is a +// placeholder that keeps local site builds (and VitePress dead-link checks) +// working and is overwritten in the deploy workspace before the build. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = path.resolve(root, "..", ".."); + +function argValue(flag) { + const i = process.argv.indexOf(flag); + return i !== -1 ? process.argv[i + 1] : undefined; +} +const reportsDir = argValue("--reports"); +const outPath = argValue("--out") ?? path.join(repoRoot, "website", "guide", "fidelity-matrix.md"); + +// Same VitePress hazards as fidelity-report.mjs: bare tags, table pipes, and +// Vue interpolation in free text break the site build. Probe names/reasons are +// corpus data, so escape anything rendered outside a code span. +const cell = (s) => + String(s ?? "") + .replace(/\|/g, "\\|") + .replace(//g, ">") + .replace(/\{\{/g, "{{") + .replace(/\}\}/g, "}}"); + +/** Recursively collect report.json files under dir. */ +function collectReports(dir) { + const out = []; + if (!dir || !fs.existsSync(dir)) return out; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...collectReports(p)); + else if (entry.name === "report.json") out.push(p); + } + return out; +} + +/** + * The artifact directory name carries the matrix axes + * (crosscheck-report-rn-); the report body carries the + * RESOLVED versions. Prefer the body, fall back to the directory name. + */ +function describeCell(reportPath, report) { + const dirName = path.basename(path.dirname(reportPath)); + const m = /^crosscheck-report-rn(.+?)-(locked|latest-supported)$/.exec(dirName); + return { + rn: report.reactNativeVersion ?? m?.[1] ?? "unknown", + vitest: report.vitestVersion ?? (m?.[2] === "latest-supported" ? "latest 4.x" : "locked"), + flavor: m?.[2] ?? "locked", + }; +} + +const HEADER = ` +# Fidelity matrix + +The [behavioral cross-check](/guide/fidelity) runs the same probe corpus under +the mock engine **and** under real React Native — and CI runs it against +**every supported React Native version** in the +[Vitest × RN matrix](https://github.com/danfry1/vitest-native/actions/workflows/native-rn-matrix.yml). +This page is generated from that matrix's own reports, so every number below +was produced by CI, not written by hand. +`; + +const reportFiles = collectReports(reportsDir); +let body; + +if (reportFiles.length === 0) { + body = ` +::: info No matrix data in this build +This page is populated from the latest CI matrix run when the site is deployed. +A local or matrix-less build shows this placeholder. The single-version +[Fidelity Report](/guide/fidelity) is always available and drift-gated in CI. +::: +`; +} else { + const cells = reportFiles + .map((p) => { + const report = JSON.parse(fs.readFileSync(p, "utf8")); + return { ...describeCell(p, report), report }; + }) + .sort( + (a, b) => + a.rn.localeCompare(b.rn, undefined, { numeric: true }) || a.flavor.localeCompare(b.flavor), + ); + + const allGreen = cells.every((c) => c.report.summary.matching === c.report.summary.total); + const rows = cells + .map(({ rn, vitest, report }) => { + const { matching, total } = report.summary; + const status = matching === total ? "✅" : "❌"; + const when = (report.generatedAt ?? "").slice(0, 10); + return `| ${cell(rn)} | ${cell(vitest)} | ${status} ${matching}/${total} | ${cell(when)} |`; + }) + .join("\n"); + + const divergences = cells.flatMap(({ rn, vitest, report }) => + report.probes + .filter((p) => !p.match) + .map( + (p) => + `| \`${p.name}\` | ${cell(rn)} | ${cell(vitest)} | ${cell(p.reason ?? "diverged")} |`, + ), + ); + + body = ` +${allGreen ? `**Every probe matches on every gated React Native version.**` : `**Divergences detected — see the table below.**`} + +| React Native | Vitest | Probes matching | Generated | +| --- | --- | --- | --- | +${rows} + +## Divergences + +${ + divergences.length + ? `| Probe | React Native | Vitest | Reason |\n| --- | --- | --- | --- |\n${divergences.join("\n")}` + : "_None. Every probe matched in every cell of the latest matrix run._" +} +`; +} + +fs.mkdirSync(path.dirname(outPath), { recursive: true }); +fs.writeFileSync(outPath, `${HEADER}${body}`); +console.log( + `✓ fidelity matrix page rendered (${reportFiles.length} report${reportFiles.length === 1 ? "" : "s"}) → ${path.relative(repoRoot, outPath)}`, +); diff --git a/packages/vitest-native/scripts/fidelity-report.mjs b/packages/vitest-native/scripts/fidelity-report.mjs index 4fc0232..8cbbeaa 100644 --- a/packages/vitest-native/scripts/fidelity-report.mjs +++ b/packages/vitest-native/scripts/fidelity-report.mjs @@ -136,6 +136,7 @@ generated from the corpus itself, so the numbers below are exactly what ships. - **${summary.matching} / ${summary.total} probes** match between the mock engine and real React Native. - ${ciLine} +- Per-version results straight from the CI matrix: [Fidelity Matrix](/guide/fidelity-matrix). - Reproduce it yourself: \`bun run crosscheck\`. The \`native\` engine needs no cross-check — it *is* real React Native. diff --git a/packages/vitest-native/tests/fidelity-matrix.test.ts b/packages/vitest-native/tests/fidelity-matrix.test.ts new file mode 100644 index 0000000..05f019e --- /dev/null +++ b/packages/vitest-native/tests/fidelity-matrix.test.ts @@ -0,0 +1,97 @@ +/** + * The fidelity-matrix renderer: aggregates per-cell CI cross-check reports + * into the published probes-across-versions page. + */ +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import fs from "node:fs"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.resolve(HERE, "../scripts/fidelity-matrix.mjs"); + +function render(args: string[]): { out: string; page: string } { + const outFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-out-")), "page.md"); + const out = execFileSync(process.execPath, [SCRIPT, ...args, "--out", outFile], { + encoding: "utf8", + }); + return { out, page: fs.readFileSync(outFile, "utf8") }; +} + +function writeReport(dir: string, artifactName: string, report: Record): void { + const d = path.join(dir, artifactName); + fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync(path.join(d, "report.json"), JSON.stringify(report)); +} + +describe("fidelity-matrix renderer", () => { + it("renders a placeholder page when no reports exist", () => { + const { page } = render([]); + expect(page).toContain("GENERATED FILE"); + expect(page).toContain("No matrix data in this build"); + expect(page).toContain("/guide/fidelity"); + }); + + it("aggregates per-cell reports into a sorted matrix with divergences escaped", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.85-locked", { + reactNativeVersion: "0.85.2", + vitestVersion: "4.1.8", + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 75, matching: 75 }, + probes: [{ name: "a11y-role", match: true }], + }); + writeReport(reports, "crosscheck-report-rn0.81-latest-supported", { + reactNativeVersion: "0.81.5", + vitestVersion: "4.2.1", + generatedAt: "2026-07-04T09:05:00.000Z", + summary: { total: 75, matching: 74 }, + probes: [ + { name: "a11y-role", match: true }, + // VitePress hazards in the reason: pipes break table cells, bare tags + // and {{ }} break the Vue compile — all must be escaped. + { name: "press-event", match: false, reason: "mock got a|b {{ nope }}" }, + ], + }); + + const { page } = render(["--reports", reports]); + // Sorted by RN version: 0.81 row before 0.85. + expect(page.indexOf("0.81.5")).toBeLessThan(page.indexOf("0.85.2")); + // Per-cell summary rows with pass state. + expect(page).toContain("| 0.85.2 | 4.1.8 | ✅ 75/75 | 2026-07-04 |"); + expect(page).toContain("| 0.81.5 | 4.2.1 | ❌ 74/75 | 2026-07-04 |"); + // Divergence table present, with hazards escaped. + expect(page).toContain("`press-event`"); + expect(page).toContain("mock <Text> got a\\|b {{ nope }}"); + expect(page).not.toContain(""); + // Not the all-green banner. + expect(page).toContain("Divergences detected"); + }); + + it("declares all-green when every cell matches fully", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.86-locked", { + reactNativeVersion: "0.86.0", + vitestVersion: "4.1.8", + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 75, matching: 75 }, + probes: [{ name: "a11y-role", match: true }], + }); + const { page } = render(["--reports", reports]); + expect(page).toContain("Every probe matches on every gated React Native version."); + expect(page).toContain("_None. Every probe matched in every cell of the latest matrix run._"); + }); + + it("falls back to artifact-name axes when the report lacks resolved versions", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.83-latest-supported", { + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 10, matching: 10 }, + probes: [], + }); + const { page } = render(["--reports", reports]); + expect(page).toContain("| 0.83 | latest 4.x | ✅ 10/10 |"); + }); +}); diff --git a/website/.vitepress/config.ts b/website/.vitepress/config.ts index 936af98..7f705af 100644 --- a/website/.vitepress/config.ts +++ b/website/.vitepress/config.ts @@ -81,6 +81,7 @@ export default defineConfig({ items: [ { text: 'API Coverage', link: '/api/coverage' }, { text: 'Fidelity Report', link: '/guide/fidelity' }, + { text: 'Fidelity Matrix', link: '/guide/fidelity-matrix' }, { text: 'Comparison with Jest', link: '/guide/comparison' }, { text: 'Troubleshooting', link: '/guide/troubleshooting' }, ], diff --git a/website/guide/fidelity-matrix.md b/website/guide/fidelity-matrix.md new file mode 100644 index 0000000..a52fee4 --- /dev/null +++ b/website/guide/fidelity-matrix.md @@ -0,0 +1,20 @@ + +# Fidelity matrix + +The [behavioral cross-check](/guide/fidelity) runs the same probe corpus under +the mock engine **and** under real React Native — and CI runs it against +**every supported React Native version** in the +[Vitest × RN matrix](https://github.com/danfry1/vitest-native/actions/workflows/native-rn-matrix.yml). +This page is generated from that matrix's own reports, so every number below +was produced by CI, not written by hand. + +::: info No matrix data in this build +This page is populated from the latest CI matrix run when the site is deployed. +A local or matrix-less build shows this placeholder. The single-version +[Fidelity Report](/guide/fidelity) is always available and drift-gated in CI. +::: diff --git a/website/guide/fidelity.md b/website/guide/fidelity.md index 1b4c60e..93c9c7b 100644 --- a/website/guide/fidelity.md +++ b/website/guide/fidelity.md @@ -15,6 +15,7 @@ generated from the corpus itself, so the numbers below are exactly what ships. - **75 / 75 probes** match between the mock engine and real React Native. - CI runs the same corpus across **React Native 0.81–0.85** on every commit. +- Per-version results straight from the CI matrix: [Fidelity Matrix](/guide/fidelity-matrix). - Reproduce it yourself: `bun run crosscheck`. The `native` engine needs no cross-check — it *is* real React Native. From cbff506ea5efe35086158c60915bd681dfcd4f10 Mon Sep 17 00:00:00 2001 From: Daniel Fry Date: Sun, 5 Jul 2026 14:50:14 +0100 Subject: [PATCH 2/3] security+review: exclude pull_request runs from artifact selection, escape all report fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-PR review findings: - BLOCKER: the artifact-selection query filtered by head branch name, and the matrix workflow runs on pull_request — a fork PR opened from a branch named 'main' (the fork default) would qualify as 'the latest successful main run', and pull_request runs execute the PR's own workflow file, so its artifacts are attacker-controlled and would be rendered into the published site. The query now selects only non-pull_request runs (push/schedule/workflow_dispatch execute trusted main-branch code), and the deploy job skips workflow_run triggers fired by pull_request matrix runs outright. - Probe names were the one report field rendered unescaped (inside a code span a backtick breaks out of); every report field now renders through the shared escape, which additionally neutralizes backticks. - Documented that the divergence table is unreachable from CI-selected data (crosscheck exits non-zero on divergence, so failed runs are never selected) — it renders for manually-supplied report directories. - fidelity:check now also gates the committed fidelity-matrix.md as exactly the placeholder render, so real data can never be committed unnoticed. Tests: hostile probe name (backtick/pipe/HTML injection), corrupt artifact JSON fails the render step (placeholder ships), --check pass/fail. --- .github/workflows/pages.yml | 16 ++++++- packages/vitest-native/package.json | 2 +- .../vitest-native/scripts/fidelity-matrix.mjs | 47 +++++++++++++++---- .../tests/fidelity-matrix.test.ts | 40 +++++++++++++++- 4 files changed, 93 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 3b03e6e..55b200f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -23,6 +23,11 @@ concurrency: jobs: deploy: + # workflow_run's `branches` filter matches the triggering run's HEAD branch, + # so a fork PR's matrix run (fork default branch is `main`) would fire this + # too. The artifact-selection step below never trusts pull_request runs, so + # such a trigger could only waste a deploy — skip it outright. + if: github.event_name != 'workflow_run' || github.event.workflow_run.event != 'pull_request' runs-on: ubuntu-latest timeout-minutes: 10 @@ -48,6 +53,14 @@ jobs: # per-cell cross-check reports. Best-effort: with no run or no artifacts # (expired retention, first deploy), the committed placeholder page ships # instead — the site build must never depend on matrix availability. + # + # SECURITY: only non-pull_request runs are eligible. `--branch main` + # matches the run's HEAD branch name, and the matrix runs on + # pull_request — a fork PR opened from a branch named `main` (the fork + # default) would otherwise qualify, and pull_request runs execute the + # PR's OWN workflow file, so its artifacts are attacker-controlled. + # push/schedule/workflow_dispatch runs execute only trusted main-branch + # code. - name: Render fidelity matrix from the latest matrix run continue-on-error: true env: @@ -55,7 +68,8 @@ jobs: run: | run_id=$(gh run list --repo "$GITHUB_REPOSITORY" \ --workflow native-rn-matrix.yml --branch main --status success \ - --limit 1 --json databaseId --jq '.[0].databaseId' || true) + --limit 10 --json databaseId,event \ + --jq 'first(.[] | select(.event != "pull_request")).databaseId' || true) if [ -n "$run_id" ]; then gh run download "$run_id" --repo "$GITHUB_REPOSITORY" \ --pattern 'crosscheck-report-*' --dir /tmp/crosscheck-reports || true diff --git a/packages/vitest-native/package.json b/packages/vitest-native/package.json index ac5455e..ae0d85f 100644 --- a/packages/vitest-native/package.json +++ b/packages/vitest-native/package.json @@ -198,7 +198,7 @@ "prepublishOnly": "bun run build && bun run test && bun run test:native && bun run test:native:navigation && bun run test:native:android && bun run test:native:hot && bun run test:native:hot:isolation && bun run test:native:hot:soak && bun run fidelity:check && bun run test:consumers && bun run lint && bun run typecheck", "crosscheck": "node scripts/crosscheck.mjs", "fidelity:report": "node scripts/fidelity-report.mjs", - "fidelity:check": "node scripts/fidelity-report.mjs --check", + "fidelity:check": "node scripts/fidelity-report.mjs --check && node scripts/fidelity-matrix.mjs --check", "validate:hot-parity": "node validation/idiomatic/scale/run-compare.mjs" }, "dependencies": { diff --git a/packages/vitest-native/scripts/fidelity-matrix.mjs b/packages/vitest-native/scripts/fidelity-matrix.mjs index 7744ed3..34dcc33 100644 --- a/packages/vitest-native/scripts/fidelity-matrix.mjs +++ b/packages/vitest-native/scripts/fidelity-matrix.mjs @@ -28,14 +28,18 @@ function argValue(flag) { const reportsDir = argValue("--reports"); const outPath = argValue("--out") ?? path.join(repoRoot, "website", "guide", "fidelity-matrix.md"); -// Same VitePress hazards as fidelity-report.mjs: bare tags, table pipes, and -// Vue interpolation in free text break the site build. Probe names/reasons are -// corpus data, so escape anything rendered outside a code span. +// Same VitePress hazards as fidelity-report.mjs — bare tags, table pipes, and +// Vue interpolation in free text break the site build — plus backticks, which +// would open a code span and turn the rest of the row into live markup. Every +// report field is treated as untrusted data (the reports are CI artifacts), +// so ALL of them render through this escape, code-span styling deliberately +// forgone. const cell = (s) => String(s ?? "") .replace(/\|/g, "\\|") .replace(//g, ">") + .replace(/`/g, "`") .replace(/\{\{/g, "{{") .replace(/\}\}/g, "}}"); @@ -114,12 +118,17 @@ A local or matrix-less build shows this placeholder. The single-version }) .join("\n"); + // NOTE: crosscheck.mjs exits non-zero on any divergence, so CI's + // artifact-selection (latest SUCCESSFUL run) never feeds this branch — the + // deployed page shows the last all-green run while CI is red. The rendering + // exists for manually-supplied report directories and forensics; do not + // remove it as dead code. const divergences = cells.flatMap(({ rn, vitest, report }) => report.probes .filter((p) => !p.match) .map( (p) => - `| \`${p.name}\` | ${cell(rn)} | ${cell(vitest)} | ${cell(p.reason ?? "diverged")} |`, + `| ${cell(p.name)} | ${cell(rn)} | ${cell(vitest)} | ${cell(p.reason ?? "diverged")} |`, ), ); @@ -140,8 +149,28 @@ ${ `; } -fs.mkdirSync(path.dirname(outPath), { recursive: true }); -fs.writeFileSync(outPath, `${HEADER}${body}`); -console.log( - `✓ fidelity matrix page rendered (${reportFiles.length} report${reportFiles.length === 1 ? "" : "s"}) → ${path.relative(repoRoot, outPath)}`, -); +const content = `${HEADER}${body}`; + +if (process.argv.includes("--check")) { + // Drift gate for the COMMITTED page: it must be exactly the placeholder + // render — real matrix data belongs only in the deploy workspace, never in + // the repository. + if (reportFiles.length > 0) { + console.error("✗ --check validates the committed placeholder; run it without --reports."); + process.exit(1); + } + const current = fs.existsSync(outPath) ? fs.readFileSync(outPath, "utf8") : null; + if (current !== content) { + console.error( + `✗ ${path.relative(repoRoot, outPath)} is not the placeholder render — regenerate with \`node scripts/fidelity-matrix.mjs\` and commit.`, + ); + process.exit(1); + } + console.log("✓ committed fidelity-matrix placeholder is up to date"); +} else { + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, content); + console.log( + `✓ fidelity matrix page rendered (${reportFiles.length} report${reportFiles.length === 1 ? "" : "s"}) → ${path.relative(repoRoot, outPath)}`, + ); +} diff --git a/packages/vitest-native/tests/fidelity-matrix.test.ts b/packages/vitest-native/tests/fidelity-matrix.test.ts index 05f019e..3d6b283 100644 --- a/packages/vitest-native/tests/fidelity-matrix.test.ts +++ b/packages/vitest-native/tests/fidelity-matrix.test.ts @@ -63,13 +63,51 @@ describe("fidelity-matrix renderer", () => { expect(page).toContain("| 0.85.2 | 4.1.8 | ✅ 75/75 | 2026-07-04 |"); expect(page).toContain("| 0.81.5 | 4.2.1 | ❌ 74/75 | 2026-07-04 |"); // Divergence table present, with hazards escaped. - expect(page).toContain("`press-event`"); + expect(page).toContain("| press-event |"); expect(page).toContain("mock <Text> got a\\|b {{ nope }}"); expect(page).not.toContain(""); // Not the all-green banner. expect(page).toContain("Divergences detected"); }); + it("escapes hostile probe names (reports are CI artifacts, treated as untrusted)", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.84-locked", { + reactNativeVersion: "0.84.0", + vitestVersion: "4.1.8", + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 2, matching: 1 }, + probes: [ + { name: "ok-probe", match: true }, + // A backtick would open a code span; a pipe breaks the row; a tag + // injects HTML into the built site. + { name: "evil` | ", match: false, reason: "r" }, + ], + }); + const { page } = render(["--reports", reports]); + expect(page).not.toContain(" { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + const d = path.join(reports, "crosscheck-report-rn0.82-locked"); + fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync(path.join(d, "report.json"), "{ not json"); + expect(() => render(["--reports", reports])).toThrow(); + }); + + it("--check passes on the committed placeholder and fails on drift", () => { + // The committed page must be exactly the placeholder render. + execFileSync(process.execPath, [SCRIPT, "--check"], { encoding: "utf8" }); + // A page with data (or any divergent content) must fail the gate. + const outFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-check-")), "p.md"); + fs.writeFileSync(outFile, "not the placeholder"); + expect(() => + execFileSync(process.execPath, [SCRIPT, "--check", "--out", outFile], { encoding: "utf8" }), + ).toThrow(); + }); + it("declares all-green when every cell matches fully", () => { const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); writeReport(reports, "crosscheck-report-rn0.86-locked", { From 779daa01824074add626c7f9db1abd9ab3ed10bc Mon Sep 17 00:00:00 2001 From: Daniel Fry Date: Sun, 5 Jul 2026 21:44:54 +0100 Subject: [PATCH 3/3] security: escape backslashes first in the markdown cell escapers CodeQL (js/incomplete-sanitization): escaping pipes/tags without first escaping backslashes lets an input backslash re-arm the escaped character ('a\' + '|' rendered a literal backslash followed by a LIVE pipe, breaking the table row). Both generated-page escapers (fidelity-matrix.mjs and fidelity-report.mjs, which had the same pre-existing gap flagged on main) now escape backslashes first; the escaping test carries a backslash-payload case. Regenerated artifacts are byte-identical for the current corpus (no probe text contains backslashes). --- packages/vitest-native/scripts/fidelity-matrix.mjs | 3 +++ packages/vitest-native/scripts/fidelity-report.mjs | 3 +++ packages/vitest-native/tests/fidelity-matrix.test.ts | 7 ++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/vitest-native/scripts/fidelity-matrix.mjs b/packages/vitest-native/scripts/fidelity-matrix.mjs index 34dcc33..70e752a 100644 --- a/packages/vitest-native/scripts/fidelity-matrix.mjs +++ b/packages/vitest-native/scripts/fidelity-matrix.mjs @@ -36,6 +36,9 @@ const outPath = argValue("--out") ?? path.join(repoRoot, "website", "guide", "fi // forgone. const cell = (s) => String(s ?? "") + // Backslashes first — escaping them after the fact would re-arm the very + // characters the later replacements neutralize (CodeQL js/incomplete-sanitization). + .replace(/\\/g, "\\\\") .replace(/\|/g, "\\|") .replace(//g, ">") diff --git a/packages/vitest-native/scripts/fidelity-report.mjs b/packages/vitest-native/scripts/fidelity-report.mjs index 8cbbeaa..f4bb131 100644 --- a/packages/vitest-native/scripts/fidelity-report.mjs +++ b/packages/vitest-native/scripts/fidelity-report.mjs @@ -105,6 +105,9 @@ const knownDiffs = fs.existsSync(knownDiffsPath) // any cell rendered as free text (not inside a code span). const cell = (s) => String(s ?? "") + // Backslashes first — escaping them after the fact would re-arm the very + // characters the later replacements neutralize (CodeQL js/incomplete-sanitization). + .replace(/\\/g, "\\\\") .replace(/\|/g, "\\|") .replace(//g, ">") diff --git a/packages/vitest-native/tests/fidelity-matrix.test.ts b/packages/vitest-native/tests/fidelity-matrix.test.ts index 3d6b283..2e54d17 100644 --- a/packages/vitest-native/tests/fidelity-matrix.test.ts +++ b/packages/vitest-native/tests/fidelity-matrix.test.ts @@ -51,8 +51,9 @@ describe("fidelity-matrix renderer", () => { probes: [ { name: "a11y-role", match: true }, // VitePress hazards in the reason: pipes break table cells, bare tags - // and {{ }} break the Vue compile — all must be escaped. - { name: "press-event", match: false, reason: "mock got a|b {{ nope }}" }, + // and {{ }} break the Vue compile, and a backslash would re-arm the + // escaped pipe — all must be escaped. + { name: "press-event", match: false, reason: "mock got a|b \\| {{ nope }}" }, ], }); @@ -64,7 +65,7 @@ describe("fidelity-matrix renderer", () => { expect(page).toContain("| 0.81.5 | 4.2.1 | ❌ 74/75 | 2026-07-04 |"); // Divergence table present, with hazards escaped. expect(page).toContain("| press-event |"); - expect(page).toContain("mock <Text> got a\\|b {{ nope }}"); + expect(page).toContain("mock <Text> got a\\|b \\\\\\| {{ nope }}"); expect(page).not.toContain(""); // Not the all-green banner. expect(page).toContain("Divergences detected");