diff --git a/.github/workflows/native-rn-matrix.yml b/.github/workflows/native-rn-matrix.yml index d7d2925..1cd5c13 100644 --- a/.github/workflows/native-rn-matrix.yml +++ b/.github/workflows/native-rn-matrix.yml @@ -103,3 +103,16 @@ jobs: - name: Cross-check (mock ≡ real RN ${{ matrix.rn }}) run: bun run crosscheck + + # The per-cell report feeds the published fidelity matrix (pages.yml pulls + # the latest successful run's artifacts and renders + # website/guide/fidelity-matrix.md at deploy time). Uploaded on failure + # too — a diverging cell's report is the forensic record. + - name: Upload cross-check report + if: always() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: crosscheck-report-rn${{ matrix.rn }}-${{ matrix.vitest }} + path: packages/vitest-native/crosscheck/.out/report.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 37b168b..55b200f 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -6,6 +6,12 @@ on: paths: - 'website/**' - '.github/workflows/pages.yml' + # Redeploy when the RN × Vitest matrix completes on main, so the published + # fidelity matrix reflects the latest run's per-cell cross-check reports. + workflow_run: + workflows: ['Native engine — Vitest × RN matrix'] + types: [completed] + branches: [main] workflow_dispatch: permissions: @@ -17,12 +23,19 @@ concurrency: jobs: deploy: + # workflow_run's `branches` filter matches the triggering run's HEAD branch, + # so a fork PR's matrix run (fork default branch is `main`) would fire this + # too. The artifact-selection step below never trusts pull_request runs, so + # such a trigger could only waste a deploy — skip it outright. + if: github.event_name != 'workflow_run' || github.event.workflow_run.event != 'pull_request' runs-on: ubuntu-latest timeout-minutes: 10 permissions: pages: write id-token: write + # Read the matrix workflow's artifacts (per-cell cross-check reports). + actions: read environment: name: github-pages @@ -36,6 +49,33 @@ jobs: - run: bun install --frozen-lockfile + # Render the fidelity matrix from the latest successful matrix run's + # per-cell cross-check reports. Best-effort: with no run or no artifacts + # (expired retention, first deploy), the committed placeholder page ships + # instead — the site build must never depend on matrix availability. + # + # SECURITY: only non-pull_request runs are eligible. `--branch main` + # matches the run's HEAD branch name, and the matrix runs on + # pull_request — a fork PR opened from a branch named `main` (the fork + # default) would otherwise qualify, and pull_request runs execute the + # PR's OWN workflow file, so its artifacts are attacker-controlled. + # push/schedule/workflow_dispatch runs execute only trusted main-branch + # code. + - name: Render fidelity matrix from the latest matrix run + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + run_id=$(gh run list --repo "$GITHUB_REPOSITORY" \ + --workflow native-rn-matrix.yml --branch main --status success \ + --limit 10 --json databaseId,event \ + --jq 'first(.[] | select(.event != "pull_request")).databaseId' || true) + if [ -n "$run_id" ]; then + gh run download "$run_id" --repo "$GITHUB_REPOSITORY" \ + --pattern 'crosscheck-report-*' --dir /tmp/crosscheck-reports || true + fi + node packages/vitest-native/scripts/fidelity-matrix.mjs --reports /tmp/crosscheck-reports + - name: Build website run: bun run build:website diff --git a/packages/vitest-native/package.json b/packages/vitest-native/package.json index ac5455e..ae0d85f 100644 --- a/packages/vitest-native/package.json +++ b/packages/vitest-native/package.json @@ -198,7 +198,7 @@ "prepublishOnly": "bun run build && bun run test && bun run test:native && bun run test:native:navigation && bun run test:native:android && bun run test:native:hot && bun run test:native:hot:isolation && bun run test:native:hot:soak && bun run fidelity:check && bun run test:consumers && bun run lint && bun run typecheck", "crosscheck": "node scripts/crosscheck.mjs", "fidelity:report": "node scripts/fidelity-report.mjs", - "fidelity:check": "node scripts/fidelity-report.mjs --check", + "fidelity:check": "node scripts/fidelity-report.mjs --check && node scripts/fidelity-matrix.mjs --check", "validate:hot-parity": "node validation/idiomatic/scale/run-compare.mjs" }, "dependencies": { diff --git a/packages/vitest-native/scripts/crosscheck.mjs b/packages/vitest-native/scripts/crosscheck.mjs index f91cdcc..ff35c58 100644 --- a/packages/vitest-native/scripts/crosscheck.mjs +++ b/packages/vitest-native/scripts/crosscheck.mjs @@ -30,6 +30,15 @@ function resolveReactNativeVersion() { } } +function resolveVitestVersion() { + try { + const req = createRequire(path.join(root, "package.json")); + return req("vitest/package.json").version; + } catch { + return null; + } +} + function runEngine(engine) { const out = path.join(outDir, `${engine}.json`); fs.rmSync(out, { force: true }); @@ -76,6 +85,7 @@ console.log(`${matched}/${names.length} probes match between mock and real React const failureByName = new Map(failures.map((f) => [f.name, f])); const report = { reactNativeVersion: resolveReactNativeVersion(), + vitestVersion: resolveVitestVersion(), generatedAt: new Date().toISOString(), summary: { total: names.length, matching: matched }, probes: names.map((name) => { diff --git a/packages/vitest-native/scripts/fidelity-matrix.mjs b/packages/vitest-native/scripts/fidelity-matrix.mjs new file mode 100644 index 0000000..70e752a --- /dev/null +++ b/packages/vitest-native/scripts/fidelity-matrix.mjs @@ -0,0 +1,179 @@ +// Fidelity matrix page: aggregate the per-cell cross-check reports produced by +// the CI matrix (one report.json per RN × Vitest cell) into a published +// probes-across-versions dashboard. +// +// CI already proves mock ≡ real-RN parity per RN version on every commit — but +// it used to throw the per-cell reports away, leaving the published fidelity +// page a single-version snapshot. This script turns those reports into the +// dashboard page: +// +// node scripts/fidelity-matrix.mjs --reports # /**/report.json +// node scripts/fidelity-matrix.mjs # placeholder page +// +// The page is rendered at DEPLOY time by pages.yml from the latest successful +// matrix run's artifacts; the committed website/guide/fidelity-matrix.md is a +// placeholder that keeps local site builds (and VitePress dead-link checks) +// working and is overwritten in the deploy workspace before the build. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = path.resolve(root, "..", ".."); + +function argValue(flag) { + const i = process.argv.indexOf(flag); + return i !== -1 ? process.argv[i + 1] : undefined; +} +const reportsDir = argValue("--reports"); +const outPath = argValue("--out") ?? path.join(repoRoot, "website", "guide", "fidelity-matrix.md"); + +// Same VitePress hazards as fidelity-report.mjs — bare tags, table pipes, and +// Vue interpolation in free text break the site build — plus backticks, which +// would open a code span and turn the rest of the row into live markup. Every +// report field is treated as untrusted data (the reports are CI artifacts), +// so ALL of them render through this escape, code-span styling deliberately +// forgone. +const cell = (s) => + String(s ?? "") + // Backslashes first — escaping them after the fact would re-arm the very + // characters the later replacements neutralize (CodeQL js/incomplete-sanitization). + .replace(/\\/g, "\\\\") + .replace(/\|/g, "\\|") + .replace(//g, ">") + .replace(/`/g, "`") + .replace(/\{\{/g, "{{") + .replace(/\}\}/g, "}}"); + +/** Recursively collect report.json files under dir. */ +function collectReports(dir) { + const out = []; + if (!dir || !fs.existsSync(dir)) return out; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...collectReports(p)); + else if (entry.name === "report.json") out.push(p); + } + return out; +} + +/** + * The artifact directory name carries the matrix axes + * (crosscheck-report-rn-); the report body carries the + * RESOLVED versions. Prefer the body, fall back to the directory name. + */ +function describeCell(reportPath, report) { + const dirName = path.basename(path.dirname(reportPath)); + const m = /^crosscheck-report-rn(.+?)-(locked|latest-supported)$/.exec(dirName); + return { + rn: report.reactNativeVersion ?? m?.[1] ?? "unknown", + vitest: report.vitestVersion ?? (m?.[2] === "latest-supported" ? "latest 4.x" : "locked"), + flavor: m?.[2] ?? "locked", + }; +} + +const HEADER = ` +# Fidelity matrix + +The [behavioral cross-check](/guide/fidelity) runs the same probe corpus under +the mock engine **and** under real React Native — and CI runs it against +**every supported React Native version** in the +[Vitest × RN matrix](https://github.com/danfry1/vitest-native/actions/workflows/native-rn-matrix.yml). +This page is generated from that matrix's own reports, so every number below +was produced by CI, not written by hand. +`; + +const reportFiles = collectReports(reportsDir); +let body; + +if (reportFiles.length === 0) { + body = ` +::: info No matrix data in this build +This page is populated from the latest CI matrix run when the site is deployed. +A local or matrix-less build shows this placeholder. The single-version +[Fidelity Report](/guide/fidelity) is always available and drift-gated in CI. +::: +`; +} else { + const cells = reportFiles + .map((p) => { + const report = JSON.parse(fs.readFileSync(p, "utf8")); + return { ...describeCell(p, report), report }; + }) + .sort( + (a, b) => + a.rn.localeCompare(b.rn, undefined, { numeric: true }) || a.flavor.localeCompare(b.flavor), + ); + + const allGreen = cells.every((c) => c.report.summary.matching === c.report.summary.total); + const rows = cells + .map(({ rn, vitest, report }) => { + const { matching, total } = report.summary; + const status = matching === total ? "✅" : "❌"; + const when = (report.generatedAt ?? "").slice(0, 10); + return `| ${cell(rn)} | ${cell(vitest)} | ${status} ${matching}/${total} | ${cell(when)} |`; + }) + .join("\n"); + + // NOTE: crosscheck.mjs exits non-zero on any divergence, so CI's + // artifact-selection (latest SUCCESSFUL run) never feeds this branch — the + // deployed page shows the last all-green run while CI is red. The rendering + // exists for manually-supplied report directories and forensics; do not + // remove it as dead code. + const divergences = cells.flatMap(({ rn, vitest, report }) => + report.probes + .filter((p) => !p.match) + .map( + (p) => + `| ${cell(p.name)} | ${cell(rn)} | ${cell(vitest)} | ${cell(p.reason ?? "diverged")} |`, + ), + ); + + body = ` +${allGreen ? `**Every probe matches on every gated React Native version.**` : `**Divergences detected — see the table below.**`} + +| React Native | Vitest | Probes matching | Generated | +| --- | --- | --- | --- | +${rows} + +## Divergences + +${ + divergences.length + ? `| Probe | React Native | Vitest | Reason |\n| --- | --- | --- | --- |\n${divergences.join("\n")}` + : "_None. Every probe matched in every cell of the latest matrix run._" +} +`; +} + +const content = `${HEADER}${body}`; + +if (process.argv.includes("--check")) { + // Drift gate for the COMMITTED page: it must be exactly the placeholder + // render — real matrix data belongs only in the deploy workspace, never in + // the repository. + if (reportFiles.length > 0) { + console.error("✗ --check validates the committed placeholder; run it without --reports."); + process.exit(1); + } + const current = fs.existsSync(outPath) ? fs.readFileSync(outPath, "utf8") : null; + if (current !== content) { + console.error( + `✗ ${path.relative(repoRoot, outPath)} is not the placeholder render — regenerate with \`node scripts/fidelity-matrix.mjs\` and commit.`, + ); + process.exit(1); + } + console.log("✓ committed fidelity-matrix placeholder is up to date"); +} else { + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, content); + console.log( + `✓ fidelity matrix page rendered (${reportFiles.length} report${reportFiles.length === 1 ? "" : "s"}) → ${path.relative(repoRoot, outPath)}`, + ); +} diff --git a/packages/vitest-native/scripts/fidelity-report.mjs b/packages/vitest-native/scripts/fidelity-report.mjs index 4fc0232..f4bb131 100644 --- a/packages/vitest-native/scripts/fidelity-report.mjs +++ b/packages/vitest-native/scripts/fidelity-report.mjs @@ -105,6 +105,9 @@ const knownDiffs = fs.existsSync(knownDiffsPath) // any cell rendered as free text (not inside a code span). const cell = (s) => String(s ?? "") + // Backslashes first — escaping them after the fact would re-arm the very + // characters the later replacements neutralize (CodeQL js/incomplete-sanitization). + .replace(/\\/g, "\\\\") .replace(/\|/g, "\\|") .replace(//g, ">") @@ -136,6 +139,7 @@ generated from the corpus itself, so the numbers below are exactly what ships. - **${summary.matching} / ${summary.total} probes** match between the mock engine and real React Native. - ${ciLine} +- Per-version results straight from the CI matrix: [Fidelity Matrix](/guide/fidelity-matrix). - Reproduce it yourself: \`bun run crosscheck\`. The \`native\` engine needs no cross-check — it *is* real React Native. diff --git a/packages/vitest-native/tests/fidelity-matrix.test.ts b/packages/vitest-native/tests/fidelity-matrix.test.ts new file mode 100644 index 0000000..2e54d17 --- /dev/null +++ b/packages/vitest-native/tests/fidelity-matrix.test.ts @@ -0,0 +1,136 @@ +/** + * The fidelity-matrix renderer: aggregates per-cell CI cross-check reports + * into the published probes-across-versions page. + */ +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import fs from "node:fs"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.resolve(HERE, "../scripts/fidelity-matrix.mjs"); + +function render(args: string[]): { out: string; page: string } { + const outFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-out-")), "page.md"); + const out = execFileSync(process.execPath, [SCRIPT, ...args, "--out", outFile], { + encoding: "utf8", + }); + return { out, page: fs.readFileSync(outFile, "utf8") }; +} + +function writeReport(dir: string, artifactName: string, report: Record): void { + const d = path.join(dir, artifactName); + fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync(path.join(d, "report.json"), JSON.stringify(report)); +} + +describe("fidelity-matrix renderer", () => { + it("renders a placeholder page when no reports exist", () => { + const { page } = render([]); + expect(page).toContain("GENERATED FILE"); + expect(page).toContain("No matrix data in this build"); + expect(page).toContain("/guide/fidelity"); + }); + + it("aggregates per-cell reports into a sorted matrix with divergences escaped", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.85-locked", { + reactNativeVersion: "0.85.2", + vitestVersion: "4.1.8", + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 75, matching: 75 }, + probes: [{ name: "a11y-role", match: true }], + }); + writeReport(reports, "crosscheck-report-rn0.81-latest-supported", { + reactNativeVersion: "0.81.5", + vitestVersion: "4.2.1", + generatedAt: "2026-07-04T09:05:00.000Z", + summary: { total: 75, matching: 74 }, + probes: [ + { name: "a11y-role", match: true }, + // VitePress hazards in the reason: pipes break table cells, bare tags + // and {{ }} break the Vue compile, and a backslash would re-arm the + // escaped pipe — all must be escaped. + { name: "press-event", match: false, reason: "mock got a|b \\| {{ nope }}" }, + ], + }); + + const { page } = render(["--reports", reports]); + // Sorted by RN version: 0.81 row before 0.85. + expect(page.indexOf("0.81.5")).toBeLessThan(page.indexOf("0.85.2")); + // Per-cell summary rows with pass state. + expect(page).toContain("| 0.85.2 | 4.1.8 | ✅ 75/75 | 2026-07-04 |"); + expect(page).toContain("| 0.81.5 | 4.2.1 | ❌ 74/75 | 2026-07-04 |"); + // Divergence table present, with hazards escaped. + expect(page).toContain("| press-event |"); + expect(page).toContain("mock <Text> got a\\|b \\\\\\| {{ nope }}"); + expect(page).not.toContain(""); + // Not the all-green banner. + expect(page).toContain("Divergences detected"); + }); + + it("escapes hostile probe names (reports are CI artifacts, treated as untrusted)", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.84-locked", { + reactNativeVersion: "0.84.0", + vitestVersion: "4.1.8", + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 2, matching: 1 }, + probes: [ + { name: "ok-probe", match: true }, + // A backtick would open a code span; a pipe breaks the row; a tag + // injects HTML into the built site. + { name: "evil` | ", match: false, reason: "r" }, + ], + }); + const { page } = render(["--reports", reports]); + expect(page).not.toContain(" { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + const d = path.join(reports, "crosscheck-report-rn0.82-locked"); + fs.mkdirSync(d, { recursive: true }); + fs.writeFileSync(path.join(d, "report.json"), "{ not json"); + expect(() => render(["--reports", reports])).toThrow(); + }); + + it("--check passes on the committed placeholder and fails on drift", () => { + // The committed page must be exactly the placeholder render. + execFileSync(process.execPath, [SCRIPT, "--check"], { encoding: "utf8" }); + // A page with data (or any divergent content) must fail the gate. + const outFile = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-check-")), "p.md"); + fs.writeFileSync(outFile, "not the placeholder"); + expect(() => + execFileSync(process.execPath, [SCRIPT, "--check", "--out", outFile], { encoding: "utf8" }), + ).toThrow(); + }); + + it("declares all-green when every cell matches fully", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.86-locked", { + reactNativeVersion: "0.86.0", + vitestVersion: "4.1.8", + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 75, matching: 75 }, + probes: [{ name: "a11y-role", match: true }], + }); + const { page } = render(["--reports", reports]); + expect(page).toContain("Every probe matches on every gated React Native version."); + expect(page).toContain("_None. Every probe matched in every cell of the latest matrix run._"); + }); + + it("falls back to artifact-name axes when the report lacks resolved versions", () => { + const reports = fs.mkdtempSync(path.join(os.tmpdir(), "vn-matrix-reports-")); + writeReport(reports, "crosscheck-report-rn0.83-latest-supported", { + generatedAt: "2026-07-04T09:00:00.000Z", + summary: { total: 10, matching: 10 }, + probes: [], + }); + const { page } = render(["--reports", reports]); + expect(page).toContain("| 0.83 | latest 4.x | ✅ 10/10 |"); + }); +}); diff --git a/website/.vitepress/config.ts b/website/.vitepress/config.ts index 936af98..7f705af 100644 --- a/website/.vitepress/config.ts +++ b/website/.vitepress/config.ts @@ -81,6 +81,7 @@ export default defineConfig({ items: [ { text: 'API Coverage', link: '/api/coverage' }, { text: 'Fidelity Report', link: '/guide/fidelity' }, + { text: 'Fidelity Matrix', link: '/guide/fidelity-matrix' }, { text: 'Comparison with Jest', link: '/guide/comparison' }, { text: 'Troubleshooting', link: '/guide/troubleshooting' }, ], diff --git a/website/guide/fidelity-matrix.md b/website/guide/fidelity-matrix.md new file mode 100644 index 0000000..a52fee4 --- /dev/null +++ b/website/guide/fidelity-matrix.md @@ -0,0 +1,20 @@ + +# Fidelity matrix + +The [behavioral cross-check](/guide/fidelity) runs the same probe corpus under +the mock engine **and** under real React Native — and CI runs it against +**every supported React Native version** in the +[Vitest × RN matrix](https://github.com/danfry1/vitest-native/actions/workflows/native-rn-matrix.yml). +This page is generated from that matrix's own reports, so every number below +was produced by CI, not written by hand. + +::: info No matrix data in this build +This page is populated from the latest CI matrix run when the site is deployed. +A local or matrix-less build shows this placeholder. The single-version +[Fidelity Report](/guide/fidelity) is always available and drift-gated in CI. +::: diff --git a/website/guide/fidelity.md b/website/guide/fidelity.md index 1b4c60e..93c9c7b 100644 --- a/website/guide/fidelity.md +++ b/website/guide/fidelity.md @@ -15,6 +15,7 @@ generated from the corpus itself, so the numbers below are exactly what ships. - **75 / 75 probes** match between the mock engine and real React Native. - CI runs the same corpus across **React Native 0.81–0.85** on every commit. +- Per-version results straight from the CI matrix: [Fidelity Matrix](/guide/fidelity-matrix). - Reproduce it yourself: `bun run crosscheck`. The `native` engine needs no cross-check — it *is* real React Native.