Withdraw conformance and benchmark claims sitewide; decouple site build from wasm

.github/workflows/ci.yml

@@ -187,9 +187,9 @@ jobs:
     # 16 MiB. The check and the deploy now build the same thing.
     env:
       RUSTFLAGS: "-D warnings -C link-arg=-zstack-size=16777216"
-    # 20, not 10: `npm run build` now compiles the checker to WebAssembly for
-    # the playground ([WASM-BUILD]) before Eleventy runs, and a cold wasm build
-    # of the parser + typeshed does not fit the old budget.
+    # 20, not 10: this job compiles the checker to WebAssembly for the
+    # playground ([WASM-BUILD]) before Eleventy runs, and a cold wasm build of
+    # the parser + typeshed does not fit the old budget.
     timeout-minutes: 20
     steps:
       # fetch-depth: 0 — mirrors deploy-pages.yml so the conformance over-time
@@ -241,6 +241,15 @@ jobs:
       - name: Check stamped conformance references and generated READMEs
         run: python3 scripts/gen_conformance_reference.py --check
 
+      # Separate from `npm run build` ON PURPOSE. The Eleventy build has no Rust
+      # dependency — every page but the playground renders from committed data —
+      # so a checker that does not compile can no longer take the whole site
+      # down, locally or here. The playground e2e below drives the real engine,
+      # so this job still builds it explicitly.
+      - name: Build playground engine (wasm)
+        working-directory: website
+        run: npm run build:wasm
+
       - name: Build site
         working-directory: website
         # GITHUB_TOKEN raises the GitHub API rate limit for _data/releases.js

.github/workflows/deploy-pages.yml

@@ -37,8 +37,8 @@ jobs:
   deploy:
     name: Build and deploy
     runs-on: ubuntu-latest
-    # 20, not 10: `npm run build` now compiles the checker to WebAssembly for
-    # the playground ([WASM-BUILD]) before Eleventy runs. Kept in step with the
+    # 20, not 10: this job compiles the checker to WebAssembly for the
+    # playground ([WASM-BUILD]) before Eleventy runs. Kept in step with the
     # same budget in ci.yml's website job.
     timeout-minutes: 20
     environment:
@@ -74,6 +74,14 @@ jobs:
         working-directory: website
         run: npm ci
 
+      # Separate from `npm run build` ON PURPOSE. The Eleventy build has no Rust
+      # dependency, so a checker that does not compile can no longer take every
+      # page down with it. The deployed site DOES ship a working playground, so
+      # this job runs the engine build explicitly and fails here if it breaks.
+      - name: Build playground engine (wasm)
+        working-directory: website
+        run: npm run build:wasm
+
       - name: Build site
         working-directory: website
         # GITHUB_TOKEN raises the GitHub API rate limit for _data/releases.js

docs/INDEX.md

@@ -68,3 +68,9 @@ Plans contain only unfinished work. Delete a plan when its acceptance gate passe
 | File | Contents |
 |---|---|
 | [Typing puzzles](puzzles/puzzles.md) | User-reported typing puzzles from X, with minimal repros, PEP-bug vs house-rule classification, and the resulting issues (#371, #378–#383). |
+
+## Conformance integrity
+
+| File | Contents |
+|---|---|
+| [Conformance integrity audit](CONFORMANCE-INTEGRITY-AUDIT.md#CHKARCH-CONFORMANCE-INTEGRITY-AUDIT) | Phase 1: the fitted alias predicates, measured impact, wider checker scan, remediation status, and process changes found by the 2026-08 audit. Linked from the site's [conformance correction](../website/src/docs/conformance.md). |

docs/specs/WASM-SPEC.md

@@ -136,6 +136,15 @@ The unoptimised release artefact measures **7.2 MB**, most of it the embedded
 typeshed. Size work (`opt-level="z"`, `wasm-opt`, lazy loading) and the ratchet
 that holds it are in [WASM-PLAN.md](../plans/WASM-PLAN.md).
 
+The engine is a **separate build step from the site**. `npm run build` is
+Eleventy alone and has no Rust dependency; `npm run build:wasm` compiles this
+crate into `website/src/assets/wasm`. The site is otherwise generated from
+committed data, so a checker that does not compile must not be able to take
+every page down with it — it can only cost the playground its engine. CI and
+the release deploy run `build:wasm` as their own explicit step, and a site
+served without one reports the missing engine on the playground page instead of
+hanging on a spinner.
+
 ## Testing {#WASM-TESTING}
 
 Because the engine is an `rlib`, every test runs on the host under the normal

docs/specs/WEBSITE-E2E-SPEC.md

@@ -23,22 +23,22 @@ CI that a visitor can navigate the site.
 - **Mobile docs submenu** — see [WEBSITE-MOBILE-DOCS-NAV].
 - **Mobile top nav** — the hamburger reveals the collapsed top nav.
 - **Homepage positioning** — the title, H1 and opening answer identify Basilisk
-  as a Python type checker and language server, with only measured, linked proof.
-- **Headline claims carry their proof** — the hero's two comparative claims (sole
-  perfect official conformance score, and lowest median cold full-file CLI time)
-  each sit beside the link that grades them: the official `python/typing` results
-  and the published benchmark. False positives are asserted at 0 — a ratchet per
-  [CHKARCH-CONFORMANCE] — while the caught-error count is left open, since
-  upstream adds test cases over time.
+  neutrally as a Python type checker and language server, without an unverified
+  speed or conformance claim.
+- **Integrity disclosure is unavoidable** — the hero states that the former
+  conformance and benchmark figures are withdrawn, the current conformance
+  percentage is temporarily unknown, Basilisk was removed from the official
+  results at its request, and clean reimplementation plus robustness/mutation
+  verification must finish before new figures are published. Both notices link
+  to their detailed correction pages.
 - **Social image matches its declared size** — the `og:image` URL resolves and
   the PNG's own IHDR dimensions equal the advertised `og:image:width`/`height`,
   so a re-exported image cannot silently desync from its metadata.
 - **The Chinese homepage is a translation, not a second pitch** — `/zh/` and `/`
   are asserted to produce an identical structural skeleton (section, stat-card,
-  bullet and button class lists, in order). The zh page repeats both gated
-  claims with the same proof links, and its `.hero__headline-accent` count must
-  equal the English page's, so one locale can never assert a comparative fact
-  the other has already retired.
+  bullet and button class lists, in order). The zh page repeats both withdrawal
+  notices and the temporarily unknown status, so one locale cannot retain a
+  claim the other has retracted.
 - **Homepage mobile usability** — no horizontal overflow and visible calls to
   action retain a minimum 48 px touch target on the iPhone SE viewport.
 

website/eleventy.config.js

@@ -135,7 +135,7 @@ export default function (eleventyConfig) {
       name: "Basilisk",
       url: "https://www.basilisk-python.dev",
       description:
-        "Open-source Python type checker and language server built in Rust, scoring 100% on the official python/typing conformance suite, with published cold-check benchmarks.",
+        "Open-source Python type checker and language server built in Rust. Conformance and benchmark results are withdrawn during an integrity review.",
       author: "The Basilisk Project",
       themeColor: "#e8500a",
       stylesheet: "/assets/css/styles.css",

website/package.json

@@ -3,9 +3,9 @@
   "version": "1.0.0",
   "private": true,
   "type": "module",
-  "description": "Basilisk website for the open-source Python type checker and language server built in Rust, scoring 100% on the official python/typing conformance suite, with published cold-check benchmarks.",
+  "description": "Website for Basilisk, the open-source Python type checker and language server built in Rust.",
   "scripts": {
-    "build": "npm run build:wasm && eleventy",
+    "build": "eleventy",
     "build:wasm": "npx --yes wasm-pack build ../crates/basilisk-wasm --target web --release --out-dir ../../website/src/assets/wasm --out-name basilisk_wasm",
     "start": "eleventy --serve --watch",
     "clean": "rm -rf _site",

website/src/_data/authors.json

@@ -8,7 +8,7 @@
       "shortName": "Basilisk Team",
       "role": "The team behind Basilisk",
       "avatar": "/assets/images/authors/basilisk-team.png",
-      "bio": "The Basilisk Project is the team voice for everyone who contributes to Basilisk, the open-source, strict-by-default Python type checker and language server built in Rust. Basilisk is built by Nimblesite as a human and AI partnership: humans own the direction, review, auditing, and testing, and nothing ships until the conformance suite, the tests, and a human all agree it is done.",
+      "bio": "The Basilisk Project is the team voice for everyone who contributes to Basilisk, an open-source Python type checker and language server built in Rust. Basilisk is built by Nimblesite through a human and AI development process. The project is currently strengthening its review, auditing, and robustness-testing practices after withdrawing its former conformance result.",
       "links": [
         { "label": "GitHub", "url": "https://github.com/Nimblesite/Basilisk" },
         { "label": "Discord", "url": "https://discord.gg/4wBDSGEZQd" },
@@ -26,7 +26,7 @@
       "shortName": "Christian Findlay",
       "role": "Director, Nimblesite",
       "avatar": "/assets/images/authors/christian-findlay.png",
-      "bio": "Christian Findlay is the director of Nimblesite and the person behind Basilisk. He has spent more than two decades building software across .NET, Dart, Flutter, and now Rust, and he writes regularly about type systems, developer experience, and building software with AI. Basilisk is his answer to a long-standing gap in Python tooling: a type checker whose default judgment you can actually trust, proven against the official conformance suite rather than asserted.",
+      "bio": "Christian Findlay is the director of Nimblesite and the person behind Basilisk. He has spent more than two decades building software across .NET, Dart, Flutter, and Rust, and writes about type systems, developer experience, and building software with AI. He is leading the project's current conformance remediation and review-process changes.",
       "links": [
         { "label": "Website", "url": "https://www.christianfindlay.com" },
         { "label": "GitHub", "url": "https://github.com/MelbourneDeveloper" },

website/src/_data/benchmarks.js

@@ -1,13 +1,11 @@
-// Eleventy global data: benchmark results, read from the git-tracked per-machine
-// CSV that `make bench` generates (benchmarks/status/<machine>.csv).
+// Eleventy global data for withdrawn historical benchmark results, read from the
+// git-tracked per-machine CSV that `make bench` generated.
 //
-// The website renders MEASURED FACTS. This loader parses the CSV's header and
-// per-file timings for the benchmark page. It also derives each tool's median
-// fresh-process check for the home pages. There are no speedup ratios, "beats M
-// of N" tallies, or arbitrary outlier thresholds: published values are either
-// CSV measurements or direct medians of those measurements.
+// The integrity review has withdrawn these measurements from comparison. This
+// loader preserves the old table for transparency; derived medians and fastest
+// fields are historical implementation details and must not drive public claims.
 //
-// Primary machine selection (what the website shows):
+// Historical primary-machine selection (what the withdrawn table preserves):
 //   1. $BASILISK_BENCH_PRIMARY (slug)   2. benchmarks/status/.primary file
 //   3. otherwise rank by tool coverage (a CSV missing competitor columns must
 //      never win), then prefer `gha-*` (stable CI hardware), then alphabetical
@@ -79,7 +77,7 @@ function parseCsv(text) {
   });
   const allTools = [...msIdx.keys()];
   // Warm-cache variants (…-warm) aren't separate checkers, so exclude them from
-  // the cold medians used on the home pages. Their per-file values stay in rows.
+  // the historical cold medians. Their per-file values stay in rows.
   const tools = allTools.filter((t) => !t.endsWith("-warm"));
   const rows = dataLines.slice(1).map((line) => {
     const parts = line.split(",");
@@ -107,14 +105,9 @@ function median(nums) {
   return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
 }
 
-// Per-checker median cold full-file time, for the "how it compares" speed row.
-// Every checker's own median over the fixtures it reported — a direct order
-// statistic of the measured CSV values, NOT a comparison number: the page shows
-// each tool's median next to the others and lets the reader compare, rather than
-// asserting a build-time "N× faster" ratio. Warm/cache variants are excluded;
-// this is the fresh-process measurement without a persistent result-cache.
-// Self-measured and reproducible with `make bench`, so it cannot drift from the
-// CSV.
+// Historical per-checker median cold full-file time. This and `fastest` remain
+// available only to preserve the old data shape; neither is publishable while
+// the benchmark methodology and results are under integrity review.
 function computeToolMedians(rows, tools) {
   const ms = {};
   const text = {};
@@ -164,7 +157,16 @@ function pickPrimary(files) {
 }
 
 export default function () {
-  const empty = { available: [], primary: null, meta: {}, tools: [], rows: [], hasData: false };
+  const empty = {
+    available: [],
+    primary: null,
+    meta: {},
+    tools: [],
+    rows: [],
+    hasData: false,
+    withdrawn: true,
+    publicationStatus: "historical-withdrawn",
+  };
   if (!existsSync(STATUS_DIR)) return empty;
 
   const files = readdirSync(STATUS_DIR).filter((f) => f.endsWith(".csv")).sort();
@@ -174,12 +176,14 @@ export default function () {
   const parsed = parseCsv(readFileSync(join(STATUS_DIR, primary), "utf-8"));
   if (!parsed) return empty;
 
-  // Everything exposed is either a CSV value or a median of CSV values.
+  // Preserve the old measurements as explicitly withdrawn historical data.
   return {
     available: files.map((f) => f.replace(/\.csv$/, "")),
     primary: primary.replace(/\.csv$/, ""),
     ...parsed,
     toolMedians: computeToolMedians(parsed.rows, parsed.tools),
     hasData: parsed.rows.length > 0,
+    withdrawn: true,
+    publicationStatus: "historical-withdrawn",
   };
 }

website/src/_data/conformance.js

@@ -1,16 +1,17 @@
-// Eleventy global data: PEP conformance results, read from committed outputs of
-// the real python/typing harness — never hand-typed. Implements
-// [CHKARCH-CONFORMANCE]; mirrors _data/benchmarks.js.
+// Eleventy global data retained for the conformance integrity audit. These are
+// historical outputs from the python/typing harness, not a current Basilisk
+// conformance result. The former result is withdrawn because fitted checker
+// logic made it untrustworthy; public pages must not present these values as a
+// score, standing, pass count, or proof of implementation quality.
 //
-//   conformance/conformance_status.csv         -> live per-file pass/fail
-//   website/src/_data/conformance_report.json  -> resolved python/typing@main commit + score metadata
-//   git log of conformance_status.csv          -> the over-time chart (real commits)
+//   conformance/conformance_status.csv         -> historical per-file output
+//   website/src/_data/conformance_report.json  -> historical run metadata
+//   git log of conformance_status.csv          -> historical audit trail
 //
-// A file passes iff the official harness reports no diff between expected and
-// observed diagnostics. Every number the website shows is whatever that harness
-// last produced and committed. The over-time chart is read straight from this
-// file's GIT history, not a hand-maintained ledger, so it cannot drift from what
-// actually happened.
+// A file was marked passing when the harness reported no diagnostic diff. That
+// records what happened in the exact fixtures; it does not establish general
+// conformance. Values are exposed only under `historical` with an explicit
+// withdrawn status.
 import { readFileSync, existsSync } from "fs";
 import { execFileSync } from "child_process";
 import { dirname, join } from "path";
@@ -21,8 +22,8 @@ const REPO_ROOT = join(__dirname, "../../..");
 const CONF_DIR = join(REPO_ROOT, "conformance");
 const STATUS_REL = "conformance/conformance_status.csv";
 const STATUS_CSV = join(CONF_DIR, "conformance_status.csv");
-// The resolved python/typing@main commit and score metadata. It lives in this
-// same _data dir.
+// The exact historical python/typing snapshot and withdrawn fixture-result
+// metadata. It lives in this same _data dir; it is not current-main data.
 const REPORT = join(__dirname, "conformance_report.json");
 
 // The day the official python/typing scoring rules replaced our earlier in-repo
@@ -53,7 +54,7 @@ function shortDate(iso) {
 }
 
 // Read the machine-readable report, which is the single source for the upstream
-// commit. Written by the conformance gate; never hand-edited.
+// commit. Written by the pristine fixture runner; never hand-edited.
 function readReport() {
   if (!existsSync(REPORT)) return null;
   try {
@@ -209,18 +210,22 @@ function buildChart(points) {
 export default function () {
   const status = parseStatus();
   if (!status) {
-    return { hasData: false, scorePct: null, categories: [], failing: [], history: [], chart: null };
+    return {
+      hasData: false,
+      withdrawn: true,
+      publicationStatus: "historical-withdrawn",
+      historical: null,
+    };
   }
 
   // The resolved upstream commit comes from the conformance report.
   const report = readReport();
   const upstream = report?.upstream ?? {};
   const pinnedRef = upstream.sha ?? null;
 
-  // [CHKARCH-CONFORMANCE] Build-time guarantee, not convention: every page that
-  // quotes the score must also carry the exact python/typing commit it was
-  // graded against. A build with score data but no commit would render blank
-  // SHAs and make the public number unreproducible, so fail it instead.
+  // Historical data still carries the exact python/typing commit so the audit
+  // can reproduce the withdrawn run. A missing commit would make that record
+  // incomplete, so fail rather than silently detach it from its source.
   if (!pinnedRef) {
     throw new Error(
       "conformance: conformance_status.csv has score data but conformance_report.json " +
@@ -231,14 +236,18 @@ export default function () {
   const history = gitHistory();
   return {
     hasData: true,
-    ...status,
-    upstreamRef: upstream.ref ?? "main",
-    pinnedRef,
-    pinnedRefShort: upstream.shortSha ?? (pinnedRef ? pinnedRef.slice(0, 7) : null),
-    commitDate: upstream.commitDate || null,
-    stale: upstream.stale ?? false,
-    officialSince: OFFICIAL_SINCE,
-    history,
-    chart: buildChart(history),
+    withdrawn: true,
+    publicationStatus: "historical-withdrawn",
+    historical: {
+      ...status,
+      upstreamRef: upstream.ref ?? "main",
+      pinnedRef,
+      pinnedRefShort: upstream.shortSha ?? (pinnedRef ? pinnedRef.slice(0, 7) : null),
+      commitDate: upstream.commitDate || null,
+      stale: upstream.stale ?? false,
+      officialHarnessSince: OFFICIAL_SINCE,
+      history,
+      chart: buildChart(history),
+    },
   };
 }