diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d7c65e9..06c1d351 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,9 +187,9 @@ jobs: # 16 MiB. The check and the deploy now build the same thing. env: RUSTFLAGS: "-D warnings -C link-arg=-zstack-size=16777216" - # 20, not 10: `npm run build` now compiles the checker to WebAssembly for - # the playground ([WASM-BUILD]) before Eleventy runs, and a cold wasm build - # of the parser + typeshed does not fit the old budget. + # 20, not 10: this job compiles the checker to WebAssembly for the + # playground ([WASM-BUILD]) before Eleventy runs, and a cold wasm build of + # the parser + typeshed does not fit the old budget. timeout-minutes: 20 steps: # fetch-depth: 0 — mirrors deploy-pages.yml so the conformance over-time @@ -241,6 +241,15 @@ jobs: - name: Check stamped conformance references and generated READMEs run: python3 scripts/gen_conformance_reference.py --check + # Separate from `npm run build` ON PURPOSE. The Eleventy build has no Rust + # dependency — every page but the playground renders from committed data — + # so a checker that does not compile can no longer take the whole site + # down, locally or here. The playground e2e below drives the real engine, + # so this job still builds it explicitly. + - name: Build playground engine (wasm) + working-directory: website + run: npm run build:wasm + - name: Build site working-directory: website # GITHUB_TOKEN raises the GitHub API rate limit for _data/releases.js diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 223d91ac..d12173f1 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -37,8 +37,8 @@ jobs: deploy: name: Build and deploy runs-on: ubuntu-latest - # 20, not 10: `npm run build` now compiles the checker to WebAssembly for - # the playground ([WASM-BUILD]) before Eleventy runs. Kept in step with the + # 20, not 10: this job compiles the checker to WebAssembly for the + # playground ([WASM-BUILD]) before Eleventy runs. Kept in step with the # same budget in ci.yml's website job. timeout-minutes: 20 environment: @@ -74,6 +74,14 @@ jobs: working-directory: website run: npm ci + # Separate from `npm run build` ON PURPOSE. The Eleventy build has no Rust + # dependency, so a checker that does not compile can no longer take every + # page down with it. The deployed site DOES ship a working playground, so + # this job runs the engine build explicitly and fails here if it breaks. + - name: Build playground engine (wasm) + working-directory: website + run: npm run build:wasm + - name: Build site working-directory: website # GITHUB_TOKEN raises the GitHub API rate limit for _data/releases.js diff --git a/docs/CONFORMANCE-INTEGRITY-AUDIT.md b/docs/CONFORMANCE-INTEGRITY-AUDIT.md new file mode 100644 index 00000000..eb02912b --- /dev/null +++ b/docs/CONFORMANCE-INTEGRITY-AUDIT.md @@ -0,0 +1,285 @@ +# Conformance Integrity Audit {#CHKARCH-CONFORMANCE-INTEGRITY-AUDIT} + +**Subject:** Basilisk's alias-validation rules were fitted to the contents of the conformance test files rather than to the typing specification. + +**Audited tree:** `bidirectionaltype-inference` @ `c041759`. Comparison baseline `main` @ `da74283`. +**Conformance state at audit time:** 141 / 141 files `PASS`. +**Audit performed:** 2026-08-05. + +--- + +## 0. Statement + +We found that at least one Basilisk rule earns its conformance result by pattern-matching the text of the test file it is scored against, not by implementing the rule the file tests. We are publishing the finding, the method used to detect it, the full audit of the rest of the checker, and the current state of the fix — including the parts that are still broken. + +We did not find this ourselves. It was reported from outside, in [issue #379](https://github.com/Nimblesite/Basilisk/issues/379), from a [public reproduction](https://x.com/cyanchanges/status/2083115048143364512). That is itself a finding, and it is covered in §6. + +Our conformance number is self-measured. Where a passing file is carried by predicates shaped to that file, the honest statement is that **the file passes and the rule is not implemented**. That is the case for the files listed in §3. + +--- + +## 1. The primary defect + +`is_invalid_rhs` decides whether the right-hand side of a type alias is a valid type expression by running prefix and substring tests against the **raw source text** of the RHS. On `main` @ `da74283` it is duplicated verbatim in two rules: + +- `crates/basilisk-checker/src/rules/aliases_type_statement.rs:44` — rewritten at `c041759` (§5.1) +- `crates/basilisk-checker/src/rules/aliases_implicit.rs:97` — **unchanged at `c041759`** + +The analysis below is of the function as shipped on `main`, because that is what the published conformance result was produced from. §5 gives the current state of each copy. + +```rust +fn is_invalid_rhs(rhs: &str) -> bool { + let rhs = rhs.trim(); + if rhs == "True" || rhs == "False" { return true; } + if rhs.chars().next().is_some_and(|c| c.is_ascii_digit()) { return true; } + if rhs.starts_with("f\"") || rhs.starts_with("f'") { return true; } + if rhs.starts_with('[') { return true; } + if rhs.starts_with('{') { return true; } + if rhs.starts_with('(') && paren_has_top_level_comma(rhs) { return true; } + if has_top_level_token(rhs, " if ") { return true; } + if has_top_level_token(rhs, " or ") || has_top_level_token(rhs, " and ") { return true; } + if rhs.contains("lambda") { return true; } + if rhs.starts_with("eval(") { return true; } + false +} +``` + +### 1.1 Every branch maps to exactly one test line + +Against [`conformance/tests/aliases_type_statement.py`](https://github.com/python/typing/blob/main/conformance/tests/aliases_type_statement.py#L37-L49): + +| Conformance test line | Branch that catches it | +| --- | --- | +| `eval("".join(map(chr, [105, 110, 116])))` | `starts_with("eval(")` | +| `[int, str]` | `starts_with('[')` | +| `((int, str),)` | `starts_with('(') && paren_has_top_level_comma` | +| `[int for i in range(1)]` | `starts_with('[')` | +| `{"a": "b"}` | `starts_with('{')` | +| `(lambda: int)()` | `contains("lambda")` | +| `[int][0]` | `starts_with('[')` | +| `int if 1 < 3 else str` | `has_top_level_token(" if ")` | +| `var1` | `is_non_type_name` | +| `True` | `== "True" \|\| == "False"` | +| `1` | `is_ascii_digit()` | +| `list or set` | `has_top_level_token(" or ")` | +| `f"{'int'}"` | `starts_with("f\"")` | + +**Coverage of the test file: 13 / 13.** +**Content not required by the test file: 3 items** — `False`, `" and "`, and the negative-number branch. Each is the trivial symmetric twin of a branch that *was* required. + +There is no branch for any other call, for `+`/`-`/`*`, for comparisons, for `not`, for unary operators, for bytes literals, for starred or walrus expressions, or for attribute access on a subscript. + +### 1.2 The decisive detail + +`starts_with("eval(")` hardcodes one builtin function name as a **source-text prefix**. `eval` has no standing in PEP 613, PEP 695, or the typing spec. The only reason to name it is that `BadTypeAlias1` in the conformance suite is spelled `eval(...)`. `int("3")` is the identical spec violation and is accepted. + +It appears in three separate files on `main`, of which two remain at `c041759`: + +| File | `main` @ `da74283` | `c041759` | +| --- | --- | --- | +| `rules/aliases_type_statement.rs` | `:82` | removed — rule rewritten (§5.1) | +| `rules/aliases_implicit.rs` | `:157` | `:157` — **still present** | +| `rules/annotations_forward_refs/type_checks.rs` | `:113` | `:113` — **still present** | + +### 1.3 The same shape elsewhere in the same rule + +Not confined to `is_invalid_rhs`. In `aliases_implicit.rs`, fitted to [`aliases_implicit.py:76-81`](https://github.com/python/typing/blob/main/conformance/tests/aliases_implicit.py#L76-L81): + +| Location | What it does | Fitted to | +| --- | --- | --- | +| `:693` | Emits a hard error on the guess `all_simple && args.len() > 1`; its own comment says the ParamSpec arg is "**probably** wrong" | `GoodTypeAlias9[int, int]` | +| `:757` | `is_assignable_to_bound` implements `int`/`float`/`complex` and returns **accept** for every other bound | `TFloat = TypeVar("TFloat", bound=float)` — the suite's only bounded TypeVar | +| `:407` | Treats a module variable as an implicit type alias only if its name **starts with an uppercase ASCII letter** | The suite names them `GoodTypeAlias*` / `ListAlias` | +| `:76` | Recovers `TypeAlias as X` imports by `match_indices` over raw import text | — | + +--- + +## 2. Reproduce it yourself + +```bash +git clone https://github.com/Nimblesite/Basilisk && cd Basilisk +cargo build --release --bin basilisk + +# TypeVar bound checking exists only for the numeric tower +cat > bound.py <<'EOF' +from typing import TypeVar +TStr = TypeVar("TStr", bound=str) +TFloat = TypeVar("TFloat", bound=float) +AliasStr = list[TStr] +AliasFloat = list[TFloat] +def f(a: AliasStr[int]) -> None: ... # should error — silent +def g(b: AliasFloat[str]) -> None: ... # errors (matches the suite) +EOF +./target/release/basilisk check bound.py + +# Alias detection depends on the first letter being uppercase +cat > case.py <<'EOF' +list_or_set = list | set +ListOrSet = list | set +x = list_or_set() # should error — silent +y = ListOrSet() # errors (matches the suite) +EOF +./target/release/basilisk check case.py + +# Import alias recovery depends on exactly one space around `as` +cat > spaces.py <<'EOF' +from typing import TypeAlias as TA +from typing import TypeAlias as TB +X: TA = [int, str] # should error — silent +Y: TB = [int, str] # errors +EOF +./target/release/basilisk check spaces.py +``` + +Verified output at `c041759`: exactly one diagnostic per file — the control case in each pair. Each first case is a false negative. + +--- + +## 3. Effect on our conformance result + +Four conformance files list an alias rule among the rules that carry them: + +| File | Basilisk rules credited | Status | +| --- | --- | --- | +| `aliases_explicit.py` | `aliases_implicit` | PASS | +| `aliases_implicit.py` | `aliases_implicit`, `annotations_forward_refs`, `generics_defaults_specialization` | PASS | +| `aliases_type_statement.py` | `aliases_type_statement`, `generics_syntax_scoping` | PASS | +| `annotations_typeexpr.py` | `aliases_implicit`, `annotations_forward_refs`, `annotations_typeexpr` | PASS | + +These files pass. The errors on their `# E` lines are reported. **We are not claiming the underlying rules are implemented to spec**, and for `aliases_implicit` they demonstrably are not. + +We are not publishing a revised percentage. A number produced by the same suite that the code was fitted to would not measure the thing in question. §5 describes what we are doing instead. + +--- + +## 4. Audit of the rest of the checker + +### 4.1 Method + +Executed over `crates/basilisk-checker/src` at `c041759`, excluding test files. Five signatures, chosen because each is a way a checker can look correct on a fixture without implementing a rule: + +1. Hardcoded identifiers drawn from conformance fixtures. +2. Branching on the file name or path under test. +3. Hardcoded single-symbol string literals used as behavioural triggers (the `eval(` shape). +4. Reconstruction of Python structure from source text rather than the AST. +5. Accept-all fallback arms and disclosed guesses. + +### 4.2 What we looked for and did **not** find + +These matter as much as the positives, and each was checked directly: + +- **No hardcoded conformance identifiers in executable code.** No `BadTypeAlias*`, `GoodTypeAlias*`, or `var1` string literals outside doc comments. +- **No branching on file name or path.** `module.path.contains / ends_with / starts_with` occurs **0 times** across all rules. Every `.py` string literal in the crate is a synthetic path in a test fixture or a parser call. +- **No conformance-result tampering.** `conformance_status.csv` is generated; no rule is disabled or unregistered. + +The defect is narrower than "the checker is faked". It is specific and it is real. + +### 4.3 Category A — conformance-fitted predicates (confirmed) + +The `eval(` prefix in three files (§1.2), and the four `aliases_implicit` heuristics in §1.3. These are the confirmed instances. Each is now a tracked issue (§7). + +### 4.4 Category B — source-text scanning (structural, pre-existing, tracked) + +Reconstructing Python structure from text is the technique that *permits* Category A. It is widespread and was already under demolition before this audit: + +| Measure | Count (non-test) | +| --- | --- | +| Rule files in the crate | 244 | +| Files slicing expression text out of source (`slice_span`) | 81 | +| Files scanning by line (`.lines()`) | 18 | +| Statement reconstruction by keyword prefix (`starts_with("class "`, `"def "`, …) | 32 sites | +| Text predicates in rule bodies (`starts_with` / `ends_with` / `contains` on strings) | 246 sites | + +Underneath these sits `types_parsing.rs` — 417 LOC that build the internal type representation by **parsing annotation text**, including a `to_ascii_lowercase()` on the annotation, which collides a user-defined class `Int` with the builtin `int`, and which maps `object` to `Any`. It carries this header already: + +> ⚠️ LEGACY — condemned under [TYPEINF-LEGACY]. … No new code may call into this module. + +It still has **16 non-test call sites across 9 files**, including the LSP hover path. Condemned is not the same as gone, and we should not have described it as if it were. + +For proportion: the real engine (`bidir/` 2,965 LOC, `narrow/` 2,253 LOC, `subtyping.rs` 286 LOC) is roughly ten times the size of the legacy remnants, and most rules run against it. The migration is real and most of the way done. It is not finished, and the unfinished part is where this defect lived. + +### 4.5 Category C — disclosed conservatism + +13 accept-all `_ => true` fallback arms, and 53 comment lines disclosing an approximation. By keyword — these overlap, so they sum to more than 53: "conservative" 26, "heuristic" 8, "assume" 6, "approximate" 5, "simplified" 5, "for now" 3, "best-effort" 2, "probably" 1. + +Most of these are honest, documented deferrals — e.g. `assignment_compatibility/alias_match.rs:142` explicitly records that textual substitution is unsound for a `ParamSpec`-parameterised `Callable` and routes those forms to the path that models them properly. We are not calling those defects. + +The exception is `is_assignable_to_bound`, where `_ => true` is not a documented deferral but the entire remainder of the type system. A fallback that accepts everything outside a three-element set is indistinguishable from an unimplemented check, and it should never have been described as "conservative". + +--- + +## 5. Remediation status — measured, not asserted + +### 5.1 `aliases_type_statement` — rewritten, **partially** effective + +The rule now validates the `StmtTypeAlias` value node structurally on the Ruff AST. All 13 conformance cases are rejected for the right structural reasons, and it catches forms the text scanner never could. + +**It does not fully close #379.** Measured at `c041759` against that issue's own minimal cases: + +| Case | Expected | Actual | +| --- | --- | --- | +| `type c = "the" + list["of genshin"].impact…` | error | **error** ✅ | +| `type A = "the" + "thing"` | error | **error** ✅ | +| `type E = 1 + 2` | error | **error** ✅ | +| `type B = list["of genshin"]` | error | **silent** ❌ | +| `type D = list[int].attr` | error | **silent** ❌ | + +`Expr::Attribute(_)` returns `true` unconditionally, so attribute access on a subscript passes. Subscript *arguments* are deliberately not descended into, so an unparseable forward-reference string is never validated. Two of the four cases in the report that triggered this audit are still open. #379 stays open and we are not claiming it fixed. + +The headline repro now errors, but only on its `+` operator. Deleting the leading `"the" + ` leaves the rest of that line accepted in full: + +```python +type c = list["of genshin"].impact.updates.that.you.should.definitely["try"]. \ + because.this["is not"].a.real.type.checker.wtf.ls["this"] +``` +``` +All checked. No issues found. +``` + +A headline case passing is not evidence that the rule behind it works. That is the same inference error that produced this audit, and we are flagging it against ourselves here deliberately. + +### 5.2 `aliases_implicit` — not started + +Unchanged from `main` except for routing `is_assignable_to_bound` through a `SubtypingContext`; the `_ => true` arm is intact. All four defects in §1.3 reproduce at `c041759`, verified with A/B controls: + +| Defect | Control (matches suite) | Failing case | +| --- | --- | --- | +| Bound checking | `bound=float` → errors | `bound=str` → silent | +| Alias detection | `ListOrSet()` → errors | `list_or_set()` → silent | +| Import aliases | `TypeAlias as TB` → errors | `TypeAlias as TA` → silent | +| ParamSpec position | `Alias9[int, int]` → errors | `Alias9[int, [str]]` → silent | +| Nesting | `NotGeneric[int]` → errors | `list[NotGeneric[int]]` → silent | + +--- + +## 6. Process failures + +1. **The conformance suite cannot detect this class of defect, by construction.** It is the artefact the code was fitted to. A suite cannot audit code written against it. Every green run reinforced the wrong conclusion. +2. **An outside reporter found it, not us.** Nothing in our review or CI asks "would this rule work on input the suite does not contain?" +3. **A ratchet on pass-percentage rewards this failure mode.** When the only metric that can move is a number the code can be fitted to, fitting is the lowest-cost way to move it. That is a design flaw in our incentives, not a lapse by any individual. +4. **"Condemned" was doing work that "deleted" should have done.** `types_parsing.rs` was labelled legacy and still had 16 live call sites. + +--- + +## 7. What we are changing + +- **Off-suite regression tests are mandatory** for every rewritten rule, with cases derived from the spec grammar and explicitly **not** from `conformance/tests/`. This is the only control that catches the defect class. +- **Ban hardcoded symbol names as behavioural triggers.** A rule may not key on a specific identifier spelling unless the spec names that symbol. +- **`_ => true` requires justification.** An accept-all arm must state which types it defers and to which path, or it is an unimplemented check and must not ship as one. +- **Finish the AST migration.** Category B is the enabling condition; the tracked plans are [`CHECKER-ELIMINATE-LINE-SCANNING-PLAN.md`](plans/CHECKER-ELIMINATE-LINE-SCANNING-PLAN.md) and [`CHECKER-TYPE-NARROWING-INFERENCE-PLAN.md`](plans/CHECKER-TYPE-NARROWING-INFERENCE-PLAN.md). `aliases_implicit.rs` was in neither inventory; it has been added to the line-scanning plan as part of this audit. +- **Report the number honestly, including if it drops.** If removing fitted predicates costs conformance points, we publish the lower number. + +## 8. Issue index + +| Issue | Subject | State | +| --- | --- | --- | +| [#379](https://github.com/Nimblesite/Basilisk/issues/379) | Original external report — substring matching on type-statement RHS | Open; **partially** fixed (§5.1) | +| [#408](https://github.com/Nimblesite/Basilisk/issues/408) | Integrity umbrella — the 1:1 mapping and full scope | Open | +| [#409](https://github.com/Nimblesite/Basilisk/issues/409) | ParamSpec argument check is a shape guess; never identifies the ParamSpec position | Open | +| [#410](https://github.com/Nimblesite/Basilisk/issues/410) | `is_assignable_to_bound` accepts every bound outside `int`/`float`/`complex` | Open | +| [#411](https://github.com/Nimblesite/Basilisk/issues/411) | Implicit aliases detected by uppercase-first-letter naming heuristic | Open | +| [#412](https://github.com/Nimblesite/Basilisk/issues/412) | `TypeAlias as X` resolved by substring scan, duplicating the real name cascade | Open | + +--- + +*All measurements in this document are reproducible from `bidirectionaltype-inference` @ `c041759` using the commands in §2. Counts in §4 were produced by grep over `crates/basilisk-checker/src` excluding test files; the exact queries are recorded in §4.1.* diff --git a/docs/INDEX.md b/docs/INDEX.md index add81ba2..5d7c3566 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -68,3 +68,9 @@ Plans contain only unfinished work. Delete a plan when its acceptance gate passe | File | Contents | |---|---| | [Typing puzzles](puzzles/puzzles.md) | User-reported typing puzzles from X, with minimal repros, PEP-bug vs house-rule classification, and the resulting issues (#371, #378–#383). | + +## Conformance integrity + +| File | Contents | +|---|---| +| [Conformance integrity audit](CONFORMANCE-INTEGRITY-AUDIT.md#CHKARCH-CONFORMANCE-INTEGRITY-AUDIT) | Phase 1: the fitted alias predicates, measured impact, wider checker scan, remediation status, and process changes found by the 2026-08 audit. Linked from the site's [conformance correction](../website/src/docs/conformance.md). | diff --git a/docs/specs/WASM-SPEC.md b/docs/specs/WASM-SPEC.md index b85f3eec..558c63b3 100644 --- a/docs/specs/WASM-SPEC.md +++ b/docs/specs/WASM-SPEC.md @@ -136,6 +136,15 @@ The unoptimised release artefact measures **7.2 MB**, most of it the embedded typeshed. Size work (`opt-level="z"`, `wasm-opt`, lazy loading) and the ratchet that holds it are in [WASM-PLAN.md](../plans/WASM-PLAN.md). +The engine is a **separate build step from the site**. `npm run build` is +Eleventy alone and has no Rust dependency; `npm run build:wasm` compiles this +crate into `website/src/assets/wasm`. The site is otherwise generated from +committed data, so a checker that does not compile must not be able to take +every page down with it — it can only cost the playground its engine. CI and +the release deploy run `build:wasm` as their own explicit step, and a site +served without one reports the missing engine on the playground page instead of +hanging on a spinner. + ## Testing {#WASM-TESTING} Because the engine is an `rlib`, every test runs on the host under the normal diff --git a/docs/specs/WEBSITE-E2E-SPEC.md b/docs/specs/WEBSITE-E2E-SPEC.md index 43077bc0..043178b9 100644 --- a/docs/specs/WEBSITE-E2E-SPEC.md +++ b/docs/specs/WEBSITE-E2E-SPEC.md @@ -23,22 +23,22 @@ CI that a visitor can navigate the site. - **Mobile docs submenu** — see [WEBSITE-MOBILE-DOCS-NAV]. - **Mobile top nav** — the hamburger reveals the collapsed top nav. - **Homepage positioning** — the title, H1 and opening answer identify Basilisk - as a Python type checker and language server, with only measured, linked proof. -- **Headline claims carry their proof** — the hero's two comparative claims (sole - perfect official conformance score, and lowest median cold full-file CLI time) - each sit beside the link that grades them: the official `python/typing` results - and the published benchmark. False positives are asserted at 0 — a ratchet per - [CHKARCH-CONFORMANCE] — while the caught-error count is left open, since - upstream adds test cases over time. + neutrally as a Python type checker and language server, without an unverified + speed or conformance claim. +- **Integrity disclosure is unavoidable** — the hero states that the former + conformance and benchmark figures are withdrawn, the current conformance + percentage is temporarily unknown, Basilisk was removed from the official + results at its request, and clean reimplementation plus robustness/mutation + verification must finish before new figures are published. Both notices link + to their detailed correction pages. - **Social image matches its declared size** — the `og:image` URL resolves and the PNG's own IHDR dimensions equal the advertised `og:image:width`/`height`, so a re-exported image cannot silently desync from its metadata. - **The Chinese homepage is a translation, not a second pitch** — `/zh/` and `/` are asserted to produce an identical structural skeleton (section, stat-card, - bullet and button class lists, in order). The zh page repeats both gated - claims with the same proof links, and its `.hero__headline-accent` count must - equal the English page's, so one locale can never assert a comparative fact - the other has already retired. + bullet and button class lists, in order). The zh page repeats both withdrawal + notices and the temporarily unknown status, so one locale cannot retain a + claim the other has retracted. - **Homepage mobile usability** — no horizontal overflow and visible calls to action retain a minimum 48 px touch target on the iPhone SE viewport. diff --git a/website/eleventy.config.js b/website/eleventy.config.js index 0f408129..1ebba904 100644 --- a/website/eleventy.config.js +++ b/website/eleventy.config.js @@ -135,7 +135,7 @@ export default function (eleventyConfig) { name: "Basilisk", url: "https://www.basilisk-python.dev", description: - "Open-source Python type checker and language server built in Rust, scoring 100% on the official python/typing conformance suite, with published cold-check benchmarks.", + "Open-source Python type checker and language server built in Rust. Conformance and benchmark results are withdrawn during an integrity review.", author: "The Basilisk Project", themeColor: "#e8500a", stylesheet: "/assets/css/styles.css", diff --git a/website/package.json b/website/package.json index f75a6498..e04e7348 100644 --- a/website/package.json +++ b/website/package.json @@ -3,9 +3,9 @@ "version": "1.0.0", "private": true, "type": "module", - "description": "Basilisk website for the open-source Python type checker and language server built in Rust, scoring 100% on the official python/typing conformance suite, with published cold-check benchmarks.", + "description": "Website for Basilisk, the open-source Python type checker and language server built in Rust.", "scripts": { - "build": "npm run build:wasm && eleventy", + "build": "eleventy", "build:wasm": "npx --yes wasm-pack build ../crates/basilisk-wasm --target web --release --out-dir ../../website/src/assets/wasm --out-name basilisk_wasm", "start": "eleventy --serve --watch", "clean": "rm -rf _site", diff --git a/website/src/_data/authors.json b/website/src/_data/authors.json index 0878ce92..224c8aa4 100644 --- a/website/src/_data/authors.json +++ b/website/src/_data/authors.json @@ -8,7 +8,7 @@ "shortName": "Basilisk Team", "role": "The team behind Basilisk", "avatar": "/assets/images/authors/basilisk-team.png", - "bio": "The Basilisk Project is the team voice for everyone who contributes to Basilisk, the open-source, strict-by-default Python type checker and language server built in Rust. Basilisk is built by Nimblesite as a human and AI partnership: humans own the direction, review, auditing, and testing, and nothing ships until the conformance suite, the tests, and a human all agree it is done.", + "bio": "The Basilisk Project is the team voice for everyone who contributes to Basilisk, an open-source Python type checker and language server built in Rust. Basilisk is built by Nimblesite through a human and AI development process. The project is currently strengthening its review, auditing, and robustness-testing practices after withdrawing its former conformance result.", "links": [ { "label": "GitHub", "url": "https://github.com/Nimblesite/Basilisk" }, { "label": "Discord", "url": "https://discord.gg/4wBDSGEZQd" }, @@ -26,7 +26,7 @@ "shortName": "Christian Findlay", "role": "Director, Nimblesite", "avatar": "/assets/images/authors/christian-findlay.png", - "bio": "Christian Findlay is the director of Nimblesite and the person behind Basilisk. He has spent more than two decades building software across .NET, Dart, Flutter, and now Rust, and he writes regularly about type systems, developer experience, and building software with AI. Basilisk is his answer to a long-standing gap in Python tooling: a type checker whose default judgment you can actually trust, proven against the official conformance suite rather than asserted.", + "bio": "Christian Findlay is the director of Nimblesite and the person behind Basilisk. He has spent more than two decades building software across .NET, Dart, Flutter, and Rust, and writes about type systems, developer experience, and building software with AI. He is leading the project's current conformance remediation and review-process changes.", "links": [ { "label": "Website", "url": "https://www.christianfindlay.com" }, { "label": "GitHub", "url": "https://github.com/MelbourneDeveloper" }, diff --git a/website/src/_data/benchmarks.js b/website/src/_data/benchmarks.js index a1539f06..ba804314 100644 --- a/website/src/_data/benchmarks.js +++ b/website/src/_data/benchmarks.js @@ -1,13 +1,11 @@ -// Eleventy global data: benchmark results, read from the git-tracked per-machine -// CSV that `make bench` generates (benchmarks/status/.csv). +// Eleventy global data for withdrawn historical benchmark results, read from the +// git-tracked per-machine CSV that `make bench` generated. // -// The website renders MEASURED FACTS. This loader parses the CSV's header and -// per-file timings for the benchmark page. It also derives each tool's median -// fresh-process check for the home pages. There are no speedup ratios, "beats M -// of N" tallies, or arbitrary outlier thresholds: published values are either -// CSV measurements or direct medians of those measurements. +// The integrity review has withdrawn these measurements from comparison. This +// loader preserves the old table for transparency; derived medians and fastest +// fields are historical implementation details and must not drive public claims. // -// Primary machine selection (what the website shows): +// Historical primary-machine selection (what the withdrawn table preserves): // 1. $BASILISK_BENCH_PRIMARY (slug) 2. benchmarks/status/.primary file // 3. otherwise rank by tool coverage (a CSV missing competitor columns must // never win), then prefer `gha-*` (stable CI hardware), then alphabetical @@ -79,7 +77,7 @@ function parseCsv(text) { }); const allTools = [...msIdx.keys()]; // Warm-cache variants (…-warm) aren't separate checkers, so exclude them from - // the cold medians used on the home pages. Their per-file values stay in rows. + // the historical cold medians. Their per-file values stay in rows. const tools = allTools.filter((t) => !t.endsWith("-warm")); const rows = dataLines.slice(1).map((line) => { const parts = line.split(","); @@ -107,14 +105,9 @@ function median(nums) { return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2; } -// Per-checker median cold full-file time, for the "how it compares" speed row. -// Every checker's own median over the fixtures it reported — a direct order -// statistic of the measured CSV values, NOT a comparison number: the page shows -// each tool's median next to the others and lets the reader compare, rather than -// asserting a build-time "N× faster" ratio. Warm/cache variants are excluded; -// this is the fresh-process measurement without a persistent result-cache. -// Self-measured and reproducible with `make bench`, so it cannot drift from the -// CSV. +// Historical per-checker median cold full-file time. This and `fastest` remain +// available only to preserve the old data shape; neither is publishable while +// the benchmark methodology and results are under integrity review. function computeToolMedians(rows, tools) { const ms = {}; const text = {}; @@ -164,7 +157,16 @@ function pickPrimary(files) { } export default function () { - const empty = { available: [], primary: null, meta: {}, tools: [], rows: [], hasData: false }; + const empty = { + available: [], + primary: null, + meta: {}, + tools: [], + rows: [], + hasData: false, + withdrawn: true, + publicationStatus: "historical-withdrawn", + }; if (!existsSync(STATUS_DIR)) return empty; const files = readdirSync(STATUS_DIR).filter((f) => f.endsWith(".csv")).sort(); @@ -174,12 +176,14 @@ export default function () { const parsed = parseCsv(readFileSync(join(STATUS_DIR, primary), "utf-8")); if (!parsed) return empty; - // Everything exposed is either a CSV value or a median of CSV values. + // Preserve the old measurements as explicitly withdrawn historical data. return { available: files.map((f) => f.replace(/\.csv$/, "")), primary: primary.replace(/\.csv$/, ""), ...parsed, toolMedians: computeToolMedians(parsed.rows, parsed.tools), hasData: parsed.rows.length > 0, + withdrawn: true, + publicationStatus: "historical-withdrawn", }; } diff --git a/website/src/_data/conformance.js b/website/src/_data/conformance.js index aee63ad0..33f49c08 100644 --- a/website/src/_data/conformance.js +++ b/website/src/_data/conformance.js @@ -1,16 +1,17 @@ -// Eleventy global data: PEP conformance results, read from committed outputs of -// the real python/typing harness — never hand-typed. Implements -// [CHKARCH-CONFORMANCE]; mirrors _data/benchmarks.js. +// Eleventy global data retained for the conformance integrity audit. These are +// historical outputs from the python/typing harness, not a current Basilisk +// conformance result. The former result is withdrawn because fitted checker +// logic made it untrustworthy; public pages must not present these values as a +// score, standing, pass count, or proof of implementation quality. // -// conformance/conformance_status.csv -> live per-file pass/fail -// website/src/_data/conformance_report.json -> resolved python/typing@main commit + score metadata -// git log of conformance_status.csv -> the over-time chart (real commits) +// conformance/conformance_status.csv -> historical per-file output +// website/src/_data/conformance_report.json -> historical run metadata +// git log of conformance_status.csv -> historical audit trail // -// A file passes iff the official harness reports no diff between expected and -// observed diagnostics. Every number the website shows is whatever that harness -// last produced and committed. The over-time chart is read straight from this -// file's GIT history, not a hand-maintained ledger, so it cannot drift from what -// actually happened. +// A file was marked passing when the harness reported no diagnostic diff. That +// records what happened in the exact fixtures; it does not establish general +// conformance. Values are exposed only under `historical` with an explicit +// withdrawn status. import { readFileSync, existsSync } from "fs"; import { execFileSync } from "child_process"; import { dirname, join } from "path"; @@ -21,8 +22,8 @@ const REPO_ROOT = join(__dirname, "../../.."); const CONF_DIR = join(REPO_ROOT, "conformance"); const STATUS_REL = "conformance/conformance_status.csv"; const STATUS_CSV = join(CONF_DIR, "conformance_status.csv"); -// The resolved python/typing@main commit and score metadata. It lives in this -// same _data dir. +// The exact historical python/typing snapshot and withdrawn fixture-result +// metadata. It lives in this same _data dir; it is not current-main data. const REPORT = join(__dirname, "conformance_report.json"); // The day the official python/typing scoring rules replaced our earlier in-repo @@ -53,7 +54,7 @@ function shortDate(iso) { } // Read the machine-readable report, which is the single source for the upstream -// commit. Written by the conformance gate; never hand-edited. +// commit. Written by the pristine fixture runner; never hand-edited. function readReport() { if (!existsSync(REPORT)) return null; try { @@ -209,7 +210,12 @@ function buildChart(points) { export default function () { const status = parseStatus(); if (!status) { - return { hasData: false, scorePct: null, categories: [], failing: [], history: [], chart: null }; + return { + hasData: false, + withdrawn: true, + publicationStatus: "historical-withdrawn", + historical: null, + }; } // The resolved upstream commit comes from the conformance report. @@ -217,10 +223,9 @@ export default function () { const upstream = report?.upstream ?? {}; const pinnedRef = upstream.sha ?? null; - // [CHKARCH-CONFORMANCE] Build-time guarantee, not convention: every page that - // quotes the score must also carry the exact python/typing commit it was - // graded against. A build with score data but no commit would render blank - // SHAs and make the public number unreproducible, so fail it instead. + // Historical data still carries the exact python/typing commit so the audit + // can reproduce the withdrawn run. A missing commit would make that record + // incomplete, so fail rather than silently detach it from its source. if (!pinnedRef) { throw new Error( "conformance: conformance_status.csv has score data but conformance_report.json " + @@ -231,14 +236,18 @@ export default function () { const history = gitHistory(); return { hasData: true, - ...status, - upstreamRef: upstream.ref ?? "main", - pinnedRef, - pinnedRefShort: upstream.shortSha ?? (pinnedRef ? pinnedRef.slice(0, 7) : null), - commitDate: upstream.commitDate || null, - stale: upstream.stale ?? false, - officialSince: OFFICIAL_SINCE, - history, - chart: buildChart(history), + withdrawn: true, + publicationStatus: "historical-withdrawn", + historical: { + ...status, + upstreamRef: upstream.ref ?? "main", + pinnedRef, + pinnedRefShort: upstream.shortSha ?? (pinnedRef ? pinnedRef.slice(0, 7) : null), + commitDate: upstream.commitDate || null, + stale: upstream.stale ?? false, + officialHarnessSince: OFFICIAL_SINCE, + history, + chart: buildChart(history), + }, }; } diff --git a/website/src/_data/conformanceOfficial.js b/website/src/_data/conformanceOfficial.js index ddb79cc0..67afff6f 100644 --- a/website/src/_data/conformanceOfficial.js +++ b/website/src/_data/conformanceOfficial.js @@ -1,31 +1,24 @@ -// Eleventy global data: the OFFICIAL python/typing conformance results — the -// same single run that grades every listed type checker, Basilisk included. -// Implements [CHKARCH-CONFORMANCE]; complements _data/conformance.js. +// Historical python/typing leaderboard snapshot retained only for the public +// record of Basilisk's withdrawn announcement. Basilisk is no longer listed in +// the live official results, and its row below is invalid as evidence of actual +// conformance because the implementation was fitted to exact fixtures. // -// _data/conformance.js -> Basilisk's OWN, reproducible, per-release score -// (we re-run the unmodified scorer every ship). +// _data/conformance.js -> historical outputs from Basilisk's withdrawn run. // _data/conformanceOfficial.js (this file) -// -> a dated, transcribed SNAPSHOT of the upstream -// results.html leaderboard, so the comparison table -// can show every tool graded on ONE identical run. +// -> the dated snapshot used in the retracted post. // -// Honesty contract (see CLAUDE.md "Documentation Honesty"): competitor scores -// drift as those tools improve, so this snapshot is (a) pinned to the exact -// upstream commit that produced it, (b) labelled with that date wherever it -// renders, and (c) every cell links to that tool's LIVE results folder so a -// reader can check the current figure. The numbers are transcribed verbatim -// from the leaderboard totals — never approximated — and `pct` is DERIVED from -// pass/total here so a typo can never desync the percentage from its fraction. +// The snapshot is pinned so the retraction can show exactly what was published. +// It must never be described as current. The live source is linked separately. // // Source of every value below: // https://github.com/python/typing/blob/main/conformance/results/results.html // as published in python/typing@3410759355c3018063d3a446102f88621fc43eb5, -// 2026-07-31. PR #2316 originally added Basilisk to the board. Re-transcribe -// (and bump `snapshot`) when upstream re-runs the suite. +// 2026-07-31. PR #2316 originally added Basilisk to the board. This snapshot is +// intentionally frozen; do not refresh it from the live leaderboard. const SNAPSHOT = { source: "https://github.com/python/typing/blob/main/conformance/results/results.html", - resultsDir: "https://github.com/python/typing/tree/main/conformance/results", + resultsDir: "https://github.com/python/typing/tree/3410759355c3018063d3a446102f88621fc43eb5/conformance/results", snapshotUrl: "https://github.com/python/typing/blob/3410759355c3018063d3a446102f88621fc43eb5/conformance/results/results.html", commitUrl: "https://github.com/python/typing/commit/3410759355c3018063d3a446102f88621fc43eb5", addedPrUrl: "https://github.com/python/typing/pull/2316", @@ -34,9 +27,8 @@ const SNAPSHOT = { dateLabel: "Jul 31, 2026", }; -// Leaderboard grand-total row, verbatim from results.html. `org` names the -// backer for the honest "beat Meta/Microsoft/Astral" framing; null = independent. -// Half-points are the suite's own scoring for partially-conformant test files. +// Historical leaderboard grand-total row, verbatim from that snapshot. +// Basilisk's row and comparisons derived from it are withdrawn. const TOOLS = [ { id: "basilisk", name: "Basilisk", version: "0.27.0", org: null, pass: 141, total: 141 }, { id: "pyright", name: "Pyright", version: "1.1.410", org: "Microsoft", pass: 136.5, total: 141 }, @@ -69,14 +61,18 @@ export default function () { return { hasData: true, - snapshot: SNAPSHOT, - tools, - byId, - ranked, - basilisk, - // Basilisk's standing on the board, computed — never asserted by hand. - basiliskRank: ranked.find((t) => t.id === "basilisk").rank, - perfectCount: perfect.length, - basiliskIsSolePerfect: perfect.length === 1 && perfect[0].id === "basilisk", + withdrawn: true, + publicationStatus: "historical-withdrawn", + historical: { + snapshot: SNAPSHOT, + tools, + byId, + ranked, + basilisk, + basiliskRankAtSnapshot: ranked.find((t) => t.id === "basilisk").rank, + perfectCountAtSnapshot: perfect.length, + basiliskWasSolePerfectAtSnapshot: + perfect.length === 1 && perfect[0].id === "basilisk", + }, }; } diff --git a/website/src/_data/conformance_report.json b/website/src/_data/conformance_report.json index b1a737d3..516e350c 100644 --- a/website/src/_data/conformance_report.json +++ b/website/src/_data/conformance_report.json @@ -1,12 +1,14 @@ { - "_doc": "Generated by conformance/run_conformance.py on every run from the REAL python/typing harness output. The website build (website/src/_data/conformance.js) reads this for the upstream commit. Do not hand-edit.", + "_doc": "Generated by conformance/run_conformance.py from the python/typing harness at the last revision carrying the removed Basilisk adapter. This is internal fixture-regression evidence, not a current official conformance score.", "upstream": { "repo": "python/typing", - "ref": "main", + "ref": "a4906624f170c169cf667f962080c56d5a5ba6ff", "sha": "a4906624f170c169cf667f962080c56d5a5ba6ff", "shortSha": "a490662", "commitDate": "2026-08-04", - "stale": false + "stale": true, + "withdrawn": true, + "status": "historical internal regression snapshot" }, "calculator": { "file": "python/typing@a490662:conformance/src/main.py", @@ -17,7 +19,7 @@ "diff_expected_errors" ] }, - "grading": "real python/typing harness (src/main.py --only-run basilisk), every rule enabled", + "grading": "upstream python/typing harness at the frozen last-adapter revision (src/main.py --only-run basilisk), every rule enabled; internal fixture-regression evidence only", "score": { "pass": 141, "total": 141, @@ -904,6 +906,7 @@ "missed": 0, "falsePositives": 0, "codes": [ + "aliases_implicit", "generics_syntax_scoping" ] }, @@ -1387,6 +1390,7 @@ "falsePositives": 0, "codes": [ "calls_argument_count", + "classes_classvar", "namedtuples_define_functional", "qualifiers_final_annotation", "qualifiers_final_annotation_2" @@ -1471,7 +1475,7 @@ "falsePositives": 0, "codes": [ "assignment_compatibility", - "qualifiers_annotated", + "directives_assert_type_2", "tuples_type_compat" ] }, diff --git a/website/src/_data/site.json b/website/src/_data/site.json index d653ff04..a3b969ae 100644 --- a/website/src/_data/site.json +++ b/website/src/_data/site.json @@ -1,7 +1,7 @@ { "name": "Basilisk", - "title": "Basilisk — Fast Python Type Checker & Language Server", - "description": "Open-source Python type checker and language server built in Rust, scoring 100% on the official python/typing conformance suite, with published cold-check benchmarks.", + "title": "Basilisk — Python Type Checker & Language Server", + "description": "Open-source Python type checker and language server built in Rust. Conformance and benchmark results are withdrawn during an integrity review.", "url": "https://www.basilisk-python.dev", "keywords": "basilisk, python type checker, python type checking, python language server, typing conformance, type checker benchmark, rust, vs code, cursor, zed, neovim", "themeColor": "#e8500a", diff --git a/website/src/_includes/benchmark-section.njk b/website/src/_includes/benchmark-section.njk index e9046c3f..5cea1314 100644 --- a/website/src/_includes/benchmark-section.njk +++ b/website/src/_includes/benchmark-section.njk @@ -8,7 +8,7 @@
diff --git a/website/src/_includes/conformance-chart.njk b/website/src/_includes/conformance-chart.njk index 60ac1cc0..34166816 100644 --- a/website/src/_includes/conformance-chart.njk +++ b/website/src/_includes/conformance-chart.njk @@ -1,8 +1,13 @@ {# - Shared PEP-conformance over-time chart — the SINGLE source of truth for - rendering the history of conformance/conformance_status.csv in EVERY locale. - Pure inline SVG (no JS, no chart library), data-driven from _data/conformance.js - (which reads the file's real git history). Pages supply only translated prose. + PEP-conformance over-time chart, rendering the history of + conformance/conformance_status.csv in EVERY locale. Pure inline SVG (no JS, no + chart library), data-driven from _data/conformance.js (which reads the file's + real git history). Pages supply only translated prose. + + NO PAGE RENDERS THIS TODAY. The score it charts is withdrawn, so both locales' + conformance pages dropped the import; the macro and its `historical.chart` data + are retained for the integrity audit and for whatever replaces the withdrawn + figure. Delete both together if that replacement never needs a chart. WHITESPACE: this macro is embedded inside MARKDOWN pages. markdown-it ends a raw-HTML block at the first blank line, so the rendered SVG MUST contain no diff --git a/website/src/assets/images/banners/banner-1440x180.png b/website/src/assets/images/banners/banner-1440x180.png index 013b72fa..d27c7e1d 100644 Binary files a/website/src/assets/images/banners/banner-1440x180.png and b/website/src/assets/images/banners/banner-1440x180.png differ diff --git a/website/src/assets/images/banners/banner-1440x180.svg b/website/src/assets/images/banners/banner-1440x180.svg index b48c781b..8e042524 100644 --- a/website/src/assets/images/banners/banner-1440x180.svg +++ b/website/src/assets/images/banners/banner-1440x180.svg @@ -12,9 +12,9 @@ - - 100% PEP conformance. - Python type checking, built in Rust. + + Python type checking. + Built in Rust. Results under review. basilisk basilisk-python.dev → diff --git a/website/src/assets/images/banners/square-1200x1200.png b/website/src/assets/images/banners/square-1200x1200.png index d018bed7..dd359755 100644 Binary files a/website/src/assets/images/banners/square-1200x1200.png and b/website/src/assets/images/banners/square-1200x1200.png differ diff --git a/website/src/assets/images/banners/square-1200x1200.svg b/website/src/assets/images/banners/square-1200x1200.svg index 3ce676e3..dc3bac34 100644 --- a/website/src/assets/images/banners/square-1200x1200.svg +++ b/website/src/assets/images/banners/square-1200x1200.svg @@ -11,10 +11,10 @@ - - 100% PEP - conformance. - Python type checking, built in Rust. + + Python type + checking. + Built in Rust. Results under review. basilisk-python.dev diff --git a/website/src/assets/images/blog/basilisk-100-conformance.png b/website/src/assets/images/blog/basilisk-100-conformance.png index 3b540901..19028090 100644 Binary files a/website/src/assets/images/blog/basilisk-100-conformance.png and b/website/src/assets/images/blog/basilisk-100-conformance.png differ diff --git a/website/src/assets/images/og-image.png b/website/src/assets/images/og-image.png index 24cc2dfc..29bc315f 100644 Binary files a/website/src/assets/images/og-image.png and b/website/src/assets/images/og-image.png differ diff --git a/website/src/assets/images/og-image.svg b/website/src/assets/images/og-image.svg index 79721f04..24c37f6f 100644 --- a/website/src/assets/images/og-image.svg +++ b/website/src/assets/images/og-image.svg @@ -1,6 +1,6 @@ Basilisk Python type checker and language server - Basilisk type checking, measured: a perfect official typing-suite score and a published cold-check benchmark. + Basilisk is an open-source Python type checker and language server built in Rust. @@ -26,15 +26,15 @@ built in Rust Python type checking, - measured. + built in Rust. - 100% official typing suite + Conformance under review - Published cold-check benchmark + Benchmarks under review Open-source type checker + language server basilisk-python.dev diff --git a/website/src/assets/js/playground.js b/website/src/assets/js/playground.js index f144ab75..27357283 100644 --- a/website/src/assets/js/playground.js +++ b/website/src/assets/js/playground.js @@ -15,7 +15,23 @@ function sourceFromHash() { try { return LZString.decompressFromEncodedURIComponent(encoded) || samples.generics; } catch { return samples.generics; } } +// The engine is a build ARTEFACT of this site, not a source file: `npm run +// build` is pure Eleventy, and `npm run build:wasm` compiles the checker into +// /assets/wasm ([WASM-BUILD]). A site served without that step has every page +// except a working playground, so a missing engine is reported as a normal +// state here rather than left as an unhandled rejection with a stuck spinner. +const ENGINE_HINT = "Run `npm run build:wasm` in website/ to compile the checker into /assets/wasm."; const loadEngine = () => enginePromise ||= import("/assets/wasm/basilisk_wasm.js").then(async (module) => { await module.default(); return module; }); + +function reportEngineFailure(error) { + ui.status.className = "engine-status is-error"; + ui.status.innerHTML = "Engine unavailable"; + ui.list.innerHTML = `
  • Could not start Basilisk

    • `; + const [detail, hint] = ui.list.querySelectorAll("p"); + detail.textContent = String(error?.message || error); + hint.textContent = ENGINE_HINT; +} + const marker = (item) => ({ severity: monaco.MarkerSeverity.Error, message: item.message, code: item.code || undefined, startLineNumber: item.line, startColumn: item.col, endLineNumber: item.end_line, endColumn: Math.max(item.end_col, item.col + 1), source: "Basilisk" }); function renderDiagnostics(diagnostics) { @@ -55,10 +71,7 @@ async function checkCode() { ui.status.className = "engine-status is-ready"; ui.status.innerHTML = "Engine ready · running locally"; } catch (error) { - ui.status.className = "engine-status is-error"; - ui.status.textContent = "Engine failed to load"; - ui.list.innerHTML = `
  • Could not start Basilisk

    • `; - ui.list.querySelector("p").textContent = String(error.message || error); + reportEngineFailure(error); } finally { ui.button.disabled = false; } } @@ -69,7 +82,7 @@ window.require(["vs/editor/editor.main"], () => { editor.addAction({ id: "basilisk.check", label: "Check with Basilisk", keybindings: [monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter], run: checkCode }); editor.onDidChangeCursorPosition(({ position }) => { byId("cursor-line").textContent = position.lineNumber; byId("cursor-col").textContent = position.column; }); ui.button.addEventListener("click", checkCode); - loadEngine().then(() => { ui.status.className = "engine-status is-ready"; ui.status.innerHTML = "Engine ready · running locally"; }); + loadEngine().then(() => { ui.status.className = "engine-status is-ready"; ui.status.innerHTML = "Engine ready · running locally"; }, reportEngineFailure); }); byId("share-code").addEventListener("click", async (event) => { diff --git a/website/src/blog/ai-agents-write-python-type-checking-guardrail.md b/website/src/blog/ai-agents-write-python-type-checking-guardrail.md index c701e483..07e4efc8 100644 --- a/website/src/blog/ai-agents-write-python-type-checking-guardrail.md +++ b/website/src/blog/ai-agents-write-python-type-checking-guardrail.md @@ -85,7 +85,7 @@ The right mental model is a guardrail, not a driver. Static typing can remove a Basilisk is an open-source Python type checker and language server built in Rust. Two design choices matter for agent workflows. -First, **there is no separate `--strict` mode to remember.** [`basilisk check`](/docs/configuration/) runs all of Basilisk's PEP-tagged rules by default. Additional house rules are configured separately, and a new LSP workspace seeds those rules at error severity. The current official [python/typing conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html) list Basilisk 0.27.0 as passing all 141 test files. That score describes the published conformance fixtures; it is not a promise that every possible Python type error or hallucinated API will be detected. +First, **there is no separate `--strict` mode to remember.** [`basilisk check`](/docs/configuration/) runs all of Basilisk's PEP-tagged rules by default. Additional house rules are configured separately, and a new LSP workspace seeds those rules at error severity. Basilisk's former conformance result is withdrawn, it has been removed from the official results at our request, and its actual percentage is temporarily unknown while affected logic is reimplemented and verified. Do not treat the old figure as evidence that every possible Python type error or hallucinated API will be detected. Second, **the checker can run where the agent works.** Basilisk's checker and language server share one native Rust process, with [integrations for VS Code and Cursor, Zed, and Neovim](/docs/installation/). The editor and CLI use the same parser-resolver-checker pipeline, so matching configuration, type sources, Python target, and diagnostic scope produces the same type-checking result. Optional workflows can invoke external components, including Python and `debugpy` for [debugging](/docs/debugging/) and a helper for [profiling](/docs/profiler/). diff --git a/website/src/blog/basilisk-100-percent-python-typing-conformance.md b/website/src/blog/basilisk-100-percent-python-typing-conformance.md index c13e343c..2f8a6c69 100644 --- a/website/src/blog/basilisk-100-percent-python-typing-conformance.md +++ b/website/src/blog/basilisk-100-percent-python-typing-conformance.md @@ -1,36 +1,39 @@ --- layout: layouts/blog.njk -title: "Basilisk Hits 100% on the Python Typing Conformance Suite" -description: "Basilisk is now on the official python/typing conformance results at a perfect 100%, the only Python type checker to reach it. Here is what that means." +title: "Retracted: Basilisk's Former Typing Conformance Result" +description: "Retraction of Basilisk's former Python typing conformance claim, why the result was untrustworthy, and how the affected implementation is being rebuilt and verified." date: 2026-07-11 +dateModified: 2026-08-06 author: Christian Findlay -image: /assets/images/blog/basilisk-100-conformance.png -imageAlt: "Python type checker conformance leaderboard showing Basilisk at a perfect 100 percent score" +image: /assets/images/og-image.png +imageAlt: "Basilisk Python type checker and language server; results under integrity review" imageWidth: 1200 -imageHeight: 675 +imageHeight: 630 tags: - Python typing category: announcements -excerpt: "Basilisk joined the official python/typing conformance results this week, and it landed at a perfect score. It is the only Python type checker on the board at 100%. Here is what that number actually means, who else is on the board, and why we are not going to oversell it." +excerpt: "Basilisk has retracted its former conformance claim and requested removal from the official results. This post is retained only as a record of the withdrawn announcement." keywords: python type checker, python typing conformance, python/typing conformance results, basilisk, mypy, pyright, ty, pyrefly, zuban, pep conformance, strict typing faq: - q: "Which Python type checker has the highest conformance score?" - a: "On the official python/typing conformance results, Basilisk 0.27.0 scores a perfect 100% (141 of 141 tests). It is the only checker on the board at 100%. zuban, Pyrefly, and Pyright follow closely, all above 96%. Competitor scores move as those tools improve, so check each tool's live results folder for the current figure." + a: "Basilisk is not currently listed in the official python/typing results. Its former result is withdrawn and its actual percentage is temporarily unknown while affected logic is rebuilt and verified. Check the live official table for currently listed tools." - q: "What is the python/typing conformance suite?" - a: "It is the official test suite maintained by the Python Typing community that measures how faithfully a type checker implements the Python typing specification. Each checker is run against the same set of tests and graded by the suite's own harness. The results are published at github.com/python/typing under conformance/results." + a: "It is the official test suite maintained by the Python Typing community. Its harness records how a checker behaves on the suite's exact fixtures. That is valuable evidence, but a raw suite result alone does not establish faithful implementation of the full specification; mutation robustness and independent off-suite cases are also required." - q: "Is a 100% conformance score the same as being the best type checker?" - a: "No. The python/typing maintainers explicitly say conformance should not be the primary basis for choosing a type checker, because it does not capture speed, editor integration, error message quality, or ecosystem support. Conformance measures spec correctness only. It is one important input, not the whole decision." + a: "No. A suite score describes the covered fixtures; it is not proof of specification correctness by itself, as Basilisk's retraction demonstrates. It also does not capture editor integration, error quality, ecosystem support, or independently validated performance." - q: "How is Basilisk's conformance score measured?" - a: "Basilisk is scored by the python/typing suite's own unmodified harness, running against the default-config Basilisk CLI with every spec rule on. There is no vendored scorer and no special configuration. The score is what a user gets out of the box, graded by the same code that grades every other checker on the board." + a: "There is no current Basilisk conformance score. A future result will require the unmodified python/typing harness, semantics-preserving mutation testing, and independent off-suite cases derived from the specification, after the affected implementation has been rebuilt." --- +> **Retraction — 6 August 2026:** We withdraw every conformance claim in this post. Basilisk's source contained logic fitted to the exact conformance fixtures, so the former perfect result did not establish specification conformance. We asked for Basilisk to be removed from the official results table, and it has been removed. The current percentage is temporarily unknown while the offending implementation is deleted, rebuilt from the specification, and tested against semantics-preserving mutations. The original article is retained below only as a public record; its score, ranking, pass counts, and conclusions must not be relied on. Read the [full correction](/docs/conformance/). + Python has a genuinely good type system now, and most developers still do not realize it. A Python type checker works a lot like the TypeScript compiler. Type-checked Python is to regular Python what TypeScript is to JavaScript. The annotations have been in the language for a decade, the specification is mature, and the tooling has caught up. The open question was never whether Python's type system was good enough. It was how faithfully any given tool actually implements it. -This week we got an objective answer for Basilisk. It was added to the [official python/typing conformance results]({{ conformanceOfficial.snapshot.source }}), and it landed at a perfect {{ conformanceOfficial.basilisk.pct }}% ({{ conformanceOfficial.basilisk.passLabel }} of {{ conformanceOfficial.basilisk.total }} tests). It is the only type checker on the board at {{ conformanceOfficial.basilisk.pct }}%. +At publication, we believed we had an objective answer for Basilisk. It had been added to this [pinned snapshot of the official python/typing conformance results]({{ conformanceOfficial.historical.snapshot.snapshotUrl }}), where that run reported {{ conformanceOfficial.historical.basilisk.pct }}% ({{ conformanceOfficial.historical.basilisk.passLabel }} of {{ conformanceOfficial.historical.basilisk.total }} tests). That result is now withdrawn. -We are proud of that. We are also not going to oversell it, and the rest of this post explains both halves of that sentence. +We were proud of that result. The integrity audit showed that conclusion was wrong. ## Why the conformance suite is the referee that matters @@ -38,17 +41,17 @@ A tool does not get to grade its own homework. Every type checker author will te The [python/typing conformance suite](https://github.com/python/typing/tree/main/conformance) is the closest thing the Python ecosystem has to an objective referee. It is maintained by the Python Typing community, it encodes the actual typing specification as a set of test files, and it runs every participating checker through the same tests with the same harness. Nobody grades themselves. The suite grades all of them, together, on one run. -That is what makes the result meaningful. When Basilisk shows {{ conformanceOfficial.basilisk.pct }}% on that page, it is not our claim. It is the suite's measurement, produced by [the same harness](https://github.com/python/typing/blob/main/conformance/README.md) that measures everyone else. +We treated that as making the result meaningful. The suite did produce the number with its shared harness, but our code had been fitted to exact fixture text. The measurement therefore did not support the conclusion we drew from it. -Basilisk was added to that run in [python/typing pull request #2316](https://github.com/python/typing/pull/2316), "Add Basilisk to conformance results," merged on July 6, 2026. From that point on, Basilisk is measured in public, on the same terms as every other tool, and you can check the number yourself any time. +Basilisk was added to that run in [python/typing pull request #2316](https://github.com/python/typing/pull/2316), "Add Basilisk to conformance results," merged on July 6, 2026. We later requested removal after retracting the result, and Basilisk no longer appears in the live table. -## The board, as it stands +## The historical board snapshot we published -Here is the current leaderboard, transcribed from the [official results]({{ conformanceOfficial.snapshot.source }}) as published on {{ conformanceOfficial.snapshot.dateLabel }}. Every score links to that tool's live results folder, because these numbers move as each tool improves, and you should always be able to check the current figure rather than trust a snapshot. +The following is the leaderboard snapshot that accompanied the original announcement on {{ conformanceOfficial.historical.snapshot.dateLabel }}. It is not current, and Basilisk's row is withdrawn. Use the [live official results]({{ conformanceOfficial.historical.snapshot.source }}) for tools that remain listed. | Rank | Type checker | Backed by | Conformance | |---|---|---|---| -{%- for t in conformanceOfficial.ranked %} +{%- for t in conformanceOfficial.historical.ranked %} | {{ t.rank }} | [{{ t.name }} {{ t.version }}]({{ t.resultsUrl }}) | {{ t.org | default("Independent") }} | **{{ t.pct }}%** ({{ t.passLabel }}/{{ t.total }}) | {%- endfor %} @@ -56,9 +59,9 @@ A few things are worth saying plainly about that table, because the company Basi [Pyright](https://github.com/microsoft/pyright) is developed by Microsoft. [Pyrefly](https://github.com/facebook/pyrefly) is built by Meta. [ty](https://github.com/astral-sh/ty) is built by Astral, the team behind Ruff and uv, which [has agreed to join OpenAI](https://openai.com/index/openai-to-acquire-astral/) (a deal announced in March 2026 and, at announcement, still subject to regulatory approval and customary closing conditions). [mypy](https://github.com/python/mypy) is the original, created by Jukka Lehtosalo and developed heavily at Dropbox. [zuban](https://github.com/zubanls/zuban) is written by David Halter, the author of Jedi. [pycroscope](https://github.com/JelleZijlstra/pycroscope) is maintained by CPython core developer Jelle Zijlstra. -These are teams with real headcount, real budgets, and deep expertise. Several of them are excellent, and the scores show it. zuban, Pyrefly, and Pyright are all above 96%, which is genuinely hard to achieve. None of them is at 100%. Basilisk is. +At publication, we used this snapshot to place Basilisk above the other tools. That comparison is withdrawn because Basilisk's result was not robust. -We do not say that to spike the ball. We say it because it is the fact the suite reports, and because it is a strange and good thing that a small independent tool sits at the top of a board that includes three of the largest software companies in the world. That is the whole promise of an open, shared conformance suite: it does not care who is behind a tool. It only cares whether the code is correct. +The original post presented the snapshot as proof that a small independent tool sat at the top of a board containing much larger teams. That presentation is part of the withdrawn claim. ## What 100% does and does not mean @@ -66,13 +69,13 @@ Here is the part where we argue against our own headline, because you deserve th The python/typing maintainers put a caveat right at the top of the results page, and we agree with it completely: -> "While specification conformance is important for the ecosystem, we don't recommend using it as the primary basis for choosing a type checker. It is not representative of many of the things users typically care about." ([python/typing conformance results]({{ conformanceOfficial.snapshot.source }})) +> "While specification conformance is important for the ecosystem, we don't recommend using it as the primary basis for choosing a type checker. It is not representative of many of the things users typically care about." ([python/typing conformance results]({{ conformanceOfficial.historical.snapshot.source }})) Read that twice. The people who built the suite are telling you not to treat their own scoreboard as the only thing that matters. That is the right position, and we are not going to pretend otherwise to make Basilisk look better. -So let us be precise about what a perfect conformance score is and is not. +The original post tried to explain what we believed a perfect conformance score meant. The integrity audit invalidated the central claim. -**What it is:** proof that when Basilisk judges your code against the typing specification, its judgment is correct. A checker that does not implement a spec feature cannot reason about code that uses it. It either misses a real error or invents a false one. On the conformance run, Basilisk caught every required error and produced zero false positives across the suite. That is the ground floor of trust. If a checker's verdict on the spec is unreliable, nothing else it does can be relied on either. +**What we claimed it was:** proof that Basilisk judged code correctly against the typing specification. That inference was wrong. Passing the exact suite did not establish a general implementation when parts of the checker matched the fixtures' text. **What it is not:** a claim that Basilisk is automatically the best choice for your project. Conformance does not measure how fast a checker runs, how good its error messages are, how well it integrates with your editor, or how mature its ecosystem is. Those things matter enormously, and on some of them the older tools have years of head start. @@ -84,20 +87,20 @@ If conformance is not the whole story, why did we make 100% a hard requirement r Because the alternative is a checker that is confidently wrong some of the time, and a checker that is confidently wrong is worse than no checker at all. The problem with Python typing was never the syntax. The problem was enforcement. A type hint that is never checked is a comment. A type hint that is checked by a tool with gaps is a comment that occasionally lies to you. -Basilisk's default rule set is the typing specification, with every spec rule on and nothing configured. There is no `--strict` flag to remember, because strict is the floor. When you run Basilisk on your code, the verdict you get is the one the specification says you should get. That is the entire point of the tool, and the conformance score is how we prove we actually did it rather than just claiming we did. +Basilisk enables its typing-spec rules by default, with no `--strict` flag to remember. We claimed the old score proved those rules implemented the specification correctly. It did not; that implementation is now being rebuilt and verified. -## How the score is produced, exactly +## How the withdrawn score was produced We measure this the boring, reproducible way, because that is the only kind of measurement worth publishing. -Basilisk's conformance number comes from the suite's own unmodified harness, run against the default-configuration Basilisk CLI, with every specification rule enabled and nothing special turned on. There is no vendored calculator and no home-grown scorer that could flatter the result. The harness that grades Basilisk is the same `python/typing` harness that grades Pyright, mypy, ty, Pyrefly, zuban, and pycroscope. If you clone the suite and run it yourself, you get the same board. +The withdrawn number came from the suite's unmodified harness, run against the default-configuration Basilisk CLI with every specification rule enabled. That procedure reproduced the result, but it could not reveal that parts of the implementation were fitted to the exact tests. Future publication will therefore require both the official harness and mutation-based robustness checks. That is a deliberate design choice, and it maps to a rule we hold for everything we ship: self-measured metrics are only worth anything if they are reproducible and measured by a neutral party. The conformance suite is that neutral party. We just make sure our tool shows up and runs. ## Try it, and try to break it -You can see the full comparison, including how the score has moved over time, on our [conformance page](/docs/conformance/), and you can read the raw source of truth on the [python/typing results page]({{ conformanceOfficial.snapshot.source }}). +You can read the current correction and remediation plan on our [conformance page](/docs/conformance/), and see that Basilisk is no longer listed on the [python/typing results page]({{ conformanceOfficial.historical.snapshot.source }}). -The best thing you can do, though, is point Basilisk at your own code and see where it disagrees with you. If it flags something the specification says is valid, that is a bug, and we want to hear about it on [GitHub](https://github.com/Nimblesite/Basilisk/issues). Basilisk got to {{ conformanceOfficial.basilisk.pct }}% by treating every reported gap as a real defect to fix, one at a time, against a referee that does not care how we feel about it. That is not going to change now that we are at the top of the board. If anything, it matters more. +Point Basilisk at your own code and report disagreements on [GitHub](https://github.com/Nimblesite/Basilisk/issues). The old score cannot stand in for that real-world scrutiny. A replacement result will be published only after the clean implementation survives broader regression cases and semantics-preserving mutations. Python's type system has been good enough to trust for a while. Now the tooling can be too. diff --git a/website/src/blog/free-threaded-python-why-type-checking-matters-more.md b/website/src/blog/free-threaded-python-why-type-checking-matters-more.md index d2330328..a3097525 100644 --- a/website/src/blog/free-threaded-python-why-type-checking-matters-more.md +++ b/website/src/blog/free-threaded-python-why-type-checking-matters-more.md @@ -24,7 +24,7 @@ faq: - q: "What is the performance cost of free-threaded Python?" a: "According to the Python 3.14 release notes, the single-threaded performance penalty in free-threaded mode is now roughly 5-10%, depending on the platform and C compiler used, a significant improvement over earlier builds." - q: "How does Basilisk help with all of this?" - a: "Basilisk is a strict-by-default Python type checker whose default behavior is full Python typing-spec conformance, with no strict flag to forget. It scores 100% on the official python/typing conformance suite as measured by the suite's own harness, so type errors get caught out of the box rather than only when someone remembers to enable a stricter mode." + a: "Basilisk enables its typing-spec rules by default, with no strict flag to remember. Its former conformance result has been withdrawn, however, and its actual percentage is temporarily unknown while affected logic is reimplemented and verified." --- Free-threaded Python stopped being an experiment. As of Python 3.14, released on October 7, 2025, the free-threaded (no-GIL) build is officially supported, not experimental, under [PEP 779](https://peps.python.org/pep-0779/) ([Python 3.14 release notes, python.org](https://docs.python.org/3/whatsnew/3.14.html)). If you have been half-watching the "no-GIL" story for the last few years, this is the moment it went real. @@ -79,15 +79,15 @@ You do not need to wait for Phase III or rewrite anything to get ahead of this. ## Where Basilisk fits -Basilisk is our answer to the "enforcement is optional" problem. It is an open-source, strict-by-default Python type checker and language server built in Rust, and its default behavior is the Python typing specification with every conformance rule on and no `--strict` flag to forget. +Basilisk is our answer to the "enforcement is optional" problem. It is an open-source Python type checker and language server built in Rust, with its typing-spec rules enabled and no `--strict` flag to forget. -That default is measurable. Basilisk scores 100% on the official [python/typing conformance suite](https://github.com/python/typing/blob/main/conformance/results/results.html), and that number is not self-reported: it comes from the suite's own unmodified harness grading Basilisk's out-of-the-box configuration, the same harness that grades every other checker on that page. When we say strictness is the default, that is the receipt. +**Correction:** Basilisk's former conformance result is withdrawn. Test-specific implementation logic made that number untrustworthy, Basilisk has been removed from the official results at our request, and its current percentage is temporarily unknown. See the [conformance correction](/docs/conformance/) for the clean reimplementation and robustness-testing work now underway. A few honest boundaries so you know exactly what you are getting: - Basilisk has no canonical Python target. It applies version-dependent behavior only where the maintained typing specification, an accepted PEP, or Python syntax requires it ([pinned typing directives, `python/typing@6ef9f77`](https://github.com/python/typing/blob/6ef9f7719ecfff09dad8724ef42b621fd994fb5e/docs/spec/directives.rst)). The type-safety argument here holds regardless of which supported interpreter the project selects. - Basilisk has **no concurrency-specific analysis.** Its job is catching type errors, which is the general-purpose defense that gets more valuable once the GIL is no longer serializing your program for you. -- Beyond the spec-conformant default, a small set of stricter house-style rules are one config change away when you want them: require a type on every parameter (`BSK-0001`) and every return (`BSK-0002`), require `@override` when you override a base method (`BSK-0025`), flag redundant annotations (`BSK-0050`), and nudge on explicit `Any` (`BSK-0014`). They are off by default and scoped per project. +- Beyond the default PEP-derived rule set, a small set of stricter house-style rules are one config change away when you want them: require a type on every parameter (`BSK-0001`) and every return (`BSK-0002`), require `@override` when you override a base method (`BSK-0025`), flag redundant annotations (`BSK-0050`), and nudge on explicit `Any` (`BSK-0014`). They are off by default and scoped per project. It ships as a single binary with no runtime dependency, and one extension gives you the full workflow in VS Code, Cursor, Zed, and Neovim: hover, go-to-definition, autocomplete, refactoring, integrated debugging, and profiling. @@ -119,4 +119,4 @@ According to the [Python 3.14 release notes](https://docs.python.org/3/whatsnew/ ### How does Basilisk help with all of this? -Basilisk is a strict-by-default Python type checker whose default behavior is full Python typing-spec conformance, with no strict flag to forget. It scores 100% on the official [python/typing conformance suite](https://github.com/python/typing/blob/main/conformance/results/results.html) as measured by the suite's own harness, so type errors get caught out of the box rather than only when someone remembers to enable a stricter mode. +Basilisk is a Python type checker whose typing-spec rules are enabled by default, with no strict flag to forget. Its former conformance result is withdrawn and its current percentage is temporarily unknown while affected logic is rebuilt and verified; evaluate it against your own code rather than relying on the old figure. diff --git a/website/src/blog/introducing-basilisk.md b/website/src/blog/introducing-basilisk.md index 4b861ef9..2c815af7 100644 --- a/website/src/blog/introducing-basilisk.md +++ b/website/src/blog/introducing-basilisk.md @@ -35,7 +35,7 @@ Python's typing tools took the opposite approach. [Four modes in Pyright](https: ## What every other tool gets wrong -The problem isn't technical capability. Pyright, at [~99% PEP conformance](https://github.com/python/typing/blob/main/conformance/results/results.html), is genuinely excellent at finding type errors when configured correctly. The problem is the default. +The problem isn't technical capability. Pyright is genuinely excellent at finding type errors when configured correctly. The problem is the default. When strictness is opt-in: - New projects start without it because there's no immediate pressure to add it @@ -48,9 +48,9 @@ The result is a codebase that *appears* to be using type checking but is actuall ## Basilisk's position -Basilisk's default *is* the Python typing spec — full PEP conformance, with no `--strict` flag to forget. And when you want more than the spec, it's one config change away: opt-in Basilisk rules require a type on every parameter, declare every return, and make `Any` explicit. +Basilisk enables its PEP-derived rules by default, with no `--strict` flag to forget. Its actual conformance is currently under integrity review after withdrawal of the former result. When you want checks beyond the spec, opt-in Basilisk rules can require a type on every parameter, declare every return, and make `Any` explicit. -This is not about making Python developers' lives harder. It's about making the safe path easy to reach. The spec-conformant baseline is the default; stricter checking is there the moment a team decides they want it — switched on in config, scoped per-project or per-path, never forced. +This is not about making Python developers' lives harder. It's about making the safe path easy to reach. The spec-derived rule set is the default; stricter checking is there the moment a team decides they want it — switched on in config, scoped per-project or per-path, never forced. Turning that stricter checking on for an existing codebase does require work — but it's work that surfaces real bugs. With Basilisk's annotation rules switched on, every BSK-0001 is a function where the type contract was never defined. A non-exhaustive `match` is a case silently ignored. These are not false positives — they are places where the type system was not being used. diff --git a/website/src/blog/openai-acquires-astral-what-it-means-for-basilisk.md b/website/src/blog/openai-acquires-astral-what-it-means-for-basilisk.md index c66794f5..b8e67b62 100644 --- a/website/src/blog/openai-acquires-astral-what-it-means-for-basilisk.md +++ b/website/src/blog/openai-acquires-astral-what-it-means-for-basilisk.md @@ -95,7 +95,7 @@ Basilisk's relationship to Astral is concrete and load-bearing: 1. **Our parser is Ruff's parser.** Basilisk depends on `ruff_python_parser`, `ruff_python_ast`, and `ruff_text_size`, pinned to an **immutable git commit** (`rev 7c645a9`, equal to tag `0.15.17`) on `astral-sh/ruff`. We pin a `rev`, not a tag, precisely so the version "can never be swapped out from under us." That code is MIT-licensed and already in our `Cargo.lock`. Nothing about this acquisition can reach back and change the bytes we build against. 2. **Our lint/format path shells out to the Ruff CLI** — `ruff==0.15.17`, pinned identically in CI and the dev container. Same story: a pinned, permissively licensed binary we control the version of. -3. **ty is now an OpenAI-backed competitor.** Astral's type checker, ty, occupies the same conceptual space as the Basilisk checker, and it will now have OpenAI's resources behind it. We take that seriously — but it sharpens, rather than threatens, what makes Basilisk different: **out-of-the-box PEP conformance**, one **complete LSP** (test explorer, debugging, profiling, autofixes) in a single extension, and a relentless march toward 100% PEP conformance. A faster-funded type checker validates the bet that Python deserves first-class, Rust-speed tooling — it doesn't make our differentiation any less true. +3. **ty is now an OpenAI-backed competitor.** Astral's type checker, ty, occupies the same conceptual space as the Basilisk checker, and it will now have OpenAI's resources behind it. We take that seriously. Basilisk combines typing-spec rules with one **complete LSP** (test explorer, debugging, profiling, autofixes) in a single extension. Its actual conformance is currently under integrity review, and the affected logic is being rebuilt rather than represented by the withdrawn result. 4. **The architecture bet is shared — and now vindicated.** Basilisk, like Astral's tools, is built in Rust on the Ruff AST with Salsa for incrementality. Astral proved that stack scales to millions of users. We made the same call independently. That's reassuring, not threatening. **Net effect on you, today:** zero. Your Basilisk install builds from pinned, MIT-licensed Ruff code and a pinned Ruff binary. The acquisition does not, and cannot, alter either. diff --git a/website/src/blog/python-315-typeform-fastapi-pydantic-annotations.md b/website/src/blog/python-315-typeform-fastapi-pydantic-annotations.md index 8cd9e9f9..2162f5be 100644 --- a/website/src/blog/python-315-typeform-fastapi-pydantic-annotations.md +++ b/website/src/blog/python-315-typeform-fastapi-pydantic-annotations.md @@ -3,7 +3,7 @@ layout: layouts/blog.njk title: "Python 3.15: The Type Hints FastAPI and Pydantic Actually Run" description: "Python 3.15 rc1 ships TypeForm, closed TypedDicts, and disjoint bases. Here is what the three new typing PEPs change for FastAPI and Pydantic code." date: 2026-08-04 -dateModified: 2026-08-04 +dateModified: 2026-08-06 author: The Basilisk Project image: /assets/images/blog/python-315-annotations-fastapi-pydantic.png imageAlt: "A translucent specification plate is scanned in cyan as it activates an orange-lit precision engine" @@ -26,7 +26,7 @@ faq: - q: "What does closed=True do on a TypedDict?" a: "PEP 728 adds closed and extra_items class arguments to TypedDict. A closed TypedDict does not allow extra keys beyond those declared in the class body, while extra_items allows arbitrary extra items whose values are of the specified type." - q: "Does Basilisk support the Python 3.15 typing features?" - a: "Yes. Basilisk passes the official python/typing conformance suite files covering all three PEPs — typeforms_typeform.py, typeddicts_extra_items.py, and directives_disjoint_base.py — as part of a 100% score across all 141 files, graded by the suite's own unmodified upstream harness." + a: "Basilisk's support status for these features is being revalidated. The previously published pass counts and overall conformance result are withdrawn because test-specific implementation logic made the result untrustworthy." --- Python 3.15 reaches its first release candidate today. [PEP 790](https://peps.python.org/pep-0790/), the 3.15 release schedule, puts rc1 on 2026-08-04 and the final release on 2026-10-01. The feature set is frozen; what is in the tree now is what ships in October. @@ -132,25 +132,15 @@ Here is the uncomfortable part. A frozen feature set in CPython is the *start* o `TypeForm`, `closed`, `extra_items`, and `disjoint_base` are runtime-importable in 3.15 whether or not the type checker in your editor understands what they mean. When it doesn't, you get the worst version of static typing: annotations that look precise, pass import, and are being interpreted by nothing. Meanwhile the runtime frameworks *are* interpreting them, so your editor and your production server now hold different beliefs about the same line of code. -That gap is exactly what the [official `python/typing` conformance suite](https://github.com/python/typing/blob/main/conformance/results/results.html) exists to measure — and it is the reason we treat that suite as the only scoreboard that counts. +That gap is part of what the [official `python/typing` conformance suite](https://github.com/python/typing/blob/main/conformance/results/results.html) exists to measure. Our integrity review has also shown that the suite alone is insufficient when an implementation has been developed against the exact fixtures; robustness and mutation testing are required as well. ## Where Basilisk stands Basilisk is an open-source Python type checker and language server that adds code intelligence, formatting, type-aware refactoring, testing, debugging, and CPU and memory profiling to VS Code, Cursor, and Windsurf, with the same Rust language-server core behind Zed and Neovim. -All three of the Python 3.15 typing PEPs have conformance test files in the official suite, and Basilisk passes all three: +All three Python 3.15 typing PEPs have conformance test files in the official suite. We previously published pass counts for those files and used them to assert support. **Those figures and that support claim are withdrawn.** Test-specific implementation logic elsewhere in the checker demonstrated that a pass against an exact fixture was not enough to establish a general implementation. -| Suite file | Feature | Required errors caught | Missed | False positives | -|---|---|---|---|---| -| `typeforms_typeform.py` | [`TypeForm`](https://typing.python.org/en/latest/spec/type-forms.html#typeform) | 16 | 0 | 0 | -| `typeddicts_extra_items.py` | `closed` / `extra_items` | 22 | 0 | 0 | -| `directives_disjoint_base.py` | [`disjoint_base`](https://typing.python.org/en/latest/spec/directives.html#disjoint-base) | 8 | 0 | 0 | - -Those three sit inside a 100% score across the whole suite: 141 of 141 files, 970 required errors caught, 0 missed, 0 false positives. - -How that number is produced matters more than the number. Every Basilisk CI run clones the tests *and* the harness fresh from the latest `python/typing` commit, builds a clean release binary from the current checkout, and runs the suite's own unmodified `conformance/src/main.py` against it in Basilisk's default configuration — no vendored scorer, no cached fixtures, no project config file in the tree. The result is graded by upstream's code, not ours, and it is published in the [official conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html). - -The false-positive column is the one to watch when a language adds features. A checker that does not know what `closed=True` means has two ways to be wrong: stay quiet about the extra key it should reject, or flag valid code it fails to understand. The second is worse, because it trains you to ignore the tool. Zero is the only acceptable figure there, and it is a ratchet in our build — it can go down, never up. +Basilisk's support status for `TypeForm`, `closed` / `extra_items`, and `disjoint_base` is therefore being revalidated as part of the clean reimplementation and integrity audit. Until those rules pass semantics-preserving mutations and broader cases that were not present in the suite, do not rely on the old table. See the [conformance correction](/docs/conformance/) for the publication bar we will apply to the replacement result. ## What to do before October @@ -165,4 +155,4 @@ The larger shift is worth naming. Python spent a decade treating annotations as Which raises the standard for the tools that read those annotations. If your type hints are going to run, something had better be checking them. -[Install Basilisk for VS Code](/docs/install-vscode/) · [See the official conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html) · [See the benchmarks](/docs/benchmarks/) +[Install Basilisk for VS Code](/docs/install-vscode/) · [Read the conformance correction](/docs/conformance/) · [Read the benchmark review notice](/docs/benchmarks/) diff --git a/website/src/blog/type-manipulation-pep-827.md b/website/src/blog/type-manipulation-pep-827.md index 5071babb..fb6f6fc2 100644 --- a/website/src/blog/type-manipulation-pep-827.md +++ b/website/src/blog/type-manipulation-pep-827.md @@ -182,7 +182,7 @@ PEP 827 also needs tightening before it is ready. The core ideas are compelling, There is precedent for splitting the work. PEP 827 builds on [`ParamSpec`](https://peps.python.org/pep-0612/), [variadic generics and `Unpack`](https://peps.python.org/pep-0646/), [TypedDict keyword arguments](https://peps.python.org/pep-0692/), and [deferred annotation evaluation](https://peps.python.org/pep-0649/). It also intersects with the draft [inline TypedDict proposal](https://peps.python.org/pep-0764/) and ongoing work on intersection types. The `**kwargs` inference and extended-callable pieces could move independently without waiting for the full recursive evaluator. -Basilisk will follow the draft, its reference implementations, and any conformance tests that emerge. The sensible order is to understand the inert introspection and construction operators first, then conditional evaluation and union normalization, and leave the most difficult recursive and class-mutating features until their semantics settle. The [`python/typing` conformance suite](https://github.com/python/typing/tree/main/conformance) remains the authority for what Basilisk claims to support. +Basilisk will follow the draft, its reference implementations, and any conformance tests that emerge. The sensible order is to understand the inert introspection and construction operators first, then conditional evaluation and union normalization, and leave the most difficult recursive and class-mutating features until their semantics settle. Any future support claim must be grounded in the typing specification, the official [`python/typing` conformance suite](https://github.com/python/typing/tree/main/conformance), and robustness testing that shows the implementation is not fitted to the exact fixtures. We are not going to claim support for a moving draft. If PEP 827, or a smaller proposal derived from it, enters the Python typing specification, Basilisk will implement it against the same public conformance process we use for the rest of the type system. diff --git a/website/src/docs/benchmarks.njk b/website/src/docs/benchmarks.njk index 37ba089f..97fae988 100644 --- a/website/src/docs/benchmarks.njk +++ b/website/src/docs/benchmarks.njk @@ -1,10 +1,10 @@ --- layout: layouts/docs.njk title: "Python Type Checker Performance Benchmarks" -description: "Measured whole-file CLI processing times for Basilisk, Pyright, mypy, ty, Pyrefly and zuban, with every benchmark fixture linked to its source." +description: "Historical Python type-checker benchmark figures, withdrawn from comparison while Basilisk audits and revalidates the methodology and results." keywords: basilisk benchmarks, python type checker speed, pyright vs basilisk, hyperfine, mypy ty pyrefly performance date: 2026-07-04 -dateModified: 2026-07-31 +dateModified: 2026-08-06 author: The Basilisk Project eleventyNavigation: key: Benchmarks @@ -13,21 +13,30 @@ permalink: /docs/benchmarks/ ---

    Performance benchmarks

    +

    + These benchmark figures are withdrawn pending an integrity review. + Do not use the values below to compare Basilisk with other tools. The conformance failure exposed + shortcomings in our review process, so we are auditing the benchmark runner, fixtures, data pipeline, + and every performance claim before publishing new results. The table remains visible only as a + historical record. We will replace it with newly measured figures when the methodology and results + have been revalidated through the integrity review. +

    + {% if benchmarks.hasData %}

    - Each row is one complete Python fixture file. Each value is the mean wall-clock + The table below is a withdrawn historical record. Each row is one complete Python fixture file. Each value is the mean wall-clock time for a fresh checker process to check that entire file, including startup, stub loading, analysis, and diagnostic output. It does not measure one typing rule. Lower is better.

    - Treat these numbers as indicative, not authoritative. They are currently produced by + Historical methodology note — not a validation of the figures. They were produced by running make bench on a contributor's local workstation, so every tool's timing carries that machine's background load — enough that a run taken under load can shift results by tens of - percent across the whole table at once. We are moving the benchmark onto dedicated, isolated - hardware in the CI pipeline so published figures come from a controlled environment and are - reproducible run to run. Until then, compare tools within a single run (they are all measured - back to back on the same machine) rather than comparing a figure here against one recorded elsewhere. + percent across the whole table at once. We will not treat that signal as publication evidence or a + CI pass/fail gate. Any replacement methodology must use controlled, repeatable conditions and state + its limits before new figures are published. No comparison should be made from this table while the + integrity review is open.

    {% import "benchmark-section.njk" as benchmark %} diff --git a/website/src/docs/comparison.md b/website/src/docs/comparison.md index 6193655c..1fdcea9e 100644 --- a/website/src/docs/comparison.md +++ b/website/src/docs/comparison.md @@ -4,7 +4,7 @@ title: "Best Python Type Checker? Basilisk vs Pyright & mypy" description: "Which Python type checker is best? Compare Basilisk, Pyright, mypy, ty, and Pyrefly on official PEP conformance, strictness, editor support, and benchmarks." keywords: best python type checker, basilisk vs pyright, python type checker comparison, mypy vs basilisk, ty, pyrefly date: 2026-02-28 -dateModified: 2026-03-31 +dateModified: 2026-08-06 author: The Basilisk Project eleventyNavigation: key: Comparison @@ -15,29 +15,17 @@ eleventyNavigation: There is no universal **best Python type checker** for every codebase. The right choice depends on what you value most: typing-spec conformance, mature framework plugins, editor integration, performance, or a complete language-server workflow. This comparison makes those tradeoffs explicit and links every changing score to its source. -The Python type checker landscape has changed significantly. The tools differ in how faithfully they implement the typing spec, in whether they're a complete language server or only a checker, and in speed, which we [measure and publish](/docs/benchmarks/) rather than assert. +The Python type checker landscape has changed significantly. The tools differ in how faithfully they implement the typing spec, in whether they're a complete language server or only a checker, and in speed. Basilisk's previously published performance measurements are currently [withdrawn pending review](/docs/benchmarks/). -On the [official python/typing conformance suite]({{ conformanceOfficial.snapshot.source }}), **Basilisk is the only type checker with a perfect {{ conformanceOfficial.byId.basilisk.pct }}% score** ({{ conformanceOfficial.byId.basilisk.passLabel }}/{{ conformanceOfficial.byId.basilisk.total }}), ahead of zuban ({{ conformanceOfficial.byId.zuban.pct }}%), Pyrefly ({{ conformanceOfficial.byId.pyrefly.pct }}%), Pyright ({{ conformanceOfficial.byId.pyright.pct }}%), ty ({{ conformanceOfficial.byId.ty.pct }}%), and mypy ({{ conformanceOfficial.byId.mypy.pct }}%), all graded on the same run. +

    Conformance correction: Basilisk's former result is withdrawn and its current percentage is temporarily unknown. Basilisk has been removed from the official results table at our request while affected logic is rebuilt and stress-tested beyond the exact suite fixtures. Do not use the old score or leaderboard position to compare these tools.

    ## The fundamental question Before comparing features and performance, there is one question that decides whether you can trust a checker's verdict at all: -**How much of the official typing specification does it actually implement?** +**How much of the official typing specification does it actually implement, beyond the exact tests used to measure it?** -| Tool | PEP conformance (official suite¹) | -|---|---| -| **Basilisk** | **{{ conformanceOfficial.byId.basilisk.pct }}% ({{ conformanceOfficial.byId.basilisk.passLabel }}/{{ conformanceOfficial.byId.basilisk.total }})** | -| zuban | {{ conformanceOfficial.byId.zuban.pct }}% | -| Pyrefly | {{ conformanceOfficial.byId.pyrefly.pct }}% | -| Pyright | {{ conformanceOfficial.byId.pyright.pct }}% | -| pycroscope | {{ conformanceOfficial.byId.pycroscope.pct }}% | -| ty | {{ conformanceOfficial.byId.ty.pct }}% | -| mypy | {{ conformanceOfficial.byId.mypy.pct }}% | - -Every score above is from **one identical run** of the official python/typing suite, which now grades Basilisk alongside every other checker. Basilisk tops it at a perfect {{ conformanceOfficial.byId.basilisk.pct }}%, the only tool on the board to do so. - -A checker that doesn't implement a spec feature can't judge code that uses it: it either misses real errors or invents false ones. Basilisk's **default** rule set *is* the typing spec: it runs the core PEP conformance rules and nothing else, and passes every file in the suite at our pinned commit. Rule selection is entirely config-driven, so the default is exactly the core PEP set, never more. +The [official results table](https://github.com/python/typing/blob/main/conformance/results/results.html) remains the source for checkers currently listed there. Basilisk is not currently listed. Its old figure failed robustness checks against semantics-preserving test mutations, so the honest answer for Basilisk is temporarily **unknown** while the affected implementation is replaced. See the [conformance correction](/docs/conformance/). Want checking stricter than the spec? Switch on the **opt-in Basilisk rules** in config. They're off by default and, by design, flag things the spec does *not* call errors (an unannotated parameter, say), so turning them on will actually *break* strict spec conformance. That's the point: they're yours to enable when your team wants more than the spec, not something forced on every project. @@ -52,7 +40,7 @@ Every tick, cross, and label below links to the primary source (official docs, r | Annotation quick-fix (inserts placeholder) | ✅ `: Any` / `-> None` ² | ❌ ³ | ❌ ⁴ | double-click inlay hint ⁵ | ❌ (code action) | | Auto-insert *inferred* types | ❌ | ❌ ³ | ❌ ⁴ | ❌ | ✅ CLI `pyrefly infer` ⁶ | | Opt-in rules beyond the spec | ✅ config | strict mode ⁷ | `--strict` ⁴ | severities only ⁸ | ✅ `strict` preset ⁹ | -| PEP conformance¹ | **{{ conformanceOfficial.byId.basilisk.pct }}%, #1, only perfect score** | {{ conformanceOfficial.byId.pyright.pct }}% | {{ conformanceOfficial.byId.mypy.pct }}% | {{ conformanceOfficial.byId.ty.pct }}% | {{ conformanceOfficial.byId.pyrefly.pct }}% | +| PEP conformance¹ | **Temporarily unknown; old result withdrawn** | See live results | See live results | See live results | See live results | | Implementation | Rust | TypeScript ³ | Python/C ⁴ | Rust ¹⁰ | Rust ¹¹ | | Runtime required | None | Node.js ³ | Python ⁴ | None ¹⁰ | None ¹¹ | | Completions, hover, goto | ✅ | ✅ ¹² | ❌ ⁴ | ✅ ¹³ | ✅ ¹⁴ | @@ -66,7 +54,7 @@ Every tick, cross, and label below links to the primary source (official docs, r **Sources:** -¹ Full-pass scores from one run of the [official python/typing conformance suite]({{ conformanceOfficial.snapshot.source }}), snapshot [python/typing@`{{ conformanceOfficial.snapshot.sha }}`]({{ conformanceOfficial.snapshot.commitUrl }}) ({{ conformanceOfficial.snapshot.dateLabel }}): basilisk {{ conformanceOfficial.byId.basilisk.version }}, pyright {{ conformanceOfficial.byId.pyright.version }}, mypy {{ conformanceOfficial.byId.mypy.version }}, ty {{ conformanceOfficial.byId.ty.version }}, pyrefly {{ conformanceOfficial.byId.pyrefly.version }}, zuban {{ conformanceOfficial.byId.zuban.version }}. Basilisk is the only checker at a perfect {{ conformanceOfficial.byId.basilisk.pct }}%. These scores drift as the tools improve, so each links to its live results folder rather than a frozen figure. +¹ See the [live official python/typing results](https://github.com/python/typing/blob/main/conformance/results/results.html) for checkers currently listed there. Basilisk requested removal after retracting its former result; its current percentage will be published only after the clean implementation passes robustness and mutation verification. ² Basilisk's quick-fix inserts a **placeholder** annotation (`: Any` on parameters and attributes, `-> None` on returns; empty-collection variables get `list[Any]` / `dict[str, Any]`) for you to replace with the real type. It does not infer types. See [Missing annotation rules](/docs/rules/missing-annotations/). @@ -116,12 +104,12 @@ Every tick, cross, and label below links to the primary source (official docs, r ## Pyright -**By Microsoft. TypeScript-based. {{ conformanceOfficial.byId.pyright.pct }}% PEP conformance on the official suite ([source]({{ conformanceOfficial.snapshot.source }})), behind Basilisk's perfect {{ conformanceOfficial.byId.basilisk.pct }}%.** +**By Microsoft. TypeScript-based. See its current entry in the [official conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html).** -Pyright was long the conformance front-runner and remains one of the most capable checkers. On the current official suite it scores {{ conformanceOfficial.byId.pyright.pct }}%, strong, but now behind Basilisk ({{ conformanceOfficial.byId.basilisk.pct }}%), zuban, and Pyrefly. It handles the vast majority of PEP typing features and has excellent performance for a TypeScript-based tool. +Pyright was long the conformance front-runner and remains one of the most capable checkers. It handles a broad range of PEP typing features and has a mature editor ecosystem. **What Pyright does well:** -- Strong PEP coverage ({{ conformanceOfficial.byId.pyright.pct }}% on the official conformance suite) +- Strong PEP coverage; see the live official conformance results - Excellent documentation and error messages - Deep VS Code integration via Pylance - Fast enough for interactive use in most codebases @@ -133,13 +121,13 @@ Pyright was long the conformance front-runner and remains one of the most capabl - Pylance (the VS Code extension) is proprietary: its richest features don't leave VS Code - No plugins, so there is no way to add framework-specific type intelligence -**When Pyright makes sense:** Basilisk now exceeds Pyright's conformance ({{ conformanceOfficial.byId.basilisk.pct }}% vs {{ conformanceOfficial.byId.pyright.pct }}% on the official suite) while adding a full LSP, integrated debugger, and profiler. Pyright remains a strong, mature option if you're already invested in the Microsoft VS Code ecosystem and don't mind the Node.js dependency. +**When Pyright makes sense:** Pyright remains a strong, mature option if you're already invested in the Microsoft VS Code ecosystem and don't mind the Node.js dependency. --- ## mypy -**The original. Python/C-based. {{ conformanceOfficial.byId.mypy.pct }}% on the official suite ([source]({{ conformanceOfficial.snapshot.source }})), versus Basilisk's perfect {{ conformanceOfficial.byId.basilisk.pct }}%.** +**The original. Python/C-based. See its current entry in the [official conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html).** mypy defined what Python type checking looks like. Its `--strict` flag was the reference implementation for what "strict" means in Python typing for years. @@ -150,7 +138,7 @@ mypy defined what Python type checking looks like. Its `--strict` flag was the r - Long history means most edge cases are handled **What mypy doesn't do:** -- Slowest cold single-file check of the tools in [our measured benchmarks](/docs/benchmarks/) (its incremental cache narrows the gap on re-checks) +- Requires a Python runtime for checking - Daemon mode (`dmypy`) is fragile under certain conditions - Not a language server, no completions, hover, or go-to-definition - Requires a Python runtime @@ -162,7 +150,7 @@ mypy defined what Python type checking looks like. Its `--strict` flag was the r ## ty (Astral) -**Built by the Ruff team. Rust + Salsa. {{ conformanceOfficial.byId.ty.pct }}% on the official suite ([source]({{ conformanceOfficial.snapshot.source }})), still maturing, well behind Basilisk's perfect {{ conformanceOfficial.byId.basilisk.pct }}%.** +**Built by the Ruff team. Rust + Salsa. See its current entry in the [official conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html).** ty is the most interesting new entrant. It's built by the same team that created Ruff (now the de facto Python linter), uses a Salsa-based incremental architecture, is built in Rust like Basilisk, and has Astral's engineering velocity behind it. @@ -173,17 +161,17 @@ ty is the most interesting new entrant. It's built by the same team that created - Sub-10ms incremental speed ([4.7ms on PyTorch](https://astral.sh/blog/ty), December 2025) **What ty doesn't do (yet):** -- Scores {{ conformanceOfficial.byId.ty.pct }}% on the [official python/typing conformance suite]({{ conformanceOfficial.snapshot.source }}), well behind Basilisk's perfect {{ conformanceOfficial.byId.basilisk.pct }}%; still maturing +- Its typing implementation is still maturing - Gradual typing by default - No integrated debugger or profiler -**When ty makes sense:** If you want to bet on Astral's velocity and can tolerate lower type coverage during the adoption period. ty may eventually become a major player; it's too early to depend on it for strict enforcement. +**When ty makes sense:** If you value Astral's tooling ecosystem and are comfortable adopting a rapidly evolving checker. --- ## Pyrefly (Meta) -**Production-tested at Instagram scale. Rust-based. {{ conformanceOfficial.byId.pyrefly.pct }}% PEP conformance on the official suite ([source]({{ conformanceOfficial.snapshot.source }})), behind Basilisk's perfect {{ conformanceOfficial.byId.basilisk.pct }}%.** +**Production-tested at Instagram scale. Rust-based. See its current entry in the [official conformance results](https://github.com/python/typing/blob/main/conformance/results/results.html).** Pyrefly was built by Meta to handle their Python codebase, one of the largest in the world. It emphasizes throughput ([1.85M LOC/sec on 166-core Meta infrastructure](https://pyrefly.org/)) over strict enforcement. @@ -206,15 +194,15 @@ Pyrefly was built by Meta to handle their Python codebase, one of the largest in Basilisk is not a faster version of an existing tool. It occupies a different position: -**Unique to Basilisk:** -1. The **only type checker with a perfect {{ conformanceOfficial.byId.basilisk.pct }}%** on the official python/typing suite, 100% PEP conformance out of the box, with **opt-in Basilisk rules** you switch on in config for checking stricter than the spec, they never run, and never touch the conformance score, unless you ask +**Basilisk combines:** +1. Typing-spec rules enabled by default, plus **opt-in Basilisk rules** for checking stricter than the spec. The conformance implementation is currently being rebuilt and its percentage is temporarily unknown. 2. Annotation quick-fixes, one-click code actions that insert a placeholder annotation (`: Any`, `-> None`) on unannotated code, so you can fill in the real type instead of finding the spot by hand 3. A complete, open-source LSP in every editor, completions, hover, go-to-definition, refactoring, debugging, and profiling, the same in VS Code, plus native Zed and Neovim extensions (Open VSX for Cursor, Windsurf, and others coming very soon; JetBrains planned), not just inside one proprietary VS Code extension 4. Integrated debugger and profiler brokered through the language server 5. WASM plugin system (planned), extensible without forking, secure by design **Where Basilisk is still growing:** -- Basilisk is under active development. It passes {{ conformance.scorePct }}% of the official suite ({{ conformance.pass }}/{{ conformance.total }}) at our [pinned commit](/docs/conformance/), counting errors *and* warnings, the strictest grading, with {{ conformance.fp }} false positives and {{ conformance.missed }} missed required errors. +- Basilisk is under active development. Its former conformance result is withdrawn; affected logic is being reimplemented from scratch and the [current percentage is temporarily unknown](/docs/conformance/). - Plugin ecosystem: mypy's Django and SQLAlchemy plugins are mature. Basilisk's WASM plugins are planned. -The recommendation: teams starting a new Python project get full PEP-conformant checking from Basilisk on day one, the only checker with a perfect score on the official suite, with the option to switch on stricter-than-spec rules whenever they're ready, and the same experience across every editor rather than one proprietary extension. +The recommendation: evaluate Basilisk for its integrated open-source editor workflow and test it against your own code. Do not choose it on the basis of the withdrawn conformance or benchmark figures. A new conformance result will be published when the clean implementation and robustness review are complete. diff --git a/website/src/docs/conformance.md b/website/src/docs/conformance.md index 55440061..bc0ec70f 100644 --- a/website/src/docs/conformance.md +++ b/website/src/docs/conformance.md @@ -1,103 +1,51 @@ --- layout: layouts/docs.njk -title: "Basilisk Scores 100% on the Official Python Typing Conformance Suite" -description: "Basilisk is the only Python type checker with a perfect 100% score — published on the official python/typing conformance results page, ahead of Pyright, mypy, Pyrefly and ty. Here's the proof and how it's measured." -keywords: pep conformance, python typing conformance results, 100% conformant type checker, best python type checker, basilisk conformance score, python/typing results +title: "Basilisk Conformance Results Are Withdrawn" +description: "Basilisk has withdrawn its former Python typing conformance claim. Its current percentage is temporarily unknown while affected logic is rebuilt and stress-tested beyond the suite." +keywords: basilisk conformance correction, python typing conformance, python/typing results, mutation testing date: 2026-06-23 -dateModified: 2026-07-07 +dateModified: 2026-08-06 author: The Basilisk Project eleventyNavigation: key: Conformance order: 10 --- -{% from "conformance-chart.njk" import chart %} -# Basilisk scores a perfect 100% +# Conformance results withdrawn -Basilisk is the **only Python type checker with a perfect {{ conformanceOfficial.byId.basilisk.pct }}% score** on the [**official `python/typing` conformance results**](https://github.com/python/typing/blob/main/conformance/results/results.html) — and it is **published right there on the Python typing repository's own results page**, graded on the same single run as every other checker. +

    Correction: Basilisk has retracted its former perfect-score claim. The result was not a trustworthy measure of specification conformance. We asked for Basilisk to be removed from the python/typing results table, and it has been removed. Basilisk's current conformance percentage is temporarily unknown.

    + +We found checker logic fitted to the exact contents of conformance test files rather than implementing the typing specification generally. For example, type-alias validation used prefixes and substrings from raw source text, including a special case for `eval(` because that spelling appeared in one test. Equivalent, valid mutations of the suite could therefore change Basilisk's result even though the typing behavior being tested had not changed. + +The official suite remains valuable, but a passing result from code developed against the exact fixtures is not enough evidence. We will not publish a replacement percentage until the affected logic has been reimplemented cleanly and shown to survive robustness testing.

    - Official python/typing results ↗ - The PR that added us ↗ + Current python/typing results ↗ + Full integrity audit ↗ + Original bug report ↗ + Integrity remediation tracker ↗ Python typing spec ↗ Conformance README ↗

    -## The official leaderboard - -Every score below comes from **one identical run** of the [official `python/typing` conformance suite](https://github.com/python/typing/blob/main/conformance/results/results.html) — the same suite and scorer the typing community uses to grade every checker. Basilisk tops it, and is the **only tool on the board at a perfect score**. - -
    -
    - Mean wall-clock time in milliseconds for each checker to process each complete fixture file + Withdrawn historical mean wall-clock times in milliseconds for each checker to process each complete fixture file; not for tool comparison
    - - -{%- for t in conformanceOfficial.ranked %} - - - - - -{%- endfor %} - -
    ToolBackerOfficial conformance
    {% if t.id == "basilisk" %}Basilisk{% else %}{{ t.name }}{% endif %}{{ t.org or "independent" }}{% if t.id == "basilisk" %}{{ t.pct }}% ({{ t.passLabel }}/{{ t.total }}){% else %}{{ t.pct }}%{% endif %}
    -
    - -

    Snapshot of results.html at python/typing@{{ conformanceOfficial.snapshot.sha }} ({{ conformanceOfficial.snapshot.dateLabel }}). These figures drift as the other tools improve, so every cell links to that tool's live results folder — check the current number yourself.

    - -## How it's measured - -We don't score ourselves against our own yardstick. The number above is produced by the **official `python/typing` harness**, run unmodified against the `basilisk` CLI built straight from the current checkout (`cargo build --release`), in its **default configuration**, with **every PEP conformance rule on and nothing else configured** — the same binary every install channel ships. A file passes only when the harness's diff is empty: every required error reported, and **nothing** reported on a line the suite doesn't mark. We count every diagnostic the checker emits — errors *and* warnings — so a single false positive fails the whole file. - -Today that is **{{ conformance.scorePct }}%** — **{{ conformance.pass }} of {{ conformance.total }}** test files passing, {{ conformance.caught }} required errors caught, **{{ conformance.fp }} false positives**, **{{ conformance.missed }} missed errors**. We run in lock step with `python/typing@main` (graded at [`{{ conformance.pinnedRefShort }}`](https://github.com/python/typing/tree/{{ conformance.pinnedRef }}/conformance){% if conformance.commitDate %}, {{ conformance.commitDate }}{% endif %}); a ratchet gate keeps the score from ever regressing, and an upstream test we fail blocks merge and release. - -Basilisk's **opt-in house-style rules** (require-annotation, redundant-annotation, missing-`@override`, explicit-`Any`) never run during scoring — a fresh install runs none of them, and enabling them would only *lower* the score, since the spec treats an unannotated value as *inferred*, not an error. "Stricter than the spec" and "conformant to the spec" are different goals; this score measures only the second. - -### Reproduce it yourself - -Basilisk is a **registered checker in the official suite** — `BasiliskTypeChecker` -lives in `python/typing`'s [`conformance/src/type_checker.py`](https://github.com/python/typing/blob/main/conformance/src/type_checker.py) — -so you run the real harness directly, with nothing to patch: - -```bash -# Clone python/typing FRESH, run its OWN harness against the basilisk binary, -# and regenerate conformance/conformance_status.csv from the real results. -python3 conformance/run_conformance.py --bin target/release/basilisk -``` - -Or drive the upstream harness by hand against any `basilisk` on your PATH: +## What is happening now -```bash -git clone --depth 1 https://github.com/python/typing -BASILISK_BIN=$(which basilisk) python typing/conformance/src/main.py --only-run basilisk -``` +The offending implementation is being removed, and the affected behavior is being rebuilt from the specification and structured syntax rather than from test-file text. The review also covers similar source-text predicates, duplicated logic, permissive fallbacks, and other places where a narrow fixture could have stood in for a general implementation. -The runner lives in [`conformance/run_conformance.py`](https://github.com/Nimblesite/Basilisk/blob/main/conformance/run_conformance.py); it clones the suite fresh, runs the unmodified upstream harness, and never scores anything itself. +This is active remediation, not an indefinite withdrawal. We expect to establish a defensible result after the clean implementation and validation work is complete. If that result is lower than the former claim, we will publish the lower result. -## How the score got honest +## The new publication bar -We'd rather say this plainly than paper over it. An earlier in-repo script inflated the figure by **excluding some diagnostic codes from the diff and not counting false positives at all**. We threw it out and adopted the official `python/typing` scoring semantics on the real default CLI. The chart is read straight from the **git history of `conformance/conformance_status.csv`** at build time — one point per commit that changed it, including that correction. +A future conformance result must satisfy all of these checks: -{{ chart(conformance, { - "label": "Conformance score over time", - "heading": "From an in-repo script to the official harness", - "prevLegend": "Earlier in-repo script — excluded codes, ignored false positives (not the official measure)", - "officialLegend": "Official python/typing harness on the real default CLI", - "dropNote": "Early points came from an in-repo script that excluded diagnostic codes and didn’t count false positives; later points use the official python/typing scoring semantics on the real default CLI. Today’s official figure is " + conformance.chart.current.score + "% — a measurement that got honest, not a checker that got worse.", - "caption": "Each dot is a real commit to conformance/conformance_status.csv, recomputed every build. Hover a point for its date, commit, score, and false-positive count." -}) }} +1. Run the official, unmodified `python/typing` harness against Basilisk's default configuration. +2. Apply AST-preserving mutations such as consistent renaming of type variables and equivalent spelling changes. A rule is not accepted if those changes move its result. +3. Pass independent off-suite cases derived from the typing specification and real-world code rather than from the upstream fixture text. +4. Add regression and mutation tests for every test-specific implementation found by the audit. +5. Publish the robustness and off-suite results alongside the suite percentage and make the methodology reproducible. -## Where each category stands today +Until that work is complete, old conformance tables, charts, category scores, pass counts, and false-positive totals are withdrawn and should not be cited as Basilisk's current state. -Read live from `conformance/conformance_status.csv` at build time: +## Related performance figures -
    - - - -{%- for cat in conformance.categories %} - -{%- endfor %} - -
    CategoryPassingScore
    {{ cat.label }}{{ cat.pass }} / {{ cat.total }}{{ cat.pct }}%
    -
    - +The same review failure means our published benchmark figures also require revalidation. They are retained only as a clearly labelled historical record on the [benchmarks page](/docs/benchmarks/) and must not be used to compare Basilisk with other tools. New performance figures will be published only after the methodology and results have passed the integrity review. diff --git a/website/src/docs/index.md b/website/src/docs/index.md index 58d6703b..208c264b 100644 --- a/website/src/docs/index.md +++ b/website/src/docs/index.md @@ -4,7 +4,7 @@ title: "Python Language Server & Type Checker — Basilisk Docs" description: "Install, configure, and use Basilisk: an open-source Python type checker and language server in Rust, with refactoring, formatting, debugging, profiling, and editor integrations." keywords: basilisk, python language server, python type checker, python typing, lsp, vs code, cursor, zed, neovim, rust date: 2026-02-28 -dateModified: 2026-08-04 +dateModified: 2026-08-06 author: The Basilisk Project eleventyNavigation: key: Introduction @@ -13,9 +13,9 @@ eleventyNavigation: # Introduction -Basilisk is an open-source **Python type checker and language server** built in Rust. It adds code intelligence, formatting, type-aware refactoring, testing, debugging, and CPU and memory profiling to your editor, and its default rule set is the Python typing specification — no `--strict` flag to remember. +Basilisk is an open-source **Python type checker and language server** built in Rust. It adds code intelligence, formatting, type-aware refactoring, testing, debugging, and CPU and memory profiling to your editor. Its default rules are intended to implement the Python typing specification, and that implementation is currently undergoing an integrity review. -It scores **{{ conformanceOfficial.byId.basilisk.pct }}%** on the [official `python/typing` conformance results]({{ conformanceOfficial.snapshot.source }}) — {{ conformanceOfficial.byId.basilisk.passLabel }} of {{ conformanceOfficial.byId.basilisk.total }} test files{% if conformanceOfficial.basiliskIsSolePerfect %}, the only checker on that page at a perfect score{% endif %}. See [how it's measured](/docs/conformance/), or the [type checker comparison](/docs/comparison/) for how the tools differ. +**Conformance correction:** Basilisk's former result is withdrawn, its current percentage is temporarily unknown, and it has been removed from the official `python/typing` results at our request. We are rebuilding affected logic from scratch and will publish a new result after robustness and mutation verification. Read the [full correction](/docs/conformance/). Extensions ship for **VS Code**, **Cursor**, **Windsurf**, **Zed**, and **Neovim**; any editor that speaks the Language Server Protocol can use the same server. JetBrains support is planned. Feature coverage varies per editor — see [the integration matrix](/docs/installation/#integration-status-by-editor). @@ -23,7 +23,7 @@ Extensions ship for **VS Code**, **Cursor**, **Windsurf**, **Zed**, and **Neovim [Pylance](https://marketplace.visualstudio.com/items?itemName=ms-python.vscode-pylance), the default Python extension in VS Code, is [proprietary](https://github.com/microsoft/pylance-release/blob/main/FAQ.md) — you cannot inspect, modify, or redistribute it. [Pyright](https://microsoft.github.io/pyright/#/features), the open-source checker underneath it, is a type checker only: completions, hover, go-to-definition, and refactoring come from the proprietary Pylance layer. mypy, ty, and Pyrefly are checkers too, so a full workflow means assembling a language server, a debugger, and a profiler alongside them, then keeping that stack in step across a team. -Basilisk puts type checking, language features, formatting, debugging, and profiling in one open-source binary, with the typing spec as the default rule set. Checking stricter than the spec is available as opt-in rules you enable in configuration. +Basilisk puts type checking, language features, formatting, debugging, and profiling in one open-source binary. Typing-spec rules are enabled by default; checking stricter than the spec is available as opt-in rules you enable in configuration. Whether those default rules implement the specification correctly is the subject of the current audit. ## What Basilisk is @@ -33,7 +33,7 @@ Basilisk puts type checking, language features, formatting, debugging, and profi - An **integrated debugger** — press F5 to debug Python with breakpoints, stepping, variable inspection, and watch expressions, brokered by the Basilisk LSP. Requires `debugpy` in your project environment. See [Debugging](/docs/debugging/) - An **integrated profiler** — sampling CPU profiler with inline heatmap annotations, flame graphs, memory leak detection, and reference graphs. See [Profiler](/docs/profiler/) - A **built-in formatter** — the Ruff formatter compiled into the binary, plus native import organizing. See [Formatting](/docs/formatting/) -- A **type checker whose default rule set is the typing spec** — every PEP rule runs on `basilisk check`; opt-in Basilisk rules run on `basilisk analyze` once you enable them +- A **type checker with PEP-derived rules enabled by default** — those rules run on `basilisk check`; opt-in Basilisk rules run on `basilisk analyze` once you enable them - **Standard-library types with no setup** — a complete typeshed `stdlib/` tree is compiled into the binary and checking never downloads anything. Pin an exact `python/typeshed` commit, or a typeshed distribution by wheel SHA-256, and it is verified offline against your local store - A **CLI for CI** — `basilisk check` exits 1 when errors are found; `basilisk format --check` exits 1 when a file would change - **uv integration** — workspace detection, lock-file parsing, and package management commands @@ -51,9 +51,9 @@ Basilisk puts type checking, language features, formatting, debugging, and profi - Not a Python runtime or package manager — running, testing, debugging, and memory profiling use your project's own interpreter - Not tied to one editor — the same server backs VS Code, Cursor, Windsurf, Zed, and Neovim, though what each editor surfaces differs -## Conformant by default, configurable from there +## Typing-spec rules by default, configurable from there -Basilisk's behaviour is decided entirely by **configuration**, and the default configuration is exactly the **core PEP conformance rule set** — the same rules the official typing-conformance suite grades. Out of the box you get a checker that follows the spec, with no flags to remember. +Basilisk's behaviour is decided entirely by **configuration**, and the default configuration enables the **core PEP rule set** — the rules the official typing-conformance suite grades. These rules aim to follow the specification with no strictness flag required, but the withdrawn result means their actual conformance is temporarily unknown while they are audited and reimplemented where necessary. Stricter-than-spec checking is **opt-in**. Basilisk also ships extra rules the spec doesn't define — *require an annotation* on every parameter and return, a redundant-annotation warning, a missing-`@override` nudge, an explicit-`Any` nudge. They stay **off** until you enable them in config. Because they flag code the spec considers valid, turning them on deliberately trades strict spec conformance for a stricter standard of your team's choosing — a per-project choice, never a default. @@ -66,17 +66,17 @@ Configuration is also where you relax rules for the paths that need it — place "imports_unresolved" = "info" ``` -This keeps the default honest — pure spec conformance — while letting each team dial strictness exactly where they want it. +This keeps the default focused on spec-derived rules while letting each team dial additional strictness exactly where they want it. ## Project status -Basilisk is under **active development** — the core checker, LSP server, and editor extensions are all working, and it is the only checker with a perfect score on the official python/typing conformance suite. Autocomplete, go-to-definition, hover, diagnostics, inlay hints, refactoring, debugging, and profiling are shipping today. +Basilisk is under **active development** — the core checker, LSP server, and editor extensions are working. Its former conformance result is withdrawn while affected checker logic is rebuilt and verified. Autocomplete, go-to-definition, hover, diagnostics, inlay hints, refactoring, debugging, and profiling are shipping today. | Phase | Milestone | Status | |---|---|---| | 1 | Parser, resolver, type checker, CLI | Complete | | 2 | LSP server, editor extensions (VS Code, Cursor, Zed, Neovim) | Complete | -| 3 | Expanded rule set, PEP conformance ({{ conformance.scorePct }}% against `python/typing@main`), gradual adoption | In progress | +| 3 | Clean PEP-rule reimplementation, robustness and mutation verification, gradual adoption | In progress | | 4 | WASM plugins, Django/Pydantic/SQLAlchemy | Planned | | 5 | SARIF/JUnit output, JetBrains extension | Planned | | 6 | Plugin marketplace, community stubs, ecosystem | Planned | diff --git a/website/src/docs/install-vscode.md b/website/src/docs/install-vscode.md index ed28d5c1..bccd263f 100644 --- a/website/src/docs/install-vscode.md +++ b/website/src/docs/install-vscode.md @@ -25,13 +25,13 @@ The extension is published to the **[VS Code Marketplace](https://marketplace.vi Open a Python file and Basilisk activates automatically — diagnostics, completions, hover, go-to-definition, rename, refactoring, formatting, debugging (F5), and profiling. -![Basilisk in VS Code — PEP-conformant type errors shown inline with red squiggles and listed in the Problems panel](/assets/images/vscode-diagnostics.png) +![Basilisk in VS Code — type diagnostics shown inline with red squiggles and listed in the Problems panel](/assets/images/vscode-diagnostics.png) -*PEP-conformant diagnostics the moment you open a file — no configuration.* +*Type diagnostics appear when you open a file, with no additional strictness configuration.* ## Is Basilisk the best Python VS Code extension for you? -No Python extension is best for every project. Basilisk is designed for developers who want one open-source extension for typing-spec-conformant checking, completions, navigation, refactoring, formatting, debugging, and profiling — with the same language server available outside VS Code. If your project depends on mature mypy framework plugins or you prefer Pylance's established VS Code-only workflow, review the [Python type checker comparison](/docs/comparison/) before switching. +No Python extension is best for every project. Basilisk is designed for developers who want one open-source extension for typing-spec rules, completions, navigation, refactoring, formatting, debugging, and profiling — with the same language server available outside VS Code. Its current conformance percentage is temporarily unknown during the [integrity review](/docs/conformance/). If your project depends on mature mypy framework plugins or you prefer Pylance's established VS Code-only workflow, review the [Python type checker comparison](/docs/comparison/) before switching. ## The binary is bundled — no separate install diff --git a/website/src/docs/migration.md b/website/src/docs/migration.md index ef0adb6d..e0c6c6ea 100644 --- a/website/src/docs/migration.md +++ b/website/src/docs/migration.md @@ -13,8 +13,10 @@ eleventyNavigation: # Migration Guide -Basilisk's unconfigured default enables its complete core PEP rule set. Extra -Basilisk rules—required annotations, explicit-`Any` policy, required +Basilisk's unconfigured default enables every currently registered PEP-tagged +rule. That is a configuration property, not a claim that the implementation is +complete or conformant; the current level is temporarily unknown during the +[integrity remediation](/docs/conformance/). Extra Basilisk rules—required annotations, explicit-`Any` policy, required `@override`, style, redundancy, dependency hygiene, and stub hygiene—are opt-in. Migration therefore means choosing the policy you want, checking the project, and recording narrow exceptions for the debt you cannot resolve yet. diff --git a/website/src/docs/quick-start.md b/website/src/docs/quick-start.md index 9635a0a0..3657f2fd 100644 --- a/website/src/docs/quick-start.md +++ b/website/src/docs/quick-start.md @@ -1,7 +1,7 @@ --- layout: layouts/docs.njk title: "Quick Start — Type-Check Your First File in 5 Minutes" -description: "Get started with Basilisk in 5 minutes. Install the VS Code extension, run your first type check, and see PEP-conformant Python diagnostics in action." +description: "Get started with Basilisk in 5 minutes. Install the VS Code extension, run your first type check, and inspect Python diagnostics in your editor." keywords: basilisk, quick start, best python type checker, python language server, type checking, tutorial, vs code date: 2026-02-28 dateModified: 2026-07-14 @@ -80,10 +80,12 @@ error[names_unbound]: Function `describe` returns `label` but `label` may be unb Found 3 diagnostics (3 errors). ``` -Out of the box, Basilisk enables the complete PEP typing-spec rule set, and -every violation is an **error**. Nothing here is house style — this is the -[Python type system specification](https://typing.python.org/en/latest/spec/index.html), -enforced. +Out of the box, Basilisk enables every currently registered PEP-tagged rule at +**error** severity; optional house rules are separate. That describes the +default configuration, not completeness or correctness. Basilisk's actual +conformance level is temporarily unknown while affected rules are rebuilt from +the [Python type system specification](https://typing.python.org/en/latest/spec/index.html) +and independently validated. See the [conformance correction](/docs/conformance/). ## Step 2 — Fix the errors diff --git a/website/src/docs/releases.njk b/website/src/docs/releases.njk index a2be5620..f050aafc 100644 --- a/website/src/docs/releases.njk +++ b/website/src/docs/releases.njk @@ -4,7 +4,7 @@ title: "Basilisk Releases — Downloads & Changelog" description: "Every published Basilisk release — version, date, release notes, and downloadable binaries and editor extensions — generated at build time straight from GitHub Releases." keywords: basilisk releases, download basilisk, changelog, release notes, python language server downloads, vsix date: 2026-06-23 -dateModified: 2026-06-23 +dateModified: 2026-08-06 author: The Basilisk Project # English-only — the notes come verbatim from GitHub Releases, so opt this page # out of the language cluster (no /zh/ hreflang or switcher link that would 404). @@ -16,6 +16,14 @@ permalink: /docs/releases/ ---

    Releases

    +

    + Historical release notes may contain withdrawn conformance or benchmark claims. + Those claims are preserved verbatim as part of the changelog, but they are not current evidence and + must not be relied on. Basilisk's conformance percentage is temporarily unknown and its benchmark + figures are under review. Read the conformance correction and + benchmark notice for the current status. +

    + {%- if releases.hasData %}

    All {{ releases.count }} published Basilisk releases, generated at diff --git a/website/src/errors/error.njk b/website/src/errors/error.njk index 180fd385..523a4410 100644 --- a/website/src/errors/error.njk +++ b/website/src/errors/error.njk @@ -16,7 +16,7 @@ pagination: permalink: "/errors/{{ rule.code }}/" eleventyComputed: title: "{{ rule.code }}: {{ rule.summary | safe }} — Basilisk" - description: "What Basilisk's {{ rule.code }} ({{ rule.summary | safe }}) diagnostic means and how to fix it — the PEP-conformant Python type checker." + description: "What Basilisk's {{ rule.code }} ({{ rule.summary | safe }}) Python diagnostic means, why it appears, and how to fix it." --- {% set stem = examples[rule.code] %} {% set ruleTag = rule.tags[1] | default('core') %} diff --git a/website/src/index.njk b/website/src/index.njk index db8f12d9..a5f1710c 100644 --- a/website/src/index.njk +++ b/website/src/index.njk @@ -1,22 +1,13 @@ --- layout: layouts/base.njk -title: "Basilisk — Fast Python Type Checker & Language Server" -description: "Basilisk is an open-source Python type checker and language server in Rust: 100% on the official Python typing conformance suite, fastest in our benchmark." +title: "Basilisk — Python Type Checker & Language Server" +description: "Basilisk is an open-source Python type checker and language server in Rust. Its conformance and benchmark results are withdrawn during an integrity review." keywords: "python type checker, python typing, python type checking, python language server, python type checker comparison, python type checker rust, typing conformance, type checker benchmark" -imageAlt: "Basilisk Python type checker and language server, with published conformance results and cold-check benchmarks" -dateModified: 2026-08-03 +imageAlt: "Basilisk Python type checker and language server" +dateModified: 2026-08-06 permalink: / --- -{% set hasCompleteBenchmark = benchmarks.hasData - and benchmarks.toolMedians.ms.basilisk - and benchmarks.toolMedians.ms.pyright - and benchmarks.toolMedians.ms.mypy - and benchmarks.toolMedians.ms.ty - and benchmarks.toolMedians.ms.pyrefly - and benchmarks.toolMedians.ms.zuban %} -{% set basiliskIsFastest = hasCompleteBenchmark and benchmarks.toolMedians.fastest == 'basilisk' %} -

    @@ -24,20 +15,14 @@ permalink: / Python type checker · language server

    - {% if conformanceOfficial.basiliskIsSolePerfect %}The only Python type checker that scores {{ conformanceOfficial.basilisk.pct }}% on the official Python typing suite.{% elif conformance.hasData %}A Python type checker that scores {{ conformance.scorePct }}% on the official Python typing suite.{% else %}A Python type checker built for speed and conformance.{% endif %} - {% if basiliskIsFastest %}And the fastest we’ve benchmarked.{% endif %} + An open-source Python type checker and language server, built in Rust.

    - Basilisk is an open-source Python type checker and language server built in Rust. - {% if conformance.hasData and conformance.pass == conformance.total %} - {% if conformanceOfficial.basiliskIsSolePerfect %}It is the only checker that passes{% else %}It passes{% endif %} - every file of the official python/typing conformance suite, - with {{ conformance.caught }} required errors caught and {{ conformance.fp }} false positives. - {% elif conformance.hasData %}It scores - {{ conformance.scorePct }}% on the official python/typing conformance suite.{% endif %} - {% if basiliskIsFastest %}It also records the - lowest median cold full-file CLI time of any checker in our published benchmark.{% endif %} + We have withdrawn both our former conformance claim and our published benchmark figures. + Basilisk was removed from the official python/typing results at our request, + and its current conformance percentage is temporarily unknown while we replace + test-specific implementations and verify the new work with robustness and mutation testing.

    @@ -67,40 +52,30 @@ permalink: /
    - {% if conformance.hasData or benchmarks.hasData %} -
    - {% if conformance.hasData %} +
    - {{ conformance.scorePct }}% - - {% if conformanceOfficial.basiliskIsSolePerfect %}Only listed checker with a perfect official score{% else %}Official typing conformance score{% endif %} - + Unknown + Current typing conformance - {{ conformance.pass }} of {{ conformance.total }} files pass the suite’s unmodified harness, - with {{ conformance.missed }} missed required errors and {{ conformance.fp }} false positives. - Official results, {{ conformanceOfficial.snapshot.dateLabel }}. + The former result is retracted. At our request, Basilisk has been + removed from the official results table. + We will publish a new result when the clean implementation is robust to semantics-preserving mutations.
    - {% endif %} - {% if benchmarks.hasData and benchmarks.toolMedians.ms.basilisk %}
    - {{ benchmarks.toolMedians.ms.basilisk }} ms - - {% if basiliskIsFastest %}Fastest in our published cold-check benchmark{% else %}Published cold-check benchmark{% endif %} - + Withdrawn + Published benchmark figures - Basilisk’s median across {{ benchmarks.rows | length }} synthetic, single-file fixtures on {{ benchmarks.meta.machine }}. - Fresh-process CLI timing; not project throughput or editor latency. + The benchmark numbers are also under integrity review and should not be used to compare tools. + The historical table remains available for transparency while the methodology and results are revalidated.
    - {% endif %}

    - Conformance uses the official Python typing harness; performance is self-measured and reproducible. - Verify the conformance score → - Read the benchmark methodology → + Both sets of figures are withdrawn pending a clean reimplementation and integrity review. + Read the conformance correction → + Read the benchmark notice →

    - {% endif %}
    diff --git a/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md b/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md index 54e6ec15..f2e2f9c6 100644 --- a/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md +++ b/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md @@ -1,37 +1,40 @@ --- layout: layouts/blog.njk -title: "Basilisk 在 Python 类型符合性测试套件上达到 100%" -description: "Basilisk 现已进入官方 python/typing 符合性结果，取得完美的 100%，是唯一达到这一成绩的 Python 类型检查器。本文解释这意味着什么。" +title: "已撤回：Basilisk 此前的类型符合性结果" +description: "撤回 Basilisk 此前的 Python typing 符合性声明，说明该结果为何不可信，以及如何重新实现并验证受影响逻辑。" date: 2026-07-11 +dateModified: 2026-08-06 author: Christian Findlay -image: /assets/images/blog/basilisk-100-conformance.png -imageAlt: "Python 类型检查器符合性排行榜，显示 Basilisk 取得完美的 100% 分数" +image: /assets/images/og-image.png +imageAlt: "Basilisk Python 类型检查器与语言服务器；结果正在接受完整性审查" imageWidth: 1200 -imageHeight: 675 +imageHeight: 630 tags: - Python typing category: announcements lang: zh -excerpt: "本周 Basilisk 加入了官方 python/typing 符合性结果，并取得了完美的分数。它是排行榜上唯一达到 100% 的 Python 类型检查器。本文解释这个数字究竟意味着什么、榜上还有谁，以及我们为什么不会过度宣传它。" +excerpt: "Basilisk 已撤回此前的符合性声明，并请求从官方结果中移除。本文仅作为已撤回公告的历史记录保留。" keywords: python类型检查器, python类型符合性, python/typing符合性结果, basilisk, mypy, pyright, ty, pyrefly, zuban, pep符合性, 严格类型 faq: - q: "哪个 Python 类型检查器的符合性分数最高？" - a: "在官方 python/typing 符合性结果中，Basilisk 0.27.0 取得完美的 100%（141 项测试中通过 141 项）。它是榜上唯一达到 100% 的检查器。zuban、Pyrefly 和 Pyright 紧随其后，均在 96% 以上。竞品分数会随着这些工具的改进而变化，因此请查看每个工具的实时结果文件夹以获取当前数字。" + a: "Basilisk 目前不在官方 python/typing 结果中。此前的结果已撤回；在受影响逻辑完成重新实现和验证之前，实际百分比暂时未知。其他工具请查看官方实时结果表。" - q: "什么是 python/typing 符合性测试套件？" - a: "这是由 Python Typing 社区维护的官方测试套件，用于衡量类型检查器对 Python 类型规范的实现有多忠实。每个检查器都针对同一组测试运行，并由该套件自己的评分工具评分。结果发布在 github.com/python/typing 的 conformance/results 下。" + a: "这是由 Python Typing 社区维护的官方测试套件。它使用自己的评分工具记录检查器在确切测试用例上的行为。这是有价值的证据，但原始套件结果本身不能证明完整规范已被忠实实现；还必须通过保持语义的变异和独立的套件外用例验证。" - q: "100% 的符合性分数是否等于是最好的类型检查器？" - a: "不。python/typing 的维护者明确表示，符合性不应成为选择类型检查器的主要依据，因为它无法反映速度、编辑器集成、错误信息质量或生态系统支持。符合性只衡量规范正确性。它是一个重要的输入，但不是全部决定因素。" + a: "不。套件分数只描述被覆盖的测试用例；正如 Basilisk 此次撤回所证明的，它本身不能证明规范实现正确。它也无法反映编辑器集成、错误信息质量、生态系统支持或经过独立验证的性能。" - q: "Basilisk 的符合性分数是如何测量的？" - a: "Basilisk 由 python/typing 套件自己未经修改的评分工具评分，针对默认配置的 Basilisk CLI 运行，开启所有规范规则。没有内置的自制评分器，也没有特殊配置。这个分数就是用户开箱即用所得到的，由为榜上所有其他检查器评分的同一套代码评出。" + a: "Basilisk 目前没有可发布的符合性分数。受影响实现完成重建后，未来结果必须同时通过未经修改的 python/typing 评分工具、保持语义的变异测试，以及依据规范独立设计的套件外用例。" --- +> **撤回说明——2026 年 8 月 6 日：**我们撤回本文中的所有符合性声明。Basilisk 源码中存在针对确切符合性测试用例实现的逻辑，因此此前的满分结果不能证明规范符合性。我们请求从官方结果表中移除 Basilisk，现已完成移除。在删除有问题的实现、根据规范重新构建并通过保持语义的变异测试之前，当前百分比暂时未知。下方原文仅作为公开历史记录保留；其中的得分、排名、通过数量和结论均不可依赖。请阅读[完整更正](/zh/docs/conformance/)。 + Python 现在拥有一个真正出色的类型系统，而大多数开发者仍然没有意识到这一点。Python 类型检查器的工作方式很像 TypeScript 编译器。经过类型检查的 Python 之于普通 Python，就如同 TypeScript 之于 JavaScript。类型注解已经存在于语言中十年了，规范已经成熟，工具也已经跟上。 从来悬而未决的问题从不是 Python 的类型系统是否足够好。而是任何一个具体工具究竟对它的实现有多忠实。 -本周我们为 Basilisk 得到了一个客观的答案。它被添加进了[官方 python/typing 符合性结果]({{ conformanceOfficial.snapshot.source }})，并取得了完美的 {{ conformanceOfficial.basilisk.pct }}%（{{ conformanceOfficial.basilisk.total }} 项测试中通过 {{ conformanceOfficial.basilisk.passLabel }} 项）。它是榜上唯一达到 {{ conformanceOfficial.basilisk.pct }}% 的类型检查器。 +发布本文时，我们以为 Basilisk 得到了一个客观答案。它被添加进这份[官方 python/typing 符合性结果的固定快照]({{ conformanceOfficial.historical.snapshot.snapshotUrl }})，当时的运行报告了 {{ conformanceOfficial.historical.basilisk.pct }}%（{{ conformanceOfficial.historical.basilisk.total }} 项测试中通过 {{ conformanceOfficial.historical.basilisk.passLabel }} 项）。该结果现已撤回。 -我们为此感到自豪。同时我们也不会过度宣传它，本文接下来的部分会同时解释这句话的两半。 +我们曾为此感到自豪。完整性审计证明这个结论是错误的。 ## 为什么符合性测试套件才是那个重要的裁判 @@ -39,17 +42,17 @@ Python 现在拥有一个真正出色的类型系统，而大多数开发者仍 [python/typing 符合性测试套件](https://github.com/python/typing/tree/main/conformance)是 Python 生态系统中最接近客观裁判的东西。它由 Python Typing 社区维护，将真正的类型规范编码为一组测试文件，并用同一套评分工具、同样的测试来运行每一个参与的检查器。没有人给自己打分。是这个套件在一次运行中把它们全部一起评分。 -这正是结果之所以有意义的原因。当 Basilisk 在那个页面上显示 {{ conformanceOfficial.basilisk.pct }}% 时，那不是我们的声称。那是套件的测量结果，由[给其他所有人评分的同一套工具](https://github.com/python/typing/blob/main/conformance/README.md)产出。 +我们曾认为这使结果具有意义。套件确实使用共享评分工具产出了该数字，但我们的代码针对确切测试文本进行了适配，因此该测量无法支撑我们得出的结论。 -Basilisk 是通过 [python/typing 拉取请求 #2316](https://github.com/python/typing/pull/2316)（"Add Basilisk to conformance results"）加入那次运行的，该请求于 2026 年 7 月 6 日合并。从那一刻起，Basilisk 就在公开场合、以与其他所有工具相同的条件被衡量，你随时都可以自己核对这个数字。 +Basilisk 是通过 [python/typing 拉取请求 #2316](https://github.com/python/typing/pull/2316)（"Add Basilisk to conformance results"）加入那次运行的，该请求于 2026 年 7 月 6 日合并。撤回结果后，我们请求将其移除；Basilisk 已不再出现在实时结果表中。 -## 当前的排行榜 +## 当时发布的历史排行榜快照 -以下是当前排行榜，转录自 {{ conformanceOfficial.snapshot.dateLabel }} 发布的[官方结果]({{ conformanceOfficial.snapshot.source }})。每个分数都链接到该工具的实时结果文件夹，因为这些数字会随着每个工具的改进而变化，你应当始终能核对当前数字，而不是相信某个快照。 +以下是原公告在 {{ conformanceOfficial.historical.snapshot.dateLabel }} 使用的排行榜快照。它不是当前结果，Basilisk 这一行也已撤回。当前仍列出的工具请查看[官方实时结果]({{ conformanceOfficial.historical.snapshot.source }})。 | 排名 | 类型检查器 | 背后团队 | 符合性 | |---|---|---|---| -{%- for t in conformanceOfficial.ranked %} +{%- for t in conformanceOfficial.historical.ranked %} | {{ t.rank }} | [{{ t.name }} {{ t.version }}]({{ t.resultsUrl }}) | {{ t.org | default("独立") }} | **{{ t.pct }}%** ({{ t.passLabel }}/{{ t.total }}) | {%- endfor %} @@ -57,9 +60,9 @@ Basilisk 是通过 [python/typing 拉取请求 #2316](https://github.com/python/ [Pyright](https://github.com/microsoft/pyright) 由微软开发。[Pyrefly](https://github.com/facebook/pyrefly) 由 Meta 构建。[ty](https://github.com/astral-sh/ty) 由 Astral 构建，也就是 Ruff 和 uv 背后的团队，该团队[已同意加入 OpenAI](https://openai.com/index/openai-to-acquire-astral/)（该交易于 2026 年 3 月宣布，在宣布时仍需监管批准和惯例性交割条件）。[mypy](https://github.com/python/mypy) 是最早的那个，由 Jukka Lehtosalo 创建，并在 Dropbox 大量开发。[zuban](https://github.com/zubanls/zuban) 由 Jedi 的作者 David Halter 编写。[pycroscope](https://github.com/JelleZijlstra/pycroscope) 由 CPython 核心开发者 Jelle Zijlstra 维护。 -这些团队拥有真实的人手、真实的预算和深厚的专业知识。其中好几个都很出色，分数也证明了这一点。zuban、Pyrefly 和 Pyright 都在 96% 以上，这真的很难做到。它们没有一个达到 100%。Basilisk 达到了。 +发布时，我们用该快照将 Basilisk 排在其他工具之前。由于 Basilisk 的结果缺乏稳健性，这项比较现已撤回。 -我们这么说不是为了炫耀。我们这么说是因为这是套件报告的事实，也是因为一件既奇特又美好的事：一个小型独立工具位居一个包含全球三大软件公司的排行榜之首。这正是一个开放、共享的符合性套件的全部承诺：它不在乎工具背后是谁。它只在乎代码是否正确。 +原文将该快照描述为一个小型独立工具位居大型团队之前的证明。这个说法属于已撤回的声明。 ## 100% 意味着什么，又不意味着什么 @@ -67,13 +70,13 @@ Basilisk 是通过 [python/typing 拉取请求 #2316](https://github.com/python/ python/typing 的维护者在结果页面的顶部就放了一个提醒，我们完全同意： -> "虽然规范符合性对生态系统很重要，但我们不建议将其作为选择类型检查器的主要依据。它并不能代表用户通常关心的许多方面。"（[python/typing 符合性结果]({{ conformanceOfficial.snapshot.source }})） +> "虽然规范符合性对生态系统很重要，但我们不建议将其作为选择类型检查器的主要依据。它并不能代表用户通常关心的许多方面。"（[python/typing 符合性结果]({{ conformanceOfficial.historical.snapshot.source }})） 请读两遍。构建这个套件的人正在告诉你，不要把他们自己的记分牌当作唯一重要的东西。这是正确的立场，我们不会为了让 Basilisk 看起来更好而假装不是这样。 -那么让我们准确地说清楚，完美的符合性分数是什么，又不是什么。 +原文试图解释我们当时认为满分意味着什么。完整性审计推翻了核心结论。 -**它是什么：**它证明当 Basilisk 依据类型规范来评判你的代码时，它的判断是正确的。一个未实现某个规范特性的检查器，无法对使用该特性的代码进行推理。它要么漏掉一个真正的错误，要么发明一个虚假的错误。在这次符合性运行中，Basilisk 捕获了每一个必需的错误，并在整个套件中产生了零误报。这是信任的底线。如果一个检查器对规范的判定不可靠，那么它所做的任何其他事情也都无法依赖。 +**我们曾声称它是什么：**证明 Basilisk 能根据类型规范正确判断代码。这个推论是错误的。如果检查器的一部分匹配了测试文本，通过确切套件并不能证明通用实现。 **它不是什么：**它不是一个声称 Basilisk 自动就是你项目最佳选择的说法。符合性不衡量检查器运行有多快、错误信息有多好、与你的编辑器集成得有多好，或者其生态系统有多成熟。这些方面极其重要，而在其中一些方面，较老的工具有多年的先发优势。 @@ -85,20 +88,20 @@ python/typing 的维护者在结果页面的顶部就放了一个提醒，我们 因为另一种情况是一个有时会自信地出错的检查器，而一个自信地出错的检查器比没有检查器更糟。Python 类型的问题从来不在于语法。问题在于强制执行。一个从不被检查的类型提示只是一句注释。一个被有漏洞的工具检查的类型提示，是一句偶尔会对你撒谎的注释。 -Basilisk 的默认规则集就是类型规范，开启所有规范规则、不做任何配置。没有要记住的 `--strict` 标志，因为严格就是底线。当你在自己的代码上运行 Basilisk 时，你得到的判定就是规范所说你应当得到的那个。这就是这个工具的全部意义，而符合性分数正是我们证明自己确实做到了、而非仅仅声称做到的方式。 +Basilisk 默认启用类型规范规则，无需记住 `--strict` 标志。我们曾声称旧分数证明这些规则正确实现了规范。事实并非如此；相关实现正在重新构建和验证。 -## 这个分数究竟是如何产生的 +## 已撤回分数如何产生 我们用那种枯燥、可复现的方式来测量它，因为那是唯一值得发布的测量方式。 -Basilisk 的符合性数字来自套件自己未经修改的评分工具，针对默认配置的 Basilisk CLI 运行，开启所有规范规则、不打开任何特殊设置。没有内置的计算器，也没有可能美化结果的自制评分器。给 Basilisk 评分的那套工具，正是给 Pyright、mypy、ty、Pyrefly、zuban 和 pycroscope 评分的同一套 `python/typing` 工具。如果你克隆该套件并自己运行，你会得到同样的榜单。 +已撤回的数字来自套件自己未经修改的评分工具，针对默认配置的 Basilisk CLI 运行并开启所有规范规则。该过程可以复现数字，却无法揭示部分实现针对确切测试进行了适配。因此，未来发布必须同时通过官方评分工具和基于变异的稳健性检查。 这是一个刻意的设计选择，它对应着我们对所交付的一切都坚持的一条规则：自我测量的指标只有在可复现、且由中立方测量时才有价值。符合性套件就是那个中立方。我们只是确保我们的工具出现并运行。 ## 试用它，并试着让它出错 -你可以在我们的[符合性页面](/docs/conformance/)上查看完整的对比，包括分数随时间的变化，也可以在 [python/typing 结果页面]({{ conformanceOfficial.snapshot.source }})上阅读最原始的事实来源。 +你可以在[符合性页面](/zh/docs/conformance/)阅读当前更正和修复计划，并在 [python/typing 结果页面]({{ conformanceOfficial.historical.snapshot.source }})上看到 Basilisk 已不再列出。 -不过，你能做的最好的事，是把 Basilisk 指向你自己的代码，看看它在哪里和你意见相左。如果它标记了规范认为合法的东西，那就是一个 bug，我们希望在 [GitHub](https://github.com/Nimblesite/Basilisk/issues) 上听到它。Basilisk 之所以达到 {{ conformanceOfficial.basilisk.pct }}%，正是通过把每一个被报告的缺口都当作一个真实的缺陷来修复，一次一个，面对一个不在乎我们感受的裁判。现在我们身处榜首，这一点不会改变。如果有什么不同，那就是它变得更重要了。 +请把 Basilisk 指向你自己的代码，并在 [GitHub](https://github.com/Nimblesite/Basilisk/issues) 上报告分歧。旧得分不能替代这种真实检验。只有在全新实现通过更广泛的回归用例和保持语义的变异后，我们才会发布替代结果。 Python 的类型系统已经足够好、值得信任有一段时间了。现在，工具也可以了。 diff --git a/website/src/zh/blog/free-threaded-python-why-type-checking-matters-more.md b/website/src/zh/blog/free-threaded-python-why-type-checking-matters-more.md index b01ea3bf..c015ec4c 100644 --- a/website/src/zh/blog/free-threaded-python-why-type-checking-matters-more.md +++ b/website/src/zh/blog/free-threaded-python-why-type-checking-matters-more.md @@ -25,7 +25,7 @@ faq: - q: "自由线程 Python 的性能代价是多少？" a: "根据 Python 3.14 发布说明，自由线程模式下单线程的性能损失现在约为 5-10%，取决于平台和 C 编译器，相较早期构建有显著改进。" - q: "Basilisk 在这一切中如何提供帮助？" - a: "Basilisk 是一个严格优先的 Python 类型检查器，其默认行为就是完全符合 Python 类型规范，没有要忘记的严格标志。它在官方 python/typing 符合性套件上取得 100%（由套件自己的评分工具测量），因此类型错误开箱即用就会被捕获，而不是只有在有人记得开启更严格的模式时才被捕获。" + a: "Basilisk 默认启用类型规范规则，没有要忘记的严格标志。但此前的符合性结果已经撤回；在受影响逻辑完成重新实现和验证之前，实际百分比暂时未知。" --- 自由线程 Python 不再是一个实验。自 2025 年 10 月 7 日发布的 Python 3.14 起，自由线程（无 GIL）构建在 [PEP 779](https://peps.python.org/pep-0779/) 下获得正式支持，而非实验性（[Python 3.14 发布说明，python.org](https://docs.python.org/3/whatsnew/3.14.html)）。如果过去几年你一直半留意着"无 GIL"这个故事，那么现在就是它成真的时刻。 @@ -80,15 +80,15 @@ faq: ## Basilisk 处在哪里 -Basilisk 是我们对"强制执行是可选的"这个问题的回答。它是一个用 Rust 构建的开源、严格优先的 Python 类型检查器和语言服务器，其默认行为就是 Python 类型规范，开启所有符合性规则，没有要忘记的 `--strict` 标志。 +Basilisk 是我们对"强制执行是可选的"这个问题的回答。它是一个用 Rust 构建的开源 Python 类型检查器和语言服务器，默认启用类型规范规则，没有要忘记的 `--strict` 标志。 -那个默认是可测量的。Basilisk 在官方 [python/typing 符合性套件](https://github.com/python/typing/blob/main/conformance/results/results.html)上取得 100%，而这个数字不是自我报告的：它来自套件自己未经修改的评分工具对 Basilisk 开箱即用配置的评分，也就是给那个页面上其他所有检查器评分的同一套工具。当我们说严格是默认时，这就是凭据。 +**更正：**Basilisk 此前的符合性结果已撤回。针对特定测试实现的逻辑使该数字不可信；应我们的请求，Basilisk 已从官方结果中移除，当前百分比暂时未知。请参阅[符合性更正](/zh/docs/conformance/)，了解正在进行的全新实现和稳健性测试。 几条诚实的边界，好让你确切知道你得到的是什么： - Basilisk 没有规范的 Python 目标版本。只有维护中的 typing 规范、已接受 PEP 或 Python 语法要求时，行为才随版本变化（[固定提交的 typing 指令规范，`python/typing@6ef9f77`](https://github.com/python/typing/blob/6ef9f7719ecfff09dad8724ef42b621fd994fb5e/docs/spec/directives.rst)）。本文的类型安全论点适用于项目选择的任何受支持解释器。 - Basilisk **没有针对并发的专门分析。** 它的工作是捕获类型错误，而这是那种在 GIL 不再为你串行化程序之后变得更有价值的通用防线。 -- 在符合规范的默认之外，一小组更严格的房屋风格规则在你需要时只差一次配置改动：要求每个参数都有类型（`BSK-0001`）、每个返回值都有类型（`BSK-0002`），在覆盖基类方法时要求 `@override`（`BSK-0025`），标记冗余注解（`BSK-0050`），并对显式 `Any` 予以提示（`BSK-0014`）。它们默认关闭，并按项目限定。 +- 在默认的规范规则之外，一小组更严格的团队风格规则在你需要时只差一次配置改动：要求每个参数都有类型（`BSK-0001`）、每个返回值都有类型（`BSK-0002`），在覆盖基类方法时要求 `@override`（`BSK-0025`），标记冗余注解（`BSK-0050`），并对显式 `Any` 予以提示（`BSK-0014`）。它们默认关闭，并按项目限定。 它作为单个二进制文件发布，没有运行时依赖，而一个扩展就能在 VS Code、Cursor、Zed 和 Neovim 中为你提供完整的工作流：悬停、跳转到定义、自动补全、重构、集成调试和性能分析。 @@ -120,4 +120,4 @@ Basilisk 是我们对"强制执行是可选的"这个问题的回答。它是一 ### Basilisk 在这一切中如何提供帮助？ -Basilisk 是一个严格优先的 Python 类型检查器，其默认行为就是完全符合 Python 类型规范，没有要忘记的严格标志。它在官方 [python/typing 符合性套件](https://github.com/python/typing/blob/main/conformance/results/results.html)上取得 100%（由套件自己的评分工具测量），因此类型错误开箱即用就会被捕获，而不是只有在有人记得开启更严格的模式时才被捕获。 +Basilisk 默认启用类型规范规则，没有要忘记的严格标志。但此前的符合性结果已撤回；在受影响逻辑完成重新实现和验证之前，当前百分比暂时未知。请在自己的代码上评估它，不要依赖旧数字。 diff --git a/website/src/zh/blog/introducing-basilisk.md b/website/src/zh/blog/introducing-basilisk.md index f410afd4..feb75ebb 100644 --- a/website/src/zh/blog/introducing-basilisk.md +++ b/website/src/zh/blog/introducing-basilisk.md @@ -35,7 +35,7 @@ Python 的类型工具采取了相反的方式。[Pyright 的四种模式](https ## 其他工具的错误所在 -问题不在于技术能力。Pyright 在正确配置时，在约 99% PEP 符合性下，确实非常擅长发现类型错误。问题在于默认值。 +问题不在于技术能力。Pyright 在正确配置时确实非常擅长发现类型错误。问题在于默认值。 当严格性是选择加入的时候： - 新项目开始时没有它，因为没有立即的压力去添加它 @@ -48,9 +48,9 @@ Python 的类型工具采取了相反的方式。[Pyright 的四种模式](https ## Basilisk 的立场 -Basilisk 的默认*就是* Python 类型规范——完全符合 PEP，没有要忘记传递的 `--strict` 标志。而当你想要超出规范的检查时，只需一次配置改动：可选的 Basilisk 规则会要求每个参数都有类型、声明每个返回值，并让 `Any` 始终显式。 +Basilisk 默认启用源自 PEP 的规则，没有要忘记传递的 `--strict` 标志。在撤回此前的结果后，其实际符合程度正在接受完整性审查。当你想要超出规范的检查时，只需一次配置改动：可选的 Basilisk 规则会要求每个参数都有类型、声明每个返回值，并让 `Any` 始终显式。 -这不是为了让 Python 开发人员的生活更艰难。这是为了让安全路径触手可及。符合规范的基线就是默认；而当团队决定想要更严格的检查时，它随时都在——在配置中开启，按项目或路径限定，绝不强加。 +这不是为了让 Python 开发人员的生活更艰难。这是为了让安全路径触手可及。默认启用的是源自规范的规则集；而当团队决定想要更严格的检查时，它随时都在——在配置中开启，按项目或路径限定，绝不强加。 为现有代码库开启这种更严格的检查确实需要工作——但这是暴露真实错误的工作。开启 Basilisk 的注解规则后，每个 BSK-0001 都是一个从未定义类型契约的函数。一个非穷举的 `match` 就是一个被静默忽略的情况。这些不是误报——它们是类型系统未被使用的地方。 diff --git a/website/src/zh/blog/openai-acquires-astral-what-it-means-for-basilisk.md b/website/src/zh/blog/openai-acquires-astral-what-it-means-for-basilisk.md index c762faf9..f45138ad 100644 --- a/website/src/zh/blog/openai-acquires-astral-what-it-means-for-basilisk.md +++ b/website/src/zh/blog/openai-acquires-astral-what-it-means-for-basilisk.md @@ -96,7 +96,7 @@ Basilisk 与 Astral 的关系是具体而承重的： 1. **我们的解析器就是 Ruff 的解析器。** Basilisk 依赖 `ruff_python_parser`、`ruff_python_ast` 与 `ruff_text_size`，并将它们钉在 `astral-sh/ruff` 上一个**不可变的 git 提交**（`rev 7c645a9`，等同于标签 `0.15.17`）。我们钉的是 `rev` 而非标签，正是为了让这个版本"永远无法被人从我们脚下换掉"。那份代码是 MIT 授权的，并且已经写进了我们的 `Cargo.lock`。这次收购无法回过头去改变我们所构建的那些字节。 2. **我们的检查/格式化路径调用的是 Ruff CLI**——`ruff==0.15.17`，在 CI 和开发容器中钉得完全一致。同样的道理：一个我们自己掌控版本的、宽松许可的二进制文件。 -3. **ty 如今是有 OpenAI 撑腰的竞争对手。** Astral 的类型检查器 ty 与 Basilisk 的检查器处在同一概念空间，如今它背后将有 OpenAI 的资源。我们认真对待这一点——但它磨砺、而非威胁了 Basilisk 的差异化所在：**开箱即用的 PEP 合规性**、集成在单一扩展中的一套**完整 LSP**（测试浏览器、调试、性能分析、自动修复），以及朝着 100% PEP 合规一路推进的执着。一个资金更雄厚的类型检查器，恰恰印证了"Python 值得拥有一流、Rust 级速度的工具"这一判断——它并不会让我们的差异化变得不再成立。 +3. **ty 如今是有 OpenAI 撑腰的竞争对手。** Astral 的类型检查器 ty 与 Basilisk 的检查器处在同一概念空间，如今它背后将有 OpenAI 的资源。我们认真对待这一点。Basilisk 将类型规范规则和一套**完整 LSP**（测试浏览器、调试、性能分析、自动修复）整合在单一扩展中。其实际符合性目前正在接受完整性审查，受影响逻辑会被重新实现，不能再由已撤回的结果代表。 4. **共同的架构押注——如今得到了印证。** 与 Astral 的工具一样，Basilisk 用 Rust 构建、基于 Ruff AST、以 Salsa 实现增量计算。Astral 已经证明这套技术栈能扩展到数以百万计的用户。我们独立地做出了同样的选择。这令人安心，而非令人不安。 **对今天的你而言，净影响：** 零。你的 Basilisk 安装构建自被钉死的、MIT 授权的 Ruff 代码和一个被钉死的 Ruff 二进制文件。这次收购不会、也不能改变其中任何一个。 diff --git a/website/src/zh/docs/comparison.md b/website/src/zh/docs/comparison.md index 3554368f..a27d0487 100644 --- a/website/src/zh/docs/comparison.md +++ b/website/src/zh/docs/comparison.md @@ -4,28 +4,22 @@ title: Python 类型检查工具对比 description: "Basilisk 与 Pyright、mypy、ty、Pyrefly 等 Python 类型检查工具的对比：严格性、PEP 符合性、性能与功能。" keywords: basilisk vs pyright, python 类型检查工具对比, mypy vs basilisk, ty, pyrefly lang: zh +dateModified: 2026-08-06 --- # Python 类型检查工具对比 -Python 类型检查器的格局已经发生了重大变化。2025 年推出了三个基于 Rust 的新工具。它们的差异在于对类型规范的实现有多忠实、究竟是一个完整的语言服务器还是仅仅一个检查器，以及速度（我们[实测并公开发布](/docs/benchmarks/)，而非空口断言）。 +Python 类型检查器的格局已经发生了重大变化。它们的差异在于对类型规范的实现有多忠实、究竟是一个完整的语言服务器还是仅仅一个检查器，以及速度。Basilisk 之前公开的性能数据目前已[撤回并等待审查](/docs/benchmarks/)。 + +

    符合性更正：Basilisk 此前的结果已撤回，当前百分比暂时未知。应我们的请求，Basilisk 已从官方结果表中移除，同时受影响的逻辑正在重新实现并接受独立稳健性验证。请勿使用旧得分或旧排名比较这些工具。

    ## 根本问题 在比较功能和性能之前，有一个问题决定了你究竟能否信任某个检查器的判断： -**它究竟实现了官方类型规范的多少？** - -| 工具 | PEP 符合性（官方套件¹） | -|---|---| -| **Basilisk** | **{{ conformanceOfficial.byId.basilisk.pct }}%（{{ conformanceOfficial.byId.basilisk.passLabel }}/{{ conformanceOfficial.byId.basilisk.total }}）** | -| zuban | {{ conformanceOfficial.byId.zuban.pct }}% | -| Pyrefly | {{ conformanceOfficial.byId.pyrefly.pct }}% | -| Pyright | {{ conformanceOfficial.byId.pyright.pct }}% | -| ty | {{ conformanceOfficial.byId.ty.pct }}% | -| mypy | {{ conformanceOfficial.byId.mypy.pct }}% | +**除了用于衡量它的固定测试之外，它究竟实现了官方类型规范的多少？** -以上每个得分都来自官方 python/typing 套件的**同一次运行**：Basilisk 以完美的 {{ conformanceOfficial.byId.basilisk.pct }}% 位居榜首，是唯一做到这一点的工具。不实现某个规范特性的检查器，就无法判断使用该特性的代码：它要么漏掉真实错误，要么凭空制造误报。Basilisk 的**默认**规则集*就是*类型规范：它只运行核心 PEP 符合性规则，别无其他，并在我们固定的提交上通过套件中的每一个文件。规则选择完全由配置驱动，因此默认就是核心 PEP 规则集，绝不更多。 +对于当前列出的检查器，请查看[官方实时结果表](https://github.com/python/typing/blob/main/conformance/results/results.html)。Basilisk 目前不在表中。旧结果无法通过保持语义不变的测试变异，因此在替换受影响实现期间，Basilisk 的诚实答案暂时是**未知**。详情见[符合性更正](/zh/docs/conformance/)。 想要比规范更严格的检查？在配置中开启**可选的 Basilisk 规则**。它们默认关闭，并且按设计会标记规范*不*视为错误的东西（比如未注解的参数）, 所以开启它们实际上会*破坏*对规范的严格符合。这正是要点：当你的团队想要超出规范的检查时再启用它们，而不是强加给每个项目。 @@ -40,7 +34,7 @@ Python 类型检查器的格局已经发生了重大变化。2025 年推出了 | 注解快速修复（插入占位符） | ✅ `: Any` / `-> None` ² | ❌ ³ | ❌ ⁴ | 双击内联提示 ⁵ | ❌（代码操作） | | 自动插入*推断*类型 | ❌ | ❌ ³ | ❌ ⁴ | ❌ | ✅ CLI `pyrefly infer` ⁶ | | 超出规范的可选规则 | ✅ 配置 | strict 模式 ⁷ | `--strict` ⁴ | 仅严重级别 ⁸ | ✅ `strict` 预设 ⁹ | -| PEP 符合性¹ | **{{ conformanceOfficial.byId.basilisk.pct }}%, 第一，唯一满分** | {{ conformanceOfficial.byId.pyright.pct }}% | {{ conformanceOfficial.byId.mypy.pct }}% | {{ conformanceOfficial.byId.ty.pct }}% | {{ conformanceOfficial.byId.pyrefly.pct }}% | +| PEP 符合性¹ | **暂时未知；旧结果已撤回** | 见实时结果 | 见实时结果 | 见实时结果 | 见实时结果 | | 实现语言 | Rust | TypeScript ³ | Python/C ⁴ | Rust ¹⁰ | Rust ¹¹ | | 需要运行时 | 无 | Node.js ³ | Python ⁴ | 无 ¹⁰ | 无 ¹¹ | | 补全、悬停、跳转 | ✅ | ✅ ¹² | ❌ ⁴ | ✅ ¹³ | ✅ ¹⁴ | @@ -54,7 +48,7 @@ Python 类型检查器的格局已经发生了重大变化。2025 年推出了 **来源：** -¹ 完全通过得分来自[官方 python/typing 符合性套件]({{ conformanceOfficial.snapshot.source }})的一次运行，快照 [python/typing@`{{ conformanceOfficial.snapshot.sha }}`]({{ conformanceOfficial.snapshot.commitUrl }})（{{ conformanceOfficial.snapshot.dateLabel }}）：basilisk {{ conformanceOfficial.byId.basilisk.version }}、pyright {{ conformanceOfficial.byId.pyright.version }}、mypy {{ conformanceOfficial.byId.mypy.version }}、ty {{ conformanceOfficial.byId.ty.version }}、pyrefly {{ conformanceOfficial.byId.pyrefly.version }}、zuban {{ conformanceOfficial.byId.zuban.version }}。Basilisk 是唯一取得完美 {{ conformanceOfficial.byId.basilisk.pct }}% 的检查器。这些得分会随工具改进而变化，因此每个都链接到其实时结果目录，而非固定数字。 +¹ 当前列出的检查器请参见[官方 python/typing 实时结果](https://github.com/python/typing/blob/main/conformance/results/results.html)。Basilisk 在撤回旧结果后请求移除；只有在全新实现通过稳健性和变异验证后，才会发布当前百分比。 ² Basilisk 的快速修复插入的是**占位符**注解（参数和属性为 `: Any`，返回值为 `-> None`；空集合变量为 `list[Any]` / `dict[str, Any]`），供你替换为真实类型。它不推断类型。参见[缺失注解规则](/zh/docs/rules/missing-annotations/)。 @@ -104,12 +98,12 @@ Python 类型检查器的格局已经发生了重大变化。2025 年推出了 ## Pyright -**由微软开发。基于 TypeScript。在官方套件上 {{ conformanceOfficial.byId.pyright.pct }}% PEP 符合性（[来源]({{ conformanceOfficial.snapshot.source }})）：落后于 Basilisk 完美的 {{ conformanceOfficial.byId.basilisk.pct }}%。** +**由微软开发，基于 TypeScript。当前符合性请参见[官方结果](https://github.com/python/typing/blob/main/conformance/results/results.html)。** -Pyright 长期是符合性的领跑者，至今仍是最强的检查器之一。在当前官方套件上它得分 {{ conformanceOfficial.byId.pyright.pct }}%, 很强，但如今落后于 Basilisk（{{ conformanceOfficial.byId.basilisk.pct }}%）。它正确处理了绝大多数 PEP 类型功能，对于基于 TypeScript 的工具来说性能出色。 +Pyright 长期是符合性的领跑者，至今仍是最强的检查器之一。它处理广泛的 PEP 类型功能，并拥有成熟的编辑器生态系统。 **Pyright 做得好的地方：** -- 强大的 PEP 覆盖率（官方符合性套件 {{ conformanceOfficial.byId.pyright.pct }}%） +- 强大的 PEP 覆盖率；请参见官方实时符合性结果 - 出色的文档和错误消息 - 通过 Pylance 深度集成 VS Code - 在大多数代码库中足够快用于交互使用 @@ -121,13 +115,13 @@ Pyright 长期是符合性的领跑者，至今仍是最强的检查器之一。 - Pylance（VS Code 扩展）是专有的：其最丰富的功能不离开 VS Code - 无插件，无法添加框架特定的类型智能 -**Pyright 何时有意义：** Basilisk 现在在符合性上超过了 Pyright（官方套件 {{ conformanceOfficial.byId.basilisk.pct }}% 对 {{ conformanceOfficial.byId.pyright.pct }}%），同时还提供完整 LSP、集成调试器和性能分析器。如果您已经深度投入微软的 VS Code 生态系统并且不介意 Node.js 依赖，Pyright 仍是一个强大、成熟的选择。 +**Pyright 何时有意义：** 如果您已经深度投入微软的 VS Code 生态系统并且不介意 Node.js 依赖，Pyright 仍是一个强大、成熟的选择。 --- ## mypy -**原创。基于 Python/C。官方套件 {{ conformanceOfficial.byId.mypy.pct }}%（[来源]({{ conformanceOfficial.snapshot.source }})）：对比 Basilisk 完美的 {{ conformanceOfficial.byId.basilisk.pct }}%。** +**原创，基于 Python/C。当前符合性请参见[官方结果](https://github.com/python/typing/blob/main/conformance/results/results.html)。** mypy 定义了 Python 类型检查的样子。多年来，其 `--strict` 标志是 Python 类型中"严格"含义的参考实现。 @@ -138,7 +132,7 @@ mypy 定义了 Python 类型检查的样子。多年来，其 `--strict` 标志 - 悠久的历史意味着处理了大多数边缘情况 **mypy 不做的事情：** -- 在[我们的实测基准](/docs/benchmarks/)中冷启动单文件检查最慢（其增量缓存可在重检时缩小差距） +- 检查需要 Python 运行时 - 守护进程模式（`dmypy`）在某些条件下不稳定 - 不是语言服务器，没有补全、悬停或跳转到定义 - 需要 Python 运行时 @@ -150,7 +144,7 @@ mypy 定义了 Python 类型检查的样子。多年来，其 `--strict` 标志 ## ty（Astral） -**由 Ruff 团队构建。Rust + Salsa。官方套件 {{ conformanceOfficial.byId.ty.pct }}%（[来源]({{ conformanceOfficial.snapshot.source }})）：仍在成熟中，远落后于 Basilisk 完美的 {{ conformanceOfficial.byId.basilisk.pct }}%。** +**由 Ruff 团队构建，使用 Rust + Salsa。当前符合性请参见[官方结果](https://github.com/python/typing/blob/main/conformance/results/results.html)。** ty 是最有趣的新入场者。它由创建 Ruff 的同一团队构建（现在是事实上的 Python linter），使用基于 Salsa 的增量架构，与 Basilisk 一样用 Rust 构建，并拥有 Astral 的工程速度支持。 @@ -161,17 +155,17 @@ ty 是最有趣的新入场者。它由创建 Ruff 的同一团队构建（现 - 亚 10 毫秒的增量速度（[PyTorch 上 4.7ms](https://astral.sh/blog/ty)，2025 年 12 月） **ty 尚不做的事情：** -- 在[官方 python/typing 符合性套件]({{ conformanceOfficial.snapshot.source }})上得分 {{ conformanceOfficial.byId.ty.pct }}%, 远落后于 Basilisk 完美的 {{ conformanceOfficial.byId.basilisk.pct }}%；仍在成熟中 +- 类型实现仍在成熟中 - 默认渐进类型 - 无集成调试器或性能分析器 -**ty 何时有意义：** 如果您愿意押注 Astral 的开发速度并能容忍采用期间较低的类型覆盖率。ty 最终可能成为主要参与者；现在依赖它进行严格执行还为时过早。 +**ty 何时有意义：** 如果您重视 Astral 的工具生态，并愿意采用一个快速发展的检查器。 --- ## Pyrefly（Meta） -**在 Instagram 规模上经过生产测试。基于 Rust。官方套件 {{ conformanceOfficial.byId.pyrefly.pct }}% PEP 符合性（[来源]({{ conformanceOfficial.snapshot.source }})）：落后于 Basilisk 完美的 {{ conformanceOfficial.byId.basilisk.pct }}%。** +**在 Instagram 规模上经过生产测试，基于 Rust。当前符合性请参见[官方结果](https://github.com/python/typing/blob/main/conformance/results/results.html)。** Pyrefly 由 Meta 构建，用于处理他们的 Python 代码库，世界上最大的代码库之一。它强调吞吐量（[1.85M LOC/秒，166 核 Meta 基础设施](https://pyrefly.org/)）而不是严格执行。 @@ -194,15 +188,15 @@ Pyrefly 由 Meta 构建，用于处理他们的 Python 代码库，世界上最 Basilisk 不是现有工具的更快版本。它占据了不同的位置： -**Basilisk 独有的：** -1. **唯一在官方 python/typing 套件上取得完美 {{ conformanceOfficial.byId.basilisk.pct }}% 的类型检查器**：开箱即 100% PEP 符合，并提供**可选的 Basilisk 规则**，可在配置中开启以获得比规范更严格的检查，除非你主动启用，它们从不运行，也从不影响符合性得分 +**Basilisk 的组合：** +1. 默认启用类型规范规则，并提供**可选的 Basilisk 规则**以实现比规范更严格的检查。符合性实现正在重新构建，当前百分比暂时未知 2. 注解快速修复，一键代码操作，为未注解的代码插入占位符注解（`: Any`、`-> None`），方便你填入真实类型，而不用手动找位置 3. 在每款编辑器中完整的开源 LSP, 补全、悬停、跳转到定义、重构、调试和性能分析，在 VS Code 以及原生 Zed 和 Neovim 扩展中相同（Cursor、Windsurf 等的 Open VSX 即将推出；JetBrains 计划中）, 不仅仅在一个专有的 VS Code 扩展内 4. 通过语言服务器代理的集成调试器和性能分析器 5. WASM 插件系统（计划中）, 无需分叉即可扩展，设计安全 **Basilisk 仍在成长的地方：** -- Basilisk 正在积极开发中。它在我们[固定的提交](/zh/docs/conformance/)上通过官方套件的 {{ conformance.scorePct }}%（{{ conformance.pass }}/{{ conformance.total }}，错误*和*警告，最严格评分），{{ conformance.fp }} 处误报、{{ conformance.missed }} 处遗漏的必需错误。 +- Basilisk 正在积极开发中。此前的符合性结果已撤回；受影响逻辑正在从头实现，[当前百分比暂时未知](/zh/docs/conformance/)。 - 插件生态系统：mypy 的 Django 和 SQLAlchemy 插件已经成熟。Basilisk 的 WASM 插件是计划中的。 -建议：开始新 Python 项目的团队从第一天起就能用 Basilisk 获得完全符合 PEP 的检查，它是官方套件上唯一取得满分的检查器，并可在准备好时开启比规范更严格的规则，且在每款编辑器中获得相同体验，而非局限于一个专有扩展。 +建议：根据 Basilisk 集成的开源编辑器工作流进行评估，并在您自己的代码上测试它。不要依据已撤回的符合性或基准测试数据做出选择。待全新实现和稳健性审查完成后，我们会发布新的符合性结果。 diff --git a/website/src/zh/docs/conformance.md b/website/src/zh/docs/conformance.md index 44004b4a..24b3da6b 100644 --- a/website/src/zh/docs/conformance.md +++ b/website/src/zh/docs/conformance.md @@ -1,96 +1,47 @@ --- layout: layouts/docs.njk -title: "Basilisk 在官方 python/typing 一致性套件中取得满分 100%" -description: "Basilisk 是唯一在官方 python/typing 一致性测试结果中取得满分 100% 的 Python 类型检查器——就发布在 Python typing 仓库的结果页面上，领先于 Pyright、mypy、Pyrefly 和 ty。这里是证据以及衡量方式。" -keywords: pep 符合性, python/typing 一致性结果, 100% 符合的类型检查器, basilisk 符合性得分, python/typing 结果 +title: "Basilisk 符合性结果已撤回" +description: "Basilisk 已撤回此前的 Python typing 符合性声明。在相关逻辑完成全新实现并通过独立稳健性验证之前，当前百分比暂时未知。" +keywords: basilisk 符合性更正, python typing 符合性, python/typing 结果, 变异测试 +dateModified: 2026-08-06 lang: zh --- -{% from "conformance-chart.njk" import chart %} -# Basilisk 取得满分 100% +# 符合性结果已撤回 -Basilisk 是**唯一在[官方 `python/typing` 一致性测试结果](https://github.com/python/typing/blob/main/conformance/results/results.html)中取得满分 {{ conformanceOfficial.byId.basilisk.pct }}%** 的 Python 类型检查器——而且它**就发布在 Python typing 仓库自己的结果页面上**，与其他所有检查器在同一次运行中评分。 +

    更正：Basilisk 已撤回此前的满分声明。该结果并不能可信地衡量规范符合性。我们请求将 Basilisk 从 python/typing 结果表中移除，现已完成移除。Basilisk 当前的符合性百分比暂时未知。

    + +我们发现，检查器中的一些逻辑针对符合性测试文件的确切内容进行了适配，而不是普遍实现类型规范。例如，类型别名验证曾对原始源代码文本执行前缀和子字符串判断，其中甚至专门判断了 `eval(`，仅仅因为某个测试使用了这种写法。因此，即使被测试的类型行为没有改变，对套件进行语义等价的变异也可能让 Basilisk 的结果发生变化。 + +官方套件仍然有价值，但针对固定测试用例开发出的代码即使通过，也不足以证明实现正确。在受影响逻辑完成全新实现并通过稳健性测试之前，我们不会发布替代百分比。

    - 官方 python/typing 结果 ↗ - 收录 Basilisk 的 PR ↗ + 当前 python/typing 结果 ↗ + 完整性审计全文 ↗ + 最初的缺陷报告 ↗ + 完整性修复跟踪 ↗ Python 类型规范 ↗ 符合性 README ↗

    -## 官方排行榜 - -下面每个得分都来自[官方 `python/typing` 一致性套件](https://github.com/python/typing/blob/main/conformance/results/results.html)的**同一次运行**——也就是类型社区用来为每个检查器打分的同一套件与评分器。Basilisk 位居榜首，也是**榜单上唯一取得满分**的工具。 - -
    - - - -{%- for t in conformanceOfficial.ranked %} - - - - - -{%- endfor %} - -
    工具背后机构官方符合性
    {% if t.id == "basilisk" %}Basilisk{% else %}{{ t.name }}{% endif %}{{ t.org or "独立" }}{% if t.id == "basilisk" %}{{ t.pct }}%（{{ t.passLabel }}/{{ t.total }}）{% else %}{{ t.pct }}%{% endif %}
    -
    - -

    results.html 在 python/typing@{{ conformanceOfficial.snapshot.sha }}（{{ conformanceOfficial.snapshot.dateLabel }}）的快照。这些数字会随其他工具的改进而变化，因此每个单元格都链接到该工具的**实时**结果目录——请自行核对当前数字。

    - -## 如何衡量 - -我们不用自己的尺子给自己打分。上面的数字由**官方 `python/typing` harness** 产生，对通过 **wheel 安装的 `basilisk` 命令**原样运行——也就是你从 PyPI 得到的同一个 CLI，在其**默认配置**下，**每条 PEP 符合性规则都开启、别无其他配置**。文件只有在 harness 的差异为空时才通过：每个必需错误都被报告，且**没有**诊断落在套件未标记的行上。我们计入检查器发出的每一个诊断——错误*和*警告——因此一处误报就会让整个文件失败。 - -目前的结果是 **{{ conformance.scorePct }}%**——{{ conformance.total }} 个测试文件中 **{{ conformance.pass }}** 个通过，捕获 {{ conformance.caught }} 个必需错误，**{{ conformance.fp }} 处误报**、**{{ conformance.missed }} 处遗漏**。我们与 `python/typing@main` 步调一致（针对 [`{{ conformance.pinnedRefShort }}`](https://github.com/python/typing/tree/{{ conformance.pinnedRef }}/conformance){% if conformance.commitDate %}，{{ conformance.commitDate }}{% endif %} 评分）；棘轮门禁防止得分回退，任何我们未通过的上游测试都会阻止合并与发布。 - -Basilisk 的**可选内建规则**（要求注解、冗余注解、缺失 `@override`、显式 `Any`）在评分中从不运行——全新安装一条都不会启用，开启它们只会*拉低*得分，因为规范将未注解的值视为*推断*而非错误。"比规范更严格"与"符合规范"是不同的目标；这个得分只衡量后者。 - -### 自己复现 - -Basilisk 是官方套件中的**已注册检查器**——`BasiliskTypeChecker` 就在 `python/typing` 的 [`conformance/src/type_checker.py`](https://github.com/python/typing/blob/main/conformance/src/type_checker.py) 中——所以你直接运行真实 harness，无需任何修补： - -```bash -# 全新克隆 python/typing，用它自己的 harness 针对 basilisk 二进制运行， -# 并从真实结果重新生成 conformance/conformance_status.csv。 -python3 conformance/run_conformance.py --bin target/release/basilisk -``` - -或针对 PATH 上任意 `basilisk` 手动驱动上游 harness： +## 当前工作 -```bash -git clone --depth 1 https://github.com/python/typing -BASILISK_BIN=$(which basilisk) python typing/conformance/src/main.py --only-run basilisk -``` +有问题的实现正在被删除，相关行为将根据规范和结构化语法重新实现，不再依赖测试文件文本。审查范围也包括类似的源文本判断、重复逻辑、过度宽松的兜底分支，以及其他可能用狭窄测试用例代替通用实现的地方。 -运行器位于 [`conformance/run_conformance.py`](https://github.com/Nimblesite/Basilisk/blob/main/conformance/run_conformance.py)；它全新克隆套件、运行未经修改的上游 harness，自身从不进行任何评分。 +这是正在进行的修复，并非无限期撤回。我们预计在全新实现和验证完成后，很快会得到一个可以辩护的结果。如果新结果低于此前的声明，我们会如实发布较低的结果。 -## 得分如何变诚实 +## 今后发布结果的门槛 -我们宁愿坦白说明也不愿掩盖。早期的一个仓库内脚本曾通过**把若干诊断代码排除在差异比对之外、且完全不计入误报**来抬高数字。我们弃用了它，改用官方 `python/typing` 评分语义，对真实的默认 CLI 运行。下面的图表在构建时直接读取 **`conformance/conformance_status.csv` 的 git 历史**——每个改动该文件的提交对应一个点，包括那次更正。 +未来的符合性结果必须通过以下全部检查： -{{ chart(conformance, { - "label": "符合性得分随时间变化", - "heading": "从仓库内脚本到官方 harness", - "prevLegend": "早期仓库内脚本——排除代码、忽略误报（并非官方衡量方式）", - "officialLegend": "官方 python/typing harness，对真实默认 CLI 运行", - "dropNote": "早期的点来自一个排除诊断代码、且不计入误报的仓库内脚本；之后的点使用官方 python/typing 评分语义、对真实默认 CLI 运行。今天的官方数字是 " + conformance.chart.current.score + "%——是衡量变诚实了，而非检查器变差了。", - "caption": "每个点都是对 conformance/conformance_status.csv 的真实提交，每次构建重新计算。悬停某点可查看其日期、提交、得分与误报数。" -}) }} +1. 使用 Basilisk 默认配置运行官方、未经修改的 `python/typing` 评分工具。 +2. 进行保持 AST 语义的变异，例如一致地重命名类型变量和采用等价写法。如果这些变化会改变结果，该规则就不能算作已实现。 +3. 通过依据类型规范和真实代码独立设计的套件外用例，而不是从上游测试文本衍生用例。 +4. 为审计发现的每一处针对测试的实现添加回归测试和变异测试。 +5. 将稳健性与套件外验证结果同套件百分比一并发布，并保证方法可复现。 -## 各类别现状 +在这项工作完成之前，旧的符合性表格、图表、分类得分、通过数量和误报统计均已撤回，不应被引用为 Basilisk 的当前状态。 -构建时从 `conformance/conformance_status.csv` 实时读取： +## 相关性能数据 -
    - - - -{%- for cat in conformance.categories %} - -{%- endfor %} - -
    类别通过得分
    {{ cat.label }}{{ cat.pass }} / {{ cat.total }}{{ cat.pct }}%
    -
    - +同样的审查失效意味着已公开的基准测试数据也必须重新验证。为保持透明，这些数据仅作为明确标注的历史记录保留在[基准测试页面](/docs/benchmarks/)上，不得用于将 Basilisk 与其他工具进行比较。只有在方法和结果通过完整性审查后，我们才会发布新的性能数据。 diff --git a/website/src/zh/docs/index.md b/website/src/zh/docs/index.md index 24ef4d2a..87a10e0f 100644 --- a/website/src/zh/docs/index.md +++ b/website/src/zh/docs/index.md @@ -8,9 +8,9 @@ lang: zh # 简介 -Basilisk 是一个**完整的开源 Python 语言服务器**。您依赖现代 Python 扩展提供的一切——自动补全、跳转到定义、悬停信息、重构、诊断、集成调试、性能分析——Basilisk 全部提供，完全开源，默认符合 Python 类型规范。 +Basilisk 是一个**完整的开源 Python 语言服务器**。您依赖现代 Python 扩展提供的一切——自动补全、跳转到定义、悬停信息、重构、诊断、集成调试、性能分析——Basilisk 全部提供，完全开源。其默认规则旨在实现 Python 类型规范，而该实现目前正在接受完整性审查。 -它也是**唯一在[官方 `python/typing` 一致性测试结果]({{ conformanceOfficial.snapshot.source }})中取得满分 100%** 的 Python 类型检查器——就发布在 Python typing 仓库自己的排行榜上，领先于 Pyright、mypy、Pyrefly 和 ty。参见[我们如何衡量](/zh/docs/conformance/)。 +**符合性更正：**Basilisk 此前的结果已撤回，当前百分比暂时未知，并已应我们的请求从官方 `python/typing` 结果中移除。我们正在从头重新实现受影响的逻辑，并会在通过稳健性测试和变异验证后发布新结果。请阅读[完整更正](/zh/docs/conformance/)。 它不仅仅是一个类型检查器。它是一个功能完整的 LSP，已为 **VS Code**、**Cursor**、**Windsurf**、**Zed** 和 **Neovim** 提供扩展——以及支持语言服务器协议的其他编辑器。JetBrains（IntelliJ / PyCharm）支持已纳入计划。无专有扩展。无 Node.js。单个 Rust 二进制文件，在每款编辑器中提供相同的体验。 @@ -20,7 +20,7 @@ Basilisk 是一个**完整的开源 Python 语言服务器**。您依赖现代 P 其他每个 Python 类型检查器（mypy、ty、Pyrefly）都*只是*检查器——没有补全、没有重构、没有调试器。你得另外搭一个语言服务器，并让两者在团队中保持同步。 -Basilisk 采取不同的立场。它的默认*就是*类型规范——开箱即完全符合 PEP——并将整个工具栈（类型检查、语言功能、调试、性能分析）整合为一个开源工具，在**每一款**编辑器中运行方式相同，而不仅仅是 VS Code。想要比规范更严格的检查？开启可选的 Basilisk 规则。类型注解是契约，不是文档。 +Basilisk 采取不同的立场。它默认启用类型规范规则，并将整个工具栈（类型检查、语言功能、调试、性能分析）整合为一个开源工具，在**每一款**编辑器中运行方式相同，而不仅仅是 VS Code。默认规则的实际符合程度暂时未知，正在审查和必要的重新实现中。想要比规范更严格的检查？开启可选的 Basilisk 规则。 ## Basilisk 是什么 @@ -29,7 +29,7 @@ Basilisk 采取不同的立场。它的默认*就是*类型规范——开箱即 - **注解快速修复**——一键代码操作，为未注解的代码插入占位注解（`: Any`、`-> None`），方便您填入真实类型 - **集成调试器**——按 F5 调试 Python，支持断点、单步执行、变量检查和监视表达式，全部通过 Basilisk LSP 代理 - **集成性能分析器**——采样式 CPU 分析器，具有内联热图注解、火焰图、内存泄漏检测和引用图可视化，全部在您的编辑器内 -- **默认符合 PEP 规范的类型检查器**——开箱即用核心规范规则集，并提供可选的 Basilisk 规则以实现比规范更严格的检查 +- **默认启用 PEP 规则的类型检查器**——开箱即启用核心规范规则集，并提供可选的 Basilisk 规则以实现比规范更严格的检查 - **开箱即得标准库类型**——二进制文件中编译进了一份完整的 typeshed `stdlib/` 树，且检查从不下载任何东西，因此无需网络、无需配置即可获得标准库类型；固定某个确切的 `python/typeshed` 提交后，它会离线对照本地存储库校验 - **用于 CI 集成的 CLI 工具**——发现错误时以代码 1 退出 - **迁移助手**，读取您现有的 `pyrightconfig.json` 或 `mypy.ini` @@ -42,9 +42,9 @@ Basilisk 采取不同的立场。它的默认*就是*类型规范——开箱即 - 不是运行时类型检查器——分析在开发时静态发生 - 不依赖特定编辑器——同一个服务器驱动 VS Code、Cursor、Windsurf、Zed 和 Neovim -## 默认符合规范，并可从此配置 +## 默认启用规范规则，并可从此配置 -Basilisk 的行为完全由**配置**决定，而默认配置恰好就是**核心 PEP 符合性规则集**——与官方类型符合性套件评分所用的规则相同。开箱即得一个遵循规范的检查器，无需记住任何标志。 +Basilisk 的行为完全由**配置**决定，而默认配置启用**核心 PEP 规则集**——与官方类型符合性套件评分所用的规则相同。这些规则旨在无需额外严格模式即可遵循规范，但在撤回旧结果后，其实际符合程度暂时未知，正在接受审查并在需要时重新实现。 比规范更严格的检查是**可选启用**的。Basilisk 还附带规范未定义的额外规则——要求每个参数和返回值都有注解、冗余注解警告、缺失 `@override` 提示、显式 `Any` 提示。在你于配置中启用之前，它们始终**关闭**。由于它们会标记规范视为有效的代码，刻意开启它们就是用严格的规范符合换取由团队自行选择的更严格标准——这是逐项目的选择，绝非默认。 @@ -57,17 +57,17 @@ Basilisk 的行为完全由**配置**决定，而默认配置恰好就是**核 "imports_unresolved" = "info" ``` -这让默认保持诚实——纯粹的规范符合——同时让每个团队在他们想要的地方精确地调节严格程度。 +这让默认设置专注于源自规范的规则，同时让每个团队在需要的地方精确调节额外严格程度。 ## 项目状态 -Basilisk 正在**积极开发中**——核心检查器、LSP 服务器和编辑器扩展都在工作，且它是官方 python/typing 符合性套件上唯一取得满分的检查器。自动补全、跳转到定义、悬停、诊断、内联提示、重构、调试和性能分析今天就在发布。 +Basilisk 正在**积极开发中**——核心检查器、LSP 服务器和编辑器扩展都在工作。此前的符合性结果已撤回，受影响的检查器逻辑正在重新实现和验证。自动补全、跳转到定义、悬停、诊断、内联提示、重构、调试和性能分析今天就在发布。 | 阶段 | 里程碑 | 状态 | |---|---|---| | 1 | 解析器、解析器、类型检查器、CLI | 完成 | | 2 | LSP 服务器、编辑器扩展（VS Code、Cursor、Zed、Neovim） | 完成 | -| 3 | 扩展规则集，PEP 符合性（对照 `python/typing@main` 达 {{ conformance.scorePct }}%），渐进式采用 | 进行中 | +| 3 | 全新实现 PEP 规则、稳健性与变异验证、渐进式采用 | 进行中 | | 4 | WASM 插件，Django/Pydantic/SQLAlchemy | 计划中 | | 5 | SARIF/JUnit 输出，JetBrains 扩展 | 计划中 | | 6 | 插件市场，社区存根，生态系统 | 计划中 | diff --git a/website/src/zh/docs/install-vscode.md b/website/src/zh/docs/install-vscode.md index e483b559..1fbd9a85 100644 --- a/website/src/zh/docs/install-vscode.md +++ b/website/src/zh/docs/install-vscode.md @@ -21,13 +21,13 @@ dateModified: 2026-03-31 打开一个 Python 文件，Basilisk 会自动激活——诊断、自动补全、悬停、跳转到定义、重命名、重构、格式化、调试（F5）和性能分析。 -![Basilisk 在 VS Code 中——符合 PEP 规范的类型错误以红色波浪线内联显示，并列在问题面板中](/assets/images/vscode-diagnostics.png) +![Basilisk 在 VS Code 中——类型诊断以红色波浪线内联显示，并列在问题面板中](/assets/images/vscode-diagnostics.png) -*打开文件的瞬间即可获得符合 PEP 规范的诊断——无需任何配置。* +*打开文件时即可看到类型诊断，无需额外配置严格模式。* ## Basilisk 是适合您的最佳 Python VS Code 扩展吗？ -没有哪个 Python 扩展适合所有项目。Basilisk 面向的开发者，是希望用一个开源扩展同时获得符合类型规范的检查、自动补全、导航、重构、格式化、调试和性能分析——并且在 VS Code 之外也能使用同一个语言服务器的人。如果您的项目依赖成熟的 mypy 框架插件，或者您更偏好 Pylance 已经成型的、仅限 VS Code 的工作流，请在切换前阅读[Python 类型检查器对比](/zh/docs/comparison/)。 +没有哪个 Python 扩展适合所有项目。Basilisk 面向的开发者，是希望用一个开源扩展同时获得类型规范规则、自动补全、导航、重构、格式化、调试和性能分析——并且在 VS Code 之外也能使用同一个语言服务器的人。在[完整性审查](/zh/docs/conformance/)期间，其当前符合性百分比暂时未知。如果您的项目依赖成熟的 mypy 框架插件，或者您更偏好 Pylance 已经成型的、仅限 VS Code 的工作流，请在切换前阅读[Python 类型检查器对比](/zh/docs/comparison/)。 ## 二进制文件已捆绑——无需单独安装 diff --git a/website/src/zh/docs/migration.md b/website/src/zh/docs/migration.md index 69a66822..4b9f97ca 100644 --- a/website/src/zh/docs/migration.md +++ b/website/src/zh/docs/migration.md @@ -9,7 +9,9 @@ dateModified: 2026-07-14 # 迁移指南 -Basilisk 的默认配置启用完整的核心 PEP 规则集。Basilisk 自有的注解、 +Basilisk 的默认配置启用当前已注册的所有 PEP 标签规则。这只是配置属性， +并不表示实现已经完整或符合规范；在[完整性修复](/zh/docs/conformance/)期间， +实际符合性水平暂时未知。Basilisk 自有的注解、 `@override`、样式、依赖和存根规则默认关闭，需要项目显式启用。迁移的 核心是先确定目标规则，再只为当前无法解决的债务记录小范围例外。 diff --git a/website/src/zh/docs/quick-start.md b/website/src/zh/docs/quick-start.md index 33e707af..e0425ede 100644 --- a/website/src/zh/docs/quick-start.md +++ b/website/src/zh/docs/quick-start.md @@ -1,7 +1,7 @@ --- layout: layouts/docs.njk title: 快速开始 -description: 5 分钟内开始使用 Basilisk。安装扩展，运行第一次类型检查，体验默认符合 PEP 规范的 Python 诊断。 +description: 5 分钟内开始使用 Basilisk。安装扩展，运行第一次类型检查，并在编辑器中查看 Python 诊断。 keywords: basilisk, 快速开始, python语言服务器, 类型检查, 教程, vs code, cursor, windsurf, zed, neovim lang: zh --- @@ -75,9 +75,11 @@ error[names_unbound]: Function `describe` returns `label` but `label` may be unb Found 3 diagnostics (3 errors). ``` -开箱即用，Basilisk 启用完整的 PEP 类型规范规则集，每个违反都是**错误**。 -这里没有任何主观风格约束——这是 -[Python 类型系统规范](https://typing.python.org/en/latest/spec/index.html)的严格执行。 +开箱即用，Basilisk 会以**错误**级别启用当前已注册的所有 PEP 标签规则； +可选的项目风格规则另行配置。这描述的是默认配置，并不证明实现完整或正确。 +在受影响规则依据 [Python 类型系统规范](https://typing.python.org/en/latest/spec/index.html) +完成重写并通过独立验证之前，Basilisk 的实际符合性水平暂时未知。请参阅 +[符合性更正](/zh/docs/conformance/)。 ## 第 2 步——修复错误 diff --git a/website/src/zh/index.njk b/website/src/zh/index.njk index e5369971..d8f3a6d1 100644 --- a/website/src/zh/index.njk +++ b/website/src/zh/index.njk @@ -1,26 +1,14 @@ --- layout: layouts/base.njk -title: "Basilisk —— 快速的 Python 类型检查器与语言服务器" -description: "Basilisk 是用 Rust 构建的开源 Python 类型检查器与语言服务器：在官方 python/typing 符合性套件中取得 100%，并在我们公开的基准测试中最快。" +title: "Basilisk —— Python 类型检查器与语言服务器" +description: "Basilisk 是用 Rust 构建的开源 Python 类型检查器与语言服务器。符合性结果和基准测试数据已撤回，目前正在进行完整性审查。" keywords: "python 类型检查器, python type checker, python 类型检查, python 语言服务器, python language server, python 类型检查器对比, rust python 类型检查器, typing conformance, 类型检查器基准测试" -imageAlt: "Basilisk Python 类型检查器与语言服务器，附公开的符合性结果与冷启动基准测试" -dateModified: 2026-08-04 +imageAlt: "Basilisk Python 类型检查器与语言服务器" +dateModified: 2026-08-06 lang: zh permalink: /zh/ --- -{# 与英文首页逐字对应的声明门控（[WEBSITE-E2E-SMOKE]）： - “最快”只有在六款工具的中位数全部在场且 Basilisk 最低时才渲染； - “唯一”只有在官方快照中 Basilisk 是唯一满分者时才渲染。 #} -{% set hasCompleteBenchmark = benchmarks.hasData - and benchmarks.toolMedians.ms.basilisk - and benchmarks.toolMedians.ms.pyright - and benchmarks.toolMedians.ms.mypy - and benchmarks.toolMedians.ms.ty - and benchmarks.toolMedians.ms.pyrefly - and benchmarks.toolMedians.ms.zuban %} -{% set basiliskIsFastest = hasCompleteBenchmark and benchmarks.toolMedians.fastest == 'basilisk' %} -
    @@ -28,14 +16,13 @@ permalink: /zh/ Python 类型检查器 · 语言服务器

    - {% if conformanceOfficial.basiliskIsSolePerfect %}唯一在官方 Python typing 套件中取得 {{ conformanceOfficial.basilisk.pct }}% 的 Python 类型检查器。{% elif conformance.hasData %}在官方 Python typing 套件中取得 {{ conformance.scorePct }}% 的 Python 类型检查器。{% else %}为速度与规范符合性打造的 Python 类型检查器。{% endif %} - {% if basiliskIsFastest %}也是我们测过最快的。{% endif %} + 用 Rust 构建的开源 Python 类型检查器与语言服务器。

    - Basilisk 是用 Rust 构建的开源 Python 类型检查器与语言服务器。 - {% if conformance.hasData and conformance.pass == conformance.total %}它是{% if conformanceOfficial.basiliskIsSolePerfect %}唯一{% endif %}通过官方 python/typing 符合性套件中每一个文件的检查器，捕获 {{ conformance.caught }} 个必需错误，{{ conformance.fp }} 处误报。{% elif conformance.hasData %}它在官方 python/typing 符合性套件中取得 {{ conformance.scorePct }}%。{% endif %} - {% if basiliskIsFastest %}在我们公开的基准测试中，它的冷启动全文件 CLI 检查中位耗时也低于所有其他检查器。{% endif %} + 我们已撤回此前的符合性声明和公开的基准测试数据。应我们的请求，Basilisk + 已从官方 python/typing 结果中移除。在重新实现针对测试特例的逻辑， + 并通过稳健性测试与变异测试验证之前，当前符合性百分比暂时未知。

    @@ -65,40 +52,30 @@ permalink: /zh/
    - {% if conformance.hasData or benchmarks.hasData %} -
    - {% if conformance.hasData %} +
    - {{ conformance.scorePct }}% - - {% if conformanceOfficial.basiliskIsSolePerfect %}官方榜单中唯一取得满分的检查器{% else %}官方类型符合性得分{% endif %} - + 暂时未知 + 当前类型符合性 - {{ conformance.total }} 个文件中有 {{ conformance.pass }} 个通过套件未经修改的评分器， - 遗漏的必需错误 {{ conformance.missed }} 处，误报 {{ conformance.fp }} 处。 - 官方结果，{{ conformanceOfficial.snapshot.date }}。 + 之前的结果已撤回。应我们的请求，Basilisk 已从 + 官方结果表中移除。 + 待全新实现通过保持测试语义的变异验证后，我们会发布新的结果。
    - {% endif %} - {% if benchmarks.hasData and benchmarks.toolMedians.ms.basilisk %}
    - {{ benchmarks.toolMedians.ms.basilisk }} ms - - {% if basiliskIsFastest %}我们公开的冷启动基准测试中最快{% else %}公开的冷启动基准测试{% endif %} - + 已撤回 + 公开的基准测试数据 - Basilisk 在 {{ benchmarks.meta.machine }} 上 {{ benchmarks.rows | length }} 个合成单文件基准用例的中位数。 - 全新进程 CLI 计时；不代表项目吞吐量或编辑器延迟。 + 基准测试数据也正在接受完整性审查，不应再用于工具比较。 + 为保持透明，历史表格会保留，直至方法与结果完成重新验证。
    - {% endif %}

    - 符合性由官方 Python typing 评分器评定；性能为自测且可复现。 - 验证符合性得分 → - 阅读基准测试方法 → + 两组数据均已撤回，等待全新实现和完整性审查。 + 阅读符合性更正 → + 阅读基准测试说明 →

    - {% endif %}
    diff --git a/website/tests/e2e/benchmarks.spec.ts b/website/tests/e2e/benchmarks.spec.ts index cf912686..9a8ef7a8 100644 --- a/website/tests/e2e/benchmarks.spec.ts +++ b/website/tests/e2e/benchmarks.spec.ts @@ -19,6 +19,13 @@ test("benchmark table reports whole-file timings and links every fixture", async }) => { await page.goto("/docs/benchmarks/"); + await expect(page.locator(".bench-caveat").first()).toContainText( + "benchmark figures are withdrawn pending an integrity review", + ); + await expect(page.locator(".bench-caveat").first()).toContainText( + "Do not use the values below to compare Basilisk with other tools", + ); + await expect(page.locator(".releases-intro")).toContainText( "Each row is one complete Python fixture file", ); diff --git a/website/tests/e2e/homepage.spec.ts b/website/tests/e2e/homepage.spec.ts index ce9c00d4..89949d9b 100644 --- a/website/tests/e2e/homepage.spec.ts +++ b/website/tests/e2e/homepage.spec.ts @@ -13,14 +13,14 @@ test.describe("homepage positioning", () => { page, }) => { await expect(page).toHaveTitle( - "Basilisk — Fast Python Type Checker & Language Server", + "Basilisk — Python Type Checker & Language Server", ); await expect(page.locator("h1")).toHaveCount(1); await expect(page.locator("h1")).toHaveText( - "The only Python type checker that scores 100% on the official Python typing suite. And the fastest we’ve benchmarked.", + "An open-source Python type checker and language server, built in Rust.", ); await expect(page.locator(".hero__subheadline")).toContainText( - "Basilisk is an open-source Python type checker and language server built in Rust.", + "withdrawn both our former conformance claim", ); await expect( page.locator('a[href="vscode:extension/Nimblesite.basilisk"]'), @@ -34,44 +34,34 @@ test.describe("homepage positioning", () => { expect(description?.length).toBeLessThanOrEqual(160); }); - test("carries a proof link beside each headline claim", async ({ page }) => { - // The two comparative claims in the hero are only publishable while they - // are linked to the source that grades them: the conformance claim to the - // official python/typing results, the speed claim to our benchmark and its - // methodology. The false-positive count is asserted at 0 because that is a - // ratchet; the caught count is left open because upstream adds test cases. + test("puts the integrity correction beside the product introduction", async ({ page }) => { await expect(page.locator(".hero__subheadline")).toContainText( - "only checker that passes every file of the official python/typing conformance suite", + "withdrawn both our former conformance claim and our published benchmark figures", ); await expect(page.locator(".hero__subheadline")).toContainText( - /\d+ required errors caught and 0 false positives/, + "removed from the official python/typing results at our request", ); await expect(page.locator(".hero__subheadline")).toContainText( - "lowest median cold full-file CLI time of any checker in our published benchmark", + "current conformance percentage is temporarily unknown", + ); + await expect(page.locator(".hero__subheadline")).toContainText( + "robustness and mutation testing", ); - await expect( - page.locator('.hero__subheadline a[href*="github.com/python/typing"]'), - ).toHaveCount(1); - await expect( - page.locator('.hero__subheadline a[href="/docs/benchmarks/"]'), - ).toHaveCount(1); }); - test("shows only linked and scoped headline proof", async ({ page }) => { + test("shows both withdrawn result notices and their detail links", async ({ page }) => { await expect(page.locator(".hero__proof .stat-card")).toHaveCount(2); await expect(page.locator(".hero__proof")).toContainText( - "Only listed checker with a perfect official score", + "Current typing conformance", ); await expect(page.locator(".hero__proof")).toContainText( - "Fastest in our published cold-check benchmark", + "Published benchmark figures", ); - // Both counts are ratchets: the suite grades us at zero on each, and the - // headline "only checker" claim is only true while they stay there. await expect(page.locator(".hero__proof")).toContainText( - "0 missed required errors and 0 false positives", + "removed from the official results table", ); await expect(page.locator(".hero__proof-cta")).toContainText( - "performance is self-measured and reproducible", + "Both sets of figures are withdrawn", ); await expect( page.locator('.hero__proof a[href*="github.com/python/typing"]'), @@ -81,6 +71,8 @@ test.describe("homepage positioning", () => { ).toBeVisible(); const body = await page.locator("body").innerText(); + expect(body).not.toContain("scores 100%"); + expect(body).not.toContain("fastest we’ve benchmarked"); expect(body).not.toContain("Strict by default"); expect(body).not.toContain("Every diagnostic"); expect(body).not.toContain("One binary"); @@ -89,7 +81,7 @@ test.describe("homepage positioning", () => { test("publishes matching social and software metadata", async ({ page }) => { await expect(page.locator('meta[property="og:title"]')).toHaveAttribute( "content", - "Basilisk — Fast Python Type Checker & Language Server", + "Basilisk — Python Type Checker & Language Server", ); await expect(page.locator('meta[property="og:image"]')).toHaveAttribute( "content", @@ -141,8 +133,8 @@ test.describe("homepage positioning", () => { }); // The Chinese homepage is a translation of the English one, not a separate -// pitch: same sections, same data gates, same proof links. These tests fail if -// the two drift apart in structure or if a zh superlative escapes its gate. +// pitch: same sections and the same integrity disclosures. These tests fail if +// one locale quietly retains a claim that the other has withdrawn. test.describe("Chinese homepage", () => { const skeleton = (page: import("@playwright/test").Page) => page.evaluate(() => @@ -165,52 +157,37 @@ test.describe("Chinese homepage", () => { expect(chinese).toEqual(english); }); - test("renders both headline claims beside their proof", async ({ page }) => { + test("renders the same correction and withdrawn-result notices", async ({ page }) => { await page.goto("/zh/"); await expect(page.locator("h1")).toHaveCount(1); await expect(page.locator("h1")).toContainText( - "唯一在官方 Python typing 套件中取得 100% 的 Python 类型检查器。", - ); - await expect(page.locator(".hero__headline-accent")).toHaveText( - "也是我们测过最快的。", + "用 Rust 构建的开源 Python 类型检查器与语言服务器。", ); await expect(page.locator(".hero__subheadline")).toContainText( - "它是唯一通过官方 python/typing 符合性套件中每一个文件的检查器", + "已撤回此前的符合性声明和公开的基准测试数据", ); await expect(page.locator(".hero__subheadline")).toContainText( - /捕获 \d+ 个必需错误，0 处误报/, + "当前符合性百分比暂时未知", ); - await expect( - page.locator('.hero__subheadline a[href*="github.com/python/typing"]'), - ).toHaveCount(1); - await expect( - page.locator('.hero__subheadline a[href="/docs/benchmarks/"]'), - ).toHaveCount(1); await expect(page.locator(".hero__proof .stat-card")).toHaveCount(2); await expect(page.locator(".hero__proof")).toContainText( - "官方榜单中唯一取得满分的检查器", + "当前类型符合性", ); await expect(page.locator(".hero__proof")).toContainText( - "我们公开的冷启动基准测试中最快", + "公开的基准测试数据", ); }); - test("keeps its comparative claims on the same gates as the English page", async ({ + test("keeps its disclosure structure aligned with the English page", async ({ page, }) => { - // A gate that fires on one locale and not the other means one of the two - // pages is asserting a comparative fact its data no longer supports. await page.goto("/"); - const englishAccents = await page - .locator(".hero__headline-accent") - .count(); + const englishCards = await page.locator(".hero__proof .stat-card").count(); await page.goto("/zh/"); - expect(await page.locator(".hero__headline-accent").count()).toBe( - englishAccents, - ); + expect(await page.locator(".hero__proof .stat-card").count()).toBe(englishCards); // Chinese readers search the English product nouns too; both must be present. const keywords = await page