From 1d6cee373a23dcc61a453194959baf1241cd370a Mon Sep 17 00:00:00 2001 From: Christian Findlay <16697547+MelbourneDeveloper@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:32:21 +1000 Subject: [PATCH 1/3] Remove the Zed extension publish job from the release workflow Its registry-listing step failed the v0.41.0 release, and Zed publishing is no longer wanted in the release path. Removed rather than disabled. Scoped to the release workflow only: the basilisk-zed tree and its CI job are untouched. --- .github/workflows/release.yml | 93 ----------------------------------- 1 file changed, 93 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1d1e984..6ca59c48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -721,99 +721,6 @@ jobs: git push origin "${GITHUB_REF_NAME}" fi - publish-zed: - name: Publish Zed extension - needs: release - runs-on: ubuntu-latest - timeout-minutes: 15 - if: startsWith(github.ref, 'refs/tags/') - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # The zed-industries/extensions registry compiles the extension to WASM - # standalone, so the mirror must build outside the monorepo. Gate the push - # on that exact build here — never publish a tree the registry cannot - # compile. Same target the `zed` CI job uses ([ZED-MIRROR]). - - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: wasm32-wasip2 - - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - with: - key: release-zed-wasm - cache-on-failure: true - - - name: Render standalone Zed extension and push to the basilisk-zed mirror - # Same write convention as publish-nvim: clone the sibling Nimblesite/* - # repo with the x-access-token credential, replace its content, then - # commit as github-actions[bot] and push. Unlike the nvim plugin, the - # Zed crate carries the 0.0.0-PLACEHOLDER version and a workspace path - # dep on basilisk-common, so it cannot be pinned or built by the registry - # as-is. render-zed-mirror.sh vendors basilisk-common, stamps the - # version, and emits a self-contained tree; the WASM build below gates - # the push. The PAT must be able to write to Nimblesite/basilisk-zed. - env: - BREW_SCOOP_PAT: ${{ secrets.BREW_SCOOP_PAT }} - run: | - set -euo pipefail - if [ -z "${BREW_SCOOP_PAT:-}" ]; then - echo "::error::BREW_SCOOP_PAT not accessible. The Nimblesite org secret exists; ensure Basilisk is in its allowed repositories AND that the token can write to Nimblesite/basilisk-zed (Org Settings → Secrets and variables → Actions → BREW_SCOOP_PAT → Repository access)." - exit 1 - fi - - export BASILISK_VERSION="${GITHUB_REF_NAME#v}" - - # Render into a staging tree, then prove it builds standalone before - # anything is pushed. cargo build also writes the Cargo.lock that the - # registry uses for a reproducible build, so commit the generated lock. - scripts/render-zed-mirror.sh staging "${BASILISK_VERSION}" - ( cd staging && cargo build --release --target wasm32-wasip2 ) - - git clone "https://x-access-token:${BREW_SCOOP_PAT}@github.com/Nimblesite/basilisk-zed.git" mirror - - # Replace the mirror's tracked content with the rendered tree, - # preserving the mirror's own .git. The build's target/ dir is large - # and machine-specific — never publish it. - rm -rf staging/target - find mirror -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + - cp -R staging/. mirror/ - - cd mirror - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - if git diff --cached --quiet; then - echo "basilisk-zed already up to date for ${BASILISK_VERSION}" - else - git commit -m "basilisk ${BASILISK_VERSION}" - git push - fi - - # Tag the mirror with the SAME tag as the monorepo. The registry pins a - # commit; the tag gives the registry PR a stable, human-readable ref - # and keeps the extension version and the binary that resolve_binary - # auto-downloads from Nimblesite/Basilisk releases identical. - # Idempotent: skip if the tag already exists. - if git ls-remote --exit-code --tags origin "refs/tags/${GITHUB_REF_NAME}" >/dev/null 2>&1; then - echo "Tag ${GITHUB_REF_NAME} already present on the mirror" - else - git tag "${GITHUB_REF_NAME}" - git push origin "${GITHUB_REF_NAME}" - fi - - # Pushing the mirror publishes NOTHING on its own — Zed installs only what - # zed-industries/extensions lists. That listing was a manual to-do nobody - # ever did, which is why Basilisk has never appeared in Zed's extensions - # view. This step performs it: first release opens the PR, later releases - # move the submodule pointer and version on the same branch. Idempotent — - # a re-run with nothing to change is a no-op. See [ZED-MIRROR]. - - name: Submit or bump the zed-industries/extensions listing - env: - GH_TOKEN: ${{ secrets.BREW_SCOOP_PAT }} - run: | - set -euo pipefail - scripts/publish_zed_registry.py "${GITHUB_REF_NAME#v}" "${GITHUB_REF_NAME}" - publish-scoop: name: Publish Scoop manifest needs: release From a47bf3c945e4dfa9be2fef27b23d28c13172f623 Mon Sep 17 00:00:00 2001 From: Christian Findlay <16697547+MelbourneDeveloper@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:36:48 +1000 Subject: [PATCH 2/3] Make VSIX publishing survive transient registry failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.41.0 published three of five VSIX targets to Open VSX, took a 503 on the fourth, and 'set -e' aborted the loop — leaving win32-arm64 and win32-x64 unpublished with no retry and no second chance. Both publish loops (Open VSX and Marketplace) now: - retry each target up to 4 times with exponential backoff, - treat 'already exists' as the desired end state, so a re-run or a partial retry no longer fails on targets that already landed, - attempt every target before failing, so one bad target cannot strand the ones behind it, - still fail the job if any target ends up unpublished — a partial push is never reported as green. The Marketplace loop additionally exits immediately on an expired PAT with the steps to mint a new one, since retrying that can only fail. Verified with a stub publisher across four paths: all-success, all- already-published, transient-then-success, and permanent-failure. The permanent-failure case attempts both targets and reports both. --- .github/workflows/release.yml | 96 ++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ca59c48..af65f84b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -511,18 +511,59 @@ jobs: # One publish per platform-specific VSIX. vsce requires a separate # publish call for each --target platform; globbing all into one # publish silently uses only the first. + # Same hardening as the Open VSX step, for the same reason: an + # unguarded loop under `set -e` aborts on the first failure and + # strands every target behind it. Retry with backoff, treat an + # already-published version as done so re-runs are safe, and try + # every target before failing. An expired PAT is NOT retried — it + # can only fail again, so it exits immediately with the fix. shopt -s globstar nullglob published=0 + skipped=0 + failed=() for vsix in **/*.vsix; do echo "Publishing $vsix" - npx --yes @vscode/vsce publish $flag --packagePath "$vsix" - published=$((published + 1)) + attempt=1 + delay=15 + while :; do + set +e + out="$(npx --yes @vscode/vsce publish $flag --packagePath "$vsix" 2>&1)" + rc=$? + set -e + echo "$out" + if [ "$rc" -eq 0 ]; then + published=$((published + 1)) + break + fi + if grep -qiE 'already exists|greater than .* version' <<<"$out"; then + echo "$vsix is already published at this version — nothing to do" + skipped=$((skipped + 1)) + break + fi + if grep -qi 'Personal Access Token used has expired' <<<"$out"; then + echo "::error::VSCODE_MARKETPLACE_PAT has EXPIRED. Mint a new Azure DevOps PAT at https://aka.ms/vscodepat — Organization: 'All accessible organizations', Scope: Marketplace → Manage — and update the Nimblesite org secret VSCODE_MARKETPLACE_PAT. Retrying cannot help." + exit 1 + fi + if [ "$attempt" -ge 4 ]; then + echo "::warning::$vsix still failing after $attempt attempts" + failed+=("$vsix") + break + fi + echo "Attempt $attempt failed (exit $rc); retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done done - if [ "$published" -eq 0 ]; then + if [ "$((published + skipped))" -eq 0 ] && [ "${#failed[@]}" -eq 0 ]; then echo "::error::No VSIX artifacts found to publish" exit 1 fi - echo "Published $published VSIX(es) to the Marketplace" + echo "Marketplace: $published published, $skipped already present, ${#failed[@]} failed" + if [ "${#failed[@]}" -gt 0 ]; then + echo "::error::Marketplace publish failed for: ${failed[*]}" + exit 1 + fi publish-vsix-ovsx: name: Publish VSIX (Open VSX) @@ -574,18 +615,57 @@ jobs: # ovsx is version-pinned: a floating `npx ovsx` would fetch and run # the latest release at publish time, inside the job that holds the # token — a supply-chain risk. Bump deliberately. + # Open VSX returns transient 5xx under load. v0.41.0 published three + # targets, took a 503 on the fourth, and `set -e` aborted the loop — + # leaving win32-arm64 and win32-x64 unpublished with no retry. So: + # retry each target with backoff, treat "already exists" as the + # desired end state (makes re-runs and partial retries safe), and + # attempt EVERY target before failing, so one bad target can never + # strand the ones behind it. The job still fails if any target is + # ultimately unpublished — a partial push is never reported as green. shopt -s globstar nullglob published=0 + skipped=0 + failed=() for vsix in **/*.vsix; do echo "Publishing $vsix" - npx --yes ovsx@1.0.0 publish $flag --packagePath "$vsix" - published=$((published + 1)) + attempt=1 + delay=15 + while :; do + set +e + out="$(npx --yes ovsx@1.0.0 publish $flag --packagePath "$vsix" 2>&1)" + rc=$? + set -e + echo "$out" + if [ "$rc" -eq 0 ]; then + published=$((published + 1)) + break + fi + if grep -qi 'already exists' <<<"$out"; then + echo "$vsix is already published at this version — nothing to do" + skipped=$((skipped + 1)) + break + fi + if [ "$attempt" -ge 4 ]; then + echo "::warning::$vsix still failing after $attempt attempts" + failed+=("$vsix") + break + fi + echo "Attempt $attempt failed (exit $rc); retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done done - if [ "$published" -eq 0 ]; then + if [ "$((published + skipped))" -eq 0 ] && [ "${#failed[@]}" -eq 0 ]; then echo "::error::No VSIX artifacts found to publish" exit 1 fi - echo "Published $published VSIX(es) to Open VSX" + echo "Open VSX: $published published, $skipped already present, ${#failed[@]} failed" + if [ "${#failed[@]}" -gt 0 ]; then + echo "::error::Open VSX publish failed for: ${failed[*]}" + exit 1 + fi deploy-pages: name: Deploy GitHub Pages From ea8ee32f32f5712e15db19717c9f3894ac737d40 Mon Sep 17 00:00:00 2001 From: Christian Findlay <16697547+MelbourneDeveloper@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:09:21 +1000 Subject: [PATCH 3/3] Cleanup --- README-pypi.md | 5 +---- README.md | 5 +---- README.zh.md | 4 +--- basilisk-zed/README.md | 2 +- basilisk-zed/README.zh.md | 2 +- basilisk.nvim/README.md | 2 +- basilisk.nvim/README.zh.md | 2 +- docs/CONFORMANCE-INTEGRITY-AUDIT.md | 2 +- docs/readme/README.src.md | 5 +---- docs/readme/README.zh.src.md | 4 +--- vscode-extension/README.md | 5 +---- vscode-extension/README.zh.md | 4 +--- ...lisk-100-percent-python-typing-conformance.md | 13 +++++-------- website/src/docs/conformance.md | 4 ++-- website/src/index.njk | 2 +- ...lisk-100-percent-python-typing-conformance.md | 16 ++++++---------- website/src/zh/docs/conformance.md | 8 ++------ website/src/zh/index.njk | 2 +- 18 files changed, 29 insertions(+), 58 deletions(-) diff --git a/README-pypi.md b/README-pypi.md index 87c753e1..56fcfa98 100644 --- a/README-pypi.md +++ b/README-pypi.md @@ -50,10 +50,7 @@ instead of implementing the typing specification generally: rules that matched the *spelling* of code rather than its meaning. Rename an import or reformat a file and the answer changed. A score produced that way is not evidence. -**This was a mistake and a failure to verify — not an attempt to game the suite.** -Nobody set out to defeat the conformance tests, and nothing was concealed from -`python/typing`: the submission ran the suite's own unmodified harness, with -default configuration and every rule enabled. Our process treated the score as +**This was a mistake and a failure to verify.** Our process treated the score as the goal, matching text raises a score faster than real analysis does, and we published without ever asking whether a rule still held when the same program was spelled differently. Basilisk's author has published a diff --git a/README.md b/README.md index 2ba2e7a5..f4b6ed1e 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,7 @@ instead of implementing the typing specification generally: rules that matched the *spelling* of code rather than its meaning. Rename an import or reformat a file and the answer changed. A score produced that way is not evidence. -**This was a mistake and a failure to verify — not an attempt to game the suite.** -Nobody set out to defeat the conformance tests, and nothing was concealed from -`python/typing`: the submission ran the suite's own unmodified harness, with -default configuration and every rule enabled. Our process treated the score as +**This was a mistake and a failure to verify.** Our process treated the score as the goal, matching text raises a score faster than real analysis does, and we published without ever asking whether a rule still held when the same program was spelled differently. Basilisk's author has published a diff --git a/README.zh.md b/README.zh.md index db0fb4f4..85ebcfe0 100644 --- a/README.zh.md +++ b/README.zh.md @@ -48,9 +48,7 @@ 那些规则匹配的是代码的**写法**,而不是代码的含义。改一个导入别名或重新格式化文件, 结论就会变。这样得出的分数并不能作为证据。 -**这是一个错误、一次验证上的失职,而不是有意操纵测试套件。** 没有人试图去糊弄一致性 -测试,也没有向 `python/typing` 隐瞒任何东西:提交时运行的是套件自己未经修改的评分 -工具,使用默认配置并启用全部规则。我们的流程把分数当成了目标,而匹配文本比真正做 +**这是一个错误、一次验证上的失职。** 我们的流程把分数当成了目标,而匹配文本比真正做 分析更快地提高分数;我们在发布之前,始终没有问过这样一个问题 —— 同一个程序换一种 写法时,这条规则是否依然成立。Basilisk 作者已发表 [个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 diff --git a/basilisk-zed/README.md b/basilisk-zed/README.md index c2a6de9b..7842079a 100644 --- a/basilisk-zed/README.md +++ b/basilisk-zed/README.md @@ -14,7 +14,7 @@ Basilisk is an open-source Python type checker and language server built in Rust > > **The type checker still contains code that isn't doing real type checking, and it is not yet trustworthy.** Some rules decide from the way code is *spelled* rather than what it means, so they can be wrong in both directions — a false error on correct code, or silence where there is a genuine bug. Don't gate CI on it, and don't read a clean run as a clean codebase. Our former conformance claim and our benchmark figures are withdrawn, and Basilisk was [removed from the official results](https://github.com/python/typing/blob/main/conformance/results/results.html) at our request. > -> **This was a mistake and a failure to verify — not an attempt to game the suite.** Nothing was concealed from `python/typing`; the submission ran the suite's own unmodified harness, and we published on a green run without ever checking whether our rules survived a semantics-preserving change. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology). +> **This was a mistake and a failure to verify.** We published on a green run without ever checking whether our rules survived a semantics-preserving change. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology). > > **We are auditing every rule and deleting the ones that don't hold up** — not rewriting them, not patching them, with a failing test left behind so the gap stays visible. Where a rule can't be made reliable in a straightforward way, we will depend on a different, established type checker rather than ship our own unreliable version of it. > diff --git a/basilisk-zed/README.zh.md b/basilisk-zed/README.zh.md index fdbf7c81..f2852dcb 100644 --- a/basilisk-zed/README.zh.md +++ b/basilisk-zed/README.zh.md @@ -16,7 +16,7 @@ Basilisk 是用 Rust 打造的开源 Python 类型检查器与语言服务器: > > **类型检查器中仍然存在没有做真正类型检查的代码,它目前还不值得信任。** 有些规则依据的是代码的**写法**而不是含义,因此两个方向上都可能出错 —— 既可能对正确的代码报出虚假错误,也可能对真实的缺陷保持沉默。请不要用它作为 CI 的门禁,也不要把一次干净的运行结果当作代码库是干净的。此前的一致性宣称与基准测试数字均已撤回,并主动请求[从官方结果中移除](https://github.com/python/typing/blob/main/conformance/results/results.html)。 > -> **这是一个错误、一次验证上的失职,而不是有意操纵测试套件。** 我们没有向 `python/typing` 隐瞒任何东西:提交时运行的是套件自己未经修改的评分工具;我们仅凭一次全绿的运行就发布了结果,却从未检查过我们的规则能否经受住保持语义的改写。Basilisk 作者已发表[个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 +> **这是一个错误、一次验证上的失职。** 我们仅凭一次全绿的运行就发布了结果,却从未检查过我们的规则能否经受住保持语义的改写。Basilisk 作者已发表[个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 > > **我们正在逐条审计规则,并删除那些站不住脚的规则** —— 不是重写,也不是打补丁,而是删除,并留下一个失败的测试,让缺口保持可见。如果一条规则无法以直截了当的方式做到可靠,我们会转而依赖另一个成熟的类型检查器,而不是端出我们自己那份不可靠的实现。 > diff --git a/basilisk.nvim/README.md b/basilisk.nvim/README.md index 0c75c310..a43155b4 100644 --- a/basilisk.nvim/README.md +++ b/basilisk.nvim/README.md @@ -14,7 +14,7 @@ Basilisk is an open-source Python type checker and language server built in Rust > > **The type checker still contains code that isn't doing real type checking, and it is not yet trustworthy.** Some rules decide from the way code is *spelled* rather than what it means, so they can be wrong in both directions — a false error on correct code, or silence where there is a genuine bug. Don't gate CI on it, and don't read a clean run as a clean codebase. Our former conformance claim and our benchmark figures are withdrawn, and Basilisk was [removed from the official results](https://github.com/python/typing/blob/main/conformance/results/results.html) at our request. > -> **This was a mistake and a failure to verify — not an attempt to game the suite.** Nothing was concealed from `python/typing`; the submission ran the suite's own unmodified harness, and we published on a green run without ever checking whether our rules survived a semantics-preserving change. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology). +> **This was a mistake and a failure to verify.** We published on a green run without ever checking whether our rules survived a semantics-preserving change. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology). > > **We are auditing every rule and deleting the ones that don't hold up** — not rewriting them, not patching them, with a failing test left behind so the gap stays visible. Where a rule can't be made reliable in a straightforward way, we will depend on a different, established type checker rather than ship our own unreliable version of it. > diff --git a/basilisk.nvim/README.zh.md b/basilisk.nvim/README.zh.md index f128a4a1..05920d9e 100644 --- a/basilisk.nvim/README.zh.md +++ b/basilisk.nvim/README.zh.md @@ -16,7 +16,7 @@ Basilisk 是用 Rust 打造的开源 Python 类型检查器与语言服务器: > > **类型检查器中仍然存在没有做真正类型检查的代码,它目前还不值得信任。** 有些规则依据的是代码的**写法**而不是含义,因此两个方向上都可能出错 —— 既可能对正确的代码报出虚假错误,也可能对真实的缺陷保持沉默。请不要用它作为 CI 的门禁,也不要把一次干净的运行结果当作代码库是干净的。此前的一致性宣称与基准测试数字均已撤回,并主动请求[从官方结果中移除](https://github.com/python/typing/blob/main/conformance/results/results.html)。 > -> **这是一个错误、一次验证上的失职,而不是有意操纵测试套件。** 我们没有向 `python/typing` 隐瞒任何东西:提交时运行的是套件自己未经修改的评分工具;我们仅凭一次全绿的运行就发布了结果,却从未检查过我们的规则能否经受住保持语义的改写。Basilisk 作者已发表[个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 +> **这是一个错误、一次验证上的失职。** 我们仅凭一次全绿的运行就发布了结果,却从未检查过我们的规则能否经受住保持语义的改写。Basilisk 作者已发表[个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 > > **我们正在逐条审计规则,并删除那些站不住脚的规则** —— 不是重写,也不是打补丁,而是删除,并留下一个失败的测试,让缺口保持可见。如果一条规则无法以直截了当的方式做到可靠,我们会转而依赖另一个成熟的类型检查器,而不是端出我们自己那份不可靠的实现。 > diff --git a/docs/CONFORMANCE-INTEGRITY-AUDIT.md b/docs/CONFORMANCE-INTEGRITY-AUDIT.md index d8b56006..c143e7d4 100644 --- a/docs/CONFORMANCE-INTEGRITY-AUDIT.md +++ b/docs/CONFORMANCE-INTEGRITY-AUDIT.md @@ -14,7 +14,7 @@ We found that at least one Basilisk rule earns its conformance result by pattern We did not find this ourselves. It was reported from outside, in [issue #379](https://github.com/Nimblesite/Basilisk/issues/379), from a [public reproduction](https://x.com/cyanchanges/status/2083115048143364512). That is itself a finding, and it is covered in §6. -**On intent.** This was a verification failure, not an attempt to game the conformance suite. Nobody set out to defeat it, and nothing was concealed from `python/typing`: the submission ran the suite's own unmodified harness, with default configuration and every specification rule enabled — no special-casing of the harness, no doctored output, no disabled rules, no edited results file. What went wrong is set out in §6: the process named the score as the target, matching source text raises a score faster than real analysis does, and the control that would have caught the drift — semantics-preserving mutation (§7) — did not exist, so every green run reinforced a conclusion we had no basis for. When the defect was demonstrated we accepted it and requested our own removal from the results table. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology). This document describes defective engineering and a failure of review; it does not describe deception, and nothing in it should be read as alleging any. +**On intent.** This was a verification failure. What went wrong is set out in §6: the process named the score as the target, matching source text raises a score faster than real analysis does, and the control that would have caught the drift — semantics-preserving mutation (§7) — did not exist, so every green run reinforced a conclusion we had no basis for. When the defect was demonstrated we accepted it and requested our own removal from the results table. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology). Our conformance number is self-measured. Where a passing file is carried by predicates shaped to that file, the honest statement is that **the file passes and the rule is not implemented**. That is the case for the files listed in §3. diff --git a/docs/readme/README.src.md b/docs/readme/README.src.md index 26348731..6dc1516e 100644 --- a/docs/readme/README.src.md +++ b/docs/readme/README.src.md @@ -61,10 +61,7 @@ instead of implementing the typing specification generally: rules that matched the *spelling* of code rather than its meaning. Rename an import or reformat a file and the answer changed. A score produced that way is not evidence. -**This was a mistake and a failure to verify — not an attempt to game the suite.** -Nobody set out to defeat the conformance tests, and nothing was concealed from -`python/typing`: the submission ran the suite's own unmodified harness, with -default configuration and every rule enabled. Our process treated the score as +**This was a mistake and a failure to verify.** Our process treated the score as the goal, matching text raises a score faster than real analysis does, and we published without ever asking whether a rule still held when the same program was spelled differently. Basilisk's author has published a diff --git a/docs/readme/README.zh.src.md b/docs/readme/README.zh.src.md index 5cc9ef96..1f634fe1 100644 --- a/docs/readme/README.zh.src.md +++ b/docs/readme/README.zh.src.md @@ -55,9 +55,7 @@ 那些规则匹配的是代码的**写法**,而不是代码的含义。改一个导入别名或重新格式化文件, 结论就会变。这样得出的分数并不能作为证据。 -**这是一个错误、一次验证上的失职,而不是有意操纵测试套件。** 没有人试图去糊弄一致性 -测试,也没有向 `python/typing` 隐瞒任何东西:提交时运行的是套件自己未经修改的评分 -工具,使用默认配置并启用全部规则。我们的流程把分数当成了目标,而匹配文本比真正做 +**这是一个错误、一次验证上的失职。** 我们的流程把分数当成了目标,而匹配文本比真正做 分析更快地提高分数;我们在发布之前,始终没有问过这样一个问题 —— 同一个程序换一种 写法时,这条规则是否依然成立。Basilisk 作者已发表 [个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 diff --git a/vscode-extension/README.md b/vscode-extension/README.md index c7c5432a..365f8f46 100644 --- a/vscode-extension/README.md +++ b/vscode-extension/README.md @@ -50,10 +50,7 @@ instead of implementing the typing specification generally: rules that matched the *spelling* of code rather than its meaning. Rename an import or reformat a file and the answer changed. A score produced that way is not evidence. -**This was a mistake and a failure to verify — not an attempt to game the suite.** -Nobody set out to defeat the conformance tests, and nothing was concealed from -`python/typing`: the submission ran the suite's own unmodified harness, with -default configuration and every rule enabled. Our process treated the score as +**This was a mistake and a failure to verify.** Our process treated the score as the goal, matching text raises a score faster than real analysis does, and we published without ever asking whether a rule still held when the same program was spelled differently. Basilisk's author has published a diff --git a/vscode-extension/README.zh.md b/vscode-extension/README.zh.md index 41ab71d5..7f115e81 100644 --- a/vscode-extension/README.zh.md +++ b/vscode-extension/README.zh.md @@ -48,9 +48,7 @@ 那些规则匹配的是代码的**写法**,而不是代码的含义。改一个导入别名或重新格式化文件, 结论就会变。这样得出的分数并不能作为证据。 -**这是一个错误、一次验证上的失职,而不是有意操纵测试套件。** 没有人试图去糊弄一致性 -测试,也没有向 `python/typing` 隐瞒任何东西:提交时运行的是套件自己未经修改的评分 -工具,使用默认配置并启用全部规则。我们的流程把分数当成了目标,而匹配文本比真正做 +**这是一个错误、一次验证上的失职。** 我们的流程把分数当成了目标,而匹配文本比真正做 分析更快地提高分数;我们在发布之前,始终没有问过这样一个问题 —— 同一个程序换一种 写法时,这条规则是否依然成立。Basilisk 作者已发表 [个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology)。 diff --git a/website/src/blog/basilisk-100-percent-python-typing-conformance.md b/website/src/blog/basilisk-100-percent-python-typing-conformance.md index 6da7d0ee..5ec0ed0c 100644 --- a/website/src/blog/basilisk-100-percent-python-typing-conformance.md +++ b/website/src/blog/basilisk-100-percent-python-typing-conformance.md @@ -19,8 +19,8 @@ faq: a: "No. Basilisk was removed from the official results table at our own request, and we have not asked to be re-listed. We will not submit again until the audit is finished, semantics-preserving mutation testing passes clean, and someone outside the project has reviewed the work." - q: "What is Basilisk's current conformance percentage?" a: "We do not publish one, and we ask that no figure be attributed to us. Any number we produced before the audit was measured against fixtures our own code had been fitted to, so it did not measure what we said it measured. We expect an honest figure to be lower than the withdrawn one, and we will say so plainly when we can measure it properly." - - q: "Was the original result an attempt to game the conformance suite?" - a: "No. It was a verification failure. Our development process named the suite score as the target to build against, and matching source text raises that score faster than real analysis does, so the code drifted that way. Nobody set out to defeat the suite and nothing was concealed from python/typing; the submission ran the suite's own unmodified harness. We did not run the check that would have caught the problem before we published, and that is our responsibility. Basilisk's author has published a personal account at https://www.christianfindlay.com/blog/basilisk-conformance-apology." + - q: "How did this happen?" + a: "It was a verification failure. Our development process named the suite score as the target to build against, and matching source text raises that score faster than real analysis does, so the code drifted that way. We did not run the check that would have caught the problem before we published, and that is our responsibility. Basilisk's author has published a personal account at https://www.christianfindlay.com/blog/basilisk-conformance-apology." - q: "Should I use Basilisk's type checker in CI?" a: "Not yet. Until the audit is finished, do not gate CI on it, do not block a merge with it, and do not read a clean run as a clean codebase. The rest of Basilisk — the language server, refactoring, formatting, debugging, and profiling — does not depend on the rules under audit." --- @@ -66,11 +66,9 @@ A suite result produced by code developed against that suite's exact fixtures do measure specification conformance. The file passes and the rule is not implemented. That is why the correction is deletion rather than a better score. -## This was a mistake, not an attempt to game the suite +## How this happened -We didn't set out to defeat the conformance suite, and nothing was concealed from -`python/typing`. The submission ran the suite's own unmodified harness, with -Basilisk's default configuration and every specification rule enabled. When the defect was demonstrated, we asked for our own removal. +When the defect was demonstrated, we asked for our own removal. Our development process named the conformance score as the thing to build against, and matching source text raises that score faster than real analysis does — so that is the direction the code drifted, one plausible-looking rule at a time. Then we published and submitted the result on the strength of a green run, without ever running the one check that would have exposed @@ -80,8 +78,7 @@ That check did not exist. It still doesn't — building it is part of the remedi The conformance suite cannot catch this class of defect by construction, because it is the very artefact the code was fitted to, so every green run reinforced a conclusion we had no basis for. We believed the number meant what we said it meant. We were -wrong, and we were wrong because we did not verify it, not because we were trying to -get away with something. +wrong, and we were wrong because we did not verify it. It was also **not us who found it**. It was reported from outside, in [issue #379](https://github.com/Nimblesite/Basilisk/issues/379), from a public diff --git a/website/src/docs/conformance.md b/website/src/docs/conformance.md index 8f7e0774..9727858e 100644 --- a/website/src/docs/conformance.md +++ b/website/src/docs/conformance.md @@ -19,9 +19,9 @@ We found checker logic fitted to the exact contents of conformance test files ra A passing result from code developed against the exact fixtures is not evidence, so the fix is not a better score. -## This was a mistake, not an attempt to game the suite +## How this happened -We didn't set out to defeat the conformance suite. What actually happened is duller. Our development process named the conformance score as the thing to build against, and matching source text raises that score faster than real analysis does, so that is the direction the code drifted — one plausible-looking rule at a time. We then published and submitted on the strength of a green run, without ever running the one check that would have exposed it: does this rule still hold when the same program is spelled differently? That check did not exist, and building it is part of the remediation below. The suite cannot catch this class of defect by construction, because it is the artefact the code was fitted to, so every green run reinforced a conclusion we had no basis for. +Our development process named the conformance score as the thing to build against, and matching source text raises that score faster than real analysis does, so that is the direction the code drifted — one plausible-looking rule at a time. We then published and submitted on the strength of a green run, without ever running the one check that would have exposed it: does this rule still hold when the same program is spelled differently? That check did not exist, and building it is part of the remediation below. The suite cannot catch this class of defect by construction, because it is the artefact the code was fitted to, so every green run reinforced a conclusion we had no basis for. We believed the number meant what we said it meant. We were wrong, and we were wrong because we failed to verify it. Basilisk's author has published a [personal account and apology](https://www.christianfindlay.com/blog/basilisk-conformance-apology) taking responsibility in his own words. diff --git a/website/src/index.njk b/website/src/index.njk index cdab1a06..9c3a3de7 100644 --- a/website/src/index.njk +++ b/website/src/index.njk @@ -78,7 +78,7 @@ permalink: /

- Both sets of figures are withdrawn while the audit runs. This was a mistake and a failure to verify, not an attempt to game the suite. + Both sets of figures are withdrawn while the audit runs. Read the conformance correction → Read the author’s apology → Read the benchmark notice → diff --git a/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md b/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md index 3a81a4b9..7c19bdc2 100644 --- a/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md +++ b/website/src/zh/blog/basilisk-100-percent-python-typing-conformance.md @@ -20,8 +20,8 @@ faq: a: "不在。我们主动请求将 Basilisk 从官方结果表中移除,并且没有申请重新列入。在审计完成、保持语义的变异测试全部通过、并且由项目之外的人完成复核之前,我们不会再次提交。" - q: "Basilisk 当前的符合性百分比是多少?" a: "我们不发布任何数字,也请不要把任何数字归到我们名下。审计之前我们得到的任何数字,都是针对我们自己的代码已经适配过的测试用例测出来的,因此它并没有衡量我们所声称的东西。我们预计诚实的数字会低于此前撤回的那个;等到能够正确测量时,我们会如实说明。" - - q: "当初的结果是不是有意操纵符合性测试套件?" - a: "不是。这是一次验证失败。我们的开发流程把套件分数定为要去达成的目标,而匹配源文本比真正做分析更快地提高这个分数,代码于是滑向了那个方向。没有人试图去糊弄测试套件,也没有向 python/typing 隐瞒任何东西——提交使用的是套件自己未经修改的评分工具。我们在发布之前没有做那项本可以发现问题的检查,这是我们的责任。Basilisk 作者已在 https://www.christianfindlay.com/blog/basilisk-conformance-apology 发表了个人说明。" + - q: "这是怎么发生的?" + a: "这是一次验证失败。我们的开发流程把套件分数定为要去达成的目标,而匹配源文本比真正做分析更快地提高这个分数,代码于是滑向了那个方向。我们在发布之前没有做那项本可以发现问题的检查,这是我们的责任。Basilisk 作者已在 https://www.christianfindlay.com/blog/basilisk-conformance-apology 发表了个人说明。" - q: "我应该在 CI 中使用 Basilisk 的类型检查器吗?" a: "暂时不要。在审计完成之前,请不要用它作为 CI 门禁,不要用它阻断合并,也不要把一次干净的运行结果当作代码库是干净的。Basilisk 的其余部分——语言服务器、重构、格式化、调试与性能分析——并不依赖正在审计的这些规则。" --- @@ -58,15 +58,11 @@ Basilisk 检查器中的一部分逻辑,依据的是源代码的**写法**而 针对某个套件的确切测试用例开发出来的代码,即使在该套件上通过,也不能衡量规范符合性。 真实情况是:文件通过了,而规则并没有实现。这正是为什么我们的纠正措施是删除,而不是去拿一个更好的分数。 -## 这是一个错误,而不是有意操纵测试套件 +## 这是怎么发生的 -我们希望在这一点上毫不含糊,因为事件的先后顺序容易让人得出一个比事实更糟的解读。 +当缺陷被演示出来之后,我们主动请求移除自己。 -**没有人试图去糊弄符合性测试套件,也没有向 `python/typing` 隐瞒任何东西。** 提交时运行的是套件 -自己未经修改的评分工具,使用 Basilisk 的默认配置并启用全部规范规则。没有针对评分工具做特殊处理, -没有伪造输出,没有禁用规则,也没有手工编辑结果文件。当缺陷被演示出来之后,我们主动请求移除自己。 - -真实经过要平淡得多,也更普通。我们的开发流程把符合性分数定为要去达成的目标,而匹配源文本比 +我们的开发流程把符合性分数定为要去达成的目标,而匹配源文本比 真正做分析更快地提高这个分数 —— 于是代码就朝那个方向滑了过去,一条看起来合理的规则接着一条。 然后我们仅凭一次全绿的运行就发布并提交了结果,却始终没有做那项本可以揭露问题的检查: 同一个程序换一种写法时,这条规则是否依然成立? @@ -74,7 +70,7 @@ Basilisk 检查器中的一部分逻辑,依据的是源代码的**写法**而 这项检查当时并不存在,现在也还没有 —— 把它建起来正是补救工作的一部分。符合性测试套件从构造上 就无法发现这一类缺陷,因为它正是代码被适配过去的那个对象,所以每一次全绿运行都在强化一个 我们根本没有依据的结论。我们当时相信那个数字就是我们所说的意思。我们错了;我们之所以错, -是因为我们没有去验证它,而不是因为我们想蒙混过关。 +是因为我们没有去验证它。 发现问题的**也不是我们**。它是由外部报告的,见 [issue #379](https://github.com/Nimblesite/Basilisk/issues/379),并附有公开的复现。 diff --git a/website/src/zh/docs/conformance.md b/website/src/zh/docs/conformance.md index 015c4f54..1716a47b 100644 --- a/website/src/zh/docs/conformance.md +++ b/website/src/zh/docs/conformance.md @@ -15,13 +15,9 @@ lang: zh 针对固定测试用例开发出的代码即使通过,也不能作为证据;因此,解决办法不是拿到一个更好的分数。 -## 这是一个错误,而不是有意操纵测试套件 +## 这是怎么发生的 -我们希望在这一点上毫不含糊,因为事件的先后顺序容易让人得出一个比事实更糟的解读。 - -**没有人试图去糊弄符合性测试套件,也没有向 `python/typing` 隐瞒任何东西。** 提交时运行的是套件自己未经修改的评分工具,使用 Basilisk 的默认配置并启用全部规范规则 —— 没有针对评分工具做特殊处理,没有伪造输出,没有禁用规则,也没有手工编辑结果文件。当缺陷被演示给我们之后,我们主动请求移除自己。 - -真实经过要平淡得多。我们的开发流程把符合性分数定为要去达成的目标,而匹配源文本比真正做分析更快地提高这个分数,于是代码就朝那个方向滑了过去 —— 一条看起来合理的规则接着一条。然后我们仅凭一次全绿的运行就发布并提交了结果,却始终没有做那项本可以揭露问题的检查:同一个程序换一种写法时,这条规则是否依然成立?这项检查当时并不存在,把它建起来正是下文补救工作的一部分。符合性测试套件从构造上就无法发现这一类缺陷,因为它正是代码被适配过去的那个对象,所以每一次全绿运行都在强化一个我们根本没有依据的结论。 +我们的开发流程把符合性分数定为要去达成的目标,而匹配源文本比真正做分析更快地提高这个分数,于是代码就朝那个方向滑了过去 —— 一条看起来合理的规则接着一条。然后我们仅凭一次全绿的运行就发布并提交了结果,却始终没有做那项本可以揭露问题的检查:同一个程序换一种写法时,这条规则是否依然成立?这项检查当时并不存在,把它建起来正是下文补救工作的一部分。符合性测试套件从构造上就无法发现这一类缺陷,因为它正是代码被适配过去的那个对象,所以每一次全绿运行都在强化一个我们根本没有依据的结论。 我们当时相信那个数字就是我们所说的意思。我们错了;我们之所以错,是因为我们没有去验证它。Basilisk 作者已就此发表[个人说明与致歉](https://www.christianfindlay.com/blog/basilisk-conformance-apology),并承担责任。 diff --git a/website/src/zh/index.njk b/website/src/zh/index.njk index 59868ad0..2f1d2def 100644 --- a/website/src/zh/index.njk +++ b/website/src/zh/index.njk @@ -77,7 +77,7 @@ permalink: /zh/

- 审计期间,两组数据均已撤回。这是一个错误、一次验证上的失职,而不是有意操纵测试套件。 + 审计期间,两组数据均已撤回。 阅读符合性更正 → 阅读作者的致歉 → 阅读基准测试说明 →