diff --git a/scripts/asvs/scorecard.py b/scripts/asvs/scorecard.py index dc978efc..f207d6f4 100644 --- a/scripts/asvs/scorecard.py +++ b/scripts/asvs/scorecard.py @@ -20,11 +20,15 @@ import argparse import ast +import datetime +import io import re import shutil import subprocess import sys import tempfile +import time +import tokenize import tomllib from collections import Counter from dataclasses import dataclass, field @@ -33,6 +37,33 @@ Verdict = Literal["pass", "partial", "fail", "na", "needs-review", "unverified"] +#: What KIND of artifact an evidence anchor resolves INTO. Derived at check time from where the token +#: lands, never authored: it is a property of the landing site, not a judgement about the cell. +#: +#: - ``code`` — a Python file, outside every docstring and ``#`` comment. +#: - ``doc`` — a Python file, inside a docstring (the first statement of a module, class or +#: function) or inside a ``#`` comment. +#: - ``foreign`` — not a Python file at all: ``.md``, ``.ts``, ``.js``, ``.yml``, ``.toml``, … +#: +#: **``doc`` is a LABEL, never a demotion**, and nothing here consumes it as one. The split is +#: rendered so the record stops presenting every anchor as if it were code evidence; it is +#: deliberately NOT wired into :func:`check_completeness`, into any verdict, or into the exit code. +#: +#: Measured 2026-08-09, vault ``origin/main`` 1a59e4a1's scorecard against engine tree c383eeab — +#: quoted as a (file x ref) pair because a count is a fact about one, and this programme has already +#: produced three wrong-base errors by dropping the qualifier: +#: +#: - 1,980 anchors; 1,979 located; **1,479 code, 233 doc (173 docstring + 60 comment), 267 foreign** +#: (197 ``.md``, 17 ``.ts``, 16 ``.js``, 13 ``.yml``, 8 ``.toml``, the rest scattered). +#: - So **500 of 1,979 (25.3%) resolve into prose or a non-Python file**, which no structural or +#: executable scheme reaches, ever. +#: - **17 of 343 evidenced cells carry NO code anchor at all** and rest entirely on documentation, +#: which for a documentation requirement is the correct ground: 1.4.3, 2.1.2, 3.1.1, 3.5.5, 11.1.1, +#: 12.2.2, 13.1.1-13.1.4, 13.4.3, 14.1.2, 15.1.1, 15.1.2, 15.1.4, 15.2.1, 16.1.1. That figure was +#: derived here independently and agrees exactly with the 2026-08-09 recut analysis, which is why +#: the label is stated as a label rather than hedged. +AnchorForm = Literal["code", "doc", "foreign"] + #: The scoring buckets. ``unverified`` is deliberately first-class (ADR 0156 §5): a cell inherited from #: an earlier assessment and never re-read against the requirement text is NOT a Pass, and conflating #: the two is what let ~219 unchecked verdicts hide inside a headline. @@ -94,11 +125,22 @@ class ScorecardError(Exception): @dataclass(frozen=True) class Anchor: - """A claim that some token exists in the tree, at roughly a known place.""" + """A claim that some token exists in the tree, at roughly a known place. + + ``sym`` and ``ctx`` are OPTIONAL and additive. ``None`` means *not asserted*; an empty string means + *asserted to be nothing* — module level for ``sym``, unnested for ``ctx``. Those are different + claims and a single ``str`` default would silently merge them, turning every un-backfilled anchor + into an assertion that it sits at module level and unnested. See :func:`derive_sym_ctx`. + """ path: str line: int expect: str + #: Enclosing symbol, dotted (``ClassName.method``). ``""`` = module level, ``None`` = not asserted. + sym: str | None = None + #: Block-node chain from the symbol inward (``Try.body``, ``Try.body>If.orelse``). ``""`` = + #: unnested, ``None`` = not asserted. + ctx: str | None = None @dataclass(frozen=True) @@ -195,12 +237,38 @@ class Findings: #: recorded line numbers rot; they just do not red the gate. advisories: list[str] = field(default_factory=list) checked_anchors: int = 0 + #: The POPULATION of absence claims a pass looked at. Set by BOTH :func:`check_absences` and + #: :func:`prove_absences` — the two passes read the same population, and without it neither one's + #: outcome counters can be reconciled against anything. A run that scanned 276 claims and a run + #: that scanned zero otherwise print counter sets that look equally plausible. checked_absences: int = 0 skipped_anchors: int = 0 + #: Derived :data:`AnchorForm` of every anchor that LOCATED, plus an ``undetermined`` bucket for a + #: Python file that would not parse or tokenize. Anchors that did not locate (GONE or AMBIGUOUS) + #: are absent from this counter entirely — there is no landing site to classify — so its total is + #: ``checked_anchors`` minus those, and the summary prints both numbers rather than either alone. + anchor_forms: Counter[str] = field(default_factory=Counter) + #: Anchors carrying a ``sym`` or a ``ctx``, so the summary can say how much of the record the + #: structural check actually reached. Backfill is Stream D's; until it lands this is small, and a + #: check that silently covers 3 of 1,980 anchors while printing like a whole-corpus result is the + #: exact overstatement this pass has been correcting everywhere else. + checked_sym_ctx: int = 0 #: Populated only by :func:`prove_absences`. ``proved_absences`` counts claims whose observable #: went red under the applied mutation (a live proof); ``static_screened`` counts claims that took #: the static backstop (a screen, not a proof); ``skipped_absences`` counts claims carrying no - #: ``mutation_path`` (nothing to apply). UNPROVEN and PROVE-ERROR outcomes go into ``problems``. + #: ``mutation_path`` (nothing to apply). + #: + #: **These three do NOT sum to the population, and that is why ``checked_absences`` above must be + #: printed beside them.** FIVE outcomes raise a problem and increment no counter at all — a + #: ``mutation_path`` that escapes the tree, one that is not a file, a baseline that is not green, + #: an UNPROVEN mutated-green, and a mutated run that errored rather than failed. The arithmetic + #: that closes is:: + #: + #: checked_absences - proved_absences - static_screened - skipped_absences + #: == claims that ended in a problem-only branch + #: + #: and note that ``len(problems)`` is NOT that number: a SUSPECT finding rides along with a claim + #: already counted in ``static_screened``, so problems and claims are different populations. proved_absences: int = 0 static_screened: int = 0 skipped_absences: int = 0 @@ -322,7 +390,17 @@ def load_scorecard(path: Path) -> list[Cell]: verified_at=str(raw.get("verified_at", "")), reviewed_by=str(raw.get("reviewed_by", "")), evidence=tuple( - Anchor(path=str(e["path"]), line=int(e["line"]), expect=str(e["expect"])) + Anchor( + path=str(e["path"]), + line=int(e["line"]), + expect=str(e["expect"]), + # `.get(...)` WITHOUT a "" default, deliberately: absent must stay None. An + # empty string is the assertion "module level" / "unnested", so defaulting to + # it would turn all 1,980 un-backfilled anchors into that claim overnight and + # light up every anchor that is legitimately inside a function. + sym=None if e.get("sym") is None else str(e["sym"]), + ctx=None if e.get("ctx") is None else str(e["ctx"]), + ) for e in raw.get("evidence", []) ), absence=tuple( @@ -404,6 +482,259 @@ def check_completeness(cells: list[Cell], corpus: dict[str, int]) -> list[str]: return problems +#: Suffixes this module will submit to Python structural analysis. Everything else is ``foreign`` by +#: construction — no `ast` and no `tokenize` reaches a `.md`, `.ts`, `.yml` or `.toml` file, ever. +PYTHON_SUFFIXES: Final[frozenset[str]] = frozenset({".py", ".pyi"}) + + +def _prose_spans(text: str) -> list[tuple[int, int]] | None: + """Character-offset spans of the docstrings and ``#`` comments in one Python source. + + ``None`` means the source could not be analysed — it does not parse, or does not tokenize. The + caller must report that as UNDETERMINED rather than defaulting, because the default that feels + natural is ``code`` and it would silently inflate the one number this split exists to deflate. + + **A docstring is identified STRUCTURALLY — the first statement of a module, class or function, via + ``ast`` — and never by looking at the string's contents.** That is the whole design, and the + alternative was measured and rejected: a token mask ("does this text look like code?") misfiles the + Content-Security-Policy fragments in ``messagefoundry_webconsole/_security.py`` and the entire SQL + Server and Postgres DDL as prose. Those are long, quoted, space-separated strings that read as + English to a mask while being the literal subject of the control their cell cites. Position cannot + make that mistake, and not because it is a better mask — because it never asks the question. A + string that is not the first statement of a scope is not a docstring, whatever it reads like. + + Comments come from ``tokenize`` rather than a ``#`` scan, so a ``#`` inside a string literal is + not mistaken for one — which the CSP and URL fragments in the live record depend on. + """ + line_starts = [0] + for i, ch in enumerate(text): + if ch == "\n": + line_starts.append(i + 1) + + def offset(row: int, col: int) -> int: + return line_starts[row - 1] + col + + try: + tree = ast.parse(text) + except (SyntaxError, ValueError): + return None + # Row pairs, not character columns: `ast` reports `col_offset` as a UTF-8 BYTE offset, which + # disagrees with the character offsets everything else here uses the moment a line holds a + # non-ASCII character. So `ast` is asked only WHICH string is a docstring, and `tokenize` — whose + # columns are characters — supplies the span. + doc_rows: set[tuple[int, int]] = set() + for node in ast.walk(tree): + if not isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + continue + if not node.body: + continue + first = node.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + and first.end_lineno is not None + ): + doc_rows.add((first.lineno, first.end_lineno)) + + spans: list[tuple[int, int]] = [] + try: + for tok in tokenize.generate_tokens(io.StringIO(text).readline): + if tok.type == tokenize.COMMENT or ( + tok.type == tokenize.STRING and (tok.start[0], tok.end[0]) in doc_rows + ): + spans.append((offset(*tok.start), offset(*tok.end))) + except (tokenize.TokenError, IndentationError, SyntaxError, ValueError): + return None + return spans + + +def _form_from_spans( + path: str, spans: list[tuple[int, int]] | None, start: int +) -> AnchorForm | None: + """Classify one landing site against pre-computed spans (see :func:`anchor_form`).""" + if Path(path).suffix not in PYTHON_SUFFIXES: + return "foreign" + if spans is None: + return None + # The token's START decides, not any overlap. An `expect` that begins in code and runs into a + # trailing lint-suppression comment is code; the OVERLAP rule calls it doc and reclassified 57 of + # 1,712 live Python anchors that way when both were measured on 2026-08-09. (The suppression + # directive is named in words, not written literally: ruff parses one in any comment, and this + # module is also inside the corpus that absence patterns grep over.) + return "doc" if any(lo <= start < hi for lo, hi in spans) else "code" + + +def anchor_form(path: str, text: str, start: int) -> AnchorForm | None: + """The derived ``form`` of one anchor: where does its token actually land? + + ``path`` is the anchor's repo-relative path, ``text`` the file's contents as the anchor check read + them, ``start`` the character offset of the match. ``None`` means undetermined — a Python file + that would not parse — and is never silently folded into ``code``. + + This answers a question the record has been getting wrong by omission: **a quarter of its anchors + resolve into prose or a non-Python file, and no structural or executable check reaches any of them, + ever.** The record presented all of them as code evidence. Naming the form does not change a single + verdict and must not: see :data:`AnchorForm` for the measurement and for why ``doc`` is a label + rather than a demotion. + """ + if Path(path).suffix not in PYTHON_SUFFIXES: + return "foreign" + return _form_from_spans(path, _prose_spans(text), start) + + +# --- sym + ctx: WHERE in the file's structure the token sits --------------------------------------- + +#: The block-bearing fields of every statement that can nest another, and **the single source of truth +#: for both derivation and validation.** A `ctx` value is well-formed exactly when every element names +#: a key here and one of its fields, so a typo cannot produce a chain that silently never matches. +#: One table, two consumers — the alternative is a validator and a deriver that disagree, which is the +#: same defect shape as a gate whose check and whose error message were written separately. +_BLOCK_FIELDS: Final[dict[str, tuple[str, ...]]] = { + "Try": ("body", "handlers", "orelse", "finalbody"), + "TryStar": ("body", "handlers", "orelse", "finalbody"), + "If": ("body", "orelse"), + "For": ("body", "orelse"), + "AsyncFor": ("body", "orelse"), + "While": ("body", "orelse"), + "With": ("body",), + "AsyncWith": ("body",), + "Match": ("cases",), +} + +#: Nodes that nest a statement but are never reached by a name of their own: an ``ExceptHandler`` +#: comes only via ``Try.handlers``, a ``match_case`` only via ``Match.cases``. +#: +#: **Kept separate from :data:`_TRANSPARENT` on purpose, and the separation was earned by a failed +#: injection.** "Can the walk descend into this?" and "does it contribute a chain element?" are two +#: questions, and a first cut answered both from one set. That made the second one UNTESTABLE: +#: deleting ``ExceptHandler`` from the transparent set silently stopped the walk DESCENDING rather +#: than starting to RECORD, the chain came out identical by a different route, and an injection that +#: should have reddened a test changed nothing observable. Two tables, two behaviours, both drivable. +_DESCEND_ONLY: Final[dict[str, tuple[str, ...]]] = { + "ExceptHandler": ("body",), + "match_case": ("body",), +} + +#: Nodes descended through WITHOUT contributing an element — recording them would double every +#: handler chain (``Try.handlers>ExceptHandler.body``) for no added discrimination. Must name every +#: key of :data:`_DESCEND_ONLY`; a test asserts that, because the two are independent by design and +#: nothing else would notice them drifting apart. +_TRANSPARENT: Final[frozenset[str]] = frozenset({"ExceptHandler", "match_case"}) + +#: Nodes that OPEN A SYMBOL: they contribute a name to ``sym`` and RESET ``ctx``, because a block +#: chain is meaningful only within the symbol that contains it. +_SCOPE_NODES: Final = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + +_SYM_RE: Final[re.Pattern[str]] = re.compile(r"[A-Za-z_]\w*(\.[A-Za-z_]\w*)*") + + +def _block_fields(node: ast.AST) -> tuple[str, ...]: + """Which fields the walk may descend into. DESCENT ONLY — whether the step is recorded is + :data:`_TRANSPARENT`'s question, asked separately in :func:`sym_ctx_at`.""" + if isinstance(node, (ast.Module, *_SCOPE_NODES)): + return ("body",) + name = type(node).__name__ + return _BLOCK_FIELDS.get(name) or _DESCEND_ONLY.get(name, ()) + + +def _next_block(node: ast.AST, line: int) -> tuple[str, ast.AST] | None: + """The (field, child) one level in that contains `line`, or ``None`` at the innermost node.""" + for fieldname in _block_fields(node): + for child in getattr(node, fieldname, None) or []: + lo = getattr(child, "lineno", None) + hi = getattr(child, "end_lineno", None) + if lo is not None and hi is not None and lo <= line <= hi: + return fieldname, child + return None + + +def derive_sym_ctx(text: str, line: int) -> tuple[str, str] | None: + """The enclosing symbol and block chain at `line`. ``None`` when the source will not parse. + + ``sym`` is dotted and outermost-first (``SqlServerStore.route_handoff``), ``""`` at module level. + ``ctx`` is the chain of block statements between the symbol and the line, joined by ``>`` + (``Try.body``, ``Try.body>If.orelse``), ``""`` when unnested. + + **``ctx``, NOT raw indentation, and the difference is the whole reason this field is shaped this + way.** Cell 12.3.5 carries the identical 4-versus-8 indent mismatch as 10.5.4 and is a non-event — + a hand-trimming slip, with ``ctx`` unchanged at both ends. Indentation flags it and would have to + be triaged; ``ctx`` correctly does not fire, because nothing about the statement's position in the + control flow changed. An indentation check would have spent a person's attention on a whitespace + edit while claiming to be a structural signal. + + Containment is decided by LINE, so a one-line compound (``if x: y = 1``) attributes the line to the + ``If`` rather than to the assignment inside it. `ast` reports ``col_offset`` as a UTF-8 BYTE offset, + which disagrees with the character offsets the rest of this module uses the moment a line holds a + non-ASCII character; line granularity has no such failure mode and every real anchor is a statement + on its own line. + + **For a multi-line ``expect`` the line is the token's FIRST line**, matching what the drift + advisory reports. Any backfill must derive at the same line or the 42 multi-line tokens in the + record will mismatch by construction — not because anything moved, but because the two ends + measured different lines. + """ + tree = parse_or_none(text) + return None if tree is None else sym_ctx_at(tree, line) + + +def parse_or_none(text: str) -> ast.Module | None: + """``ast.parse`` that reports failure as a value. Callers must surface it, never default it.""" + try: + return ast.parse(text) + except (SyntaxError, ValueError): + return None + + +def sym_ctx_at(tree: ast.Module, line: int) -> tuple[str, str]: + """The symbol/chain walk itself, split from parsing so a file is parsed once per run and not + once per anchor — ``settings.py`` alone carries 218 anchors.""" + sym: list[str] = [] + ctx: list[str] = [] + node: ast.AST = tree + while True: + step = _next_block(node, line) + if step is None: + break + fieldname, child = step + name = type(node).__name__ + if not isinstance(node, (ast.Module, *_SCOPE_NODES)) and name not in _TRANSPARENT: + ctx.append(f"{name}.{fieldname}") + if isinstance(child, _SCOPE_NODES): + sym.append(child.name) + ctx.clear() # a block chain is meaningful only INSIDE the symbol that holds it + node = child + return ".".join(sym), ">".join(ctx) + + +def malformed_sym_ctx(sym: str | None, ctx: str | None) -> str | None: + """Why these values could never match anything, or ``None`` if they are well-formed. + + Well-formedness is checked against :data:`_BLOCK_FIELDS`, the same table the deriver walks, so a + chain that is accepted here is one the deriver can actually produce. This is the only part of the + sym/ctx feature that is FATAL rather than advisory, and the reason is that it is a defect in the + RECORD rather than a fact about the code: no amount of engine movement can cause it, the fix is + unambiguous, and an unmatchable chain otherwise sits there advising forever. + """ + if sym is not None and sym and not _SYM_RE.fullmatch(sym): + return f"sym {sym!r} is not a dotted Python identifier path" + if ctx is None or not ctx: + return None + for element in ctx.split(">"): + node, _, fieldname = element.partition(".") + if node not in _BLOCK_FIELDS: + return ( + f"ctx element {element!r} names {node!r}, which is not a block statement this " + f"deriver can produce (known: {', '.join(sorted(_BLOCK_FIELDS))})" + ) + if fieldname not in _BLOCK_FIELDS[node]: + return ( + f"ctx element {element!r} names field {fieldname!r}, which {node} does not have " + f"(known: {', '.join(_BLOCK_FIELDS[node])})" + ) + return None + + def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: """Open every evidence anchor and assert its token still resolves, and resolves UNAMBIGUOUSLY. @@ -434,14 +765,26 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: this file, once* — not *this control operates on the path the cell describes*. A cell can therefore be green here and wrong: measured 2026-08-09 on 15.3.1, which was ``pass`` with every anchor resolving while the control it named had a hole, found only by executing the code. + + Each anchor that LOCATES is also given a derived :data:`AnchorForm` (:func:`anchor_form`) and + counted into ``findings.anchor_forms``. That is reporting only — no branch here reads it, and no + verdict depends on it. """ + # Per-run caches. 1,980 live anchors concentrate onto ~224 distinct files (`settings.py` alone + # carries 218), so both the read and the parse are re-done an order of magnitude more often than + # there are files to do them on. + text_cache: dict[Path, str] = {} + spans_cache: dict[Path, list[tuple[int, int]] | None] = {} + ast_cache: dict[Path, ast.Module | None] = {} for c in cells: for a in c.evidence: target = root / a.path if not target.is_file(): findings.problems.append(f"{c.id}: evidence path {a.path} does not exist") continue - text = target.read_text(encoding="utf-8", errors="replace") + if target not in text_cache: + text_cache[target] = target.read_text(encoding="utf-8", errors="replace") + text = text_cache[target] findings.checked_anchors += 1 occurrences = text.count(a.expect) if occurrences > 1: @@ -490,13 +833,108 @@ def check_anchors(cells: list[Cell], root: Path, findings: Findings) -> None: # tokens span a newline, because the old check matched against joined text and nothing # forbade it. A per-line scan finds none of those and raises on the lookup; counting # newlines before the match handles a multi-line token as naturally as a single-line one. - actual = text.count("\n", 0, text.index(a.expect)) + 1 + start = text.index(a.expect) + # Derived form of the landing site. Only anchors that reached here are classified: a GONE + # or AMBIGUOUS token has no single landing site, so inventing a form for it would be a + # made-up number in a split whose whole purpose is to stop the record overstating itself. + if target not in spans_cache: + spans_cache[target] = ( + _prose_spans(text) if Path(a.path).suffix in PYTHON_SUFFIXES else None + ) + findings.anchor_forms[ + _form_from_spans(a.path, spans_cache[target], start) or "undetermined" + ] += 1 + actual = text.count("\n", 0, start) + 1 if actual != a.line: findings.advisories.append( f"{c.id}: {a.path} — {a.expect!r} is unique and present, recorded at line " f"{a.line} but actually at {actual} (offset {actual - a.line:+d}). " "Advisory: the line is a navigation aid, not the proof" ) + _check_sym_ctx(c.id, a, text, actual, ast_cache, target, findings) + + +def _check_sym_ctx( + cell_id: str, + a: Anchor, + text: str, + actual: int, + ast_cache: dict[Path, ast.Module | None], + target: Path, + findings: Findings, +) -> None: + """Validate a recorded ``sym``/``ctx`` against where the token now sits. + + **THIS IS A DISPLACEMENT SIGNAL, NOT A DEFECT DETECTOR; its security-relevant precision measured + 0 of 1 on the only datum the corpus offers, because 10.5.4's red was a HARDENING.** The change + that moved that statement into a ``try`` made the code safer, and a structural check fired on it. + Read this field as "your reasoning about this cell is stale, go re-read it" — never as "something + is wrong here". A signal described as a defect detector, in the place people read it, will be + quoted as one, and this one would then be quoted at 0% precision. + + That is why a mismatch is an ADVISORY and not a problem. The only fatal outcome is a value that is + malformed (:func:`malformed_sym_ctx`) — a defect in the record, which no engine movement can cause. + + **ADDITIVE to the line-drift advisory, never a replacement.** The two catch different things and + neither dominates: a token can move hundreds of lines without leaving its symbol (drift fires, + sym/ctx does not), and a token can be welded into a new ``try`` or ``if`` without moving at all + (sym/ctx fires, drift does not). + + Measured 2026-08-09, vault ``origin/main`` 1a59e4a1's anchors against engine tree ``4667e945``, + over the 1,712 Python anchors that locate and parse. **The region is the innermost node the + ``(sym, ctx)`` PAIR pins** — both fields must match, so the pair is only as loose as its tighter + half: + + - **536 of 1,712 (31.3%)** sit in a region spanning MORE than the 81 lines the retired +/-40 + window covered. For those, sym/ctx alone is the LOOSER of the two signals. + - Region spans: median 33, p90 567, max 9,799. + - 1,403 of 1,712 are unnested (``ctx == ""``) and 280 are at module level (``sym == ""``), so for + most anchors the pair reduces to "which function is this in". + + **The 38.4% / 639 figure this work was briefed with is reproducible, under a LOOSER definition:** + scoring the region as the enclosing SYMBOL only, ignoring the ``ctx`` refinement, gives **634** at + this ref against that brief's 639. Both definitions support the same conclusion and it is the only + one that matters here — replacing drift with sym/ctx would lose detection on roughly a third of + the record — so this stays additive either way. + """ + if a.sym is None and a.ctx is None: + return # not asserted. Absence of the field is not an assertion that the region is empty. + findings.checked_sym_ctx += 1 + problem = malformed_sym_ctx(a.sym, a.ctx) + if problem: + findings.problems.append( + f"{cell_id}: {a.path}:{a.line} {problem}. A chain this deriver cannot produce would " + "advise forever without ever matching, so it is refused rather than left to rot" + ) + return + if Path(a.path).suffix not in PYTHON_SUFFIXES: + findings.problems.append( + f"{cell_id}: {a.path}:{a.line} records sym/ctx on a non-Python file, which has no " + "enclosing symbol and no block structure — no derivation can ever confirm or deny it" + ) + return + if target not in ast_cache: + ast_cache[target] = parse_or_none(text) + tree = ast_cache[target] + if tree is None: + findings.advisories.append( + f"{cell_id}: {a.path} — sym/ctx recorded but the file will not parse, so neither could " + "be derived. Advisory: UNDETERMINED, which is not the same as agreeing" + ) + return + got_sym, got_ctx = sym_ctx_at(tree, actual) + if a.sym is not None and a.sym != got_sym: + findings.advisories.append( + f"{cell_id}: {a.path} — {a.expect!r} recorded sym={a.sym!r} but now sits in " + f"sym={got_sym!r} (line {actual}). Advisory: DISPLACEMENT, not a defect — re-read the " + "cell's reasoning, do not assume anything is wrong" + ) + if a.ctx is not None and a.ctx != got_ctx: + findings.advisories.append( + f"{cell_id}: {a.path} — {a.expect!r} recorded ctx={a.ctx!r} but now sits in " + f"ctx={got_ctx!r} (line {actual}). Advisory: DISPLACEMENT, not a defect — the control " + "flow around this token changed, which may be a HARDENING (10.5.4 was)" + ) def check_absences(cells: list[Cell], root: Path, findings: Findings) -> None: @@ -772,6 +1210,9 @@ def prove_absences( for c in cells: for a in c.absence: i += 1 + # The POPULATION, recorded before any outcome branch, so it is right whichever branch + # this claim takes. Without it the outcome counters float free of what was scanned. + findings.checked_absences += 1 _prove_one( a, c.id, @@ -986,6 +1427,283 @@ def render_current(cells: list[Cell], *, anchor_sha: str) -> str: return chr(10).join(lines) + chr(10) +# --- provenance: every answer carries the refs it was measured at --------------------------------- +# +# A count is a fact about a (file x ref) PAIR. Drop the ref and two readings taken from different +# places print identically — which is exactly how this programme produced three wrong-base errors in +# one working thread on 2026-08-08/09: one in an adjudication, one in the correction of that +# adjudication, and one in a DAG-ancestry check that cannot answer "did this land" under squash-merge. +# Each cost a full re-measurement cycle. A query tool without ref stamping would INDUSTRIALISE that +# failure, because cheap answers get quoted more, not less. +# +# THE REQUIREMENT IS NOT FRESHNESS. IT IS THAT THE FRESHNESS CLAIM IS NEVER SILENT. Those come apart, +# and separating them dissolves the problem with zero network: the error that motivated this was not +# reading a stale ref, it was reading a stale ref while the output said nothing about it. + +#: Seconds any single git probe may take before it is abandoned. `--status` is meant to be run in a +#: loop; a probe that can hang is a probe that gets removed. +_GIT_TIMEOUT: Final[float] = 5.0 + + +@dataclass(frozen=True) +class RepoStamp: + """Where a tree actually is, and how much it knows about where it should be. + + Every field is ALWAYS POPULATED. Degradation is a loud labelled value in the field, never an + omitted field — an absent qualifier is what produced all three wrong-base errors, where the number + was right and the ref was unnamed. + """ + + #: Short commit, or ``NO-GIT`` when the path is not inside a work tree. + sha: str + #: Whole-tree dirty. A measurement taken against uncommitted changes is not reproducible and must + #: not look like one that is. + dirty: bool + #: ``CURRENT`` | ``BEHIND `` | ``AHEAD `` | ``DIVERGED`` | ``NO-UPSTREAM`` | ``NO-GIT`` | + #: ``UNRESOLVED``. The last two are extensions and they exist BECAUSE of the never-silent rule: + #: calling a non-repo ``NO-UPSTREAM`` would be a false statement (it implies a repo), and calling a + #: comparison that git refused ``CURRENT`` would be the exact silence this field exists to break. + freshness: str + #: The ref the freshness was measured AGAINST, or ``none``. Named because "BEHIND 37" is not a + #: claim until you know behind WHAT: on a feature branch the branch's own upstream and the + #: canonical line are different questions with different answers. + upstream: str + #: Humanised age of ``FETCH_HEAD``, or ``NEVER-FETCHED``. NOT decoration: ``BEHIND 0`` from a + #: six-hour-old fetch and ``BEHIND 0`` from a one-minute-old fetch are different claims and must + #: not print identically. + remote_knowledge: str + + def ref(self) -> str: + return f"{self.sha}+dirty" if self.dirty else self.sha + + +def _git(repo: Path, *args: str) -> str | None: + """One read-only git probe. ``None`` on any failure — missing git, not a repo, non-zero, timeout. + + NEVER runs a command that writes. There is deliberately no ``--fetch`` mode in this module: a + fetch mutates remote-tracking refs, which a query tool run in a loop has no business doing, and on + this machine the vault remote is intermittently unauthenticated, so a network dependency here + would fire constantly and the tool would be bypassed inside a day. A bypassed tool is worse than + none, because its absence gets read as nobody needing it. Refreshing is ``git fetch``, by hand, + which is a different act performed on purpose. + """ + try: + proc = subprocess.run( # nosec B603 B607 - fixed argv, no shell; every subcommand is read-only + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=False, + timeout=_GIT_TIMEOUT, + ) + except (OSError, subprocess.SubprocessError): + return None + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def _humanise_age(seconds: float) -> str: + if seconds < 90: + return f"{int(seconds)}s" + if seconds < 90 * 60: + return f"{int(seconds / 60)}m" + if seconds < 48 * 3600: + return f"{int(seconds / 3600)}h" + return f"{int(seconds / 86400)}d" + + +def _remote_knowledge(repo: Path, now: float) -> str: + """How old the last-fetched remote knowledge is, from the mtime of ``FETCH_HEAD``. No network. + + ``FETCH_HEAD`` can live in either the per-worktree git dir or the common one depending on git + version and who last fetched, so both are probed and the NEWEST is reported. Reporting the older + of the two would overstate staleness; reporting only one would miss a fetch entirely, and a missed + fetch reads as ``NEVER-FETCHED``, which is the loudest possible wrong answer. + """ + newest: float | None = None + for which in ("--git-dir", "--git-common-dir"): + # `--path-format` is git 2.31+. Falling back to the relative form matters more than it looks: + # without it an older git would yield no path, `NEVER-FETCHED`, and a confidently wrong + # "nobody has ever fetched here" — the loudest possible wrong answer from this field. + out = _git(repo, "rev-parse", "--path-format=absolute", which) or _git( + repo, "rev-parse", which + ) + if not out: + continue + try: + mtime = (repo / out / "FETCH_HEAD").stat().st_mtime + except OSError: + continue + newest = mtime if newest is None else max(newest, mtime) + if newest is None: + return "NEVER-FETCHED" + return _humanise_age(max(0.0, now - newest)) + + +def _freshness(repo: Path) -> tuple[str, str]: + """``(freshness, upstream)`` for a work tree, using ZERO network. + + ``git rev-list --left-right --count HEAD...`` counts against the LAST-FETCHED + remote-tracking ref, which is a purely local object. Measured against the vault checkout whose + stale ref caused the original error in this programme: ``BEHIND 37``, remote knowledge 23 minutes + old. That pair would have stopped the error dead, with no network call. + + The upstream is the branch's own ``@{upstream}`` when it has one, and ``origin/main`` otherwise — + and WHICHEVER was used is returned, because on a feature branch those are different questions. + """ + upstream = _git(repo, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}") + if not upstream: + upstream = "origin/main" if _git(repo, "rev-parse", "--verify", "origin/main") else "" + if not upstream: + return "NO-UPSTREAM", "none" + counts = _git(repo, "rev-list", "--left-right", "--count", f"HEAD...{upstream}") + parts = counts.split() if counts else [] + if len(parts) != 2 or not all(p.isdigit() for p in parts): + # The ref name exists but git would not compare against it (a pruned or corrupt + # remote-tracking ref). Labelled, never quietly rendered as CURRENT. + return "UNRESOLVED", upstream + ahead, behind = int(parts[0]), int(parts[1]) + if ahead and behind: + return "DIVERGED", upstream + if behind: + return f"BEHIND {behind}", upstream + if ahead: + return f"AHEAD {ahead}", upstream + return "CURRENT", upstream + + +def repo_stamp(path: Path, *, now: float | None = None) -> RepoStamp: + """Stamp the work tree containing `path`. Pure read: no fetch, no write, no network.""" + repo = path if path.is_dir() else path.parent + sha = _git(repo, "rev-parse", "--short", "HEAD") + if sha is None: + # Not a work tree at all. NOT reported as NO-UPSTREAM: that would imply a repo exists and is + # simply untracked, which is a different and false statement. + return RepoStamp("NO-GIT", False, "NO-GIT", "none", "NEVER-FETCHED") + porcelain = _git(repo, "status", "--porcelain") + freshness, upstream = _freshness(repo) + return RepoStamp( + sha=sha, + dirty=bool(porcelain), + freshness=freshness, + upstream=upstream, + remote_knowledge=_remote_knowledge(repo, time.time() if now is None else now), + ) + + +def provenance_lines(scorecard: Path, root: Path, *, now: float | None = None) -> list[str]: + """The non-suppressible provenance header. Every query answer carries the refs it was measured at. + + **One deviation from the spec's literal shape, and it is deliberate — do not "fix" it back.** The + spec draws ONE ``freshness`` and ONE ``remote-knowledge`` under a line naming TWO repositories. + The scorecard and the engine are separate checkouts (the vault's own CI checks out the engine into + a subdirectory and runs with ``--root engine``), and a single freshness field covering both would + itself be an unnamed qualifier — the reader could not tell which repo it described. That is the + precise defect the field exists to prevent, so the group is emitted PER REPO and labelled. No + mandated field is renamed, dropped, or given a value outside its stated set. + + ``upstream=`` is likewise additive: "BEHIND 37" is not a claim until you know behind what. + """ + sc = repo_stamp(scorecard, now=now) + en = repo_stamp(root, now=now) + generated = datetime.datetime.now(datetime.UTC).isoformat(timespec="seconds") + return [ + f"# asvs-status scorecard={sc.ref()} engine={en.ref()}", + f"# scorecard: freshness={sc.freshness} upstream={sc.upstream} " + f"remote-knowledge={sc.remote_knowledge}", + f"# engine: freshness={en.freshness} upstream={en.upstream} " + f"remote-knowledge={en.remote_knowledge}", + f"# generated={generated}", + ] + + +def status_lines(cells: list[Cell]) -> list[str]: + """``--status`` proper: what the scorecard SAYS, computed on every call and cached nowhere. + + Nothing here is persisted, because a cached query result would be document number 69 in a corpus + where 68 documents assert a tally and approximately one is correct — stale in the same way, for + the same reason, with more authority because a tool produced it. + + **This is a pure read of the scorecard and it names what it therefore cannot see.** Whether an + anchor still RESOLVES is a fact about the engine tree, costs a 40-second pass, and is what + :func:`verify` is for. Printing a structural tally under a heading that implies resolution health + would be the same overstatement the summary line was just corrected for. + """ + n = count(cells) + total = sum(n.values()) + examined = sum(1 for c in cells if c.verdict in EXAMINED_VERDICTS and c.last_verified) + inherited = sum(1 for c in cells if c.verdict in DECIDED_VERDICTS and not c.last_verified) + closed = sum(1 for c in cells if c.decision_closed) + anchors = sum(len(c.evidence) for c in cells) + anchored_cells = sum(1 for c in cells if c.evidence) + paths = {a.path for c in cells for a in c.evidence} + absences = sum(len(c.absence) for c in cells) + provable = sum(1 for c in cells for a in c.absence if a.observable) + unevidenced = sum( + 1 for c in cells if c.verdict in DECIDED_VERDICTS and not c.evidence and not c.absence + ) + pct = (100.0 * examined / total) if total else 0.0 + return [ + f"cells {total}: {n['pass']} pass, {n['partial']} partial, {n['fail']} fail, {n['na']} na, " + f"{n['needs-review']} needs-review, {n['unverified']} unverified", + f"examined {examined} of {total} ({pct:.1f}%) against the pinned text; " + f"{inherited} decided with no last_verified; {closed} closed by owner decision", + f"evidence {anchors} anchors in {anchored_cells} cells over {len(paths)} paths; " + f"{unevidenced} decided cells carry neither an anchor nor an absence claim", + f"absence {absences} claims, {provable} of them carrying an observable " + "(the rest cannot be proved by execution)", + "NOT CHECKED here: whether any anchor still resolves, whether any absence claim is still " + "true, and completeness against the corpus. --status is a pure read of the scorecard; run " + "verify for those", + ] + + +def form_summary(findings: Findings) -> list[str]: + """The derived-``form`` split, printed beside the resolved count — WITH its denominator. + + A bare "1,479 code" is unreadable: unreadable against what? So this prints the population it + classified, the population it could not, and the population it never saw, on the principle that a + broken run and a clean run must not look alike. The parts sum to ``checked_anchors`` by + construction and the reader can check that without leaving the line. + + **Nothing downstream consumes this.** It is the record's rendered face telling the truth about + what its evidence is made of, which is not the same act as scoring it. ``doc`` and ``foreign`` + anchors are not weaker claims — they are claims no structural or executable check can ever reach, + which is a fact about the CHECK, not about the cell. + """ + n = findings.anchor_forms + located = sum(n.values()) + unlocated = findings.checked_anchors - located + prose = n["doc"] + n["foreign"] + pct = (100.0 * prose / located) if located else 0.0 + out = [ + f" form of the {located} anchor(s) that located: {n['code']} code, " + f"{n['doc']} doc (docstring or # comment), {n['foreign']} foreign (not a Python file), " + f"{n['undetermined']} undetermined (Python that would not parse)" + ] + if unlocated: + out.append( + f" {unlocated} further anchor(s) did NOT locate (GONE or AMBIGUOUS, reported below) and " + "carry no form: there is no landing site to classify" + ) + covered = findings.checked_sym_ctx + out.append( + f" sym/ctx asserted on {covered} of {findings.checked_anchors} anchor(s)" + + ( + "; the rest assert neither, and absence of the field is NOT agreement" + if covered < findings.checked_anchors + else "" + ) + ) + out.append( + f" {prose} of {located} ({pct:.1f}%) resolve into prose or a non-Python file, which no " + "structural or executable check reaches, ever. That is a LABEL and not a demotion -- " + "documentation is legitimate ground for a documentation requirement -- and it feeds no " + "verdict, no completeness check and no exit code here" + ) + return out + + def _run_prove_absences(scorecard: Path, root: Path) -> int: """The ``--prove-absences`` entry point: execute-prove every absence claim (see :func:`prove_absences`). Needs no corpus — it applies mutations, it does not grep for patterns.""" @@ -995,8 +1713,14 @@ def _run_prove_absences(scorecard: Path, root: Path) -> int: print(f"error: {exc}", file=sys.stderr) return 2 # could not measure — never 0, never confused with "clean" findings = prove_absences(cells, root) + # `saw N` is the denominator, and it comes FIRST because the parts are unreadable without it: a + # run over 276 claims and a run over zero otherwise print counter sets that look equally + # plausible. The three outcome counters deliberately do not sum to it -- five problem-only + # outcomes increment nothing -- so the remainder is derivable and the gap is the point rather + # than a rounding error. See `Findings.proved_absences` for the closing arithmetic. print( - f"prove-absences: proved {findings.proved_absences} by mutation; " + f"prove-absences: saw {findings.checked_absences} absence claim(s); " + f"proved {findings.proved_absences} by mutation; " f"{findings.static_screened} static-screened; {findings.skipped_absences} skipped; " f"{len(findings.problems)} problem(s)" ) @@ -1005,6 +1729,26 @@ def _run_prove_absences(scorecard: Path, root: Path) -> int: return 0 if findings.ok else 1 +def _run_status(scorecard: Path, root: Path) -> int: + """The ``--status`` entry point: provenance first, then what the scorecard says. No corpus needed. + + Exit 0 on a successful read and 2 when the scorecard cannot be loaded. NEVER 1 — this is a query, + not a gate, and a query that borrows the gate's failure code will eventually be wired into CI as + one. The gate is :func:`verify`. + """ + for line in provenance_lines(scorecard, root): + print(line) + try: + cells = load_scorecard(scorecard) + except ScorecardError as exc: + # The provenance header is already out, so even this failure is attributable to a ref. + print(f"error: {exc}", file=sys.stderr) + return 2 + for line in status_lines(cells): + print(line) + return 0 + + def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description="Verify or render the ASVS scorecard (ADR 0156).") ap.add_argument("--scorecard", type=Path, required=True) @@ -1023,12 +1767,24 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="execute-prove absence claims (apply mutation to a scratch tree, require observable red)", ) + # A QUERY, not a gate: a ref-stamped read of the scorecard, no corpus, no engine tree, no network, + # nothing cached to disk. It exists because the dominant token cost of ASVS work is not reading + # the record, it is reconciling two readings of it that were taken at different refs and printed + # identically. There is deliberately NO --fetch: see `_git` on why a query must not mutate. + ap.add_argument( + "--status", + action="store_true", + help="print the provenance line and the scorecard's own counts, then exit (no corpus needed)", + ) # NO --anchor-sha injected by CI. The anchor is the commit the EVIDENCE was read on — a property # of the assessment, recorded in [scorecard].anchor_commit. Passing ${{ github.sha }} made the # rendered file differ on every run, so the drift check could never pass: a gate that cannot go # green is as useless as one that cannot go red, and this one shipped that way. args = ap.parse_args(argv) + if args.status: + return _run_status(args.scorecard, args.root) + if args.prove_absences: return _run_prove_absences(args.scorecard, args.root) @@ -1063,6 +1819,11 @@ def main(argv: list[str] | None = None) -> int: f"(token present and unique -- NOT proof the control operates) " f"and checked {findings.checked_absences} absence claims" ) + # Beside the resolved count, and deliberately not folded into it: WHAT those anchors resolve into. + # "Resolved 1,980" reads as 1,980 pieces of code evidence; roughly a quarter of them are prose or a + # non-Python file that no structural or executable check will ever reach. See `form_summary`. + for line in form_summary(findings): + print(line) for a in findings.advisories: print(f" DRIFT {a}", file=sys.stderr) if findings.advisories: diff --git a/tests/test_asvs_scorecard.py b/tests/test_asvs_scorecard.py index b81f5884..564f041f 100644 --- a/tests/test_asvs_scorecard.py +++ b/tests/test_asvs_scorecard.py @@ -13,28 +13,42 @@ from __future__ import annotations import json +import os +import subprocess +import time from pathlib import Path +from typing import Any import pytest from scripts.asvs.scorecard import ( + _DESCEND_ONLY, + _TRANSPARENT, Absence, Anchor, Cell, Findings, ScorecardError, _copy_scratch, + _humanise_age, + anchor_form, check_absences, check_anchors, check_completeness, check_pinning, corpus_digest, count, + derive_sym_ctx, + form_summary, load_corpus, load_scorecard, main, + malformed_sym_ctx, prove_absences, + provenance_lines, render_current, + repo_stamp, + status_lines, verify, ) @@ -345,6 +359,286 @@ def test_anchor_goes_red_when_the_file_is_gone(tmp_path: Path) -> None: assert not f.ok and "does not exist" in f.problems[0] +# --- derived anchor `form`: what does the evidence actually resolve INTO? -------------------------- +# +# Measured 2026-08-09, vault `origin/main` (1a59e4a1) scorecard against engine tree `c383eeab`: +# 1,980 anchors, 1,979 located, split 1,479 code / 233 doc / 267 foreign / 0 undetermined. So 500 of +# the located 1,979 (25.3%) resolve into prose or a non-Python file that no structural or executable +# check reaches, and the record presented all 1,980 as code evidence. +# +# Every test below names the classification rule it pins, because the rule is where the judgement is. +# The load-bearing one is `..._that_is_not_a_docstring_stays_code`: it is the case a token mask gets +# wrong, and it is the reason this classifies by POSITION rather than by what the text looks like. + +_CSP_MODULE = '''\ +"""Security headers for the web console.""" + +# The report path is public; the nonce is not. +CSP_REPORT_PATH = "/ui/csp-report" + +_POLICY = ( + "default-src 'self'; script-src 'nonce-{nonce}' 'strict-dynamic'; " + "base-uri 'none'; form-action 'self'; object-src 'none'; " + "img-src 'self' data:; connect-src 'self'; font-src 'self'" +) + + +def header(nonce: str) -> str: + """Return the Content-Security-Policy header value.""" + return _POLICY.format(nonce=nonce) +''' + +_DDL_MODULE = '''\ +"""SQL Server store.""" + + +def _schema() -> str: + return """ + CREATE TABLE sessions ( + token_hash NVARCHAR(64) NOT NULL PRIMARY KEY, + user_id NVARCHAR(255) NOT NULL, + revoked_at DATETIME2 NULL + ) + """ +''' + + +def test_form_classifies_module_class_and_function_docstrings_as_doc() -> None: + src = '"""Module prose here."""\n\n\nclass C:\n """Class prose here."""\n\n def m(self):\n """Method prose here."""\n return 1\n' + for token in ("Module prose", "Class prose", "Method prose"): + assert anchor_form("messagefoundry/m.py", src, src.index(token)) == "doc", token + + +def test_form_classifies_a_hash_comment_as_doc() -> None: + src = "# why this constant is 64 and not 32\nSIZE = 64\n" + assert anchor_form("messagefoundry/m.py", src, src.index("why this constant")) == "doc" + assert anchor_form("messagefoundry/m.py", src, src.index("SIZE = 64")) == "code" + + +def test_form_keeps_a_prose_shaped_string_that_is_not_a_docstring_as_code() -> None: + """THE case that decides the classification rule, and the reason a token mask was rejected. + + A Content-Security-Policy fragment and a block of SQL DDL are long, quoted, space-separated and + operator-free, so a "does this look like code?" mask reads them as English -- while they are the + literal subject of the control the cell cites. Measured, a token mask misfiled every CSP fragment + in `messagefoundry_webconsole/_security.py` and the whole SQL Server and Postgres DDL as prose. + + Position cannot make that mistake: neither string is the first statement of any scope, so neither + is a docstring, whatever it reads like. Falsified by relaxing `_prose_spans` to treat ANY string + token as a docstring (drop the `doc_rows` membership test): all four assertions here go RED while + the docstring tests above stay green. Restored. + """ + for token in ( + "script-src 'nonce-{nonce}' 'strict-dynamic'", + "base-uri 'none'; form-action 'self'; object-src 'none'; ", + ): + assert ( + anchor_form( + "messagefoundry_webconsole/_security.py", _CSP_MODULE, _CSP_MODULE.index(token) + ) + == "code" + ), token + for token in ("token_hash NVARCHAR(64) NOT NULL PRIMARY KEY", "CREATE TABLE sessions"): + assert ( + anchor_form("messagefoundry/store/sqlserver.py", _DDL_MODULE, _DDL_MODULE.index(token)) + == "code" + ), token + + +def test_form_does_not_mistake_a_hash_inside_a_string_for_a_comment() -> None: + """`#` is a comment character to a `#`-scan and an ordinary byte to `tokenize`. + + A CSP fragment or a URL fragment carries `#` routinely. Comments therefore come from `tokenize`, + which knows it is inside a string literal, rather than from a scan of the line. + """ + src = 'URL = "https://example.test/page#anchor-name"\nVALUE = 1\n' + assert anchor_form("messagefoundry/m.py", src, src.index("anchor-name")) == "code" + + +def test_form_classifies_a_non_python_file_as_foreign_without_parsing_it() -> None: + """198 `.md`, 17 `.ts`, 16 `.js` and 13 `.yml` anchors are in the live record. No `ast` reaches + any of them, ever, so the honest label is `foreign` rather than a Python verdict about them.""" + md = "# SECURITY.md\n\nThe engine binds 127.0.0.1 by default.\n" + assert anchor_form("docs/SECURITY.md", md, md.index("binds 127")) == "foreign" + # It is the SUFFIX that decides, not the content: this would parse as Python and must not be tried. + assert anchor_form("scripts/x.yml", "VALUE = 1\n", 0) == "foreign" + + +def test_form_is_undetermined_when_the_python_will_not_parse_never_code() -> None: + """The dangerous default is `code`, because it inflates the exact number this split deflates. + + Falsified by changing `_form_from_spans` to `return "code"` when `spans is None`: this test goes + RED and the "no prose" negative control below stays green. Restored. + """ + broken = "def f(\n" # unterminated: neither ast nor tokenize can complete it + assert anchor_form("messagefoundry/m.py", broken, 0) is None + + +def test_form_is_decided_by_the_tokens_start_not_by_any_overlap() -> None: + """An `expect` that begins in code and runs into a trailing comment is CODE. + + Measured on the live record: a START rule and an OVERLAP rule disagree on 57 of 1,712 Python + anchors, and every one of those is a code statement whose recorded token happens to run past the + end of the statement. Falsified by dropping the LOWER bound from `_form_from_spans` -- + `any(start < hi ...)`, the left-overlap rule, and a plausible slip -- which classifies this + fixture as `doc`: this test goes RED. Restored. + """ + src = "VALUE = 64 # tuned against the 2026-08 bench\n" + token = "VALUE = 64 # tuned" + assert src.count(token) == 1 # the token really does straddle the boundary + assert anchor_form("messagefoundry/m.py", src, src.index(token)) == "code" + + +def test_form_negative_control_a_file_with_no_prose_yields_no_doc(tmp_path: Path) -> None: + """The count's negative control: a form that CANNOT occur must come back zero. + + A classifier that returned `doc` on some fixed fraction, or that leaked spans between files + through the per-run cache, would show up here and nowhere else -- the positive tests above only + assert that `doc` appears. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + "SIZE = 64\nNAME = 'x'\n\n\ndef f():\n return SIZE\n", encoding="utf-8" + ) + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=( + Anchor("messagefoundry/m.py", 1, "SIZE = 64"), + Anchor("messagefoundry/m.py", 2, "NAME = 'x'"), + Anchor("messagefoundry/m.py", 5, "def f():"), + ), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.anchor_forms["code"] == 3 + assert f.anchor_forms["doc"] == 0 + assert f.anchor_forms["foreign"] == 0 + assert f.anchor_forms["undetermined"] == 0 + + +def test_check_anchors_counts_a_form_for_every_anchor_that_located(tmp_path: Path) -> None: + """The split's parts must sum to the located population, or the printed denominator is a fiction.""" + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "docs").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + '"""Prose about the gate."""\n\nSIZE = 64\n', encoding="utf-8" + ) + (tmp_path / "docs" / "SECURITY.md").write_text( + "The gate is deny-by-default.\n", encoding="utf-8" + ) + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=( + Anchor("messagefoundry/m.py", 3, "SIZE = 64"), + Anchor("messagefoundry/m.py", 1, "Prose about the gate"), + Anchor("docs/SECURITY.md", 1, "deny-by-default"), + ), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.ok + assert dict(f.anchor_forms) == {"code": 1, "doc": 1, "foreign": 1} + assert sum(f.anchor_forms.values()) == f.checked_anchors + + +def test_an_anchor_that_did_not_locate_gets_no_form_at_all(tmp_path: Path) -> None: + """GONE and AMBIGUOUS anchors have no landing site, so classifying them would be an invented + number inside the one figure whose whole purpose is to stop the record overstating itself. + + Falsified by moving the `anchor_forms` increment above the occurrence guards in `check_anchors`: + the sum then reaches 3 and both assertions here go RED. Restored. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + "SIZE = 64\ndupe\nfiller\ndupe\n", encoding="utf-8" + ) + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=( + Anchor("messagefoundry/m.py", 1, "SIZE = 64"), + Anchor("messagefoundry/m.py", 2, "dupe"), # AMBIGUOUS + Anchor("messagefoundry/m.py", 3, "vanished_token"), # GONE + ), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert len(f.problems) == 2 + assert sum(f.anchor_forms.values()) == 1 + assert f.checked_anchors == 3 # scanned three; classified one. Both numbers get printed. + + +def test_form_doc_is_a_label_and_never_a_demotion(tmp_path: Path) -> None: + """17 cells rest genuinely on documentation, which is legitimate ground for a documentation + requirement. A cell evidenced ONLY by prose must therefore stay green and stay complete. + + This is the fence against the obvious next move -- wiring `form` into the gate. If someone adds + `if form != "code": problems.append(...)`, or teaches `check_completeness` that a doc-only cell + is unevidenced, this test goes RED and says why. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "docs").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + '"""Session records are never written to the general log."""\n', encoding="utf-8" + ) + (tmp_path / "docs" / "PHI.md").write_text( + "Full payloads go only to the store.\n", encoding="utf-8" + ) + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=( + Anchor("messagefoundry/m.py", 1, "never written to the general log"), + Anchor("docs/PHI.md", 1, "Full payloads go only to the store"), + ), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.ok and f.problems == [] and f.advisories == [] + assert f.anchor_forms["code"] == 0 and f.anchor_forms["doc"] + f.anchor_forms["foreign"] == 2 + # ... and the completeness check, which decides what counts as evidence, is untouched by form. + assert check_completeness(cells, {"1.1.1": 1}) == [] + + +def test_form_summary_prints_its_denominator_and_the_population_it_never_saw() -> None: + """A broken run and a clean run must not look alike, so the split prints what it SCANNED. + + Parts, the located total, the unlocated remainder, and the derived percentage all appear. A bare + "1,479 code" is unreadable: unreadable against what? + """ + f = Findings(checked_anchors=10) + f.anchor_forms.update({"code": 5, "doc": 2, "foreign": 2}) + text = "\n".join(form_summary(f)) + assert "form of the 9 anchor(s) that located" in text + assert "5 code" in text and "2 doc" in text and "2 foreign" in text + assert "1 further anchor(s) did NOT locate" in text + assert "4 of 9 (44.4%)" in text + assert "LABEL and not a demotion" in text + + +def test_form_summary_negative_control_says_zero_rather_than_dividing_by_zero() -> None: + """Nothing located: every part is zero, the percentage is 0.0, and no line is silently omitted.""" + text = "\n".join(form_summary(Findings())) + assert "form of the 0 anchor(s) that located" in text + assert "0 code" in text and "0 doc" in text and "0 foreign" in text + assert "0 of 0 (0.0%)" in text + assert "did NOT locate" not in text # nothing was scanned, so nothing went unclassified + + # --- absence claims: the class this project is worst at ------------------------------------------- @@ -1210,6 +1504,49 @@ def test_main_prove_absences_returns_1_on_a_nonbiting_claim(tmp_path: Path) -> N assert rc == 1 +def test_main_summary_says_RESOLVED_not_VERIFIED_and_carries_the_form_split( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The rendered face of the record must not assert something the check does not establish. + + The summary line used to read "verified N evidence anchors". The run did not verify them; it + RESOLVED them -- the token is present and unique in the file, which is not evidence that the + control operates. Measured instance: 15.3.1 sat at `pass` with every anchor resolving while the + control it named had a hole, found only by executing the code. That distinction lives on the one + line most readers will ever read, so it is pinned here end-to-end through `main`, not asserted of + a helper. + + The split rides beside it for the same reason: "resolved 1,980" reads as 1,980 pieces of code + evidence, and roughly a quarter of them are prose or a non-Python file. + + Falsified by restoring the word "verified" in `main`'s summary: the first two assertions go RED. + Falsified independently by deleting the `for line in form_summary(findings)` loop: the split + assertions go RED while the wording ones stay green. Both restored. + """ + corpus = _corpus_file(tmp_path, {"1.1.1": 1}) + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + '"""Prose about the gate."""\n\nSIZE = 64\n', encoding="utf-8" + ) + sc = _scorecard_file( + tmp_path, + f'[scorecard]\nasvs_version = "5.0.0"\ncorpus_sha256 = "{corpus_digest(corpus)}"\n' + '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "pass"\n' + " [[cell.evidence]]\n" + ' path = "messagefoundry/m.py"\n line = 3\n expect = "SIZE = 64"\n' + " [[cell.evidence]]\n" + ' path = "messagefoundry/m.py"\n line = 1\n expect = "Prose about the gate"\n', + ) + rc = main(["--scorecard", str(sc), "--corpus", str(corpus), "--root", str(tmp_path)]) + out = capsys.readouterr().out + assert rc == 0 + assert "resolved 2 evidence anchors" in out + assert "verified 2 evidence anchors" not in out + assert "NOT proof the control operates" in out + assert "form of the 2 anchor(s) that located: 1 code, 1 doc" in out + assert "1 of 2 (50.0%) resolve into prose or a non-Python file" in out + + def test_main_verify_without_corpus_returns_exit_2(tmp_path: Path) -> None: """Verify mode needs the corpus; omitting `--corpus` (without `--prove-absences`) must exit 2 -- could-not-measure, never confused with a clean 0. Proves the argparse-independent guard in `main`. @@ -1229,3 +1566,860 @@ def test_main_verify_without_corpus_returns_exit_2(tmp_path: Path) -> None: ) rc = main(["--scorecard", str(sc), "--root", str(tmp_path)]) assert rc == 2 + + +# --- provenance and `--status`: a count is a fact about a (file x ref) PAIR ----------------------- +# +# Three wrong-base errors occurred in one working thread on 2026-08-08/09. In every one the NUMBER was +# right and the REF was unnamed, so two readings taken from different places printed identically. +# These tests exist to make that specific silence impossible, and they lean on real git repositories +# rather than a mocked one: the failure being prevented is a fact about remote-tracking refs, fetch +# recency and worktree layout, so a stub would reproduce my assumptions instead of git's behaviour. + +_GIT_ID = ("-c", "user.name=t", "-c", "user.email=t@t.invalid", "-c", "commit.gpgsign=false") + + +def _git(repo: Path, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", str(repo), *_GIT_ID, *args], + capture_output=True, + text=True, + check=True, + ) + return proc.stdout.strip() + + +def _commit(repo: Path, name: str) -> None: + (repo / name).write_text(name, encoding="utf-8") + _git(repo, "add", name) + _git(repo, "commit", "-m", name) + + +def _origin_and_clone(tmp_path: Path) -> tuple[Path, Path]: + """A real bare origin plus a real clone tracking `origin/main`, both on disk.""" + origin = tmp_path / "origin.git" + seed = tmp_path / "seed" + seed.mkdir() + _git(seed, "init", "--quiet") + _git(seed, "checkout", "--quiet", "-b", "main") + _commit(seed, "a.txt") + subprocess.run( + ["git", "init", "--bare", "--quiet", str(origin)], check=True, capture_output=True + ) + _git(seed, "remote", "add", "origin", str(origin)) + _git(seed, "push", "--quiet", "-u", "origin", "main") + clone = tmp_path / "clone" + subprocess.run( + ["git", "clone", "--quiet", str(origin), str(clone)], check=True, capture_output=True + ) + _git(clone, "checkout", "--quiet", "main") + return seed, clone + + +def test_git_is_available_so_no_provenance_test_can_silently_skip() -> None: + """None of the tests below is marked skipif, on purpose. A provenance guard that quietly + evaporates on a machine without git is the same class of defect as a gate that cannot go red.""" + assert subprocess.run(["git", "--version"], capture_output=True, check=True).returncode == 0 + + +def test_provenance_stamps_a_clean_checkout_with_its_sha_and_a_named_upstream( + tmp_path: Path, +) -> None: + _, clone = _origin_and_clone(tmp_path) + stamp = repo_stamp(clone) + assert stamp.sha == _git(clone, "rev-parse", "--short", "HEAD") + assert stamp.dirty is False + assert stamp.ref() == stamp.sha and "+dirty" not in stamp.ref() + assert stamp.freshness == "CURRENT" + assert stamp.upstream == "origin/main" # named, because BEHIND n is not a claim without it + + +def test_provenance_marks_a_dirty_tree_so_it_cannot_pass_for_a_reproducible_one( + tmp_path: Path, +) -> None: + """A measurement taken against uncommitted changes is not reproducible and must not look like one + that is. Falsified by hardcoding `dirty=False` in `repo_stamp`: this goes RED. Restored.""" + _, clone = _origin_and_clone(tmp_path) + assert repo_stamp(clone).dirty is False # control: clean first, so the flag means something + (clone / "a.txt").write_text("edited", encoding="utf-8") + stamp = repo_stamp(clone) + assert stamp.dirty is True + assert stamp.ref().endswith("+dirty") + + +def test_freshness_reports_BEHIND_with_its_count_and_needs_no_network(tmp_path: Path) -> None: + """THE measured case. The vault checkout that caused the original wrong-base error in this + programme reads BEHIND 37 with remote knowledge 23 minutes old; re-measured while building this, + the same checkout read BEHIND 38 at 36 minutes. Either line stops the error dead. + + The count comes from `git rev-list --left-right --count HEAD...origin/main`, which counts against + the LAST-FETCHED remote-tracking ref -- a purely local object. The fetch below is the test setting + up remote knowledge; the tool itself never fetches (see the no-mutation test further down). + """ + seed, clone = _origin_and_clone(tmp_path) + _commit(seed, "b.txt") + _commit(seed, "c.txt") + _git(seed, "push", "--quiet", "origin", "main") + _git(clone, "fetch", "--quiet", "origin") + stamp = repo_stamp(clone) + assert stamp.freshness == "BEHIND 2" + assert stamp.upstream == "origin/main" + + +def test_freshness_reports_AHEAD_with_its_count(tmp_path: Path) -> None: + _, clone = _origin_and_clone(tmp_path) + _commit(clone, "local.txt") + assert repo_stamp(clone).freshness == "AHEAD 1" + + +def test_freshness_reports_DIVERGED_when_both_sides_moved(tmp_path: Path) -> None: + """DIVERGED is its own value and must not collapse into AHEAD or BEHIND: a branch that is both is + the state in which "am I on the right base?" is hardest and most often answered wrongly.""" + seed, clone = _origin_and_clone(tmp_path) + _commit(seed, "remote.txt") + _git(seed, "push", "--quiet", "origin", "main") + _git(clone, "fetch", "--quiet", "origin") + _commit(clone, "local.txt") + assert repo_stamp(clone).freshness == "DIVERGED" + + +def test_freshness_is_NO_UPSTREAM_and_is_never_an_omitted_field(tmp_path: Path) -> None: + """A repo with no remote at all. The field is POPULATED, not dropped: an absent qualifier is + exactly what produced all three wrong-base errors. + + Falsified by returning an empty string from `_freshness` in this branch: the emptiness assertion + goes RED and the printed line silently loses its qualifier. Restored. + """ + solo = tmp_path / "solo" + solo.mkdir() + _git(solo, "init", "--quiet") + _git(solo, "checkout", "--quiet", "-b", "main") + _commit(solo, "a.txt") + stamp = repo_stamp(solo) + assert stamp.freshness == "NO-UPSTREAM" + assert stamp.upstream == "none" + assert stamp.freshness != "" and stamp.remote_knowledge != "" + + +def test_a_path_outside_any_work_tree_is_NO_GIT_and_not_NO_UPSTREAM(tmp_path: Path) -> None: + """The two are different statements and conflating them is a lie in the safer-sounding direction. + + NO-UPSTREAM says "this repo tracks nothing"; NO-GIT says "this is not a repo, so no ref exists to + quote at all". A copy of the scorecard extracted by `git show` into a temp directory reads the + second, and reporting it as the first would let it pass for a checkout. + """ + loose = tmp_path / "loose" + loose.mkdir() + stamp = repo_stamp(loose) + assert stamp.sha == "NO-GIT" + assert stamp.freshness == "NO-GIT" + assert stamp.upstream == "none" + + +def test_remote_knowledge_is_NEVER_FETCHED_before_any_fetch_has_happened(tmp_path: Path) -> None: + solo = tmp_path / "solo" + solo.mkdir() + _git(solo, "init", "--quiet") + _git(solo, "checkout", "--quiet", "-b", "main") + _commit(solo, "a.txt") + assert repo_stamp(solo).remote_knowledge == "NEVER-FETCHED" + + +def test_remote_knowledge_reports_the_AGE_of_the_last_fetch_not_merely_that_one_happened( + tmp_path: Path, +) -> None: + """BEHIND 0 from a six-hour-old fetch and BEHIND 0 from a one-minute-old fetch are DIFFERENT + CLAIMS and must not print identically. That is the whole reason this field sits beside the count. + + Falsified by returning a constant from `_remote_knowledge`: the two ages below become equal and + this goes RED. Restored. + """ + seed, clone = _origin_and_clone(tmp_path) + _git(clone, "fetch", "--quiet", "origin") + fresh = repo_stamp(clone).remote_knowledge + assert fresh.endswith("s") # seconds old, just now + + head = Path(_git(clone, "rev-parse", "--path-format=absolute", "--git-dir")) / "FETCH_HEAD" + assert head.is_file() + old = time.time() - 6 * 3600 + os.utime(head, (old, old)) + stale = repo_stamp(clone).remote_knowledge + assert stale == "6h" + assert stale != fresh + + +def test_humanise_age_covers_each_unit_boundary() -> None: + assert _humanise_age(0) == "0s" + assert _humanise_age(89) == "89s" + assert _humanise_age(90) == "1m" + assert _humanise_age(23 * 60) == "23m" + assert _humanise_age(90 * 60) == "1h" + assert _humanise_age(47 * 3600) == "47h" + assert _humanise_age(48 * 3600) == "2d" + + +def test_status_issues_no_write_and_no_network_git_subcommand( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """THE NEGATIVE CONTROL, and the one that matters most here. + + A query tool that fetches mutates repo state as a side effect of being asked a question, and on + this machine the vault remote is intermittently unauthenticated, so a network dependency would + fire constantly and the tool would be bypassed inside a day. So every git subcommand this mode + issues is captured and checked against a read-only allowlist, and the mutating verbs must appear + ZERO times: a pattern that cannot occur must come back zero. + + It also checks the OUTCOME and not only the intent -- the remote-tracking refs and the FETCH_HEAD + mtime are compared across the run. Asserting on the argv alone would pass a tool that reached the + network some other way. + + Falsified by adding a fetch to `repo_stamp`: the allowlist assertion goes RED naming the verb, and + the FETCH_HEAD mtime assertion goes RED independently of it. Restored. + """ + seed, clone = _origin_and_clone(tmp_path) + _commit(seed, "b.txt") + _git(seed, "push", "--quiet", "origin", "main") + _git(clone, "fetch", "--quiet", "origin") + + refs_before = _git(clone, "for-each-ref", "refs/remotes") + head = Path(_git(clone, "rev-parse", "--path-format=absolute", "--git-dir")) / "FETCH_HEAD" + mtime_before = head.stat().st_mtime + + seen: list[list[str]] = [] + real_run = subprocess.run + + def spy(cmd: Any, *a: Any, **kw: Any) -> Any: + seen.append(list(cmd)) + return real_run(cmd, *a, **kw) + + # Patched on the `subprocess` module itself, which is the same object the verifier imported. That + # also means the helper `_git` below is spied, so `verbs` is snapshotted before any helper call. + monkeypatch.setattr(subprocess, "run", spy) + + sc = _scorecard_file(clone, '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "unverified"\n') + rc = main(["--status", "--scorecard", str(sc), "--root", str(clone)]) + capsys.readouterr() + + assert rc == 0 + assert seen, "the spy captured nothing -- this test would otherwise pass vacuously" + verbs = {cmd[3] for cmd in seen if len(cmd) > 3} + assert verbs <= {"rev-parse", "status", "rev-list"}, f"non-read-only git verb: {verbs}" + for forbidden in ("fetch", "pull", "push", "remote", "gc", "prune", "commit", "checkout"): + assert forbidden not in verbs + + assert _git(clone, "for-each-ref", "refs/remotes") == refs_before + assert head.stat().st_mtime == mtime_before + + +def test_provenance_lines_always_carry_every_mandated_field(tmp_path: Path) -> None: + """Non-suppressible and never partially populated. Each field is checked by NAME, so dropping one + is a red rather than a shorter line nobody notices.""" + _, clone = _origin_and_clone(tmp_path) + sc = _scorecard_file(clone, '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "unverified"\n') + text = "\n".join(provenance_lines(sc, clone)) + assert text.startswith("# asvs-status scorecard=") + assert " engine=" in text + assert "scorecard: freshness=" in text and "engine:" in text + assert text.count("freshness=") == 2 # one per repo: a single field could not say WHICH repo + assert text.count("upstream=") == 2 + assert text.count("remote-knowledge=") == 2 + assert "generated=" in text + + +def test_two_readings_of_the_same_filename_at_different_refs_do_not_print_identically( + tmp_path: Path, +) -> None: + """THE POINT OF THE WHOLE FEATURE, reproduced in miniature. + + Live instance measured 2026-08-09 while building this: the vault working tree and vault + `origin/main` both hold a file called `asvs-scorecard.toml`, and they disagree -- 105 partial / + 3 fail / 1,978 anchors against 106 partial / 2 fail / 1,980 anchors. Without a ref stamp those are + two plausible readings of "the scorecard" and nothing in either output says which is which. That + is exactly how three wrong-base errors happened in one thread. + + Here the same file name is read from a stale checkout and a current one. The COUNTS are identical + by construction; only the provenance differs, which is the property under test. + """ + seed, clone = _origin_and_clone(tmp_path) + body = '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "unverified"\n' + sc_current = _scorecard_file(seed, body) + sc_stale = _scorecard_file(clone, body) + _commit(seed, "b.txt") + _git(seed, "push", "--quiet", "origin", "main") + _git(clone, "fetch", "--quiet", "origin") + + current = "\n".join(provenance_lines(sc_current, seed)) + stale = "\n".join(provenance_lines(sc_stale, clone)) + + assert sc_current.name == sc_stale.name # identical file names, as in the live instance + assert status_lines(load_scorecard(sc_current)) == status_lines(load_scorecard(sc_stale)) + assert "BEHIND 1" in stale + assert "BEHIND" not in current + assert current != stale + + +def test_status_reports_every_verdict_including_needs_review() -> None: + """`needs-review` is absent from the verify summary line and present here. A cell that was read + and then parked on purpose is not a cell that does not exist, and the record's own renderer once + contradicted itself over exactly this distinction.""" + cells = [ + Cell(id="1.1.1", level=1, verdict="pass", last_verified="2026-08-09"), + Cell(id="1.1.2", level=2, verdict="needs-review", last_verified="2026-08-09"), + Cell(id="2.1.1", level=3, verdict="unverified"), + ] + text = "\n".join(status_lines(cells)) + assert "cells 3: 1 pass, 0 partial, 0 fail, 0 na, 1 needs-review, 1 unverified" in text + assert "examined 2 of 3 (66.7%)" in text + + +def test_status_names_what_it_did_NOT_check() -> None: + """A cheap answer that looks like a full one is worse than no answer. `--status` never opens the + engine tree, so it must say so rather than let a structural tally read as anchor health. + + Falsified by deleting the final line of `status_lines`: this goes RED. Restored. + """ + text = "\n".join(status_lines([Cell(id="1.1.1", level=1, verdict="unverified")])) + assert "NOT CHECKED here" in text + assert "whether any anchor still resolves" in text + assert "run verify" in text + + +def test_main_status_needs_no_corpus_and_prints_provenance_first( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """`--status` is a pure scorecard read, so requiring `--corpus` would be friction with no purpose, + and the provenance header is the FIRST thing on stdout -- before any number it qualifies.""" + _, clone = _origin_and_clone(tmp_path) + sc = _scorecard_file( + clone, + '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "pass"\nlast_verified = "2026-08-09"\n', + ) + rc = main(["--status", "--scorecard", str(sc), "--root", str(clone)]) + out = capsys.readouterr().out.splitlines() + assert rc == 0 + assert out[0].startswith("# asvs-status scorecard=") + assert any(line.startswith("cells 1:") for line in out) + + +def test_main_status_exits_2_on_an_unreadable_scorecard_and_still_prints_provenance( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Exit 2 is could-not-measure. NEVER 1: a query that borrows the gate's failure code gets wired + into CI as a gate, and this one checks nothing about the posture. + + The header still prints, so even the failure is attributable to a ref. + """ + _, clone = _origin_and_clone(tmp_path) + rc = main(["--status", "--scorecard", str(clone / "absent.toml"), "--root", str(clone)]) + captured = capsys.readouterr() + assert rc == 2 + assert captured.out.startswith("# asvs-status scorecard=") + assert "refusing to report a pass on a missing file" in captured.err + + +def test_status_does_not_run_the_gate_and_cannot_return_the_gates_exit_code( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Negative control on the exit code: a scorecard whose anchor is broken still exits 0 under + `--status`, because `--status` does not check anchors. If someone later wires verification into + this path, the 41-second cost and this assertion land at the same moment.""" + _, clone = _origin_and_clone(tmp_path) + sc = _scorecard_file( + clone, + '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "pass"\n' + " [[cell.evidence]]\n" + ' path = "nowhere/absent.py"\n line = 1\n expect = "token_that_does_not_exist"\n', + ) + rc = main(["--status", "--scorecard", str(sc), "--root", str(clone)]) + capsys.readouterr() + assert rc == 0 + + +# --- prove-absences: the counters must be reconcilable against the population they came from ------ + + +def test_prove_absences_records_the_population_it_saw(tmp_path: Path) -> None: + """`checked_absences` is set BEFORE any outcome branch, so it is right whichever branch is taken. + + Falsified by moving the increment inside `_prove_one` after the `mutation_path` guard: the skipped + claim then goes uncounted and the first assertion goes RED. Restored. + """ + _module(tmp_path, "scanner.py", _SCANNER) + _obs_test(tmp_path, "test_scanner.py", _OBS_TEST) + proved = _live_claim( + 'def scan(p): return "infected"', "scanner.py", "test_scanner.py::test_clean" + ) + skipped = Cell( + id="1.1.2", + level=1, + verdict="fail", + absence=(Absence(pattern="x", positive_control="y", mutation="import x"),), + ) + findings = prove_absences([proved, skipped], tmp_path) + assert findings.checked_absences == 2 + assert findings.proved_absences == 1 + assert findings.skipped_absences == 1 + + +def test_prove_absences_counters_close_against_the_population(tmp_path: Path) -> None: + """The arithmetic that makes the summary readable, asserted rather than asserted-in-prose. + + FIVE outcomes raise a problem and increment no counter (an escaping `mutation_path`, one that is + not a file, a baseline that is not green, an UNPROVEN mutated-green, and a mutated run that + errored). So the closing identity is population minus the three counters, and this fixture drives + one claim into each of four different branches to check it holds across them. + + Note what is NOT asserted: that `len(problems)` equals the problem-only count. It does not, and + cannot -- a SUSPECT finding rides along with a claim already counted in `static_screened`, so + problems and claims are different populations. Asserting that equality would pin a false identity. + """ + _module(tmp_path, "scanner.py", _SCANNER) + _module(tmp_path, "quiet.py", "VALUE = 1\n") + _obs_test(tmp_path, "test_scanner.py", _OBS_TEST) + + proved = _live_claim( + 'def scan(p): return "infected"', "scanner.py", "test_scanner.py::test_clean" + ) + skipped = Cell( + id="1.1.2", + level=1, + verdict="fail", + absence=(Absence(pattern="x", positive_control="y", mutation="import x"),), + ) + screened = Cell( + id="1.1.3", + level=1, + verdict="fail", + absence=( + Absence( + pattern="x", positive_control="y", mutation="VALUE = 2", mutation_path="quiet.py" + ), + ), + ) + problem_only = Cell( + id="1.1.4", + level=1, + verdict="fail", + absence=( + Absence( + pattern="x", + positive_control="y", + mutation="import x", + mutation_path="not_a_file.py", + ), + ), + ) + + f = prove_absences([proved, skipped, screened, problem_only], tmp_path) + assert f.checked_absences == 4 + remainder = f.checked_absences - f.proved_absences - f.static_screened - f.skipped_absences + assert remainder == 1 # exactly the PROVE-ERROR claim, derived rather than counted + assert f.proved_absences == 1 and f.skipped_absences == 1 and f.static_screened == 1 + + +def test_prove_absences_summary_prints_the_population_before_the_parts( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A run that scanned N claims and a run that scanned zero must not print equally plausible + counter sets. Without `saw N` they do: all four numbers are zero in both. + + Falsified by deleting the `saw ... absence claim(s);` term from `_run_prove_absences`: both + assertions go RED. Restored. + """ + _module(tmp_path, "scanner.py", _SCANNER) + _obs_test(tmp_path, "test_scanner.py", _OBS_TEST) + sc = tmp_path / "sc.toml" + _biting_scorecard(sc, "scanner.py") + rc = main(["--scorecard", str(sc), "--root", str(tmp_path), "--prove-absences"]) + out = capsys.readouterr().out + assert rc == 0 + assert "prove-absences: saw 1 absence claim(s);" in out + assert "proved 1 by mutation" in out + + +def test_prove_absences_summary_negative_control_an_empty_run_says_saw_zero( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The negative control: a scorecard with NO absence claims must say so, not print four zeroes + that read like a clean pass over a real population.""" + sc = tmp_path / "sc.toml" + sc.write_text( + '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "unverified"\n', + encoding="utf-8", + ) + rc = main(["--scorecard", str(sc), "--root", str(tmp_path), "--prove-absences"]) + out = capsys.readouterr().out + assert rc == 0 + assert "saw 0 absence claim(s)" in out + + +# --- sym + ctx: WHERE in the structure, and it is a DISPLACEMENT signal --------------------------- +# +# Measured 2026-08-09, vault origin/main 1a59e4a1's anchors against engine tree 4667e945: over the +# 1,712 Python anchors that locate and parse, 536 (31.3%) sit in a (sym, ctx) region wider than the +# 81 lines the retired +/-40 window covered -- so sym/ctx alone is LOOSER for those, and this is +# additive to the drift advisory rather than a replacement for it. The brief's 38.4%/639 reproduces +# at 634 under the looser symbol-only region definition; both support the same conclusion. + +_NESTED = '''\ +"""Module docstring.""" + +TOP = 1 + + +class Store: + """A store.""" + + def revoke(self, keep): + rows = [] + try: + rows = self.query() + except OSError: + rows = [] + if keep: + for r in rows: + if r.stale: + self.drop(r) + return rows + + +def loose(): + return TOP +''' + + +def _sym_ctx(text: str, token: str) -> tuple[str, str] | None: + return derive_sym_ctx(text, text.count("\n", 0, text.index(token)) + 1) + + +def test_sym_ctx_derives_module_level_as_two_empty_strings() -> None: + """`""` is the assertion "module level, unnested" -- a real claim, not a missing value.""" + assert _sym_ctx(_NESTED, "TOP = 1") == ("", "") + + +def test_sym_ctx_derives_the_enclosing_symbol_dotted() -> None: + assert _sym_ctx(_NESTED, "rows = self.query()") == ("Store.revoke", "Try.body") + assert _sym_ctx(_NESTED, "return TOP") == ("loose", "") + + +def test_sym_ctx_names_the_handler_limb_not_the_body_limb() -> None: + """`Try.body` and `Try.handlers` are different regions and a statement moving between them is + exactly the displacement this field exists to notice. + + The handler's own `ExceptHandler.body` is deliberately NOT a second chain element: it is reachable + only through `Try.handlers`, so recording it would double every handler chain for no added + discrimination. + """ + assert _sym_ctx(_NESTED, "rows = self.query()") == ("Store.revoke", "Try.body") + assert _sym_ctx(_NESTED, "except OSError") == ("Store.revoke", "Try.handlers") + + +def test_sym_ctx_chains_nested_blocks_outermost_first() -> None: + assert _sym_ctx(_NESTED, "self.drop(r)") == ("Store.revoke", "If.body>For.body>If.body") + + +def test_sym_ctx_resets_the_chain_at_a_symbol_boundary() -> None: + """A block chain is meaningful only inside the symbol holding it. A nested function inside a + `try` must not inherit `Try.body`, or every helper defined in a guarded block reads as guarded. + + Falsified by deleting the `ctx.clear()` in `sym_ctx_at`: the inner function's ctx becomes + `Try.body` and this goes RED. Restored. + """ + src = "def outer():\n try:\n def inner():\n return 1\n except OSError:\n pass\n" + assert derive_sym_ctx(src, 4) == ("outer.inner", "") + + +def test_sym_ctx_is_None_when_the_file_will_not_parse() -> None: + assert derive_sym_ctx("def f(\n", 1) is None + + +def test_sym_ctx_is_not_indentation_the_12_3_5_non_event(tmp_path: Path) -> None: + """CELL 12.3.5's SHAPE, and the reason this field is `ctx` rather than an indent check. + + 12.3.5 carries the identical 4-versus-8 indent mismatch as 10.5.4 and is a NON-EVENT: a + hand-trimming slip, with the statement's position in the control flow unchanged at both ends. + Here the same statement is re-indented under a block that already contained it. Indentation + changed; `ctx` did not, and correctly does not fire. + + Falsified by deriving from `len(line) - len(line.lstrip())` instead of the block chain: the two + derivations differ and this goes RED. Restored. + """ + before = "def f():\n if x:\n do_it()\n" + after = "def f():\n if x:\n do_it()\n" # slipped indent, same block + assert _sym_ctx(before, "do_it()") == _sym_ctx(after, "do_it()") == ("f", "If.body") + + +def test_sym_ctx_DOES_fire_when_a_statement_is_welded_into_a_try_the_10_5_4_shape() -> None: + """The other half: 10.5.4's shape, where the statement really did change control-flow region. + + And the honest reading of it -- that change was a HARDENING. The signal fired correctly and the + finding was "your reasoning is stale", not "something is broken". Its security-relevant precision + on the only datum the corpus offers is 0 of 1. + """ + before = "def f():\n conn.rollback()\n" + after = "def f():\n try:\n conn.rollback()\n except OSError:\n pass\n" + assert _sym_ctx(before, "conn.rollback()") == ("f", "") + assert _sym_ctx(after, "conn.rollback()") == ("f", "Try.body") + + +# --- validation: malformed is FATAL, mismatched is ADVISORY --------------------------------------- + + +def test_malformed_ctx_names_an_unknown_node_type() -> None: + assert "not a block statement" in (malformed_sym_ctx(None, "Tyr.body") or "") + + +def test_malformed_ctx_names_a_field_the_node_does_not_have() -> None: + """`With` has no `orelse`. The table that validates is the table the deriver walks, so an accepted + chain is one the deriver can actually produce.""" + assert "does not have" in (malformed_sym_ctx(None, "With.orelse") or "") + + +def test_malformed_sym_rejects_a_non_identifier() -> None: + assert "dotted Python identifier" in (malformed_sym_ctx("Store revoke()", None) or "") + + +def test_wellformed_sym_and_ctx_pass_validation() -> None: + assert malformed_sym_ctx("Store.revoke", "Try.body>If.orelse") is None + assert malformed_sym_ctx("", "") is None # module level, unnested: a claim, and a valid one + assert malformed_sym_ctx(None, None) is None # not asserted + + +def test_a_malformed_ctx_is_FATAL_because_no_code_movement_can_cause_it(tmp_path: Path) -> None: + """The only fatal outcome in this feature. A chain the deriver cannot produce would advise + forever without ever matching, and the fix is unambiguous. + + Falsified by downgrading the `malformed_sym_ctx` branch in `_check_sym_ctx` to an advisory: + `not f.ok` goes RED. Restored. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text("def f():\n do_it()\n", encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 2, "do_it()", ctx="Nope.body"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert not f.ok + assert "not a block statement" in f.problems[0] + + +def test_sym_ctx_on_a_non_python_file_is_FATAL(tmp_path: Path) -> None: + """Markdown has no enclosing symbol. Recording one is an authoring error no derivation can ever + confirm or deny, so it is refused rather than left as a permanent silent pass.""" + (tmp_path / "docs").mkdir() + (tmp_path / "docs" / "SECURITY.md").write_text("deny by default\n", encoding="utf-8") + cells = [ + Cell( + id="1.1.1", + level=1, + verdict="pass", + evidence=(Anchor("docs/SECURITY.md", 1, "deny by default", sym="f"),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert not f.ok and "non-Python file" in f.problems[0] + + +def test_a_MISMATCHED_ctx_is_ADVISORY_never_fatal(tmp_path: Path) -> None: + """THE severity decision, and it is load-bearing. + + A mismatch means the token moved into a different control-flow region. On the only datum the + corpus offers -- 10.5.4 -- that movement was a HARDENING. A check that redded the gate on it + would have demanded a rollback of a security improvement. So: advisory. + + Falsified by appending to `problems` instead of `advisories` in `_check_sym_ctx`: `f.ok` goes RED. + Restored. + """ + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + "def f():\n try:\n conn.rollback()\n except OSError:\n pass\n", + encoding="utf-8", + ) + cells = [ + Cell( + id="10.5.4", + level=3, + verdict="pass", + evidence=(Anchor("messagefoundry/m.py", 3, "conn.rollback()", sym="f", ctx=""),), + ) + ] + f = Findings() + check_anchors(cells, tmp_path, f) + assert f.ok and f.problems == [] + assert len(f.advisories) == 1 + assert "DISPLACEMENT, not a defect" in f.advisories[0] + assert "HARDENING" in f.advisories[0] + assert "ctx=''" in f.advisories[0] and "ctx='Try.body'" in f.advisories[0] + + +def test_sym_and_ctx_are_validated_INDEPENDENTLY(tmp_path: Path) -> None: + """An anchor may assert one and not the other, and asserting neither is not agreement.""" + (tmp_path / "messagefoundry").mkdir() + (tmp_path / "messagefoundry" / "m.py").write_text( + "def f():\n if x:\n do_it()\n", encoding="utf-8" + ) + sym_only = Anchor("messagefoundry/m.py", 3, "do_it()", sym="WRONG") + ctx_only = Anchor("messagefoundry/m.py", 3, "do_it()", ctx="If.body") + neither = Anchor("messagefoundry/m.py", 3, "do_it()") + + f = Findings() + check_anchors([Cell(id="1.1.1", level=1, verdict="pass", evidence=(sym_only,))], tmp_path, f) + assert len(f.advisories) == 1 and "sym=" in f.advisories[0] + + g = Findings() + check_anchors([Cell(id="1.1.1", level=1, verdict="pass", evidence=(ctx_only,))], tmp_path, g) + assert g.advisories == [] # ctx is right, so nothing to say + + h = Findings() + check_anchors([Cell(id="1.1.1", level=1, verdict="pass", evidence=(neither,))], tmp_path, h) + assert h.advisories == [] and h.checked_sym_ctx == 0 + + +def test_sym_ctx_is_ADDITIVE_to_the_drift_advisory_and_neither_replaces_the_other( + tmp_path: Path, +) -> None: + """THE MANDATED PROPERTY, asserted rather than asserted-in-prose. Three anchors, three outcomes: + + - moved a long way, same region -> drift only (sym/ctx would have missed it) + - did not move, region changed -> sym/ctx only (drift would have missed it) + - moved AND region changed -> both + + Measured, 536 of 1,712 Python anchors sit in a (sym, ctx) region wider than the retired 81-line + window, so replacing drift with this loses detection on roughly a third of the record. + + Falsified by making `_check_sym_ctx` return early whenever the line drifted (i.e. treating them as + alternatives): the second and third counts go RED. Restored. + """ + (tmp_path / "messagefoundry").mkdir() + body = ( + "def f():\n" + + "".join(f" pad_{i} = {i}\n" for i in range(200)) + + " moved_far = 1\n" + + "def g():\n try:\n welded = 2\n except OSError:\n pass\n" + ) + (tmp_path / "messagefoundry" / "m.py").write_text(body, encoding="utf-8") + + # moved a long way, still in `f` and still unnested: drift fires, sym/ctx does not. + drift_only = Anchor("messagefoundry/m.py", 3, "moved_far = 1", sym="f", ctx="") + # Recorded at its TRUE line, so drift stays silent and only the region change speaks. + # def f=1, 200 pads=2..201, moved_far=202, def g=203, try=204, welded=205. + region_only = Anchor("messagefoundry/m.py", 205, "welded = 2", sym="g", ctx="") + + f1 = Findings() + check_anchors([Cell(id="1.1.1", level=1, verdict="pass", evidence=(drift_only,))], tmp_path, f1) + assert len(f1.advisories) == 1 and "navigation aid" in f1.advisories[0] + + f2 = Findings() + check_anchors( + [Cell(id="1.1.2", level=1, verdict="pass", evidence=(region_only,))], tmp_path, f2 + ) + assert len(f2.advisories) == 1 and "DISPLACEMENT" in f2.advisories[0] + + both = Anchor("messagefoundry/m.py", 1, "welded = 2", sym="g", ctx="") + f3 = Findings() + check_anchors([Cell(id="1.1.3", level=1, verdict="pass", evidence=(both,))], tmp_path, f3) + assert len(f3.advisories) == 2 + assert any("navigation aid" in x for x in f3.advisories) + assert any("DISPLACEMENT" in x for x in f3.advisories) + + +def test_load_reads_sym_and_ctx_and_absent_stays_None(tmp_path: Path) -> None: + """ABSENT and EMPTY are different claims. Defaulting absent to `""` would turn every one of the + 1,980 un-backfilled anchors into an assertion of "module level, unnested" overnight. + + Falsified by changing the loader to `str(e.get("sym", ""))`: the `is None` assertions go RED. + Restored. + """ + sc = _scorecard_file( + tmp_path, + '[[cell]]\nid = "1.1.1"\nlevel = 1\nverdict = "pass"\n' + " [[cell.evidence]]\n" + ' path = "m.py"\n line = 1\n expect = "x"\n' + " [[cell.evidence]]\n" + ' path = "m.py"\n line = 2\n expect = "y"\n sym = "f"\n ctx = "Try.body"\n' + " [[cell.evidence]]\n" + ' path = "m.py"\n line = 3\n expect = "z"\n sym = ""\n ctx = ""\n', + ) + absent, filled, empty = load_scorecard(sc)[0].evidence + assert absent.sym is None and absent.ctx is None + assert filled.sym == "f" and filled.ctx == "Try.body" + assert empty.sym == "" and empty.ctx == "" + + +def test_the_summary_reports_how_much_of_the_record_sym_ctx_actually_reached( + tmp_path: Path, +) -> None: + """Coverage is printed because backfill has not happened yet. A structural check that reaches 1 of + 3 anchors while printing like a whole-corpus result is the overstatement this pass keeps fixing. + + Falsified by deleting the `sym/ctx asserted on` line from `form_summary`: this goes RED. Restored. + """ + f = Findings(checked_anchors=3, checked_sym_ctx=1) + f.anchor_forms.update({"code": 3}) + text = "\n".join(form_summary(f)) + assert "sym/ctx asserted on 1 of 3 anchor(s)" in text + assert "absence of the field is NOT agreement" in text + + +def test_summary_sym_ctx_negative_control_says_nothing_extra_at_full_coverage() -> None: + """The qualifier appears only when coverage is partial, so it cannot become wallpaper.""" + f = Findings(checked_anchors=2, checked_sym_ctx=2) + f.anchor_forms.update({"code": 2}) + text = "\n".join(form_summary(f)) + assert "sym/ctx asserted on 2 of 2 anchor(s)" in text + assert "NOT agreement" not in text + + +def test_the_walk_descends_THROUGH_a_handler_into_its_nested_blocks() -> None: + """DESCENT, which is a different property from whether the handler is RECORDED. + + Added after an injection that should have failed did not: deleting `ExceptHandler` from + `_TRANSPARENT` silently stopped the walk descending instead of starting to record, so the chain + came out identical by a different route and no test could tell. Descent and recording now come + from two tables, and this test drives descent -- a statement nested inside an `if` inside an + `except` body is only reachable if the walk goes THROUGH the handler. + + Falsified by removing `ExceptHandler` from `_DESCEND_ONLY`: the chain truncates to + `Try.handlers` and this goes RED. Restored. + """ + src = ( + "def f():\n" + " try:\n" + " risky()\n" + " except OSError:\n" + " if retry:\n" + " recover()\n" + ) + assert derive_sym_ctx(src, 6) == ("f", "Try.handlers>If.body") + + +def test_a_handler_contributes_no_chain_element_of_its_own() -> None: + """RECORDING, the other half. `ExceptHandler.body` would double every handler chain for no added + discrimination, because a handler is reachable by exactly one route its parent already names. + + Falsified by removing `ExceptHandler` from `_TRANSPARENT`: the chain becomes + `Try.handlers>ExceptHandler.body>If.body` and this goes RED. Restored. + """ + src = ( + "def f():\n" + " try:\n" + " risky()\n" + " except OSError:\n" + " if retry:\n" + " recover()\n" + ) + _, ctx = derive_sym_ctx(src, 6) or ("", "") + assert "ExceptHandler" not in ctx + + +def test_descent_and_transparency_tables_do_not_drift_apart() -> None: + """They are separate by design, so nothing else would notice one gaining an entry the other + lacks. A node in `_DESCEND_ONLY` but not `_TRANSPARENT` would start emitting a chain element + nobody authored; the reverse would make a transparent node undescendable.""" + assert frozenset(_DESCEND_ONLY) == _TRANSPARENT