diff --git a/docs/status.md b/docs/status.md index 7a45bf1..b64abf3 100644 --- a/docs/status.md +++ b/docs/status.md @@ -58,6 +58,22 @@ read is a policy nobody has agreed to. its suite runs against a blank one per phase. That path has been exercised by **one** project, and `alembic upgrade head` in its test command is what made it necessary. +**A verified upgrade can be opened from the page, and only by a person.** As of item 245: an upgrade +this instance verified green — applied in a clone, your suite run before and after — carries the files +it passed with and the commit it ran at, so pressing the control on its row opens a draft pull request +holding exactly that, rooted at exactly that commit. Nothing is opened on a clock: this instance +verifies on its own schedule and **never** opens by itself (DR-0026). + +Two things that follow, and both are refusals you will meet: + +- **The half that renders the page cannot push.** It writes down that you asked; the other process, + the one with no socket and the code credential, opens it on its next turn. So the pull request + appears seconds later and the page says which state the row is in rather than pretending the click + was the act. +- **Your manifest outranks the button.** `autofix.open_upgrades` is `false` by default, and while it + is, the page tells you how many passed and that none can be opened. Having the credential is not the + same as having agreed, and the report is what there is to act on either way. + ## What does not exist yet - **Sentry's webhook route is enabled since `0.1.0a8`, and its signature is not verified.** It is diff --git a/hullwork.yml b/hullwork.yml index f555613..4990469 100644 --- a/hullwork.yml +++ b/hullwork.yml @@ -27,6 +27,14 @@ autofix: # does not exist yet, and it refuses without a code credential the live instance does not hold. agent: claude-code sandbox: docker + # Permission, not capability (DR-0019). Turned on 2026-08-13 by the operator, once item 245 made + # the request a person's act: a verified-green upgrade carries the files its suite passed with and + # the commit it ran at, and pressing the control on the page opens a draft pull request holding + # exactly that. Nothing here opens on a clock — the instance verifies on its own schedule and + # never opens by itself. What this line says to a contributor reading it is: on this repository, + # a human may ask for that, and every one of them still arrives as a draft nobody but a human + # merges. + open_upgrades: true lanes: green: - typeerror diff --git a/hullwork/advisories.py b/hullwork/advisories.py new file mode 100644 index 0000000..df5b0c5 --- /dev/null +++ b/hullwork/advisories.py @@ -0,0 +1,126 @@ +"""What OSV has published against what a project pins. DR-0024, item 230. + +The half of the product that needs **no model, no write credential and no Docker** — the half an +evaluator can use on their first day — left no trace in a running instance until this: `hullwork +deps` opened no session, stored nothing, and could not even run inside the container. + +**What this module is and is not.** It reads, asks and returns; it writes no rows and knows nothing +about pages or clocks. The caller stores the answer, because *when it was asked* is half of it and +that belongs with the row rather than in here. + +**The verification half stays where it is.** Applying an upgrade and running a suite needs the +Docker socket, and DR-0005 gives the receiver none. This can say what is published; only the +dispatcher can say whether the fix survives your tests. +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from typing import Any, Protocol + +from hullwork import dependencies +from hullwork.osv import Finding, Osv + + +class Tree(Protocol): + """A listing. **Read-only properties, not attributes**: a `Protocol` declaring a mutable + attribute is invariant, so the forge's own `Tree` — whose `paths` is a `tuple` — does not + satisfy `paths: Sequence[str]` however obviously it does in practice.""" + + @property + def paths(self) -> Sequence[str]: ... + + @property + def truncated(self) -> bool: ... + + +class Reads(Protocol): + def tree(self, repo: str) -> Tree: ... + def read_file(self, repo: str, path: str) -> str | None: ... + + +@dataclass(frozen=True) +class Report: + """What was found, and whether the question was asked at all. + + **`asked=False` with a `note` is the answer, not the absence of one.** An advisory list that + silently reads empty when OSV was unreachable says *you are fine* on no evidence, which is the + worst failure this feature can have — and it is the operator's own condition on DR-0024. + """ + + asked: bool + pinned: int = 0 + findings: list[dict[str, Any]] = field(default_factory=list) + note: str | None = None + + +def as_rows(found: Sequence[Finding]) -> list[dict[str, Any]]: + """The findings as the row stores them. One shape, so the page never sees an `Advisory`.""" + return [ + { + "package": one.dependency.name, + "version": one.dependency.version, + "source": one.dependency.source, + "advisories": [ + {"id": a.id, "summary": a.summary, "fixed": list(a.fixed)} for a in one.advisories + ], + } + for one in found + if one.advisories + ] + + +def about(repo: str, forge: Reads, ask: Callable[[Sequence[Any]], list[Finding]]) -> Report: + """Read what this repository pins, and ask what is published against it. + + Every failure is a `Report` rather than an exception, and each says which half failed: a forge + that will not list a tree and a database that will not answer are different problems with + different fixes, and *something went wrong* is neither. + """ + try: + listing = forge.tree(repo) + except Exception as exc: + return Report(asked=False, note=f"could not list {repo}: {exc}") + + pinned = dependencies.read_lockfiles( + list(listing.paths), lambda path: _read(forge, repo, path) + ) + if not pinned: + return Report( + asked=True, + note=( + "nothing here pins a version: no lock file and no `==` in a requirements file, so " + "there is nothing to ask about. A declaration is a range, and a range is not a " + "fact about what your build resolved to" + ), + ) + try: + found = ask(pinned) + except Exception as exc: + return Report( + asked=False, + pinned=len(pinned), + note=f"read {len(pinned)} pinned version(s) and could not reach OSV: {exc}", + ) + return Report(asked=True, pinned=len(pinned), findings=as_rows(found)) + + +def _read(forge: Reads, repo: str, path: str) -> str | None: + """One file, or `None`. A file that will not read costs its own contribution and no more — + `read_lockfiles` already treats that as *this file said nothing*, which is the honest reading + of a `package-lock.json` the forge refused while `uv.lock` came back fine.""" + try: + return forge.read_file(repo, path) + except Exception: + return None + + +def asking(timeout: float = 20.0) -> Callable[[Sequence[Any]], list[Finding]]: + """A callable that asks the real OSV and closes after itself.""" + + def _ask(deps: Sequence[Any]) -> list[Finding]: + with Osv(timeout=timeout) as osv: + return osv.affected(deps) + + return _ask diff --git a/hullwork/cli.py b/hullwork/cli.py index c01b64d..85122be 100644 --- a/hullwork/cli.py +++ b/hullwork/cli.py @@ -14,13 +14,12 @@ import json import logging import os -import shutil import signal import subprocess import sys import threading from collections.abc import Callable, Sequence -from contextlib import ExitStack, suppress +from contextlib import suppress from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import Path @@ -47,7 +46,6 @@ readiness, recurrence, refit, - resolve, spend, territory, triage, @@ -55,7 +53,6 @@ work, ) from hullwork import decisions as decide -from hullwork import dispatch as dispatch_module from hullwork import features as features_module from hullwork import upstream as upstream_module from hullwork.config import ConfigError, Settings, get_settings @@ -545,7 +542,7 @@ def _verify_upgrades( ) ) continue - report = _verify_one(checkout, paths, read, manifest, dep, versions, out) + report = upgrades.verify_one(checkout, paths, read, manifest, dep, versions, out) if report is not None: reports.append(report) @@ -894,146 +891,6 @@ def _print_the_queue(reports: Sequence[bump.Report], out: TextIO) -> None: print("", file=out) -def _verify_one( - checkout: Path, - paths: Sequence[str], - read: Callable[[str], str | None], - manifest: Manifest, - dep: dependencies.Dependency, - versions: list[str], - out: TextIO, -) -> bump.Report | None: - """One package, every candidate, each in its own sandbox.""" - from hullwork import trial - from hullwork.sandbox import image as image_module - from hullwork.sandbox.run import Sandbox - - runtime = manifest.runtime - assert runtime is not None # noqa: S101 - refused above, and mypy cannot see that - tests = manifest.tests or "" - source = dep.source - - # Which candidate `verify` is on, so a resolver-backed mover knows what to ask for. - _pending: dict[str, str] = {"version": ""} - - with ExitStack() as stack: - worktree = dispatch_module.prepare_worktree(checkout) - stack.callback(shutil.rmtree, worktree, ignore_errors=True) - - def files_now() -> dict[str, bytes]: - """The declared dependency files as they are in the worktree right now. - - Read per build rather than once: the rewrite happens between the two, and the second - build has to see it — `image.dependency_digest` then makes the tag differ by itself, - which is what turns the second build into a real rebuild. - """ - found: dict[str, bytes] = {} - for path in runtime.dependencies or [source]: - whole = worktree / path - if whole.exists(): - found[path] = whole.read_bytes() - return found - - built: dict[str, str] = {} - # **The commit the source is at, when the source goes into the build at all** (item 182). - # Read once: it is what `image_tag` hashes to decide whether an image can be reused, and the - # source does not move between candidates — only the dependency files do, and those are - # hashed separately by `dependency_digest`. - source_ref = trial.head_sha(checkout) if runtime.install_needs_source else None - - def build_now() -> str | None: - try: - image = image_module.build( - runtime, files_now(), None, - # **Item 113's fix, which this path never inherited** (found by item 182, on - # the first third-party tree it was pointed at). The build context holds the - # declared dependency files and never the source, and three ordinary installers - # read the source anyway: a `requirements.txt` beginning `-e .`, a `Gemfile` - # that says `gemspec`, and `mvn test`. Measured on `encode/httpx`, whose first - # requirement is `-e .[brotli,cli,http2,socks,zstd]`: - # - # ERROR: file:///work does not appear to be a Python project: - # neither 'setup.py' nor 'pyproject.toml' found. - # - # Reported as *your own environment does not build*, which was true of what we - # built and false of the project. Ruby, Java and PHP are on the roadmap as - # stacks whose attempts work; every one of them reaches this the same way. - source=worktree if runtime.install_needs_source else None, - source_ref=source_ref, - ) - except image_module.ImageBuildError as failed: - return str(failed) - built["tag"] = image.tag - return None - - # The baseline image, before anything is rewritten. A failure here is the project's - # environment, not the upgrade's, so it is said as that. - problem = build_now() - if problem is not None: - print(f" {dep.name}: your own environment does not build — {problem}\n", file=out) - return None - - made = {"n": 0} - - def make_box(_version: str) -> bump.Box: - """A box on **whatever image `built` holds right now**. - - Called once per run rather than once per candidate, because the second run has to - happen on the rebuilt image — reusing the first box measures the upgraded project's - suite against the environment it replaced, and reports `clean` for a version that was - never installed. Found by a real Docker run; see item 174. - """ - made["n"] += 1 - # Built from the worktree **as it is now**, which is what makes each run happen in the - # environment its own tree describes. Cheap when nothing changed: the digest is the - # content, so `build` reuses the existing image rather than making another. - build_now() - box = Sandbox(image=built["tag"], worktree=worktree) - stack.callback(box.cleanup) - box.ensure_volume( - f"hullwork-deps-{os.getpid()}-{made['n']}", - # **Item 114's fix, which this path never inherited either** (item 182). Anything - # the build installed under `/work` is erased by the worktree volume unless the - # image goes down first — which is what `vendor/` is for PHP, and the reason that - # item exists. Off unless the project asks, so every other project takes the path - # it took yesterday. - seed_from_image=runtime.install_needs_source, - ) - return box # type: ignore[return-value] - - # How this file is moved, and everything moving it can touch (items 175 and 176). For a - # list the line is the pin; for a resolved graph only the ecosystem's own tool may move it, - # and `touches` is what stops one candidate leaving a widened range behind for the next. - resolver = resolve.resolver_for(source) - mover = None - guarded: tuple[str, ...] = (source,) - if resolver is not None: - guarded = resolve.touches(resolver) - here = [p for p in paths if p.rsplit("/", 1)[-1] in set(resolver.needs)] - - def mover(worktree: Path, _r: resolve.Resolver = resolver) -> str | None: - outcome = resolve.upgrade( - resolver=_r, worktree=worktree, package=dep.name, version=_pending["version"], - present=here, run=resolve.in_a_container, - ) - return None if outcome.ok else f"{outcome.outcome.value}: {outcome.detail}" - - report = bump.verify( - tests=tests, source=source, package=dep.name, - was=dep.version, versions=versions, - make_box=make_box, rebuild=lambda _text: build_now(), - mover=mover, touches=guarded, pending=_pending, - ) - - for answer in report.answers: - print(f" {answer.says}", file=out) - if answer.detail: - for line in answer.detail.splitlines()[:8]: - print(f" {line}", file=out) - print("", file=out) - return report - - def _cmd_features(args: argparse.Namespace, settings: Settings, out: TextIO) -> int: """What this can do for your project, and what it cannot. Item 186. @@ -1448,7 +1305,7 @@ def refresh_manifest( return manifest -def prune(session: Session, older_than_days: int) -> int: +def prune(session: Session, older_than_days: int, *, dry_run: bool = False) -> int: """Forget the raw bodies of deliveries older than N days. Returns how many were cleared. The payload is kept so a delivery accepted before a restart can still be processed after one — @@ -1471,6 +1328,16 @@ def prune(session: Session, older_than_days: int) -> int: looking untried when it is spent. """ cutoff = datetime.now(UTC) - timedelta(days=older_than_days) + # **Counting is the same query as clearing, minus the write** (item 219). The page shows what it + # would drop before it drops it, and a preview computed by a second query is a preview that can + # disagree with the thing it previews — which on the only destructive control here would be + # worse than having no preview at all. + if dry_run: + return int( + session.query(Delivery) + .filter(Delivery.received_at < cutoff, Delivery.payload_json != "") + .count() + ) cleared = ( session.query(Delivery) .filter(Delivery.received_at < cutoff, Delivery.payload_json != "") @@ -1521,6 +1388,79 @@ def disable_project(session: Session, slug: str) -> Project: return project +def enable_project(session: Session, slug: str) -> Project: + """Watch it again. The counterpart `disable` did not have. Item 226. + + **Reversible in principle and irreversible in practice is the worst of both.** `disable` deletes + nothing — that is its whole design — and until this existed the only way to undo it was an + `UPDATE` against a SQLite file inside a Docker volume. Found by doing it by accident to a real + instance, from a button sitting beside `refresh`. + + Nothing is re-validated here: the manifest, the secret and every item are exactly where they + were, which is what made stopping safe in the first place. + """ + project = _require(session, slug) + project.active = True + session.commit() + return project + + +def ask_to_open(session: Session, slug: str, verdict_id: int) -> str: + """Record that a person wants this upgrade opened. Item 245, DR-0026. + + **This is the button, and pressing it opens nothing.** The receiver renders the page and holds + no credential that can push — that refusal is the property the whole two-process design rests + on (DR-0009) — so the act of a person is written down here and the dispatcher carries it out on + its next turn. What comes back is the sentence the page shows, and it says the pull request does + not exist yet, because a control that implies otherwise is one somebody presses twice. + + Every refusal below is a `ValueError` the page renders as-is: the caller is a form, and a form + posting an id nobody offered is the case this exists for. + """ + from hullwork.models import UpgradeVerdict + + project = _require(session, slug) + verdict = session.get(UpgradeVerdict, verdict_id) + if verdict is None or verdict.project_id != project.id: + msg = f"there is no verdict {verdict_id} for {slug!r}. Nothing was asked for." + raise ValueError(msg) + if verdict.opened_where: + msg = ( + f"{verdict.package} {verdict.was} → {verdict.to} is already open at " + f"{verdict.opened_where}." + ) + raise ValueError(msg) + if verdict.outcome != "clean": + # **Only a verdict that passed may be asked for**, and this is the guard rather than the + # template: a page that offers the control correctly today is not a reason for the write + # path to accept anything posted at it. + msg = ( + f"{verdict.package} {verdict.was} → {verdict.to} is {verdict.outcome}, so there is " + f"nothing to open. Only an upgrade this project's own suite passed can be." + ) + raise ValueError(msg) + if not verdict.artefact: + msg = ( + f"the files {verdict.package} {verdict.to} passed with are not kept any more, so what " + f"would be opened is not what was verified. It is measured again on the next report." + ) + raise ValueError(msg) + if verdict.asked_to_open_at is not None: + return ( + f"{verdict.package} {verdict.was} → {verdict.to} was already asked for. The dispatcher " + f"opens it on its next turn; nothing is lost by waiting." + ) + verdict.asked_to_open_at = datetime.now(UTC) + verdict.open_note = None + session.commit() + return ( + f"Asked for {verdict.package} {verdict.was} → {verdict.to}. **No pull request exists " + f"yet**: this half of the instance cannot push, so the dispatcher opens it on its next " + f"turn — rooted at the commit the suite ran against, with the files it passed with. " + f"Reload to see where it went." + ) + + def _require(session: Session, slug: str) -> Project: project = session.query(Project).filter(Project.slug == slug).one_or_none() if project is None: @@ -1662,7 +1602,23 @@ def _cmd_disable( args: argparse.Namespace, session: Session, settings: Settings, out: TextIO ) -> int: project = disable_project(session, args.slug) - print(f"Disabled '{project.slug}'. Its events and items are kept.", file=out) + print( + f"Disabled '{project.slug}'. Its events and items are kept, and " + f"`hullwork projects enable {project.slug}` watches it again.", + file=out, + ) + return 0 + + +def _cmd_enable( + args: argparse.Namespace, session: Session, settings: Settings, out: TextIO +) -> int: + project = enable_project(session, args.slug) + print( + f"Watching '{project.slug}' again. Nothing was re-validated: its manifest, its secret and " + f"every item are where they were.", + file=out, + ) return 0 @@ -3178,6 +3134,87 @@ def _cmd_work(args: argparse.Namespace, session: Session, settings: Settings, ou LOOP_CEILING_SECONDS = 300 +def _verify_one_upgrade( + session: Session, settings: Settings, *, say: Callable[[str | None], None] = lambda _: None +) -> str | None: + """One published fix, applied and measured, writing nothing anywhere. DR-0026. + + **The read credential.** A verification writes to no repository, so it clones with the token + that cannot push and the property holds by construction — the same reasoning that keeps `work` + from ever holding one it does not need. + """ + from hullwork import upgrades, work + + def clone(where: Settings, project: Project, into: Path) -> Path: + token = where.forge_token.get_secret_value() if where.forge_token else None + if token is None: + msg = "HULLWORK_FORGE_TOKEN is not set, so no repository can be read" + raise work.WiringError(msg) + return work.checkout(work.clone_url(where, project), token, into=into).path + + try: + return upgrades.verify_next(session, settings, clone=clone, say=say) + except (work.WiringError, SandboxError, ImageBuildError) as exc: + log.warning("an upgrade could not be verified", extra={"error": str(exc)}) + return None + + +def _open_one_requested(session: Session, settings: Settings) -> str | None: + """One upgrade somebody asked for from the page, opened. Item 245. + + **The code credential, and it is built here rather than passed in** for the same reason + `_verify_one_upgrade` builds the read one: the credential belongs to the process, and `upgrades` + stays testable against a double. `make_code_forge` answers `None` until the code token is set, + which is the ordinary state of the receiver and a misconfiguration in the dispatcher — + `open_requested` tells those apart and leaves the request pending either way. + + **Closed even when nothing was opened.** A forge client left open per turn is a socket per + minute, and this runs in a resident process. + """ + from hullwork import upgrades + + code_forge = make_code_forge(settings) + try: + return upgrades.open_requested( + session, code_forge, secrets=_redactions(settings) + ) + except ForgeError as exc: + # The verdict and its artefact are untouched, so the request stays pending and the next turn + # tries again: a forge that is down for a minute must not spend somebody's button press. + log.warning("an upgrade could not be opened", extra={"error": str(exc)}) + return None + finally: + close = getattr(code_forge, "close", None) if code_forge is not None else None + if close is not None: + close() + + +def _watch_one_opened(session: Session, settings: Settings) -> str | None: + """What became of one pull request this instance opened for an upgrade. Item 253. + + **The read credential, because this is a read.** Asking a forge about a pull request writes to + no repository, so it uses the token that cannot push — the same reasoning `_verify_one_upgrade` + gives, and it means this half of the watch would work in a process that may not push at all. + + Closed per turn for the reason `_open_one_requested` is: a client left open per turn is a socket + a minute in a resident process. + """ + from hullwork import upgrades + + forge = make_forge(settings) + try: + return upgrades.watch_opened(session, forge) + except ForgeError as exc: + # Nothing is written, so the row keeps saying what it last knew and the next turn asks + # again: a verdict recorded for one bad afternoon is worse than a stale one. + log.warning("an opened upgrade could not be asked about", extra={"error": str(exc)}) + return None + finally: + close = getattr(forge, "close", None) if forge is not None else None + if close is not None: + close() + + def _work_loop( args: argparse.Namespace, session: Session, settings: Settings, out: TextIO ) -> int: @@ -3309,6 +3346,10 @@ def _stop(signum: int, _frame: object) -> None: file=out, ) + def say(what: str | None) -> None: + """What this dispatcher is doing, for the page. Item 242.""" + lease.doing(session, holder, what) + print(f"Dispatching continuously as {holder}. Nothing listens on any port.", file=out) wait = LOOP_FLOOR_SECONDS try: @@ -3350,6 +3391,7 @@ def _stop(signum: int, _frame: object) -> None: print("The model credential works again. Claiming resumes.", file=out) try: + lease.doing(session, holder, "looking for something to do") outcomes = work.run( session, settings, limit=args.limit, slug=args.project, rehearse_into=None ) @@ -3366,6 +3408,33 @@ def _stop(signum: int, _frame: object) -> None: for outcome in outcomes: where = f" → {outcome.pull_request}" if outcome.pull_request else "" print(f"item {outcome.item_id}: {outcome.outcome.value}{where}", file=out) + # **Only when no bug was waiting** (DR-0026, item 233). A production error outranks a + # dependency upgrade, and one verification is a clone, an image build and a suite run — + # so it happens in the gap, one at a time, and never instead of the work. + if not outcomes: + # **What a person asked for goes before what the clock asked for** (item 245). This + # is one forge round trip against an artefact already on disk; a verification is a + # clone, an image build and two suite runs. Doing them the other way round would + # mean a button whose answer arrives five minutes later because the instance chose + # to start something nobody was waiting for. + opened = _open_one_requested(session, settings) + if opened: + print(opened, file=out) + else: + # **One round trip, and before the five-minute one** (item 253). Asking what + # became of a pull request already opened is cheaper than verifying a new + # upgrade, and until this existed the page said *a draft pull request is + # waiting for a person* about two that had been merged for a day. + became = _watch_one_opened(session, settings) + if became: + print(became, file=out) + tried = _verify_one_upgrade(session, settings, say=say) + if tried: + print(tried, file=out) + + # **Idle is a thing to say, not a thing to leave stale** (item 242). A dispatcher that + # stopped mid-sentence would leave the page claiming it is still building an image. + lease.doing(session, holder, None) # Work found → look again at once, because a queue drains fastest when nothing sleeps on # it. Nothing found → back off, so an idle instance is idle. @@ -3641,6 +3710,10 @@ def build_parser() -> argparse.ArgumentParser: disable.add_argument("slug") disable.set_defaults(func=_cmd_disable) + enable = actions.add_parser("enable", help="watch a project again after `disable`") + enable.add_argument("slug") + enable.set_defaults(func=_cmd_enable) + rotate = actions.add_parser("rotate-secret", help="issue a new webhook token") rotate.add_argument("slug") rotate.add_argument( diff --git a/hullwork/dependencies.py b/hullwork/dependencies.py index b3e25f2..fbb417f 100644 --- a/hullwork/dependencies.py +++ b/hullwork/dependencies.py @@ -56,6 +56,33 @@ _A_REQUIREMENT = re.compile(r"^\s*[A-Za-z0-9._]") +#: The leading numeric core of a version — `5.0.6` out of `5.0.6-rc1+build.7`. Both ecosystems this +#: product reads spell that part the same way, which is why one function serves npm and PyPI. +_CORE = re.compile(r"^\s*v?(\d+(?:\.\d+)*)") + + +def newer(candidate: str, than: str) -> bool | None: + """Whether `candidate` is a later version than `than`, or `None` when it cannot be told. + + **`None` is the point of this signature.** A version neither side can parse is not a version to + drop silently: OSV carries `1.2.3.RELEASE`, `2024-11-01` and `0.9.0.beta` among the ordinary + ones, and a comparison that guessed would either try nonsense or hide a real fix. Unknown means + *try it*, and the caller says so. + + Numbers as numbers, because `5.0.10` sorts before `5.0.9` as a string, and a rule built on that + would skip the one upgrade that mattered. + """ + here, there = _CORE.match(candidate), _CORE.match(than) + if here is None or there is None: + return None + left = [int(part) for part in here.group(1).split(".")] + right = [int(part) for part in there.group(1).split(".")] + width = max(len(left), len(right)) + left += [0] * (width - len(left)) + right += [0] * (width - len(right)) + return left > right + + @dataclass(frozen=True) class Dependency: """One pinned package, and which file said so. diff --git a/hullwork/features.py b/hullwork/features.py index e9030c4..82e52a1 100644 --- a/hullwork/features.py +++ b/hullwork/features.py @@ -53,6 +53,19 @@ class Need: #: What to do about it, in the words a person would type. Never "configure it correctly". fix: str met: Callable[[Checkout], bool] + #: **Which of the three things answers this**, named rather than inferred. Item 220: a page + #: serving one instance can supply the manifest and its own variable names, and cannot supply a + #: checkout — item 142 forbids a forge request per render. So *unmet* means three different + #: things there, and only one of them is a defect: + #: + #: * `manifest` — the instance holds it (DR-0012), so unmet is a fact about the project; + #: * `checkout` — nothing on the instance can answer it, so unmet reads *not asked yet*; + #: * `instance` — a credential, which on the receiver may be the dispatcher's by design + #: (DR-0005) and is downgraded exactly as `doctor.not_from_here` downgrades it. + #: + #: `hullwork features` ignores it: run against a real checkout all three are answerable, which + #: is why the distinction never had to exist until a page needed it. + reads: str = "manifest" @dataclass(frozen=True) @@ -159,6 +172,7 @@ def _pins_anything(checkout: Checkout) -> bool: fix="commit one, or pin with `==` — a declaration is a range and a range is not a " "fact about what your build resolved to", met=_pins_anything, + reads="checkout", ), ), limits=( @@ -176,6 +190,7 @@ def _pins_anything(checkout: Checkout) -> bool: what="a lock file or a pinned requirements file, committed", fix="commit one, or pin with `==`", met=_pins_anything, + reads="checkout", ), Need( what="a hullwork.yml naming an image (`runtime.base`) and your test command", @@ -218,6 +233,7 @@ def _pins_anything(checkout: Checkout) -> bool: what="a model credential on the instance that runs it", fix=f"set {MODEL_KEY} to an API key from any provider (DR-0004)", met=lambda c: MODEL_KEY in c.configured, + reads="instance", ), Need( what="`autofix.agent` naming an engine this instance holds", @@ -247,12 +263,14 @@ def _pins_anything(checkout: Checkout) -> bool: what="a credential able to write to your repository", fix=f"set {CODE_TOKEN}. It is the one thing here that writes anything anywhere", met=lambda c: CODE_TOKEN in c.configured, + reads="instance", ), Need( what="an `origin` remote, so the repository can be named", fix="add one — a coordinate cannot be guessed from a directory name, and a wrong " "guess opens a pull request somewhere else", met=lambda c: "origin" in c.configured, + reads="checkout", ), ), permits=( @@ -292,6 +310,7 @@ def _pins_anything(checkout: Checkout) -> bool: what="a model credential on the instance that runs it", fix=f"set {MODEL_KEY} to an API key from any provider (DR-0004)", met=lambda c: MODEL_KEY in c.configured, + reads="instance", ), ), limits=( diff --git a/hullwork/lease.py b/hullwork/lease.py index 32b5375..6bfbfda 100644 --- a/hullwork/lease.py +++ b/hullwork/lease.py @@ -149,6 +149,27 @@ def renew(session: Session, holder: str) -> bool: return True +def doing(session: Session, holder: str, what: str | None) -> None: + """Record what this dispatcher is doing, or `None` for idle. Item 242. Never raises. + + **Best effort, and deliberately so.** This is a page's trace, not the work: a database that + refuses this write must not take down a verification that is already running. It is also why + the timestamp moves only when the sentence changes — a step that has been going for nine + minutes should read as nine minutes, not reset every turn of the loop. + """ + try: + lease = session.get(DispatcherLease, 1) + if lease is None or lease.holder != holder: + return + if lease.doing != what: + lease.doing = what + lease.doing_since = _now() if what else None + session.commit() + except Exception: # a trace for a page is never worth a rollback of the work + log.warning("could not record what this dispatcher is doing", extra={"holder": holder}) + session.rollback() + + def release(session: Session, holder: str) -> None: """Give the lease up on the way out, so the next start does not wait for it to expire. @@ -158,6 +179,9 @@ def release(session: Session, holder: str) -> None: lease = session.get(DispatcherLease, 1) if lease is not None and lease.holder == holder: lease.renewed_at = RELEASED + # A stopped dispatcher is not still doing the last thing it was doing (item 242). + lease.doing = None + lease.doing_since = None session.commit() log.info("dispatcher lease released", extra={"holder": holder}) diff --git a/hullwork/main.py b/hullwork/main.py index 2880d9a..17dd83d 100644 --- a/hullwork/main.py +++ b/hullwork/main.py @@ -10,6 +10,7 @@ from fastapi import Depends, FastAPI, HTTPException, Query, Request, Response, status from fastapi.responses import HTMLResponse, RedirectResponse from pydantic import BaseModel +from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker from hullwork import __version__, operator, page, readiness @@ -17,7 +18,7 @@ from hullwork.config import ConfigError, Settings, get_settings from hullwork.db import get_engine, make_session_factory from hullwork.forge.factory import make_forge -from hullwork.ingest import sweep +from hullwork.ingest import sweep, sweep_inventory from hullwork.logging import configure_logging from hullwork.models import Item from hullwork.readiness import record_sweep_ok @@ -47,6 +48,144 @@ def _readiness_session( session.close() +def _measure_what_the_ingest_credential_may_do( + factory: sessionmaker[Session], settings: Settings +) -> None: + """Ask the forge whether the ingest credential can push, on this instance's own clock. Item 228. + + **The most important thing a project's page can say was waiting for a person.** DR-0009 forbids + the receiver holding a credential that can push, `credentials.audit` measures it, and its + docstring says *only ever run on request* — so an instance running for weeks answered *not asked + yet*. Worse: the key the page read was written by nothing at all, so asking would not have + helped either. + + **A signal that depends on somebody remembering is not a signal.** That is item 073's rule + arriving from the other side: it deleted a check that was permanently on; this one was + permanently unknown. + + Throttled by `forge_recheck_seconds`, the number this instance already uses for *how often may I + ask the forge about a thing again*. The cost is two calls per active project per interval — 12 + an hour for two projects at the default — and it is spent by the clock, never by a page render. + """ + from datetime import UTC, datetime, timedelta + + from hullwork import credentials + from hullwork.cli import _scope_probe + from hullwork.forge.factory import make_permission_reader + from hullwork.models import Project as ProjectRow + + if not settings.forge_url or not settings.forge_token: + return + due = datetime.now(UTC) - timedelta(seconds=max(settings.forge_recheck_seconds, 60)) + with factory() as session: + waiting = [ + one + for one in session.scalars( + select(ProjectRow).where(ProjectRow.active.is_(True)) + ).all() + if one.ingest_checked_at is None or one.ingest_checked_at < due + ] + if not waiting: + return + try: + found = credentials.audit( + session, make_permission_reader(settings), probe=_scope_probe(settings) + ) + except Exception: # a forge having a bad minute is not this loop's problem + log.debug("could not measure the ingest credential", exc_info=True) + return + answered = {one.slug: one for one in found} + now = datetime.now(UTC) + for project in waiting: + verdict = answered.get(project.slug) + if verdict is None: + continue + # **Both fields together, or neither.** A verdict with no timestamp is the + # permanently-on signal again, and a timestamp with no verdict is worse: it says the + # question was answered when it was not. + # **The conclusion, not one field.** `credentials.audit` only probes the token where + # the account can push, because a project whose account cannot has nothing for the + # probe to disprove — so `token_can_push` is `None` there and it is not *unknown*, it + # is *not in question*. `None` survives only when the forge would not say at all. + project.ingest_token_can_push = ( + verdict.token_can_push + if verdict.token_can_push is not None + else (False if verdict.can_push is False else None) + ) + project.ingest_checked_at = now + session.commit() + + +#: How often a project's dependencies are asked about. **Not `forge_recheck_seconds`**: advisories +#: are published on a human's schedule, and asking OSV every ten minutes would be spending somebody +#: else's public API to learn nothing. Six hours is four answers a day, which is more than the +#: publication rate of the thing being asked about. +ADVISORIES_EVERY_SECONDS = 6 * 60 * 60 + + +def _ask_what_is_published_against_what_they_pin( + factory: sessionmaker[Session], settings: Settings +) -> None: + """Read what each project pins and ask OSV what is published against it. DR-0024, item 230. + + **The receiver may do this and the dispatcher must do the rest.** Reading a lock file is the + same forge call `projects refresh` already makes, and OSV takes no credential — so this needs + nothing DR-0005 withholds. Applying an upgrade and running a suite needs the Docker socket, and + that half stays where it is. + + **Stored with when it was asked, and with whether it was asked at all.** Those are the two + conditions the operator put on accepting DR-0024, and they are the same sentence: a report + rendered without its timestamp is a claim about a moment presented as a standing fact, and an + empty advisory list from a failed request says *you are fine* on no evidence. + """ + from datetime import UTC, datetime, timedelta + + from hullwork import advisories, upgrades + from hullwork.models import DependencyReport + from hullwork.models import Project as ProjectRow + + if not settings.forge_url or not settings.forge_token: + return + due = datetime.now(UTC) - timedelta(seconds=ADVISORIES_EVERY_SECONDS) + with factory() as session: + waiting = [ + one + for one in session.scalars( + select(ProjectRow).where(ProjectRow.active.is_(True)) + ).all() + if (report := session.get(DependencyReport, one.id)) is None + or report.taken_at < due + ] + if not waiting: + return + forge = make_forge(settings) + if forge is None: + return + ask = advisories.asking() + try: + for project in waiting: + found = advisories.about(project.repo, forge, ask) + session.merge( + DependencyReport( + project_id=project.id, + taken_at=datetime.now(UTC), + asked=found.asked, + note=found.note, + pinned=found.pinned, + findings=found.findings, + ) + ) + if found.asked: + # **A new report is the only event that can make a kept artefact stale** (item + # 245), so this is the only place the forgetting can go. Guarded on `asked` + # because a request that never reached OSV answers nothing: dropping artefacts + # on an empty finding list would forget everything whenever the network blinked. + upgrades.forget_stale(session, project.id, found.findings) + finally: + forge.close() + session.commit() + + def _sweep_once(factory: sessionmaker[Session], settings: Settings) -> None: """One pass over everything outstanding. Sync, so it runs in a worker thread.""" with factory() as session: @@ -59,6 +198,8 @@ def _sweep_once(factory: sessionmaker[Session], settings: Settings) -> None: ) if not result.skipped: record_sweep_ok() + _measure_what_the_ingest_credential_may_do(factory, settings) + _ask_what_is_published_against_what_they_pin(factory, settings) if result.deliveries or result.filed or result.resolved or result.fetched or result.swept: log.info( "sweep finished outstanding work", @@ -318,14 +459,12 @@ def page_instance_index( if shut is not None: return shut acting = _may_read(session, request, token) + # **The door answers a question and lists the projects** (item 237). It was every item on the + # instance in one table, which with two projects is two projects' bugs interleaved and *whose* + # as the column to scan for. The list of every item is still `/items`. return HTMLResponse( - page.items( - session, - acting=acting, - here="./", - settings=settings, - front=True, - error_reporting=_reporting_enabled, + page.front_door( + session, settings, acting=acting, error_reporting=_reporting_enabled ), headers=page.HEADERS, ) @@ -375,6 +514,130 @@ def page_items( ) +#: The five views a project has, by the last segment of the URL each is served from. **A `POST` +#: answers with the document for the URL it posted to** (item 250), so the handler that acts needs +#: the same map the handlers that read use. +_THE_VIEWS: dict[str, Any] = { + "errors": page.errors, + "fixes": page.fixes, + "dependencies": page.dependencies, + "deliveries": page.deliveries, + "settings": page.settings_for, +} + + +def _a_project_view( + view: object, session: Session, settings: Settings, slug: str, acting: page.Acting, + **answered: str | tuple[str, str | None] | None, +) -> HTMLResponse: + """Render one of a project's feature pages, or `404` for a slug that is not one. + + The `404` is the same one an unknown path gets, on purpose: a distinct body would let somebody + with a valid read token enumerate the slugs this instance serves. + + `answered` is what the last press said — a sentence, a refusal, a rotated secret — and it is + passed through rather than composed here, for the reason `_outcome` gives. + """ + shown = view(session, settings, slug, acting=acting, **answered) # type: ignore[operator] + if shown is None: + raise HTTPException(status_code=404) + return HTMLResponse(shown, headers=page.HEADERS) + + +@app.get( + f"{page.PREFIX}/{{token}}/projects/{{slug}}/errors", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +def page_project_errors( + token: str, + slug: str, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """This project's bugs, newest first. Item 237.""" + _may_read(session, request, token) + return _a_project_view(page.errors, session, settings, slug, _acting(session, request)) + + +@app.get( + f"{page.PREFIX}/{{token}}/projects/{{slug}}/fixes", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +def page_project_fixes( + token: str, + slug: str, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """What this instance attempted on this project, and what it cost. Item 237.""" + _may_read(session, request, token) + return _a_project_view(page.fixes, session, settings, slug, _acting(session, request)) + + +@app.get( + f"{page.PREFIX}/{{token}}/projects/{{slug}}/dependencies", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +def page_project_dependencies( + token: str, + slug: str, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """What is published against what this project pins. Item 237, DR-0024. + + **Inside the project** (item 237), because a page holding every project's advisories one after + another is a wall at two projects and unusable at ten: nobody works by feature across clients. + """ + _may_read(session, request, token) + return _a_project_view(page.dependencies, session, settings, slug, _acting(session, request)) + + +@app.get( + f"{page.PREFIX}/{{token}}/projects/{{slug}}/deliveries", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +def page_project_deliveries( + token: str, + slug: str, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """What this project's tracker sent, and whether it was understood. Item 237.""" + _may_read(session, request, token) + return _a_project_view(page.deliveries, session, settings, slug, _acting(session, request)) + + +@app.get( + f"{page.PREFIX}/{{token}}/projects/{{slug}}/settings", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +def page_project_settings( + token: str, + slug: str, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """Everything this instance will do to this project on command. Item 237.""" + _may_read(session, request, token) + return _a_project_view(page.settings_for, session, settings, slug, _acting(session, request)) + + @app.get( f"{page.PREFIX}/{{token}}/projects", tags=["page"], @@ -413,7 +676,7 @@ def page_project( about a consultancy's customers as much as about this deployment. """ _may_read(session, request, token) - rendered = page.project(session, settings, slug) + rendered = page.project(session, settings, slug, acting=_acting(session, request)) if rendered is None: raise HTTPException(status.HTTP_404_NOT_FOUND, "Not Found") return HTMLResponse(rendered, headers=page.HEADERS) @@ -489,10 +752,45 @@ def _may_read(session: Session, request: Request, token: str) -> page.Acting: """ acting = _acting(session, request) if not page.opens(session, token, acting=acting): + # **A `404` on the operator's own path locks them out of every URL but one** (item 224). + # The reason the refusals here are indistinguishable is DR-0021's: a distinct answer would + # tell somebody holding a *read link* which doors exist. `me` is not a read link — it is a + # literal anybody can type, and the front door already answers it with a login. So on that + # path, and only there, the answer is the login rather than a wall. + if page.offers_a_login(token, acting): + raise _NeedsTheLoginError(request.url.path) raise HTTPException(status.HTTP_404_NOT_FOUND, "Not Found") return acting +class _NeedsTheLoginError(Exception): + """Not a failure: a request that may sign in and has not. Item 224. + + An exception rather than a return, because `_may_read` is called by nine routes as a gate and + turning it into something every one of them has to inspect is the *one place, not nine* problem + its own docstring is about. + """ + + def __init__(self, going_to: str) -> None: + super().__init__(going_to) + self.going_to = going_to + + +@app.exception_handler(_NeedsTheLoginError) +def _answer_with_the_login(request: Request, exc: Exception) -> HTMLResponse: + """The login, for a request that may sign in and has not. Item 224. + + **Where they were going travels with it**, so signing in lands on the view they opened rather + than on the front door — `page.where_it_may_land` decides what that may be, from a list. + """ + return HTMLResponse( + page.just_the_login( + page.Acting(csrf=None, offered=True), going_to=getattr(exc, "going_to", "") + ), + headers=page.HEADERS, + ) + + def _the_login_if_offered( session: Session, request: Request, token: str ) -> HTMLResponse | None: @@ -633,7 +931,7 @@ async def page_login( supplied = await _field(request, "password") issued = operator.sign_in(session, supplied) if supplied else None - redirect = _to_page(token) + redirect = _to_page(token, page.where_it_may_land(await _field(request, "going_to"))) if issued is not None: cookie, _csrf = issued redirect.set_cookie( @@ -701,11 +999,12 @@ async def page_connect_project( @app.post( - f"{page.PREFIX}/{{token}}/projects/{{slug}}", tags=["page"], include_in_schema=False + f"{page.PREFIX}/{{token}}/projects/{{slug}}/{{feature}}", tags=["page"], include_in_schema=False ) async def page_project_action( token: str, slug: str, + feature: str, request: Request, session: Annotated[Session, Depends(_readiness_session)], settings: Annotated[Settings, Depends(get_settings)], @@ -718,9 +1017,22 @@ async def page_project_action( **No default branch.** An action nobody recognises does nothing and says so — a form field that fell through to whichever branch was last is how a typo becomes a disabled project. + + **`feature` names the document that comes back, and nothing else** (item 250). A `POST` answers + at the URL its form posted to, and every relative link in that answer resolves against that URL + — so the document has to be the one that URL serves. This route used to be `projects/` + and answered three of its four branches with a document written for somewhere else: the list of + projects on a refusal and on `rotate-secret`, the dependency view on `open-upgrade`. Eight of + the page's thirty-five buttons came back with navigation that 404'd. + + It moved rather than five routes being added, so the write surface is the same size. `feature` + is checked against the five views a project has, for the reason `_a_project_view` gives: an + unknown one is the same `404` an unknown path gets. """ from hullwork import cli + if feature not in _THE_VIEWS: + raise HTTPException(status.HTTP_404_NOT_FOUND, "Not Found") _may_read(session, request, token) expected = operator.acting(session, request.cookies.get(operator.COOKIE)) if expected is None: @@ -730,32 +1042,418 @@ async def page_project_action( what = await _field(request, "action") rotated: str | None = None + swept: str | None = None try: - if what == "disable": - cli.disable_project(session, slug) + # **Each of them says what it did** (item 223). Three completed in silence: a control that + # appears to do nothing is a control somebody presses again, which on `refresh` is a second + # forge request and on `disable` is a moment of wondering whether the first one worked. + if what == "disable-preview": + # **The second control here that quietly changes what the instance does** (item 226). + # It deletes nothing, which is what made it feel safe to put beside `refresh` — and + # then there was no way back, so reversible in principle was irreversible in practice. + swept = ( + f"Stopping means no error from '{slug}' becomes an item, no issue is filed for it, " + f"and the sweep skips it. Nothing is deleted, and watching it again is one button." + ) + elif what == "enable": + watched = cli.enable_project(session, slug) + swept = ( + f"Watching '{watched.slug}' again. Nothing was re-validated: its manifest, its " + f"secret and every item are where they were." + ) + elif what == "disable": + stopped = cli.disable_project(session, slug) + swept = ( + f"'{stopped.slug}' is no longer watched. Nothing was deleted — its items, " + f"fingerprints and issue references are all still here, and connecting it again " + f"picks them up." + ) elif what == "refresh": cli.refresh_manifest(session, settings, slug) + swept = f"Read {slug}'s manifest again from its repository. It validates." elif what == "set-tracker": - cli.set_tracker(session, slug, await _field(request, "tracker_project")) + named = cli.set_tracker(session, slug, await _field(request, "tracker_project")) + swept = ( + f"'{named.slug}' is {named.tracker_project!r} in the tracker." + if named.tracker_project + else f"'{named.slug}' has no name in the tracker now, so nothing sweeps it." + ) + elif what == "open-upgrade": + # **The button DR-0026 always described** (item 245). It opens nothing here: this + # process refuses to hold a credential that can push, so what this does is write down + # that a person asked, and the dispatcher acts on it. The sentence says so. + asked = await _field(request, "verdict") + if asked is None or not asked.isdigit(): + raise ValueError( + f"{asked!r} is not a verdict this page offered. Nothing was asked for." + ) + swept = cli.ask_to_open(session, slug, int(asked)) elif what == "rotate-secret": rotated = cli.rotate_secret(session, slug) + elif what in ("sweep", "sweep-confirm"): + swept = _sweep_one(session, settings, slug, confirm=what == "sweep-confirm") + elif what == "propose": + # **In a `pre`, because the whole value of it is copying it** (item 223). A `

` + # collapses newlines, and forty-one lines of YAML arrived as one run-on blob. + swept = page.as_a_block(_propose_one(session, settings, slug)) + elif what == "lanes": + swept = _lanes_of(session, settings, slug) else: raise ValueError( f"{what!r} is not something this page does. Nothing was changed." ) except Exception as exc: # every refusal already carries its own sentence session.rollback() - shown = page.projects( - session, settings, acting=_acting(session, request), refused=str(exc) + # **The view they were on, carrying the reason** (item 250). This answered with the list of + # projects, rendered at this project's URL — so a forge that had just gone down was + # reported on a page whose every link 404'd. The refusal is the common path here, not the + # exotic one. + return _a_project_view( + _THE_VIEWS[feature], session, settings, slug, _acting(session, request), + refused=str(exc), ) - return HTMLResponse(shown, headers=page.HEADERS) - shown = page.projects( - session, settings, acting=_acting(session, request), rotated=(slug, rotated) + # **The document for the URL this posted to**, which is the view the button is on. Answering + # with any other one leaves the URL saying one view and the body showing another, and — because + # every link here is relative on purpose — resolves all of them from the wrong depth. + return _a_project_view( + _THE_VIEWS[feature], session, settings, slug, _acting(session, request), + said=swept, + # **The whole token, not `rotated[1]`** (item 250). This passed `(slug, rotated[1])` into a + # parameter typed `tuple[str, str | None]`, and `rotated` is the token itself — so the one + # answer in this product that can never be repeated printed **a single character of it** + # and type-checked. Only the hash is kept, so the secret was gone: the button stopped the + # tracker's working URL and gave nothing back to replace it with. + **({"rotated": (slug, rotated)} if rotated is not None else {}), ) - return HTMLResponse(shown, headers=page.HEADERS) +def _propose_one(session: Session, settings: Settings, slug: str) -> str: + """A manifest read from the repository's own CI configuration. Item 107, on the page (item 222). + + **It prints and does not write**, here as in the terminal: a manifest belongs in the project's + repository, committed by somebody who read it, and DR-0006's rule that what was inferred stays + commented only means anything if a person is the one who uncomments it. + + One forge read, when a person asks for it. Item 142 forbids a request per *render* — a reader + refreshing would spend one each time — and says nothing about an action somebody pressed, which + is the same shape `projects refresh` has had since item 206. + """ + from hullwork.cli import _forge_for, propose_from_ci + from hullwork.models import Project as ProjectRow + + project = session.scalars(select(ProjectRow).where(ProjectRow.slug == slug)).one_or_none() + if project is None: + raise ValueError(f"no project called {slug!r}.") + forge = _forge_for(settings, project.forge) + try: + proposed = propose_from_ci(forge, project.repo) + finally: + forge.close() + if proposed is None: + raise ValueError( + f"nothing in {project.repo} proposes a manifest: no CI configuration was found, or " + f"the one there says nothing this reader recognises. That is not a refusal to connect " + f"the project — it means the manifest has to be written by hand, and the field that " + f"decides whether anything can be built is `runtime.base`: an image your tests already " + f"run in." + ) + return proposed + + +def _lanes_of(session: Session, settings: Settings, slug: str) -> str: + """The lane policy, applied to this repository's own directories. M8, item 104, on the page. + + **An operator who cannot see the policy applied to their code is being asked to trust a + paragraph**, and this product's first principle is that trust is the product. + + Read-only and stores nothing, deliberately: a derived policy kept on disk would be a snapshot of + *which code is dangerous*, and `territory.py` explains why that fails in the direction that + matters. So this is an action and never a cache. + """ + from hullwork import territory + from hullwork.cli import _forge_for + from hullwork.models import Project as ProjectRow + + project = session.scalars(select(ProjectRow).where(ProjectRow.slug == slug)).one_or_none() + if project is None: + raise ValueError(f"no project called {slug!r}.") + forge = _forge_for(settings, project.forge) + try: + listing = forge.tree(project.repo) + except Exception as exc: + raise ValueError(f"could not read the tree of {project.repo}: {exc}") from exc + finally: + forge.close() + + claimed = territory.sensitive_tree(list(listing.paths)) + said = [ + f"{project.repo} at {listing.ref[:12]} — {len(listing.paths)} file(s), " + f"{len(claimed)} that this instance keeps a human on." + ] + if listing.truncated: + said.append( + "The forge did not serve the whole tree, so this list is incomplete — what is missing " + "is unclassified here, not classified as ordinary." + ) + by_rule: dict[str, list[str]] = {} + for path, rule in claimed: + by_rule.setdefault(rule.pattern, []).append(path) + for pattern, paths in by_rule.items(): + said.append(f"`{pattern}` — {len(paths)} file(s): {', '.join(sorted(paths)[:4])}") + return " · ".join(said) + + +def _sweep_one(session: Session, settings: Settings, slug: str, *, confirm: bool) -> str: + """Read the tracker's unresolved list for one project. DR-0011, item 219. + + **The count comes before the writing and the number you confirm is the number you were shown.** + A project with three hundred open issues becomes three hundred forge issues in one pass, and a + tool that does that on a first afternoon is uninstalled that evening — which is the whole reason + `sweep` has a `--confirm` in the terminal. On a page that is two submissions, and computing the + preview from a different query than the write would let them disagree. + """ + inventory = make_inventory(settings) + if inventory is None: + raise ValueError( + "no tracker inventory is configured. It needs HULLWORK_TRACKER_URL, " + "HULLWORK_TRACKER_TOKEN and HULLWORK_TRACKER_ORG — the organisation cannot be " + "discovered, because the least-privilege token is refused the route that would list it." + ) + results = sweep_inventory( + session, inventory, slug=slug, first_pass=True, dry_run=not confirm + ) + if not results: + return f"'{slug}' has no tracker project set, so there is nothing to sweep." + + said = [] + for result in results: + if result.error: + said.append(f"{result.project}: could not read the tracker — {result.error}") + elif confirm: + said.append( + f"{result.project}: filed {result.created} issue(s); " + f"{result.deduplicated} were already known." + ) + else: + said.append( + f"{result.project}: {result.created} issue(s) would be filed and " + f"{result.deduplicated} are already known. Nothing was written." + ) + return " · ".join(said) + + +@app.post( + f"{page.PREFIX}/{{token}}/instance", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +async def page_instance_action( + token: str, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """The instance's own housekeeping, from its own view. Item 219, item 218 §1. + + **One route with an action rather than three names**, which is item 207's rule applied to the + second noun that needed one. Each action calls what the terminal calls; none is implemented + here. + + **`prune` is the only destructive control on this page**, so it has two submissions and the + first one writes nothing: `prune-preview` says how many bodies it would clear, and the number a + person confirms is the number they were just shown. + """ + from hullwork import cli + + acting = _the_operators(session, request, token) + if not operator.csrf_ok( + operator.acting(session, request.cookies.get(operator.COOKIE)), + await _field(request, "csrf"), + ): + raise HTTPException(status.HTTP_403_FORBIDDEN, "Forbidden") + + what = await _field(request, "action") + said: str | None = None + try: + if what == "lease-release": + said = cli.release_lease(session) + elif what == "prune-preview": + days = _days(await _field(request, "older_than_days")) + said = ( + f"{cli.prune(session, days, dry_run=True)} delivery body(s) and fetched event(s) " + f"older than {days} days would be cleared. Every row, fingerprint and issue " + f"reference stays. Nothing has been cleared yet." + ) + session.rollback() + elif what == "prune": + days = _days(await _field(request, "older_than_days")) + said = ( + f"Cleared {cli.prune(session, days)} delivery body(s) older than {days} days. " + f"Every row, fingerprint and issue reference is intact." + ) + elif what == "republish": + said = _republish_all(session, settings) + elif what == "page-token": + # **The read link, re-keyed from the page** (DR-0025, item 229). Strictly less than what + # this session already does: it revokes a URL rather than granting anything. The + # password is the other half of that decision and is deliberately not here. + from hullwork.security import generate_token, hash_token + + minted = generate_token() + page.issue(session, hash_token(minted)) + # **Including the page this answer is on, when it is one of them** (item 250). Read at + # a minted URL, this button revokes the URL the answer is served from: every link on + # the page that comes back is correctly written and every one of them answers `404`. + # Read through a session it is not the token that opens the door, so nothing here + # breaks — and saying so unconditionally would be false half the time. + here_too = ( + " That includes the URL you are reading this on, so every link on this page " + "answers 404 now: open the one above." + if page.the_url_is_the_credential(token) + else "" + ) + said = page.as_a_block( + f"{settings.base_url.rstrip('/')}{page.PREFIX}/{minted}/\n\n" + f"This URL is the credential and it is shown once — only its hash is kept, so no " + f"later view can print it again. Every URL handed out before this moment has " + f"stopped working.{here_too}" + ) + else: + raise ValueError(f"{what!r} is not something this page does. Nothing was changed.") + except Exception as exc: # every refusal already carries its own sentence + session.rollback() + said = str(exc) + else: + session.commit() + + return HTMLResponse( + page.instance( + session, + settings, + error_reporting=_reporting_enabled, + acting=acting, + said=said, + ), + headers=page.HEADERS, + ) + + +def _days(raw: str | None) -> int: + """The retention window, refused rather than defaulted when it is not a number. + + A blank field becoming `0` would clear everything, which is the one mistake this control must + not make quietly. + """ + try: + days = int(raw or "") + except (TypeError, ValueError): + raise ValueError(f"{raw!r} is not a number of days. Nothing was changed.") from None + if days < 1: + raise ValueError("the window has to be at least one day. Nothing was changed.") + return days + + +def _republish_all(session: Session, settings: Settings) -> str: + """Finish every verdict the dispatcher reached and could not send. Item 077, on the page. + + The receiver has the forge credential this needs and provably not the one that can push, so a + `pr-open` verdict is refused here exactly as it is refused in the terminal: it needs the files + the agent wrote and nothing stores them (item 079). + """ + from hullwork import work as work_module + from hullwork.cli import _redactions + from hullwork.models import Item as ItemRow + from hullwork.models import Project as ProjectRow + + stranded = work_module.unpublished_verdicts(session) + if not stranded: + return "No verdict is waiting to be published." + + forge = make_forge(settings) + done: list[str] = [] + try: + for attempt in stranded: + item = session.get(ItemRow, attempt.item_id) + project = session.get(ProjectRow, item.project_id) if item else None + if project is None: # pragma: no cover - a foreign key makes this unreachable + continue + try: + where = work_module.republish( + session, + attempt, + forge=forge, + repo=project.repo, + secrets=_redactions(settings), + ) + except work_module.PublicationError as exc: + done.append(f"attempt {attempt.id}: {exc}") + continue + done.append(f"attempt {attempt.id}: published to {project.repo}{where}") + finally: + if forge is not None: + forge.close() + return " · ".join(done) + + +@app.post( + f"{page.PREFIX}/{{token}}/items/{{item_id}}", + tags=["page"], + response_class=HTMLResponse, + include_in_schema=False, +) +async def page_item_action( + token: str, + item_id: int, + request: Request, + session: Annotated[Session, Depends(_readiness_session)], + settings: Annotated[Settings, Depends(get_settings)], +) -> HTMLResponse: + """What an operator does to one item that is not a decision about it. Item 219. + + The two decisions — approve, human-only — keep their own routes: they are the product's gate and + they are pressed by somebody who may be reading nothing else. This is the housekeeping beside + them, and it takes an action field for item 207's reason. + """ + from hullwork import cli + from hullwork.models import Item as ItemRow + from hullwork.models import Project as ProjectRow + + acting = _the_operators(session, request, token) + if not operator.csrf_ok( + operator.acting(session, request.cookies.get(operator.COOKIE)), + await _field(request, "csrf"), + ): + raise HTTPException(status.HTTP_403_FORBIDDEN, "Forbidden") + + found = session.get(ItemRow, item_id) + if found is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "Not Found") + + what = await _field(request, "action") + said: str | None = None + try: + if what == "requeue": + project = session.get(ProjectRow, found.project_id) + back = cli.requeue(session, project.slug if project else "", item_id) + said = ( + f"Item {back.id} ({back.lane.value}) is now '{back.state.value}'. Its attempt was " + f"never spent, so it still has one." + ) + else: + raise ValueError(f"{what!r} is not something this page does. Nothing was changed.") + except Exception as exc: # every refusal already carries its own sentence + session.rollback() + said = str(exc) + else: + session.commit() + + shown = page.item(session, settings, item_id, acting=acting, said=said) + if shown is None: # pragma: no cover - it existed a line ago + raise HTTPException(status.HTTP_404_NOT_FOUND, "Not Found") + return HTMLResponse(shown, headers=page.HEADERS) + @app.post(f"{page.PREFIX}/{{token}}/logout", tags=["page"], include_in_schema=False) async def page_logout( token: str, diff --git a/hullwork/models.py b/hullwork/models.py index 431dd57..98a6495 100644 --- a/hullwork/models.py +++ b/hullwork/models.py @@ -227,6 +227,19 @@ class Project(Base): ) #: Deactivated rather than deleted: unregistering a project must not destroy its history. + #: **Whether the ingest _token_ can write code**, measured by asking (item 228), and when. + #: + #: The **token**, not the account. `PushCapability.can_push` is what the account may do, and a + #: token scoped to reads and issues is refused regardless — measured on this project's own + #: instance, where that flag was `True` for both projects while `POST /branches` answered + #: `403 … scope(s): [write:repository]`. Recording the account's answer here and painting it + #: red would rebuild the permanently-on signal item 073 deleted a whole check for. + #: + #: `None` is *not measured* and never a `False`. Columns rather than keys inside `manifest`, + #: which is the project's own document adopted verbatim (DR-0012) and no place for a fact the + #: instance measured — the page read exactly such a key for two items and nothing wrote it. + ingest_token_can_push: Mapped[bool | None] = mapped_column(Boolean, default=None) + ingest_checked_at: Mapped[datetime | None] = mapped_column(UtcDateTime(), default=None) active: Mapped[bool] = mapped_column(Boolean, default=True) created_at: Mapped[datetime] = mapped_column(UtcDateTime(), default=_now) @@ -774,6 +787,22 @@ class DispatcherLease(Base): #: column — and it must not read as `off`, which is the defect item 105 was closed for. error_reporting: Mapped[bool | None] = mapped_column(Boolean, default=None) + #: What this dispatcher is doing at this moment, in a person's words, or `None` when it is idle. + #: Item 242. + #: + #: **Written by the process rather than deduced by the page.** Everything a reader could infer + #: — this heartbeat, an attempt without a `finished_at`, a verdict appearing — has the same hole + #: in the middle: between two writes there is nothing to read, and that gap is the four minutes + #: somebody is trying to watch. The instance report called it *nothing in progress* while a + #: verification built an image and ran a suite twice. + #: + #: On the lease and not in a table of its own, because it is the same fact: *who is dispatching + #: now*. Two rows could disagree about whether one exists, and then neither is worth reading. + doing: Mapped[str | None] = mapped_column(String(200), default=None) + #: When that started, so the page can say how long it has been going on rather than only what + #: it is. A step that has taken nine minutes is the interesting one. + doing_since: Mapped[datetime | None] = mapped_column(UtcDateTime(), default=None) + class Installation(Base): """A name this deployment can be counted by. One row, id 1. Item 151. @@ -801,3 +830,113 @@ class Installation(Base): identifier: Mapped[str] = mapped_column(String(32)) created_at: Mapped[datetime] = mapped_column(UtcDateTime(), default=_now) + + +class DependencyReport(Base): + """What OSV had published against what a project pins, and **when that was asked**. DR-0024. + + One row per project, overwritten: this is a claim about a moment, not a history. A history would + be a different feature with a different cost, and nobody has asked for one. + + **`asked` is why the row exists.** An advisory list that silently reads empty when the network + was down is the worst failure this feature can have — it says *you are fine* on no evidence. So + the row records whether the question reached OSV at all, and `note` says what stopped it. That + is the operator's own condition on accepting DR-0024, and it is the same *I could not verify + this* that has been a first-class answer here since item 199. + """ + + __tablename__ = "dependency_reports" + + project_id: Mapped[int] = mapped_column(ForeignKey("projects.id"), primary_key=True) + taken_at: Mapped[datetime] = mapped_column(UtcDateTime(), default=_now) + #: Whether OSV answered. `False` with a `note` is a report; `False` with none is a bug. + asked: Mapped[bool] = mapped_column(Boolean, default=False) + note: Mapped[str | None] = mapped_column(Text, default=None) + #: How many pinned versions were read, so *nothing published* can be told apart from *nothing + #: read*. A project with no lock file has 0 here and no findings, and those are not the same + #: sentence. + pinned: Mapped[int] = mapped_column(Integer, default=0) + #: `[{package, version, source, advisories: [{id, summary, fixed: []}]}]`. JSON rather than two + #: more tables, because nothing queries inside it: the page renders it and the next report + #: replaces it whole. + findings: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list) + + +class UpgradeVerdict(Base): + """What happened when this instance tried one published fix. DR-0026, item 233. + + **One row per (project, package, from, to)**, overwritten. The question is *does this upgrade + hold today*, and yesterday's answer about the same pair is not a second fact — it is the same + fact, stale. + + `outcome` is `bump.Verdict`'s own vocabulary and the four states do not collapse: `clean` is + *your suite passed before and after* and **not** *this is safe*; `breaks` is the finding; + `will-not-install` is the build failing, which is a different fact from the suite failing; and + `already-red` is a suite that was failing before anything was touched, so no claim can be made + either way. + + **Nothing here was written anywhere else.** DR-0026 stops the queue at `verify`: no branch, no + pull request, no comment. This table is the whole of what the attempt produced. + """ + + __tablename__ = "upgrade_verdicts" + __table_args__ = ( + UniqueConstraint("project_id", "package", "was", "to", name="uq_upgrade_verdict"), + ) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + project_id: Mapped[int] = mapped_column(ForeignKey("projects.id"), index=True) + package: Mapped[str] = mapped_column(String(200)) + #: The version that was pinned when this was tried. A verdict about a version no longer pinned + #: is worse than no verdict, because it reads as current — so the page checks this against the + #: report before showing anything. + was: Mapped[str] = mapped_column(String(100)) + to: Mapped[str] = mapped_column(String(100)) + outcome: Mapped[str] = mapped_column(String(30)) + #: What the suite or the build said, trimmed. The evidence, not a summary of it. + detail: Mapped[str | None] = mapped_column(Text, default=None) + tried_at: Mapped[datetime] = mapped_column(UtcDateTime(), default=_now) + #: What the passing run produced: `{"files": {path: text}, "runs": {...} | None}`. Item 245. + #: + #: Only a `clean` verdict has one, and it is the whole of what a pull request is opened with. + #: Neither half is a convenience. **The files**: `bump.attempt` restores every file it moved, so + #: reconstructing the diff later means running the resolver again, and a lock regenerated twice + #: can differ — a version published in between, a different ordering, a registry that answered + #: differently. Publishing files the suite did not pass against is the defect item 045 is named + #: after. **The runs**: they are the evidence the artefact is *for* — the two exit codes and the + #: runner's own summary lines — and by the time anybody renders one the containers are gone. A + #: pull request opened from this row without them would carry strictly less than the one the + #: terminal opens, which is a degradation nobody would see. + #: + #: **`none_as_null` because emptying it has to be visible from SQL.** Without it, assigning + #: `None` writes the JSON text `null` — four bytes, so the storage *is* released, and a row that + #: reads `None` in Python. But `WHERE artefact IS NOT NULL` then counts it, which is how the + #: check run minutes after the first pull request was opened reported two artefacts where the + #: database held one. `forget_stale` filters on exactly that predicate. + artefact: Mapped[dict[str, Any] | None] = mapped_column( + JSON(none_as_null=True), default=None + ) + #: The commit the suite ran against, which is where the branch is rooted — never wherever the + #: default branch points when somebody presses the button. The base can move freely in between. + base_sha: Mapped[str | None] = mapped_column(String(64), default=None) + #: When a person asked for this one to be opened. DR-0026: *open stays a button somebody + #: presses*, and the receiver that renders the button cannot open anything, so the request is + #: written here and the dispatcher acts on it. + asked_to_open_at: Mapped[datetime | None] = mapped_column(UtcDateTime(), default=None) + #: Where the pull request went. Its presence is what *opened* means. + opened_where: Mapped[str | None] = mapped_column(Text, default=None) + #: What became of it: `None` until the forge has been asked, then `open`, `merged` or `closed`. + #: Item 253. + #: + #: **Written once was the defect.** `opened_where` alone meant *a draft pull request is waiting + #: for a person*, for ever — so a merged one kept asking for a review that had happened, and one + #: a person closed without merging displayed their "no" as work they owed. Item 138 split the + #: same two facts on `Item` and this is that split, one noun along. + opened_state: Mapped[str | None] = mapped_column(String(10), default=None) + #: When the forge was last asked about it, so a pull request that sits open for a week costs one + #: request per report cycle rather than one per turn. `Item.merge_checked_at`'s counterpart. + open_checked_at: Mapped[datetime | None] = mapped_column(UtcDateTime(), default=None) + #: Why a request produced no pull request — already open from an earlier run, or the forge + #: refused. Never silence: a row that was asked for and shows neither outcome is a row somebody + #: presses again. + open_note: Mapped[str | None] = mapped_column(Text, default=None) diff --git a/hullwork/page.py b/hullwork/page.py index a9072f3..4dfa494 100644 --- a/hullwork/page.py +++ b/hullwork/page.py @@ -31,15 +31,16 @@ from __future__ import annotations import html +import json import re -from collections.abc import Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from urllib.parse import urlsplit -from sqlalchemy import select +from sqlalchemy import func, select from hullwork import __version__, spend from hullwork.models import Attempt as _Attempt @@ -124,6 +125,22 @@ def opens(session: Session, token: str, *, acting: Acting | None = None) -> bool return verify_token(token, expected) and access is not None +def the_url_is_the_credential(token: str) -> bool: + """Whether this request is reading through a minted URL rather than through its session. + + **Asked by the one action that revokes it** (item 250). `page-token` mints a new read link and + stops every old one — including, when the answer is served at one of them, the URL it is being + read on: every link on that answer is correctly written and every one of them answers `404`. + Saying so is the difference between a reader who opens the new link and one who clicks. + + Here rather than in the route, because `MINE` is this module's reserved word and a route that + knows it is a route with an opinion about the gate — which is the property + `test_there_is_one_gate_and_not_ten` exists to keep, at four days' cost across items 193, 194, + 200 and 203. + """ + return token != MINE + + def offers_a_login(token: str, acting: Acting | None) -> bool: """Whether a request that may not read should be shown the login rather than a `404`. @@ -305,6 +322,12 @@ def _link(url: str | None, text: str | None = None) -> str: to `--measure`, because the fix for dead margins is not 120-character lines. */ .wrap { max-width: 108rem; margin: 0 auto; padding: 0 2rem 4rem; } .sheet > p, .sheet > .sub, .sheet .why, footer { max-width: var(--measure); } +/* And the same measure inside a fold. A child selector cannot see them: everything a `

` + holds is one level deeper than the sheet, so the prose under an open fold ran the full 1500px + while the identical sentence above it was held to 68ch. + `.folded` and not `details >`, because `_fold` wraps its body in that div — the first version of + this rule named a structure the page does not have, and the browser found it in ten seconds. */ +.sheet .folded > p, .sheet .folded > .sub { max-width: var(--measure); } /* A headline needs a shorter measure than a paragraph, and `ch` is relative to the element's own size: 68ch at 28px came out 1170px wide, which is a measure in name only. */ .sheet .lede { max-width: 34ch; } @@ -437,8 +460,18 @@ def _link(url: str | None, text: str | None = None) -> str: border-top: 1px solid var(--rule); } .standing li:first-child { border-top: 0; } +/* A disclosure inside a row spans it (item 236). Left in the first column it inherits the pill's + 6.2rem and sets its summary five words deep, one word per line. */ +.standing li > details { grid-column: 1 / -1; margin: .5rem 0 0; border: 0; background: none; } +.standing li > details > summary { padding: 0; font-size: var(--t-md); } +.standing li > details > .folded { padding: .4rem 0 0; } .standing .pill { justify-self: start; color: var(--c, var(--faint)); border-color: currentColor; } .standing .name { font-weight: 550; color: var(--ink); } +/* A sentence is not a label (item 242). `.name` is set in small caps because it holds a thing's + name — `CRYPTOGRAPHY 48.0.1` — and the history holds whole sentences, which small caps makes + slower to read and louder than the thing they describe. */ +.standing .said { color: var(--ink); text-transform: none; letter-spacing: 0; + font: 400 var(--t-md)/1.45 var(--sans); } .standing .why { grid-column: 2; color: var(--muted); @@ -463,6 +496,126 @@ def _link(url: str | None, text: str | None = None) -> str: .standing .why { grid-column: 1; } } +/* --- the subject table (DR-0028) ------------------------------------------------------------ + A component and not this view's markup: the same shape is going to the door, to Errors, to This + instance and to Projects, and a table invented four times is four tables that drift. + + The row is the subject. Everything known about it is in it, at a fixed height, aligned in + columns — 42px against the 164px the card-paragraph it replaces spent on three facts. What made + that view 12.6 screens was not the amount of information; it was that each fact was a paragraph + and the outcome lived in a second list. */ +.tally { + display: flex; flex-wrap: wrap; gap: .3rem 1.1rem; + font-size: var(--t-sm); color: var(--muted); + padding: .55rem .8rem; margin: 0 0 1.3rem; + background: var(--raise); border: 1px solid var(--rule); border-radius: var(--r); +} +.tally b { color: var(--ink); font-variant-numeric: tabular-nums; margin-right: .15rem; } +.band { margin: 0 0 1.5rem; } +/* The heading carries the sentence, once. A row that repeats it is the column that must not + exist. */ +.band h3 { + display: flex; align-items: baseline; gap: .6rem; + margin: 0 0 .2rem; font: 600 var(--t-md)/1.4 var(--sans); color: var(--ink); +} +.band h3 em { font-style: normal; font-weight: 400; color: var(--faint); font-size: var(--t-sm); + flex: 1; } +.band h3 > b { font: 400 var(--t-sm)/1 var(--sans); color: var(--faint); + font-variant-numeric: tabular-nums; } +/* Fixed, so one long row cannot set the width of the table. A package pinned three times and fixed + on four branches stretched it to 7,208px before the model stopped pairing every version with + every destination — and a table that can be widened by its contents will be, eventually. */ +.subjects { width: 100%; border-collapse: collapse; table-layout: fixed; } +.subjects tr { border-top: 1px solid var(--rule); } +.subjects tr:hover { background: var(--sunk); } +.subjects td { padding: 0 .6rem; height: 2.6rem; vertical-align: middle; } +.subjects td:first-child { padding-left: 0; } +.subjects td:last-child { padding-right: 0; text-align: right; } +.dot { width: 7px; height: 7px; border-radius: 50%; display: inline-block; margin-right: .55rem; + background: var(--c, var(--faint)); vertical-align: 1px; } +.dot.passed { --c: var(--passed); } +.dot.refused { --c: var(--refused); } +.dot.human { --c: var(--human); } +.dot.working { --c: var(--working); } +.dot.waiting { --c: var(--waiting); } +.dot.faint { --c: var(--faint); } +.who { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.who .thing { font-weight: 550; margin-right: .5rem; } +/* Versions and paths are compared, not read: monospaced so a digit sits under a digit. */ +.who .was, .who .to, .at { font: var(--t-sm)/1.4 var(--mono); } +.who .was { color: var(--muted); } +.who .to { color: var(--ink); } +.who .arr { color: var(--faint); margin: 0 .35rem; } +.who .note { color: var(--refused); font-size: var(--t-sm); margin-left: .55rem; } +/* A row whose subject is a sentence rather than a name: the item's own title. It takes the space + the versions take on a dependency row, and it is the one thing in the row allowed to be long. */ +.who .said { color: var(--muted); margin-left: .55rem; } +.subjects .do a { font-size: var(--t-sm); } +/* Context, and it has a known maximum — a lock file's path, or a slug and a lane. Given a width, + the subject takes everything else; without one a fixed table split the remainder evenly and cut + the item's title at eight words while `hullwork · amber` sat in twelve rem of air. */ +.at { color: var(--faint); text-align: right; white-space: nowrap; width: 15rem; + overflow: hidden; text-overflow: ellipsis; } +.subjects .fold { width: 3rem; text-align: right; } +.subjects details.adv { border: 0; background: none; margin: 0; display: inline-block; + position: relative; } +.subjects details.adv > summary { + padding: .05rem .4rem; font: var(--t-xs)/1.5 var(--sans); color: var(--faint); + font-variant-numeric: tabular-nums; border: 1px solid var(--rule); border-radius: var(--r-chip); +} +.subjects details.adv[open] > summary { color: var(--ink); border-color: var(--faint); } +.subjects details.adv > .folded { + position: absolute; right: 0; z-index: 3; width: min(30rem, 70vw); text-align: left; + margin-top: .35rem; padding: .6rem .8rem; + background: var(--raise); border: 1px solid var(--rule); border-radius: var(--r); + box-shadow: 0 10px 28px light-dark(rgba(16,19,25,.13), rgba(0,0,0,.5)); + font-size: var(--t-sm); list-style: none; +} +.subjects details.adv > .folded li { margin: .35rem 0; color: var(--muted); } +.subjects details.adv > .folded li a { font: var(--t-xs)/1.4 var(--mono); margin-right: .4rem; } +.subjects .do { width: 9.5rem; } +/* A subject that is a sentence needs the width a button was holding. An item's title is the + most informative thing in its row and eighteen of twenty-eight were being cut at eight words, + while the column beside them held `#24`. The dependency table keeps the wider one: its action is + a button with words in it. Two contents, two widths, one component. */ +.subjects.narrow .do { width: 4.5rem; } + +/* A figure and what it counts (item 248). The number is the thing being compared, so it is + tabular, right-aligned in its own column, and set at the size the strip uses — the six sections + that were prose bullets put it mid-sentence, where two of them never line up. */ +.figures { width: 100%; border-collapse: collapse; } +.figures tr { border-top: 1px solid var(--rule); } +.figures tr:first-child { border-top: 0; } +.figures td { padding: .22rem 0; vertical-align: baseline; } +/* Set at body size rather than at the strip's: the strip has five numbers and this has five rows, + so what makes one scannable here is the column and the tabular figures, not the size. Rendered + large with the caveat on its own line it made this view *taller* than the bullets it replaced, + which is the opposite of the point. */ +.fig { width: 3.2rem; text-align: right; padding-right: .8rem !important; + font: 600 var(--t-base)/1.45 var(--sans); font-variant-numeric: tabular-nums; + color: var(--ink); } +.means { color: var(--ink); } +.caveat { color: var(--faint); font-size: var(--t-sm); margin-left: .5rem; } +.subjects.narrow .at { width: 13rem; } +.subjects .do form { margin: 0; } +.subjects .do button { font-size: var(--t-sm); padding: .25rem .6rem; } +/* Below this the row stops being a row (item 221's rule, kept). Hiding the context instead was + tried and is worse: the project, the lane and the time are how a reader tells two failures of the + same kind apart, and a list that drops them is a list of titles. Two lines rather than the seven + the old table stacked — the subject, then everything about it, then anything to do. */ +@media (max-width: 46rem) { + .subjects, .subjects tbody, .subjects tr, .subjects td { display: block; } + .subjects tr.subject { padding: .5rem 0; } + .subjects td { height: auto; padding: 0; width: auto; text-align: left; } + .subjects .who { white-space: normal; overflow: visible; } + .subjects .at, .subjects .fold { + display: inline-block; width: auto; text-align: left; margin: .25rem .7rem 0 0; + font-size: var(--t-xs); + } + .subjects .do { margin-top: .35rem; } + .subjects .do:empty { margin: 0; } +} + /* --- the rail (item 212, DR-0023) ----------------------------------------------------------- Furniture. It does not scroll away and it does not move between pages, so what exists is never something a person has to remember. @@ -526,6 +679,25 @@ def _link(url: str | None, text: str | None = None) -> str: .sheet { grid-column: 2; min-width: 0; } } +/* How much is behind each name (item 235). Tabular so the column of numbers lines up, and pushed + to the right edge of the link so the words stay left-aligned and scannable. */ +.rail a { display: flex; align-items: baseline; gap: .5rem; } +.rail .count { + margin-left: auto; + font: 500 var(--t-2xs)/1 var(--sans); + font-variant-numeric: tabular-nums; + color: var(--faint); +} +.rail a[aria-current="page"] .count, .rail a:hover .count { color: var(--muted); } + +/* A feature's section (item 235, DR-0027): a label, the sentence under it, then the thing. + The gap above a section is what separates two features on a page that no longer folds them. */ +.feature { margin: 2.4rem 0 0; } +.feature:first-of-type { margin-top: 1.6rem; } +.feature > h2 { margin-top: 0; } +.says { margin: -.35rem 0 .9rem; max-width: var(--measure); + font-size: var(--t-md); color: var(--muted); } + /* The heading row and the primary action on it (item 214). The action wraps under the title on a narrow window rather than squeezing both, because a button that is half a word wide is not a button. */ @@ -541,7 +713,7 @@ def _link(url: str | None, text: str | None = None) -> str: } details.primary > summary::-webkit-details-marker { display: none; } details.primary > summary::before { content: "+"; margin-right: .4rem; font-weight: 600; } -details.primary[open] > summary::before { content: "\\2212"; } +details.primary[open] > summary::before { content: "\\2212 "; } details.primary > summary:hover { background: var(--sunk); } details.primary[open] { flex: 1 1 100%; background: var(--raise); border: 1px solid var(--rule); @@ -562,6 +734,10 @@ def _link(url: str | None, text: str | None = None) -> str: } .new button { align-self: end; } +/* A limit reads as description, not as a blocker (item 220). They sit beside the reason a feature + is off and they are a different kind of sentence: true either way, and the honest half. */ +.why.limit { color: var(--muted); padding-left: .8rem; border-left: 2px solid var(--rule); } + .decisions { list-style: none; padding: 0; margin: 0 0 1.4rem; background: var(--raise); border: 1px solid var(--rule); border-radius: var(--r); @@ -685,7 +861,11 @@ def _link(url: str | None, text: str | None = None) -> str: details > summary::before { content: "+"; font: 400 var(--t-md)/1 var(--mono); color: var(--faint); width: .7rem; text-align: center; } /* The typographic minus, which pairs the + above rather than a hyphen. */ -details[open] > summary::before { content: "\2212 "; } +/* The escape is doubled for Python before it is an escape for CSS (item 219): this is an ordinary + triple-quoted string, so a single-backslash 2212 is read as an octal escape first. The served + stylesheet carried a raw U+0091 control character and a literal digit — the stray 2 beside every + open fold, on every release so far. Comments here reach the browser: no markdown, no escapes. */ +details[open] > summary::before { content: "\\2212 "; } details > summary:hover { color: var(--ink); background: var(--sunk); } .folded { padding: 0 var(--pad) 1.1rem; } .folded > :first-child { margin-top: 0; } @@ -756,6 +936,10 @@ def _link(url: str | None, text: str | None = None) -> str: border: 1px solid var(--rule); border-radius: var(--r-chip); background: var(--canvas); color: var(--ink); } +/* WCAG 2.2 AA 2.5.8 asks 24x24, and a row's link measured 17x17 (item 221). The *Inline* exception + covers a link inside a sentence; the only way to open an item is not that. */ +.list td a { display: inline-block; min-width: 24px; min-height: 24px; line-height: 24px; } + .stuck { font: 600 var(--t-2xs)/1 var(--sans); letter-spacing: .05em; text-transform: uppercase; color: var(--refused); border: 1px solid currentColor; border-radius: var(--r-chip); padding: .16rem .34rem; margin-left: .35rem; } @@ -778,6 +962,34 @@ def _link(url: str | None, text: str | None = None) -> str: a, button, summary { transition: color 120ms ease, background 120ms ease; } } +/* The item list stops being a table where a table stops working (item 221). Seven columns on a + 390px screen wrapped titles to five lines and pushed `issue / pull` behind a sideways scroll + nobody discovers — which is not the same as not breaking the page, and is a lower bar. + + One markup, two shapes: the labels come from `data-label`, so nothing here is a second template + that has to be kept in step with the first. */ +@media (max-width: 46rem) { + .list, .list tbody { display: block; width: auto; } + .list tr:first-child { display: none; } + .list tr { + display: flex; flex-direction: column; + background: var(--raise); border: 1px solid var(--rule); border-radius: var(--r); + padding: .7rem var(--pad); margin: 0 0 .6rem; + } + .list td { border: 0; padding: .15rem 0; } + /* The title identifies the item, so it leads — `order` rather than a second markup, which is the + whole reason the cells carry their own labels. */ + .list td[data-label="title"] { + order: -1; font: 550 var(--t-base)/1.35 var(--sans); padding-bottom: .35rem; + } + .list td:not([data-label="title"])::before { + content: attr(data-label) " "; + display: inline-block; min-width: 6.5rem; + font: 550 var(--t-2xs)/1.6 var(--sans); letter-spacing: .06em; + text-transform: uppercase; color: var(--faint); + } +} + @media (max-width: 40rem) { .wrap { padding: 0 1rem 3rem; } .lede { font-size: var(--t-xl); } @@ -841,6 +1053,9 @@ def _document( up: str = "", state: tuple[str, str] | None = None, here: str = "", + counts: Counts | None = None, + inside: str | None = None, + projects: Sequence[tuple[str, int]] = (), ) -> str: """The whole page. No script, no external asset, one inlined stylesheet. @@ -872,7 +1087,7 @@ def _document( # **Every page, from here** (item 212). Rendering it per view is four chances to grow four # opinions about what this product contains, which is the drift five items this week each # cost a day to. - f"{_rail(acting, here=here, up=up)}\n" + f"{_rail(acting, here=here, up=up, counts=counts, inside=inside, projects=projects)}\n" # **And the way in, for the same reason.** It lived inside the instance report, so moving # the front door left a locked-out operator landing on a page that said nothing about the # lockout — a working lockout looking like a broken login, which is the exact failure item @@ -890,28 +1105,190 @@ def _document( #: operator's — DR-0021 gives a read link the instance and nothing that administers it — and a #: control that leads to a `404` is worse than one that is not there, so a reader is shown neither. _NOUNS: tuple[tuple[str, str, bool], ...] = ( - ("./", "Items", False), - ("projects", "Projects", False), + ("./", "What needs you", False), + ("projects", "All projects", False), ("instance", "This instance", False), - ("doctor", "Why it will not work", True), - ("config", "What it received", True), + ("doctor", "Diagnostics", True), + ("config", "Configuration", True), ) +#: A project's features, in the order somebody works through them, and where each lives **under** +#: `projects//`. Item 237, and it is the operator's own correction of item 235: +#: +#: *¿No será mejor plantear esto mismo, pero a nivel de proyecto? Es decir, tú entras en un +#: proyecto, y ves todas las features. Así no mezclamos cosas.* +#: +#: He is right, and the reason is the one item 235 got wrong: a page called *Dependencies* holding +#: every project's advisories one after another is a wall at two projects and unusable at ten. +#: Nobody works by feature across clients — they work on a client. The feature names were the fix; +#: the axis was not. +_IN_A_PROJECT: tuple[tuple[str, str, bool], ...] = ( + ("", "Overview", False), + ("errors", "Errors", False), + ("fixes", "Fixes", False), + ("dependencies", "Dependencies", False), + ("deliveries", "Deliveries", False), + ("settings", "Settings", True), +) + + +@dataclass(frozen=True) +class Counts: + """How much there is of each of a project's features, for the rail. Item 235, item 237. + + **A number in the navigation is what makes it a map rather than a list of words.** It answers + *is there anything in there* before a click, which is the question that had a reader opening + four folds to find out that three of them were empty. + + Zero renders as nothing at all, not as `0`: item 073's rule is that a signal which is always on + is not a signal, and its corollary is that a badge reading zero on every page is furniture + pretending to be information. + """ + + errors: int = 0 + fixes: int = 0 + dependencies: int = 0 + deliveries: int = 0 + + def of(self, where: str) -> int: + return { + "errors": self.errors, + "fixes": self.fixes, + "dependencies": self.dependencies, + "deliveries": self.deliveries, + }.get(where, 0) + + +def how_much_of_each(session: Session, project_id: int) -> Counts: + """Four counts for one project's rail. Item 237. + + Read here rather than passed in by every view, because a rail that is right on three pages and + stale on the fourth is worse than no rail: five views growing five opinions about what this + product contains is the drift DR-0023 exists to stop. + """ + from hullwork.models import Delivery, DependencyReport + + def count(what: object, where: object) -> int: + try: + return int( + session.scalar(select(func.count()).select_from(what).where(where)) or 0 # type: ignore[arg-type] + ) + except Exception: # pragma: no cover - a rail must never be what breaks a page + return 0 + + report = session.get(DependencyReport, project_id) + # **The number in the rail has to be the number in the view** (item 247). It counted *findings* + # beside a view that counts *packages* — `Dependencies 25` next to `20 packages`, two true + # numbers of two different things in one eye-line. A package pinned three times is three + # findings and one row, and the row is what a click leads to. + findings = ( + len({str(one.get("package") or "") for one in (report.findings or [])}) + if report is not None and report.asked + else 0 + ) + return Counts( + errors=count(_Item, _Item.project_id == project_id), + fixes=count( + _Attempt, + _Attempt.item_id.in_(select(_Item.id).where(_Item.project_id == project_id)), + ), + dependencies=findings, + deliveries=count(Delivery, Delivery.project_id == project_id), + ) + + +def each_project(session: Session) -> list[tuple[str, int]]: + """Every project and how much is waiting in it, for the rail outside a project. Item 237. + + The number is **what needs a person**, not how much exists: a count of items would read the + same on a project that is fine and one that is stuck, and the whole of the front door is + *which of these wants me*. + """ + waiting = (ItemState.WAITING_APPROVAL, ItemState.HUMAN_ONLY) + out: list[tuple[str, int]] = [] + for project in session.scalars(select(_Project).order_by(_Project.slug)).all(): + how_many = int( + session.scalar( + select(func.count()) + .select_from(_Item) + .where(_Item.project_id == project.id, _Item.state.in_(waiting)) + ) + or 0 + ) + out.append((project.slug, how_many)) + return out + -def _rail(acting: Acting, *, here: str = "", up: str = "") -> str: +def _rail( + acting: Acting, + *, + here: str = "", + up: str = "", + counts: Counts | None = None, + inside: str | None = None, + projects: Sequence[tuple[str, int]] = (), +) -> str: """The sidebar every page carries, from one function. Four pages growing four opinions about what this product contains is the drift items 193, 194, 200, 203 and 211 each cost a day to. This is that lesson applied before it happens rather than after. + + **Two modes, and the project is the axis** (item 237). Outside a project it lists the projects + and what belongs to the instance; `inside` a project it lists that project's features and + nothing else, so nothing on screen is ever about two projects at once. + + Item 235 named the features and put them on pages that spanned every project, which is the same + mistake one level up: a reader does not go looking for *dependencies*, they go looking for + *simplecheck*. + """ + if inside is not None: + return _the_projects_rail(acting, here=here, up=up, counts=counts, slug=inside) + links = "" + for slug, how_many in projects: + badge = f'{how_many}' if how_many else "" + links += f'{_h(slug)}{badge}' + if projects: + links = ( + f'Projects{links}' + 'This instance' + ) + for where, name, operators_only in _NOUNS: + if operators_only and not acting.csrf: + continue + links += ( + f'{_h(name)}" + ) + return f'' + + +def _the_projects_rail( + acting: Acting, *, here: str, up: str, counts: Counts | None, slug: str +) -> str: + """Inside a project: its features, and the way back out. Item 237. + + The way out is first and it is a link rather than a heading, because a rail that replaces itself + has to say what it replaced — otherwise somebody two levels in has no cue that the other + projects still exist. """ - links = "".join( - f'{_h(name)}" - for where, name, operators_only in _NOUNS - if not operators_only or acting.csrf + links = ( + f'← All projects' + f'{_h(slug)}' ) + for where, name, operators_only in _IN_A_PROJECT: + if operators_only and not acting.csrf: + continue + how_many = counts.of(where) if counts is not None else 0 + # Zero is rendered as nothing, not as `0`. Item 073's rule, one turn further: a badge that + # is on every row all the time is furniture pretending to be information. + badge = f'{how_many}' if how_many else "" + links += ( + f'{_h(name)}{badge}" + ) return f'' @@ -963,7 +1340,12 @@ def _what_this_instance_allows(settings: Settings) -> str: """ from hullwork.doctor import policies - return f"

What this instance allows

{_h(policies(settings).detail)}

" + return _section( + "What it allows", + "The three policies an attempt runs under, which is what a reviewer is judging when they " + "judge its artefact.", + f"

{_h(policies(settings).detail)}

", + ) def _the_credential_split(session: Session) -> str: @@ -978,7 +1360,7 @@ def _the_credential_split(session: Session) -> str: for project in projects: # Stored by `status`'s audit rather than probed here: a page render must not spend a forge # request, and a reader refreshing would spend one each time. - verdict = (project.manifest or {}).get("__ingest_can_push__") if project.manifest else None + verdict = project.ingest_token_can_push if verdict is not None: measured.append(f"{project.slug}: {'CAN push' if verdict else 'cannot push, measured'}") tail = ( @@ -988,7 +1370,12 @@ def _the_credential_split(session: Session) -> str: 'not exist. It is not asked from this page: a render must not spend somebody\'s forge ' 'quota.

' ) - return f"

Which half holds what

{_own_prose(_SPLIT)}

{tail}" + return _section( + "The two halves", + "Which credential each half holds, and what this instance measured about them rather than " + "what the design promises.", + f"

{_own_prose(_SPLIT)}

{tail}", + ) #: The twelve states, grouped into the six questions a reader actually has. Item 143. @@ -1056,6 +1443,16 @@ def _ago(when: datetime | None) -> str: _DISPATCHABLE = (ItemState.READY, ItemState.WAITING_APPROVAL) +def _since(when: datetime | None) -> str: + """`_ago`'s answer as a phrase that reads after a verb: *3h ago*, or *just now*. + + Item 247, found on the deployed door: `_ago` answers `just now` for anything under a minute, and + every caller appending " ago" to it produced **just now ago**. + """ + said = _ago(when) + return said if said in ("just now", "not recorded") else f"{said} ago" + + def _stuck(item: _Item) -> str | None: """Why this item can **never** be attempted, or `None` if nothing stops it. Item 166. @@ -1114,6 +1511,143 @@ def _proof(session: Session, *, merged: int, holding: int, recurred: int, watch: return f'
{figures}
' +def _figures(rows: Sequence[tuple[int, str, str]]) -> str: + """A count, what it counts, and the caveat under it. DR-0028, item 248. + + **The second skin of a structure the terminal already prints as sentences** — the same pattern + item 050 used for the artefact, and for the same reason: one construction, two skins, because a + second *computation* is what comes apart. `outcomes.Desk` and `outcomes.Funnel` are the + construction; `desk_lines` is the terminal's skin and this is the page's. + + Rows with a zero are not passed in. A row of noughts reads like a failure rather than like a + beginning, which is `desk_lines`'s own rule and the one thing both skins must agree on. + """ + if not rows: + return "" + lines = "".join( + f'{count}' + f'{_h(means)}' + + (f'{_h(caveat)}' if caveat else "") + + "" + for count, means, caveat in rows + ) + # In the container the rule asks for (item 215): every table on this page sits in something + # that scrolls on its own, and a two-column table being unable to overflow is a reason to + # believe it rather than a reason to exempt it. + return f'
{lines}
' + + +def _desk_figures(counted: object) -> str: + """What arrived and how much left, as figures rather than as four sentences. Item 248.""" + rows: list[tuple[int, str, str]] = [] + arrived = int(getattr(counted, "arrived", 0)) + if not arrived: + return "" + rows.append((arrived, "claims arrived", "")) + left = int(getattr(counted, "left_with_evidence", 0)) + if left: + change = int(getattr(counted, "with_a_change", 0)) + refusal = int(getattr(counted, "with_a_refusal", 0)) + how = [] + if change: + how.append(f"{change} with a change") + if refusal: + how.append(f"{refusal} with a reasoned refusal and the runs behind it") + rows.append((left, "left your desk with evidence attached", " · ".join(how))) + for field, means in ( + ("still_waiting", "still in the queue"), + ("running", "being attempted now"), + ): + value = int(getattr(counted, field, 0)) + if value: + rows.append((value, means, "")) + handed = int(getattr(counted, "handed_over", 0)) + if handed: + # **Never rounded into good news** (`desk_lines`'s rule): this is the one figure here that + # can embarrass the product, and it keeps its own words. + rows.append(( + handed, "went onto your desk rather than off it", + "red lane, or a pull request somebody read and refused", + )) + return _figures(rows) + + +def _funnel_figures(counted: object) -> str: + """Every attempt, by how it ended. Item 248.""" + rows: list[tuple[int, str, str]] = [] + fair = int(getattr(counted, "fair_try", 0)) + if fair: + parts = [] + for field, said in ( + ("pull_requests", "opened a pull request"), + ("not_reproducible", "found nothing to reproduce"), + ("failed", "could not produce a passing suite"), + ): + value = int(getattr(counted, field, 0)) + if value: + parts.append(f"{value} {said}") + rows.append((fair, "attempts got a fair try", " · ".join(parts))) + pulls = int(getattr(counted, "pull_requests", 0)) + merged = int(getattr(counted, "merged", 0)) + if pulls: + # **Counts, never a percentage** (item 119): four attempts are not a rate, and a percentage + # invites comparing instances running different code over different repositories. + rows.append((merged, f"of those {pulls} pull request(s) were merged", "")) + never = getattr(counted, "never_counted", None) + if isinstance(never, Mapping) and never: + why = " · ".join(f"{n} {_h(str(name))}" for name, n in never.items()) + rows.append((sum(int(n) for n in never.values()), "never counted against an item", why)) + rehearsals = int(getattr(counted, "rehearsals", 0)) + if rehearsals: + rows.append(( + rehearsals, "rehearsals", + "they publish nothing and are counted in none of the above", + )) + running = int(getattr(counted, "in_flight", 0)) + if running: + rows.append((running, "started and not finished", "")) + return _figures(rows) + + +def _review_figures(counted: object, said: Sequence[str]) -> str: + """What became of the pull requests, as figures where they are counts. Item 248. + + **Two of its lines are counts and one is a duration**, so this keeps the sentences for what is + not a count rather than forcing a table onto a median. The rule the whole item runs on: a figure + goes in the column, and prose that is genuinely prose stays prose. + """ + rows: list[tuple[int, str, str]] = [] + merged = int(getattr(counted, "merged", 0)) + if merged: + rows.append((merged, "merged by a human", "")) + waiting = int(getattr(counted, "waiting", 0)) + if waiting: + rows.append((waiting, "waiting for a human", "this is the review debt")) + rejected = getattr(counted, "rejected", None) + if isinstance(rejected, Mapping) and rejected: + why = " · ".join(f"{n} {_h(str(name))}" for name, n in rejected.items()) + rows.append((sum(int(n) for n in rejected.values()), "refused, with a reason", why)) + rest = "".join( + f"

{_h(line)}

" for line in said if line[:1].isalpha() + ) + return _figures(rows) + rest + + +def _section(label: str, says: str, body: str) -> str: + """A feature's section: a label you can scan, the sentence under it, then the thing. Item 235. + + **The headings were sentences** — *What is published against what it pins*, *What arrived, and + how much left your desk* — and an eye moving down the page could not use one of them. They were + accurate and they were load-bearing for navigation, which prose cannot do. + + Nothing this product says about what it measured is deleted here. The sentence moves one line + down and into grey, where it explains rather than labels, and the label is a word somebody would + have searched for. + """ + said = f'

{says}

' if says else "" + return f'

{_h(label)}

{said}{body}
' + + def _fold(summary: str, body: str) -> str: """A closed disclosure, native, no JavaScript. Item 167. @@ -1353,6 +1887,134 @@ def _strip(session: Session) -> str: return f'
{"".join(cells)}
' +def what_it_has_been_doing(session: Session, *, limit: int = 8) -> str: + """The last few things this instance did, newest first. Item 242. + + **Merged from what is already stored** — a verdict's `tried_at`, an attempt's `finished_at`, a + dependency report's `taken_at` — rather than from a log table nobody writes to. A second record + of the same events could disagree with them, and then a reader has to decide which to believe. + + It is the answer to the question a verdict cannot answer on its own: *when did that happen, and + what else was going on around it*. + """ + from hullwork.models import DependencyReport, UpgradeVerdict + + where: dict[int, str] = { + one.id: one.slug for one in session.scalars(select(_Project)).all() + } + events: list[tuple[datetime, str, str]] = [] + for verdict in session.scalars( + select(UpgradeVerdict).order_by(UpgradeVerdict.tried_at.desc()).limit(limit) + ).all(): + said, colour = _WHAT_IT_MEANT.get( + verdict.outcome, (f"ended as {verdict.outcome}", "c-idle") + ) + events.append(( + verdict.tried_at, + colour, + f"{_h(where.get(verdict.project_id, '?'))} · tried " + f"{_h(verdict.package)} {_h(verdict.was)} → {_h(verdict.to)} — {_h(said)}", + )) + for attempt, item_row in session.execute( + select(_Attempt, _Item) + .join(_Item, _Attempt.item_id == _Item.id) + .order_by(_Attempt.id.desc()) + .limit(limit) + ).all(): + outcome = getattr(attempt.outcome, "value", attempt.outcome) + events.append(( + attempt.finished_at or attempt.started_at, + "c-passed" if str(outcome) == "pr-open" else "c-idle", + f"{_h(where.get(item_row.project_id, '?'))} · attempted " + f"{_h(item_row.title[:48])} — {_h(str(outcome or 'still running'))}", + )) + for report in session.scalars(select(DependencyReport)).all(): + events.append(( + report.taken_at, + "c-idle" if report.asked else "c-refused", + f"{_h(where.get(report.project_id, '?'))} · " + + ( + f"asked OSV about {report.pinned} pinned version(s), " + f"{len(report.findings or [])} with something published" + if report.asked + else "could not ask OSV" + ), + )) + if not events: + return "" + # **A thing that happened is a subject too** (DR-0028), so it is a row of the same table the + # other views use rather than a card with a paragraph in it. + rows = "".join( + f'' + f'' + f'{said}' + f'" + for when, colour, said in sorted(events, key=lambda one: one[0], reverse=True)[:limit] + ) + return _section( + "What it has been doing", + "Newest first, across every project.", + f'{rows}
', + ) + + +def _what_it_says_it_is_doing(session: Session) -> str: + """The dispatcher's own sentence about what it is doing, or `""` when there is none. Item 242. + + **Its word, not an inference.** Everything a page could deduce — the heartbeat, an attempt with + no `finished_at`, a verdict appearing — has the same hole in the middle: between two writes + there is nothing to read, and that gap is exactly the four minutes somebody is watching. + + A stale heartbeat is not busy. A dispatcher killed mid-sentence leaves its last one behind, and + rendering that as *now* would be this page's own version of the defect it is fixing — so the + lease's own reading of itself decides, and a stale one says so instead. + """ + from hullwork import lease as lease_module + from hullwork.models import DispatcherLease + + row = session.get(DispatcherLease, 1) + state, when = lease_module.state(session) + if row is None or not row.doing: + # **Idle is a fact and being unreachable is a different one.** A door that renders nothing + # in both cases answers *is it running?* with silence, which is the question the operator + # opened it to ask. + if state == "alive": + return ( + '
nothing running' + # **`_ago` returns a phrase, not a duration**, and one of its answers is + # `just now` — which this rendered as *just now ago*, on the door, for anybody + # looking at an idle instance. + f'

The dispatcher answered ' + f"{_h(_since(when))}.

" + ) + if state in ("stale", "never"): + return ( + '
no dispatcher' + '

' + + ( + f"Nothing has claimed the lease since {_h(_since(when))}. " + if state == "stale" + else "No dispatcher has ever run against this instance. " + ) + + "Nothing will be attempted or verified until one does.

" + ) + return "" + + if state != "alive": + return ( + '
dispatcher not answering' + f'

The last thing it said it was doing was ' + f"{_h(row.doing)}, {_h(_since(row.doing_since))}. Its heartbeat has " + "stopped, so that is what it was doing rather than what it is doing.

" + ) + return ( + '
working' + f'

{_h(row.doing)}

' + f'

for {_h(_ago(row.doing_since))}

' + ) + + def _now(session: Session, prices: Prices | None) -> str: """The attempt in flight, or a sentence saying there is none. @@ -1379,13 +2041,21 @@ def _now(session: Session, prices: Prices | None) -> str: else: outcome = getattr(last.outcome, "value", last.outcome) tail = ( - f"The last one finished {_h(_ago(last.finished_at or last.started_at))} ago: " + f"The last one finished {_h(_since(last.finished_at or last.started_at))}: " f"{_h(str(outcome))}." ) + # **What the dispatcher says it is doing, before concluding it is doing nothing** (item + # 242). This band read `Item.state == IN_PROGRESS`, and a dependency verification is not an + # item — so it printed *nothing running* through five minutes of building an image and + # running somebody's suite twice. A page that reports calm during four minutes of work is + # not missing a feature; it is answering wrongly. + busy = _what_it_says_it_is_doing(session) + if busy: + return busy return ( '
nothing running' f'

{tail}' - + (f' {queued} item(s) are ready and waiting.' if queued else "") + + (f" {queued} item(s) are ready and waiting." if queued else "") + "

" ) @@ -1409,7 +2079,7 @@ def _now(session: Session, prices: Prices | None) -> str: cost = "" if attempt is not None: money = spend.cost_of(spend.tokens_of(attempt.seal), prices) if prices else None - parts = [f"started {_h(_ago(attempt.started_at))} ago"] + parts = [f"started {_h(_since(attempt.started_at))}"] if money is not None: parts.append(_h(str(money))) cost = f'

{" · ".join(parts)}

' @@ -1468,7 +2138,11 @@ def _disagreements(session: Session, settings: Settings) -> str: if not found: return '

Nothing disagrees: the three checks ran and found nothing.

' rows = "".join(f'
  • {_h(line)}
    • ' for line in found) - return f'

    What does not add up

      {rows}
    ' + return _section( + "What disagrees", + "Where two things this instance recorded cannot both be true.", + f'
      {rows}
    ', + ) def _violations_in(seal: object) -> bool: @@ -1508,145 +2182,991 @@ def _titled(one: object) -> str: return str(getattr(one, "check", None) or getattr(one, "name", "")) -def _rows_for_standing(rows: Sequence[object]) -> str: - """The panel's rows, for both views. Items 203 and 208. +def as_a_block(text: str) -> str: + """Text whose line breaks are the point, marked so `_outcome` keeps them. Item 223. - **One renderer, because two that happen to look alike is how the borrowed list ended up in - both of them** — and how a fix to one would leave the other painting a `cannot` amber. + `_outcome` renders a sentence, and a sentence in a paragraph is right for every other answer on + this page. A proposed manifest is not a sentence: it is forty-one lines somebody copies. + """ + return _BLOCK + text - Takes anything with `check`/`name`, a state and a `detail`, which is what `doctor.Finding` and - `features.Standing` both are. A decision reads quiet and a fault reads red: DR-0019 in colour, - because painting a choice somebody made as a defect tells them to go and repair it. + +#: The marker, rather than a second parameter threaded through four routes for one caller. +_BLOCK = "\x00block\x00" + + +def _outcome(said: str | None) -> str: + """What the last action answered, verbatim. Item 219. + + **Every refusal in this product already carries its own sentence**, so this renders whatever it + was given rather than composing one. A page that replaced *`--give-up` needs `--why`* with + *something went wrong* would be throwing away the only part worth reading. """ - said = [] - for one in rows: - state = getattr(one, "state", "") - word = getattr(state, "value", state) - broken = word in ("broken", "cannot") - said.append( - f'
  • ' - f'{_h(word)}' - f'{_h(_titled(one))}' - f'

    {_as_code(getattr(one, "detail", ""))}

    • ' - ) - return "".join(said) + if not said: + return "" + if said.startswith(_BLOCK): + return f'
    {_h(said[len(_BLOCK):])}
    ' + return f'

    {_as_code(said)}

    ' -def why_it_will_not_work( - session: Session, settings: Settings, *, acting: Acting = READING -) -> str: - """`doctor`, for somebody without a shell. Item 208, DR-0022. +def what_this_can_do(session: Session, project: _Project, settings: Settings) -> str: + """`hullwork features`, for the project a reader is looking at. Item 220, item 218 §2. - **The findings, not a second diagnosis.** `doctor.examine` already returns them and item 199's - pre-flight already renders them elsewhere; a page that asked its own questions would drift from - the command an operator quotes in a bug report. + The command answers for the checkout it is run in; a page serves an instance that may watch + somebody else's repositories entirely, so the answer has to be per project. The manifest is the + instance's own copy (DR-0012) and the variable names are this process's. - `not_from_here`'s downgrade comes with them and must: the receiver is not the dispatcher, and a - page reporting the model credential missing — on an instance where it is present in the half - that uses it — sends somebody to repair a working machine. + **Unmet means three different things here and only one is a defect**, which is what `Need.reads` + names. A checkout-shaped requirement cannot be answered at all — item 142 forbids a forge + request per render — so it says *not asked yet* rather than *no*, the same answer the credential + audit gives one section up. And a credential the dispatcher owns is downgraded exactly as + `doctor.not_from_here` downgrades it, because reporting the model key missing on an instance + where the half that uses it holds it sends somebody to repair a working machine. """ - from hullwork import doctor as doctor_module + from hullwork import features as features_module + from hullwork.manifest import parse_manifest - found = doctor_module.examine( - session, - settings, - code_forge=None, - env_file=Path(settings.deployment_env_file or ".env"), - compose_file=( - Path(settings.deployment_compose_file) if settings.deployment_compose_file else None - ), + manifest = None + if project.manifest: + try: + manifest = parse_manifest(json.dumps(project.manifest)) + except Exception: # a stored manifest that no longer parses is `projects refresh`'s to say + manifest = None + checkout = features_module.Checkout( + paths=(), + manifest=manifest, + configured=frozenset(_configured_here(settings)), ) - worrying = [one for one in found if one.state is not doctor_module.State.OK] - rows = _rows_for_standing(worrying) - body = ( - "

    Why it will not work

    " - + ( - f'
      {rows}
    ' - f'

    {len(found) - len(worrying)} of {len(found)} check(s) are fine.

    ' - if worrying - else f'

    All {len(found)} check(s) are fine.

    ' - ) + elsewhere = _the_dispatchers_to_answer(session) + + rows = [] + for answer in features_module.examine(checkout): + rows.append(_one_feature(answer, elsewhere=elsewhere)) + return _fold( + f"What Hullwork can do for {project.slug}, and what it cannot", + f'
      {"".join(rows)}
    ', ) - return _document("Hullwork — doctor", body, acting=acting, here="doctor") -def what_it_received(settings: Settings, *, acting: Acting = READING) -> str: - """`config`, for somebody without a shell. Item 208. +def _configured_here(settings: Settings) -> list[str]: + """The **names** of the variables this process received, never their values. - **No credential is printed**, and that is `settings_report`'s property rather than this - function's: a secret reads `set` or `not set` before it ever reaches here. Worth saying because - `config` reads like the most disclosing thing in the product and is in fact the most carefully - disclosing thing in it. + `Checkout.configured` is names only by design — that is what lets this say *needs a model + credential, and none is configured* without ever holding one. """ - from hullwork import settings_report + from hullwork.features import CODE_TOKEN, MODEL_KEY - rows = "".join( - f"{_h(name)}{_h(value)}" - f"{_h(source)}{_h(reaches)}" - for name, value, source, reaches in settings_report.rows(settings) - ) - body = ( - "

    What it received

    " - '

    What this process was handed, which is a different question from what you ' - "wrote in a file. No credential is printed: a secret reads set or " - "not set.

    " - '
    ' - f"{rows}
    variablevaluefromreaches
    " - ) - return _document( - "Hullwork — configuration", body, acting=acting, here="config" - ) + here = [] + if settings.model_key is not None: + here.append(MODEL_KEY) + if settings.forge_code_token is not None: + here.append(CODE_TOKEN) + return here -def _what_this_instance_has_switched_on(session: Session, settings: Settings) -> str: - """The feature-by-feature standing, worst first. Item 203. +def _the_dispatchers_to_answer(session: Session) -> bool: + """Whether a dispatcher is alive, which is the only thing that licenses a downgrade. - **From `features.on_this_instance`, which the terminal prints too**, so the page and - `hullwork status` cannot come to disagree about the same instance — the rule `instance`'s own - docstring states about its numbers, applied to its states. + `doctor.not_from_here`'s rule, not a second one: ownership rather than location. With no + dispatcher alive nothing is downgraded — the absence of one is exactly when somebody needs to + know what is missing. + """ + from hullwork import lease as lease_module - An instance with everything on says so in one line: fourteen green rows is a wall a reader stops - looking at, and the thing they came for is whichever one is not green. + try: + state, _ = lease_module.state(session) + except Exception: # any failure here means "cannot tell", whatever its class + return False + return state == "alive" + + +def _one_feature(answer: object, *, elsewhere: bool) -> str: + """One feature, its state, what is in the way, and its limits — which are printed either way. + + **A limit is true whether or not the feature is available.** Rendering them only for what is off + would turn them into excuses instead of the description they are. """ - from hullwork import features + feature = answer.feature # type: ignore[attr-defined] + missing = list(answer.missing) + list(answer.withheld) # type: ignore[attr-defined] + unanswerable = [need for need in missing if need.reads == "checkout"] + theirs = [need for need in missing if need.reads == "instance"] if elsewhere else [] + real = [need for need in missing if need not in unanswerable and need not in theirs] + + if real: + word, tone = "no", "refused" + elif theirs or unanswerable: + word, tone = "not from here" if theirs else "not asked yet", "idle" + else: + word, tone = "yes", "idle" - standing = features.on_this_instance(session, settings) - worrying = [one for one in standing if one.state is not features.ON] - if not worrying: + said = [ + f'
  • {_h(word)}' + f'{_h(feature.name)}' + f'

    {_as_code(feature.does)}

    ' + ] + for need in real: + said.append(f'

    needs {_as_code(need.what)} — {_as_code(need.fix)}

    ') + for need in theirs: + said.append( + f'

    {_as_code(need.what)}: not from here — a dispatcher is running and ' + f"this is a resource it uses, not one this process does.

    " + ) + for need in unanswerable: + said.append( + f'

    {_as_code(need.what)}: not asked yet — nothing on this instance ' + f"reads your tree, and a page render does not spend a forge request to find out. " + f"{_as_code('`hullwork features --checkout .`')} answers it where the code is.

    " + ) + for limit in feature.limits: + said.append(f'

    {_as_code(limit)}

    ') + said.append("
    • ") + return "".join(said) + + +def _why_it_is_empty(session: Session) -> str: + """Why there are no items, from the table that knows. Item 231. + + **The old sentence was a guess.** *Nothing has arrived from the error tracker on this instance* + was written before anything asked, and it is one of three states an empty list is consistent + with — the other two being *things arrived carrying no error* and *things arrived and could not + be understood*, which have different causes and different fixes. + """ + from hullwork.models import Delivery + + arrived = session.scalar(select(func.count()).select_from(Delivery)) or 0 + if not arrived: return ( - '

    Every feature this instance can have is on. ' - f"{len(standing)} of {len(standing)}.

    " + "No items yet, and no delivery has ever been accepted. A call with the wrong secret is " + "refused before anything is written down, so this says none arrived with a working " + "secret rather than that nobody knocked" + ) + unread = ( + session.scalar( + select(func.count()).select_from(Delivery).where(Delivery.error.is_not(None)) + ) + or 0 + ) + if unread: + return ( + f"No items yet. {arrived} delivery(s) arrived and {unread} could not be understood — " + f"each carries its own reason on its project" ) - rows = _rows_for_standing(worrying) - on = len(standing) - len(worrying) return ( - f'
      {rows}
    ' - f'

    {on} of {len(standing)} feature(s) on.

    ' + f"No items yet. {arrived} delivery(s) arrived and none of them carried an error, which is " + f"a tracker sending something that is not one" ) -def instance( - session: Session, settings: Settings, *, error_reporting: bool, acting: Acting = READING -) -> str: - """What `hullwork status` says, for somebody who does not have a terminal on this host. +def what_arrived_for(session: Session, project: _Project) -> str: + """What the tracker actually sent, and what became of it. Item 231. - **Every number comes from the function `status` calls**, never from a second query written for - this page: `readiness.check`, `outcomes.desk`, `outcomes.funnel`, `recurrence.counted` and - `undecided`, `lease.state` and `reporting_of`. A page that recomputed them would drift, and the - first anybody would know is a reader and an operator disagreeing about the same instance. - """ - from hullwork import lease, outcomes, readiness, recurrence + **The page kept saying `nothing has arrived` without asking.** Three states are consistent with + an empty front door — nothing arrived, things arrived carrying no error, things arrived and + could not be understood — and only the first is what that sentence claims. - report = readiness.check(session, settings, error_reporting=error_reporting) - merged, holding, recurred = recurrence.counted(session) - undecided = recurrence.undecided(session) - loop_state, loop_seen = lease.state(session) - reporting = lease.reporting_of(session) + **A refused secret leaves no row.** `webhooks.py` answers `401` before anything is written, so + an empty list here means nobody knocked *with a working secret*, and saying more than that would + be the same lie as an advisory list rendered empty after a failed request. - rows = [ - ("state", "ready" if report.ready else "degraded"), - ("version", report.version), - ("forge", report.forge), + No body and no hash: this table keeps payloads verbatim, and a page whose whole audience is + people who are not the operator has no business rendering somebody else's error payload. + """ + from hullwork.models import Delivery, Event + + rows = list( + session.scalars( + select(Delivery) + .where(Delivery.project_id == project.id) + .order_by(Delivery.received_at.desc()) + .limit(20) + ).all() + ) + total = session.scalar( + select(func.count()).select_from(Delivery).where(Delivery.project_id == project.id) + ) or 0 + if not rows: + return ( + "

    No delivery has ever been accepted for this project. That is not the same as " + "nobody having knocked: a call with the wrong secret is refused before anything is " + "written down, so this list can only say that none arrived with a working " + "secret. The rejection is in this instance's log.

    " + ) + + counted = session.execute( + select(Event.delivery_id, func.count()) + .where(Event.delivery_id.in_([one.id for one in rows])) + .group_by(Event.delivery_id) + ).all() + facts: dict[int, int] = dict(counted) # type: ignore[arg-type] + listed = "".join( + '' + f'" + f'' + + ( + f'{_h(one.error[:80])}' + if one.error + else ("yes" if one.processed_at else "not yet") + ) + + f'{_h(facts.get(one.id, 0))}' + f'{_h(one.attempts)}' + for one in rows + ) + carried = sum(facts.values()) + return ( + f'

    {total} delivery(s) accepted, carrying {carried} fact(s).

    ' + '
    ' + f"{listed}
    arrivedunderstoodfacts in ittries
    " + '

    A delivery carrying no fact is a tracker sending something that is not an ' + "error, which is ordinary. One that was never understood carries its own reason.

    " + ) + + +#: What each verdict means **in the reader's terms**, and the colour it is allowed to wear. +#: +#: **Four states that do not collapse into two** (DR-0026). `clean` is *your suite passed before and +#: after* and not *this is safe*. `will-not-install` is the build refusing, which is a different +#: fact from the suite failing — painting it red would be this product telling somebody their code +#: is broken when what broke was an install. `already-red` is a suite that was failing before +#: anything was touched, so no claim can be made either way, and the colour says *this needs a +#: person* rather than *this is bad news*. +_WHAT_IT_MEANT: dict[str, tuple[str, str]] = { + "clean": ("your suite passed, before and after", "c-passed"), + "breaks": ("your suite fails on it", "c-refused"), + "will-not-install": ("the build refused it, so your suite never ran", "c-idle"), + "already-red": ("your suite was already failing, so nothing can be claimed", "c-human"), + "cannot-rewrite": ("the pin could not be rewritten without breaking the install", "c-idle"), + "cannot-move": ("the pin could not be moved to it", "c-idle"), +} + + +def _may_open_upgrades(project: _Project) -> bool: + """Whether this project's manifest permits opening an upgrade. DR-0019, item 245. + + Read from the copy the instance holds, which is the copy every other decision reads (DR-0012). + A project that declares nothing permits nothing: `open_upgrades` is `false` by default because + having the credential is not the same as having agreed. + """ + manifest = getattr(project, "manifest", None) + if not isinstance(manifest, Mapping): + return False + autofix = manifest.get("autofix") + return bool(isinstance(autofix, Mapping) and autofix.get("open_upgrades")) + + +#: The states a pinned package can be in, in the order a reader can act on them. DR-0028. +#: +#: **The order is the grouping.** A reader opens this view to find out what to do, so what can be +#: done comes first and what nobody can do comes last — never the order the report happens to list. +#: The sentence beside each state is said **once**, in the heading: a column repeating the same +#: sentence twenty-seven times is a column that should not exist, which is the fault the first +#: prototype of this decision found in itself. +_BANDS: tuple[tuple[str, str, str, str], ...] = ( + ("ready", "Ready to open", "passed your suite before the change and after it", "passed"), + ("asked", "Opening", "asked for — the dispatcher opens it on its next turn", "human"), + ("open", "Already open", "a draft pull request is waiting for a person", "working"), + # **The two the forge answers, which nothing used to ask** (item 253). Without them a merged + # pull request sat in the band above asking for a review that had already happened, and a + # closed one sat there for ever displaying somebody's "no" as work they still owed. + ("merged", "Merged", + "the pull request was merged; the advisory goes when the next report is taken", "passed"), + ("declined", "You closed these", + "opened, and closed by a person without merging — each row says what they gave as the reason", + "refused"), + # **The one band whose reason is per row rather than per group** (item 178's rule, kept): a + # request that produced nothing is either already open from an earlier run or something the + # forge refused, and those are different sentences. Never silence — a row that was asked for + # and shows neither outcome is a row somebody presses again. + ("refused", "Asked for, and not opened", "each row says what stopped it", "refused"), + ("breaks", "Breaks your suite", "the upgrade applied and your tests stopped passing", + "refused"), + ("install", "Would not install", "the build refused it, so your suite never ran", "faint"), + ("stuck", "The pin would not move", + "the resolver refused the version, or your manifest forbids it", "faint"), + ("stale", "Verified before this instance kept the files", + "passed your suite before the change and after it, and is re-measured on the next report " + "so it can be opened", "faint"), + # **Item 234, as a band.** The honest answer about a project whose own test suite was already + # failing is *nothing can be claimed either way*, and it is about the project rather than about + # any upgrade — so it is said once, in this heading, and the packages it covers are still named. + # Rendering it per row is what produced fifty identical lines in an hour. + ("baseline", "Nothing could be claimed", + "this project's own test suite was already failing before anything was touched", "human"), + ("untried", "Not tried yet", "the queue reaches one per idle turn", "faint"), + ("nofix", "Nothing published to upgrade to", + "an advisory with no fixed version — a person decides", "refused"), +) + +#: Which state wins when one package is pinned at versions in different states: **the one that most +#: needs a person**. A row is a place to act, and a reader scanning for what to do must not have a +#: package hidden under its quietest version. +_BAND_RANK: dict[str, int] = {key: n for n, (key, _, _, _) in enumerate(_BANDS)} + +#: The states whose row has a pull request behind it. `open` invites a review; the other two are +#: evidence of one that happened, and both are worth a click (item 253). +_CARRY_A_LINK = ("open", "merged", "declined") + + +def _state_of(verdicts: Sequence[Any], fixed: Sequence[str]) -> tuple[str, str | None, str | None]: + """The state of one pinned version, what it would move to, and where its pull request went. + + Every rule the two lists this replaces had encoded, now in one place: + + * a build refusing is **not** a broken suite — `will-not-install` is its own state (item 233); + * `already-red` claims nothing either way and is said about the **project**, so it never reaches + a row (item 234); + * a `clean` verdict with nothing kept cannot be opened and says so, rather than offering a + control the write path would refuse (item 245); + * an outcome this page does not recognise falls through to `stuck`, which says *the pin would + not move* — and the verdict itself is still rendered in the row, so nothing is swallowed. + """ + # **What the forge last said, before what this instance did** (item 253). `opened_where` alone + # meant *waiting for a person* for ever: a merged pull request kept asking for a review that had + # happened, and one a person closed without merging displayed their "no" as work they owed. + # Both are terminal and neither is *open*, so both are read first. + settled = [v for v in verdicts if getattr(v, "opened_state", None) in ("merged", "closed")] + if settled: + one = settled[0] + where = str(one.opened_where) if one.opened_where else None + return ("merged" if one.opened_state == "merged" else "declined"), str(one.to), where + opened = [v for v in verdicts if getattr(v, "opened_where", None)] + if opened: + return "open", str(opened[0].to), str(opened[0].opened_where) + refused = [v for v in verdicts if getattr(v, "open_note", None)] + if refused: + return "refused", str(refused[0].to), None + asked = [v for v in verdicts if getattr(v, "asked_to_open_at", None) is not None] + if asked: + return "asked", str(asked[0].to), None + clean = [v for v in verdicts if v.outcome == "clean"] + openable = [v for v in clean if getattr(v, "artefact", None)] + if openable: + return "ready", str(openable[0].to), None + if clean: + return "stale", str(clean[0].to), None + breaks = [v for v in verdicts if v.outcome == "breaks"] + if breaks: + return "breaks", str(breaks[0].to), None + refused = [v for v in verdicts if v.outcome == "will-not-install"] + if refused: + return "install", str(refused[0].to), None + if not fixed: + return "nofix", None, None + moved = [v for v in verdicts if v.outcome in ("cannot-move", "cannot-rewrite")] + if moved: + return "stuck", str(moved[0].to), None + unknown = [v for v in verdicts if v.outcome != "already-red"] + if unknown: + # **A verdict `bump` adds tomorrow must not vanish because this table was not updated.** An + # unknown state is still a state, and rendering nothing would report *not tried yet*, which + # is the one thing it is not. The outcome goes in the row, in its own words. + return "stuck", str(unknown[0].to), f"it ended as {unknown[0].outcome}" + if verdicts: + return "baseline", str(verdicts[0].to), None + return "untried", str(fixed[0]), None + + +def _packages_of( + session: Session, project: _Project, findings: Sequence[Mapping[str, Any]] +) -> tuple[list[dict[str, Any]], int]: + """One row per package, and how many pairs said the project's own suite was already failing. + + **The package is the subject** (DR-0028). `brace-expansion` pinned at three versions in one lock + is one row carrying three versions, not three rows that share a name — and the report itself + hands the same `(package, version)` twice, which this collapses on the way in. + """ + from hullwork.models import UpgradeVerdict + + held = { + (one.package, one.was, one.to): one + for one in session.query(UpgradeVerdict) + .filter(UpgradeVerdict.project_id == project.id) + .all() + } + rows: dict[str, dict[str, Any]] = {} + already_red = 0 + for finding in findings: + package = str(finding.get("package") or "") + was = str(finding.get("version") or "") + source = str(finding.get("source") or "") + raw = finding.get("advisories") + advisories = raw if isinstance(raw, list) else [] + # De-duplicated for the reason two identifiers are one advisory: *fixed in 49.0.0, 49.0.0, + # 50.0.0* is a list that has been counted wrong. + fixed = list( + dict.fromkeys( + str(version) for one in advisories for version in (one.get("fixed") or []) + ) + ) + mine = [held[(package, was, to)] for to in fixed if (package, was, to) in held] + already_red += sum(1 for one in mine if one.outcome == "already-red") + state, to, said = _state_of(mine, fixed) + # A merged or closed pull request still has somewhere to go, and that is where the reader + # goes to see what happened (item 253) — the link is the evidence, not the invitation. + where = said if state in _CARRY_A_LINK else None + row = rows.setdefault(package, { + "package": package, "sources": [], "pinned": [], "to": [], "states": [], + "advisories": [], "where": None, "verdict": None, "note": None, + }) + if source and source not in row["sources"]: + row["sources"].append(source) + # **Every published destination, not only the one whose verdict won the row.** OSV publishes + # two fixed versions when an advisory was fixed on two release branches, and they are two + # answers; showing the winner's alone loses one of them from the page. + # **Two sets, never their product.** Pairing every pinned version with every published + # destination is a cartesian explosion the moment a package is pinned three times and fixed + # on four branches: `brace-expansion` rendered its version thirty times and stretched the + # table to 7,208px. What a reader needs is *these versions are pinned, and they move here*. + # + # The pinned version is kept even when there is nowhere to move it: the one row nobody can + # act on was also the one rendering an empty span where its version should be. + if was and was not in row["pinned"]: + row["pinned"].append(was) + for destination in fixed or ([to] if to else []): + if destination and destination not in row["to"]: + row["to"].append(destination) + row["states"].append(state) + row["where"] = row["where"] or where + if not row.get("note"): + if state in ("refused", "declined"): + # `declined` keeps its reason on the same column `refused` does: the reviewer's + # words, or the fact that they gave none. Item 178's rule, item 253's state. + row["note"] = next( + (str(one.open_note) for one in mine if getattr(one, "open_note", None)), None + ) + elif state not in _CARRY_A_LINK and said: + row["note"] = said + for one in advisories: + if one.get("id") not in {seen.get("id") for seen in row["advisories"]}: + row["advisories"].append(one) + if state == "ready" and row["verdict"] is None: + openable = [ + one for one in mine + if one.outcome == "clean" and getattr(one, "artefact", None) + and not one.opened_where and one.asked_to_open_at is None + ] + row["verdict"] = openable[0].id if openable else None + + ordered = [] + for row in rows.values(): + row["state"] = min(row["states"], key=lambda s: _BAND_RANK[s]) + ordered.append(row) + ordered.sort(key=lambda r: r["package"]) + return ordered, already_red + + +def _subject(row: Mapping[str, Any], colour: str, *, project: _Project, acting: Acting, + permitted: bool) -> str: + """One package, one row: what it is, where, what it would move to, and the one thing to do.""" + # Both sides are capped for the same reason: a row is scanned, and the enumeration is what the + # fold and the project's own lock file are for. + pinned = [_h(one) for one in row["pinned"][:2]] + if len(row["pinned"]) > 2: + pinned.append(f"+{len(row['pinned']) - 2}") + froms = " · ".join(pinned) + # **Three destinations and a count, not eleven.** OSV publishes a fixed version per release + # branch, so a package pinned across a major version can land in a dozen places — and a reader + # deciding whether to look needs *there is somewhere to go*, not the enumeration. + landings = [_h(one) for one in row["to"][:2]] + rest = len(row["to"]) - len(landings) + if rest > 0: + landings.append(f"+{rest}") + move = ( + f'{froms}→' + f'{" · ".join(landings)}' + if landings + else f'{froms}' + ) + says = "".join( + f'
  • ' + f'{_h(one.get("id"))} {_h(one.get("summary") or "")}
    • ' + for one in row["advisories"][:8] + ) + rest = len(row["advisories"]) - 8 + if rest > 0: + says += f'
  • and {rest} more
    • ' + fold = ( + f'
    {len(row["advisories"])}' + f'
      {says}
    ' + if row["advisories"] else "" + ) + # **The one thing a row says in prose**, because each refusal differs: *already open from an + # earlier run* and *the forge refused it* are not the same sentence, so this cannot move to a + # heading the way every other state's explanation did. + note = ( + f'{_h(row["note"])}' if row.get("note") else "" + ) + return ( + f'' + f'' + f'{_h(row["package"])}{move}{note}' + f'{_h(" · ".join(row["sources"]))}' + f'{fold}' + f'' + f"{_action_for(row, project=project, acting=acting, permitted=permitted)}" + f"" + ) + + +def _action_for(row: Mapping[str, Any], *, project: _Project, acting: Acting, + permitted: bool) -> str: + """The one control this row has, or nothing at all. + + **No column of empty cells** (DR-0028): the action column was empty in twenty-five rows out of + twenty-six, paying width to say nothing. It renders where it exists and the cell collapses + everywhere else. + """ + if row["state"] in _CARRY_A_LINK and row["where"]: + return ( + f'' + f'#{_h(str(row["where"]).rsplit("/", 1)[-1])} ↗' + ) + if row["state"] != "ready" or not permitted or not acting.csrf or not row["verdict"]: + return "" + csrf = f'' + return ( + # **`dependencies`, which is the URL this view is served from** (item 250). It posted to + # `../` and the handler answered with this view — a document written for + # `projects//dependencies`, carrying `../../`, returned from one level up. Every link + # on the page that came back resolved outside the token's prefix entirely. + f'
    {csrf}' + f'' + '
    ' + ) + + +def _the_packages( + session: Session, + project: _Project, + findings: Sequence[Mapping[str, Any]], + *, + acting: Acting, +) -> str: + """Every package with something published against it, grouped by what can be done. DR-0028.""" + rows, already_red = _packages_of(session, project, findings) + if not rows: + return "" + permitted = _may_open_upgrades(project) + bands, tally = [], [] + for key, title, says, colour in _BANDS: + here = [row for row in rows if row["state"] == key] + if not here: + continue + tally.append( + f'' + f'{len(here)} {_h(title.lower())}' + ) + lines = "".join( + _subject(row, colour, project=project, acting=acting, permitted=permitted) + for row in here + ) + bands.append( + f'

    {_h(title)}' + f'{_h(says)}{len(here)}

    ' + f'{lines}
    ' + ) + ready = sum(1 for row in rows if row["state"] == "ready") + return ( + f'

    {"".join(tally)}

    {"".join(bands)}' + f"{_baseline_note(already_red, rows)}" + f"{_about_opening(ready, permitted=permitted, acting=acting)}" + # **Said once, at the end, and it is not decoration**: the claim this half of the product + # rests on is that nothing here happens on a clock. DR-0026 is the decision; this is where a + # reader who never opens a decision record finds out. + '

    Each upgrade was applied in a clone, built, and measured against this ' + "project's own suite. This instance verifies on its own clock and never opens one " + "by itself — what gets opened, a person asks for, and nobody merges it but " + "you.

    " + ) + + +def _baseline_note(already_red: int, rows: Sequence[Mapping[str, Any]]) -> str: + """How many pairs the red-baseline band covers, said once. Item 234. + + **About the project, not about the upgrade.** A repository whose own tests are red gives one + answer to every pair in the queue, and rendering it per row buries whatever else is there — + which is why the sentence lives in that band's heading. This adds only the arithmetic the + heading cannot carry: a band lists *packages*, and the queue answered *pairs*. + """ + if not already_red or not any(row["state"] == "baseline" for row in rows): + return "" + return ( + f'

    That covers {already_red} upgrade(s), measured once and asked ' + f"again when the next dependency report is taken.

    " + ) + + +def _about_opening(ready: int, *, permitted: bool, acting: Acting) -> str: + """What a reader can do with what passed, or why they cannot. Item 245, DR-0019. + + **The count goes in front of the refusal**, which is the order the terminal uses and the reason + it reads as a decision rather than as a part that is missing. + """ + if not ready: + return "" + if not acting.csrf: + return ( + f'

    {ready} of these can be opened as draft pull requests. Signing in is ' + f"what offers the control.

    " + ) + if not permitted: + return ( + f'

    {ready} passed your suite and none can be ' + f"opened: this project has not permitted it. Set " + f"autofix: {{open_upgrades: true}} in its manifest if you want that " + f"button. It is false by default because having the credential is not the same as " + f"having agreed.

    " + ) + return "" + + +def what_is_published_against_it( + session: Session, project: _Project, *, acting: Acting = READING +) -> str: + """What OSV had published against what this project pins, and when that was asked. DR-0024. + + **The half an evaluator can use on their first day**, which until item 230 left no trace in a + running instance at all: `hullwork deps` opened no session, stored nothing, and could not run + inside the container. + + Three states and they are not two. *Nothing published* is good news; *nothing pinned* is a + different sentence about a different problem; and **could not ask** is neither — an advisory + list that silently reads empty when OSV was unreachable says *you are fine* on no evidence, and + is the failure this feature must not have. + + **No longer folded** (item 235, DR-0027). This is a feature, and a feature one click away is + what made this page difficult three times running: it returns the block, and the page it belongs + to decides where the block goes. + """ + from hullwork.models import DependencyReport + + report = session.get(DependencyReport, project.id) + if report is None: + return ( + "

    Not asked yet. This instance reads what you pin and asks OSV on its own clock, " + "within six hours of a project being connected.

    " + ) + + when = f'

    Asked {_h(_ago(report.taken_at))}.

    ' + if not report.asked: + return ( + f'

    Could not ask: ' + f'{_as_code(report.note or "the reason was not recorded")}

    {when}' + "

    This is not an empty report. Nothing here says your dependencies are fine; it " + "says the question did not reach an answer.

    " + ) + if not report.pinned: + return f'

    Nothing pins a version. {_as_code(report.note or "")}

    {when}' + if not report.findings: + return ( + f"

    OSV has nothing published against any of the {report.pinned} pinned version(s) " + f"this repository declares.

    {when}" + '

    It reads what you pinned, so a dependency your build resolves at ' + "install time is invisible to it — and it asks one database.

    " + ) + # **One list, not two** (DR-0028). What OSV publishes about a package and what this instance did + # about it were two sections six screens apart, so the questions a reader asks — what is + # wrong with this one, was it tried, what happened, can I do anything — were answered in two + # places that had to be joined from memory. + packages, _ = _packages_of(session, project, report.findings) + return ( + f'

    {report.pinned} versions pinned · {len(packages)} package(s) with ' + f"something published · asked {_h(_ago(report.taken_at))}.

    " + f"{_the_packages(session, project, report.findings, acting=acting)}" + ) + + +def _the_rest_of_its_life(project: _Project, acting: Acting) -> str: + """The four things item 207 built routes for and buttons for nothing. Item 223. + + **Every one of them was reachable only by `curl`.** The route took `refresh`, `disable`, + `set-tracker` and `rotate-secret`; the tests posted straight at it, which is a fair test of a + route and no test at all of a page. An operator on this view could do none of it. + + Rotating is separated from the other three and says what it breaks **before** it is pressed: the + tracker's current webhook URL stops working the moment it succeeds, and the new secret is shown + once. + """ + if not acting.csrf: + return "" + csrf = f'' + # **`settings`, which is the URL this view is served from** (items 249 and 250). Item 249 found + # these posting to `../projects/` — `projects/projects/`, a route that does not + # exist, so all five controls posted into a 404 — and corrected them to `../`. That was + # the minimum: a route that exists. This is the right one, because the answer to a press is + # served at the URL it posted to, and the document for that URL is this view. `../` + # answered with the project's overview and bounced a reader out of settings after every action. + where = "settings" + tracker = _h(project.tracker_project or "") + return ( + f'
    {csrf}' + '' + + "
    " + + ( + # Two submissions, like `prune`: the first says what stopping means, the second does + # it. And a disabled project is offered the way back rather than the way out. + f'
    {csrf}' + '" + '
    ' + if project.active + else f'

    Not watched. No error from it becomes an item and the sweep ' + "skips it; nothing was deleted.

    " + f'
    {csrf}' + '
    ' + ) + + f'
    {csrf}' + '

    ' + f'

    ' + '
    ' + '

    Rotating the webhook secret stops the URL your tracker is using ' + "now, and the new one is shown once and never again — only its hash is kept.

    " + f'
    {csrf}' + '' + "
    " + ) + + +def _sweeping(project: _Project, acting: Acting) -> str: + """The tracker's unresolved list, for a project the webhook cannot have told the whole truth + about. DR-0011, item 219. + + **Counted before it is filed, and the count is what you confirm.** The webhook fires when an + issue is created and never again, so a bug that was already failing when Hullwork was installed + never arrives by that door. Sweeping is how it does — and a project with three hundred open + issues becomes three hundred forge issues in one pass unless somebody sees the number first. + """ + if not acting.csrf or not project.tracker_project: + return "" + csrf = f'' + return ( + # **`settings`, the URL this view is served from** (items 249, 250). It posted to + # `../projects/` — `projects/projects/`, which is not a route, so every control + # in this section was dead — and then to `../`, which exists but is a different view. + f'
    {csrf}' + '' + '
    ' + ) + + +def _reading_the_repository(project: _Project, acting: Acting) -> str: + """The two answers that come from the repository itself, on request. Item 222. + + **Each spends one forge read, and only when somebody presses it.** Item 142's rule is about a + *render* — a reader refreshing would spend one each time — and these are actions, the same shape + `projects refresh` has had since item 206. Reading that rule as a ban on both was mine, and it + parked two commands behind a decision they never needed. + + Neither stores anything. The lane policy especially: a derived policy kept on disk would be a + snapshot of *which code is dangerous*, and `territory.py` says why that fails in the direction + that matters. + """ + if not acting.csrf: + return "" + csrf = f'' + return ( + # **`settings`, the URL this view is served from** (items 249, 250). Same two corrections as + # the block above: first to a route that exists, then to the one this view is. + f'
    {csrf}' + '' + '' + "
    " + ) + + +def _housekeeping(session: Session, acting: Acting) -> str: + """The instance's own upkeep: the lease, and the only destructive control on this page. + + Folded, because none of it is what somebody came for — item 203's rule about what is above a + fold and what is below one. Shown to nobody without a session (DR-0021). + """ + if not acting.csrf: + return "" + from hullwork import lease as lease_module + + holder = lease_module.holder_of(session) + who = ( + f"Held by {_h(holder)}." + if holder + else "No dispatcher holds it." + ) + csrf = f'' + body = ( + f"

    {who} Releasing it means the next dispatcher does not wait for the expiry.

    " + f'
    {csrf}' + '' + "
    " + '

    Verdicts the dispatcher reached and could not send are finished by ' + "publishing them again. The attempt is already spent either way.

    " + f'
    {csrf}' + '' + "
    " + "

    The read link is a shared key: anyone holding it reads every item and captured " + "output here. Issuing a new one stops the URL anybody is using now, " + "which is what you press it for — and the new one is shown once.

    " + f'
    {csrf}' + '' + '
    ' + "

    Forgetting the verbatim bodies of old deliveries keeps every row, fingerprint and " + "issue reference. It is the only thing on this page that destroys anything, so it " + "says what it would drop first.

    " + f'
    {csrf}' + '

    ' + '

    ' + '' + '
    ' + ) + return _fold("Upkeep: the lease, stranded verdicts, and forgetting old bodies", body) + + +def _rows_for_standing(rows: Sequence[object]) -> str: + """The panel's rows, for both views. Items 203 and 208. + + **One renderer, because two that happen to look alike is how the borrowed list ended up in + both of them** — and how a fix to one would leave the other painting a `cannot` amber. + + Takes anything with `check`/`name`, a state and a `detail`, which is what `doctor.Finding` and + `features.Standing` both are. A decision reads quiet and a fault reads red: DR-0019 in colour, + because painting a choice somebody made as a defect tells them to go and repair it. + """ + said = [] + for one in rows: + state = getattr(one, "state", "") + word = getattr(state, "value", state) + broken = word in ("broken", "cannot") + said.append( + f'
  • ' + f'{_h(word)}' + f'{_h(_titled(one))}' + f'

    {_as_code(getattr(one, "detail", ""))}

    • ' + ) + return "".join(said) + + +def why_it_will_not_work( + session: Session, settings: Settings, *, acting: Acting = READING +) -> str: + """`doctor`, for somebody without a shell. Item 208, DR-0022. + + **The findings, not a second diagnosis.** `doctor.examine` already returns them and item 199's + pre-flight already renders them elsewhere; a page that asked its own questions would drift from + the command an operator quotes in a bug report. + + `not_from_here`'s downgrade comes with them and must: the receiver is not the dispatcher, and a + page reporting the model credential missing — on an instance where it is present in the half + that uses it — sends somebody to repair a working machine. + """ + from hullwork import doctor as doctor_module + + found = doctor_module.examine( + session, + settings, + code_forge=None, + env_file=Path(settings.deployment_env_file or ".env"), + compose_file=( + Path(settings.deployment_compose_file) if settings.deployment_compose_file else None + ), + ) + worrying = [one for one in found if one.state is not doctor_module.State.OK] + rows = _rows_for_standing(worrying) + body = ( + '

    Diagnostics

    Every check this instance runs on itself, and what ' + "each one would stop working if it failed.

    " + + ( + f'
      {rows}
    ' + f'

    {len(found) - len(worrying)} of {len(found)} check(s) are fine.

    ' + if worrying + else f'

    All {len(found)} check(s) are fine.

    ' + ) + ) + return _document( + "Hullwork — doctor", body, acting=acting, here="doctor", + projects=each_project(session), + ) + + +def what_it_received(settings: Settings, *, acting: Acting = READING) -> str: + """`config`, for somebody without a shell. Item 208. + + **No credential is printed**, and that is `settings_report`'s property rather than this + function's: a secret reads `set` or `not set` before it ever reaches here. Worth saying because + `config` reads like the most disclosing thing in the product and is in fact the most carefully + disclosing thing in it. + """ + from hullwork import settings_report + + rows = "".join( + f"{_h(name)}{_h(value)}" + f"{_h(source)}{_h(reaches)}" + for name, value, source, reaches in settings_report.rows(settings) + ) + body = ( + "

    What it received

    " + '

    What this process was handed, which is a different question from what you ' + "wrote in a file. No credential is printed: a secret reads set or " + "not set.

    " + '
    ' + f"{rows}
    variablevaluefromreaches
    " + ) + return _document( + "Hullwork — configuration", body, acting=acting, here="config" + ) + + +def _what_this_instance_has_switched_on(session: Session, settings: Settings) -> str: + """The feature-by-feature standing, worst first. Item 203. + + **From `features.on_this_instance`, which the terminal prints too**, so the page and + `hullwork status` cannot come to disagree about the same instance — the rule `instance`'s own + docstring states about its numbers, applied to its states. + + An instance with everything on says so in one line: fourteen green rows is a wall a reader stops + looking at, and the thing they came for is whichever one is not green. + """ + from hullwork import features + + standing = features.on_this_instance(session, settings) + worrying = [one for one in standing if one.state is not features.ON] + if not worrying: + return ( + '

    Every feature this instance can have is on. ' + f"{len(standing)} of {len(standing)}.

    " + ) + rows = _rows_for_standing(worrying) + on = len(standing) - len(worrying) + return ( + f'
      {rows}
    ' + f'

    {on} of {len(standing)} feature(s) on.

    ' + ) + + +def instance( + session: Session, + settings: Settings, + *, + error_reporting: bool, + acting: Acting = READING, + said: str | None = None, +) -> str: + """What `hullwork status` says, for somebody who does not have a terminal on this host. + + **Every number comes from the function `status` calls**, never from a second query written for + this page: `readiness.check`, `outcomes.desk`, `outcomes.funnel`, `recurrence.counted` and + `undecided`, `lease.state` and `reporting_of`. A page that recomputed them would drift, and the + first anybody would know is a reader and an operator disagreeing about the same instance. + """ + from hullwork import lease, outcomes, readiness, recurrence + + report = readiness.check(session, settings, error_reporting=error_reporting) + merged, holding, recurred = recurrence.counted(session) + undecided = recurrence.undecided(session) + loop_state, loop_seen = lease.state(session) + reporting = lease.reporting_of(session) + + rows = [ + ("state", "ready" if report.ready else "degraded"), + ("version", report.version), + ("forge", report.forge), ("error reporting (this service)", "on" if report.error_reporting else "off"), ( "error reporting (dispatcher)", @@ -1676,8 +3196,12 @@ def instance( # which is the same defect item 136 already found on this page once: a fact the instance knew, # put where nobody reading would find it. The interface design says this surface exists # to show what was verified and what was not; a count of attempts is not that, and this is. - desk = "".join(f"
  • {_h(line)}
    • " for line in outcomes.desk_lines(outcomes.desk(session))) - attempts = "".join(f"
  • {_h(line)}
    • " for line in outcomes.lines(outcomes.funnel(session))) + # **The page's skin of the same structure the terminal prints as sentences** (item 248). Six + # sections of prose bullets were 500 of this view's 779 words, and every bullet was a number + # with a sentence wrapped around it — so a reader comparing this week to last had to parse eight + # of them to find two figures. + desk = _desk_figures(outcomes.desk(session)) + attempts = _funnel_figures(outcomes.funnel(session)) spent = "".join( f"
  • {_h(line.strip())}
    • " for line in spend.lines( @@ -1688,9 +3212,8 @@ def instance( ) ) - reviewed = "".join( - f"
  • {_h(line)}
    • " for line in outcomes.review_lines(outcomes.reviewed(session)) - ) + counted_reviews = outcomes.reviewed(session) + reviewed = _review_figures(counted_reviews, outcomes.review_lines(counted_reviews)) prices = spend.Prices.from_settings(settings) @@ -1700,10 +3223,14 @@ def instance( #: context second: a problem or a decision is something to *do*, and what the machine is busy #: with is something to *know*. body = ( + _outcome(said) + # Every other view has one, and this is the busiest (item 223): a page whose first landmark + # is missing is the one where a screen reader has furthest to go. + + "

    This instance

    " # The answer and the decisions are the front door's now (item 212). They are still here, # from the same function, because an operator who opens the report on a bad morning should # not have to go back to learn whether anything wants them. - does_this_need_you(session, settings, acting, error_reporting=error_reporting)[0] + + does_this_need_you(session, settings, acting, error_reporting=error_reporting)[0] + _proof( session, merged=merged, @@ -1720,32 +3247,62 @@ def instance( # The link row that used to live here is the rail now (item 212): two sets of navigation # on one page is two places to add the next noun to, and one of them will be forgotten. + '
    ' - + _fold( + # **Sections rather than folds** (item 235, DR-0027). Five disclosures titled with + # sentences — *What arrived, and how much left your desk*, *Which half holds what, and what + # this instance allows* — is a page where every answer is one click and one guess away, and + # the guess is the part a reader cannot make. + + _section( # **Renamed once a real configuration page existed** (item 211). These seven rows are # state — version, forge, sweep, backlog — and calling them *configured* was harmless # while nothing else claimed the word. `/config` claims it now, and two things with one # name is the drift this repository has spent a week removing. - "How it is right now", + "How it is now", + "What this process is, and what it is wired to, at this moment.", f'
    {table}
    ', ) # Before the attempts block, exactly as `status` orders them: this one has *what arrived* # as its denominator and that one has *what was attempted*, so a reader who opens one # should meet the wider question first. - + (_fold("What arrived, and how much left your desk", f"
      {desk}
    ") if desk else "") - + (_fold("What its attempts came to", f"
      {attempts}
    ") if attempts else "") - + (_fold("What they cost", f"
      {spent}
    ") if spent else "") - + (_fold("What reviewers did", f"
      {reviewed}
    ") if reviewed else "") - + _fold( - "Which half holds what, and what this instance allows", - _the_credential_split(session) + _what_this_instance_allows(settings), + + ( + _section( + "What left your desk", + "Of everything that arrived, how much this instance took off you.", + desk, + ) + if desk + else "" + ) + + ( + _section( + "What attempts came to", + "Every attempt this instance has made, by how it ended.", + attempts, + ) + if attempts + else "" + ) + + ( + _section("What they cost", "In tokens and in money, through one arithmetic.", + f"
      {spent}
    ") + if spent + else "" + ) + + ( + _section("What reviewers did", "What became of the pull requests it opened.", reviewed) + if reviewed + else "" ) + + _the_credential_split(session) + + _what_this_instance_allows(settings) + + _housekeeping(session, acting) + "
    " ) # The instance's own state, on the bar rather than in a folded table: it is the second question # a reader has, and item 167 had buried it under a disclosure. badge = ("ready", "ok") if report.ready else ("degraded", "bad") return _document( - "Hullwork — this instance", body, acting=acting, state=badge, here="instance" + "Hullwork — this instance", body, acting=acting, state=badge, here="instance", + projects=each_project(session), ) @@ -1755,13 +3312,21 @@ def instance( MAX_ITEMS = 200 -def _project_health(project: _Project) -> tuple[str, str]: - """Two sentences a reader looking at one client needs, and neither was on any page. Item 142. +def _project_health(project: _Project, settings: Settings) -> tuple[str, str]: + """What is wrong with this project, and nothing else. Items 142 and 228. + + **The credential**, from the column the sweep writes rather than from a forge request here: a + page render must not spend one, and a reader refreshing would spend one each time. `None` is + *not measured* and is not a pass — item 073's rule, and the same `None != False` this project + has got wrong three times. - **The credential**, from the audit `status` already stores on the project row rather than from a - forge request here: a page render must not spend one, and a reader refreshing would spend one - each time. `None` is *not asked yet* and is not a pass — item 073's rule, and the same - `None != False` this project has got wrong three times. + **It said *not asked yet* forever**, because it read a key inside the manifest JSON that nothing + ever wrote — so the command it told you to run would not have changed it either. The sweep + measures it now, on `forge_recheck_seconds`. + + **And a state that is fine says nothing at all.** `cached manifest validates` was three internal + words describing a normal state at the volume of a fault; the operator asked what it meant, + which is the answer. **The manifest**, by validating the cached copy. A project whose stored manifest no longer parses has every incoming error land red (`ingest._manifest_for` degrades to that silently, by @@ -1770,11 +3335,23 @@ def _project_health(project: _Project) -> tuple[str, str]: """ from hullwork.manifest import Manifest - pushes = (project.manifest or {}).get("__ingest_can_push__") if project.manifest else None + pushes = project.ingest_token_can_push + # **Only what is not fine, and a count of what is** (item 203, applied here by item 228). A + # project whose credential is correctly narrow does not need a line saying so at the same + # volume as everything else; a project whose credential can push needs a loud one. credential = { - None: ("unknown", "not asked yet — `hullwork status` records this when it runs"), - True: ("bad", "its ingest credential CAN push, which DR-0009 forbids"), - False: ("good", "ingest credential reaches the repository and cannot push"), + None: ( + "unknown", + "not measured yet — this instance asks the forge on its own clock, within " + f"{max(settings.forge_recheck_seconds, 60) // 60} minute(s) of a project being " + "connected", + ), + True: ( + "bad", + "the ingest **token** can write code to this repository — measured, not inferred: a " + "request only a code scope allows was accepted. DR-0009 is what that breaks", + ), + False: ("good", ""), }[pushes if pushes is None else bool(pushes)] if not project.manifest: @@ -1784,7 +3361,7 @@ def _project_health(project: _Project) -> tuple[str, str]: Manifest.model_validate( {k: v for k, v in project.manifest.items() if not k.startswith("__")} ) - manifest = ("good", "cached manifest validates") + manifest = ("good", "") except Exception: manifest = ( "bad", @@ -1794,11 +3371,10 @@ def _project_health(project: _Project) -> tuple[str, str]: # `_as_code`, not `_h` (item 213). These two sentences were written for a terminal — one of # them names `hullwork projects refresh` — and escaping them served the backticks, which beside # a command name reads as a typo in the product rather than as a quotation of it. - return ( - f'
  • {_as_code(credential[1])}
    • ' - f'
  • {_as_code(manifest[1])}
    • ', - credential[0] + manifest[0], + said = "".join( + f'
  • {_as_code(text)}
    • ' for tone, text in (credential, manifest) if text ) + return said, credential[0] + manifest[0] def _project_columns(session: Session, project_id: int) -> str: @@ -1857,7 +3433,7 @@ def _what_was_rotated(rotated: tuple[str, str | None] | None) -> str: return "" slug, token = rotated return ( - f"

    {_h(slug)} has a new webhook secret

    " + f'

    {_h(slug)} has a new webhook secret

    ' "

    The URL your tracker was posting to has stopped working — update it before the " "next error, or nothing arrives. This is the only time the new one is shown: only " "its hash is stored.

    " @@ -1936,7 +3512,7 @@ def _what_was_just_made(made: object) -> str: token = getattr(made, "token", "") slug = getattr(project, "slug", "") return ( - f'

    {_h(slug)} is connected

    ' + f'

    {_h(slug)} is connected

    ' "

    Point your error tracker's webhook at this. This is the only time it is shown — " "only its hash is stored, so nobody, including this page, can print it again. Lose it and " "hullwork projects rotate-secret issues another, which stops the old one.

    " @@ -1944,6 +3520,254 @@ def _what_was_just_made(made: object) -> str: ) +def _a_project_page( + session: Session, + slug: str, + where: str, + label: str, + says: str, + body: Callable[[_Project], str], + *, + acting: Acting, + said: str | None = None, + refused: str | None = None, + rotated: tuple[str, str | None] | None = None, +) -> str | None: + """One feature, one project, one page. Item 237. + + **The operator's own correction of item 235**: that item named the features and then put each + on a page holding every project's, which is a wall at two projects and unusable at ten. Nobody + works by feature across clients; they work on a client. + + `None` when there is no project with that slug, so the route answers the same `404` an unknown + path gets — a distinct body would let somebody enumerate the slugs an instance serves. + """ + found = session.scalars(select(_Project).where(_Project.slug == slug)).one_or_none() + if found is None: + return None + return _document( + f"Hullwork — {found.slug} {label.lower()}", + f'

    {_h(found.slug)} {_h(label)}

    ' + # **Above the body, because it is the answer to what was just pressed** (item 245). A + # sentence below eight hundred words of report is a sentence nobody sees, and this one says + # the pull request does not exist yet — which is the part a reader has to read. + # + # **A refusal answers here too** (item 250). It used to answer with the list of projects, + # rendered at this URL: six dead links handed to somebody whose forge had just gone down. + f'

    {says}

    ' + f"{_refusal(refused)}{_what_was_rotated(rotated)}{_outcome(said)}" + body(found), + acting=acting, + here=where, + # **Three levels down from `/page//`** — `projects//` — and every URL + # on this page is relative on purpose, which is what keeps the token out of the HTML. Item + # 227 was this arithmetic being wrong by one; it 404s every link in the rail when it is. + up="../../", + inside=found.slug, + counts=how_much_of_each(session, found.id), + ) + + +def dependencies( + session: Session, + settings: Settings, + slug: str, + *, + acting: Acting = READING, + said: str | None = None, + refused: str | None = None, +) -> str | None: + """What is published against what this project pins, and what came of trying the fixes.""" + del settings + return _a_project_page( + session, + slug, + "dependencies", + "Dependencies", + "What OSV publishes against the versions this project pins, and what happened when this " + "instance tried the published fix. Asked on its own clock, every six hours.", + lambda one: what_is_published_against_it(session, one, acting=acting), + acting=acting, + said=said, + refused=refused, + ) + + +def deliveries( + session: Session, settings: Settings, slug: str, *, acting: Acting = READING, + said: str | None = None, refused: str | None = None, +) -> str | None: + """What this project's tracker sent, and whether it was understood. Item 231's feature.""" + del settings + return _a_project_page( + session, + slug, + "deliveries", + "Deliveries", + "What this project's error tracker has sent, and whether it could be read. A call with the " + "wrong secret is refused before anything is written, so an empty list means nobody knocked " + "with a working secret — not that nobody knocked.", + lambda one: what_arrived_for(session, one), + acting=acting, + said=said, + refused=refused, + ) + + +def _fixes_for(session: Session, project: _Project, prices: Prices | None) -> str: + """Every attempt this project has had, with what it reached and what it cost. + + **`consumed` is the column that matters** and it is not derivable from the outcome (DR-0003): a + run that never reached the model must not spend the item's one attempt, whatever went wrong + afterwards, and a page that inferred it from the verdict would tell somebody their one try is + gone when it is not. + """ + rows = list( + session.execute( + select(_Attempt, _Item) + .join(_Item, _Attempt.item_id == _Item.id) + .where(_Item.project_id == project.id) + .order_by(_Attempt.id.desc()) + .limit(MAX_ITEMS) + ).all() + ) + if not rows: + return ( + "

    No attempt has run for this project. An item is attempted when it is queued and " + "the dispatcher is running; an amber one waits for you first.

    " + ) + listed = "".join( + # **Two levels up, because this view is three deep** (item 249). `projects//fixes` + # resolving `items/27` gives `projects//items/27`, which is a 404 — the arithmetic + # item 227 was written about, on the one view its guard did not reach. `_its_items`, one + # route along, has had `../../` since it was written. + f'' + f"{_h(bug.title[:70])}" + f'{_h(one.phase_reached.value)}' + f'{_h(one.outcome.value if one.outcome else "still running")}' + f'{"spent" if one.consumed else "not spent"}' + f'{_h(_ago(one.started_at))}' + for one, bug in rows + ) + cost = _project_cost(session, project.id, prices) + return ( + '
    ' + f"{listed}
    itemreachedendedits one trywhen
    {cost}" + ) + + +def fixes( + session: Session, settings: Settings, slug: str, *, acting: Acting = READING, + said: str | None = None, refused: str | None = None, +) -> str | None: + """What this instance tried on this project, how far each got, and what it cost. Item 235. + + The attempts existed only as a cost summary and a phase strip on an item. *What has this thing + actually done* had no page, which for a product whose claim is **a fix that was run** is the + page a reader wants second. + """ + prices = spend.Prices.from_settings(settings) + return _a_project_page( + session, + slug, + "fixes", + "Fixes", + "Every attempt this instance has made on this project, how far it got through the gates, " + "and whether it spent the item's one try. An attempt that never reached the model does not " + "spend it (DR-0003), which is why that column is recorded rather than inferred.", + lambda one: _fixes_for(session, one, prices), + acting=acting, + said=said, + refused=refused, + ) + + +def errors( + session: Session, settings: Settings, slug: str, *, acting: Acting = READING, + said: str | None = None, refused: str | None = None, +) -> str | None: + """This project's bugs, newest first. Item 237. + + Separated from the board above it because they answer different questions: the board is *who is + blocked*, this is *what is there*, and a view holding both is the one the operator kept saying + he could not read. + """ + del settings + return _a_project_page( + session, + slug, + "errors", + "Errors", + "The bugs this project's error tracker sent. Hullwork calls each one an item: the " + "bug, plus everything it has done about it.", + lambda one: _its_items(session, one), + acting=acting, + said=said, + refused=refused, + ) + + +def _its_items(session: Session, project: _Project) -> str: + """The project's items as a table, bounded and saying so.""" + rows = list( + session.scalars( + select(_Item) + .where(_Item.project_id == project.id) + .order_by(_Item.id.desc()) + .limit(MAX_ITEMS) + ).all() + ) + if not rows: + return "

    No item has arrived for this project.

    " + total = int( + session.scalar( + select(func.count()).select_from(_Item).where(_Item.project_id == project.id) + ) + or 0 + ) + listed = "".join( + f'{one.id}' + f'{_h(one.title)}' + f'{_h(one.state.value)}' + f'{_h(one.lane.value)}' + f'{_h(_ago(one.state_since))}' + for one in rows + ) + bound = f"

    Showing {len(rows)} of {total}.

    " if total > len(rows) else "" + return ( + bound + '
    ' + f"{listed}
    idtitlestatelanesince
    " + ) + + +def settings_for( + session: Session, settings: Settings, slug: str, *, acting: Acting = READING, + said: str | None = None, refused: str | None = None, + rotated: tuple[str, str | None] | None = None, +) -> str | None: + """Everything this instance will do to this project on command. Item 237. + + **Where a rotated secret is shown** (item 250). It was shown on the list of every project, + rendered at this project's URL — so the one answer in this product that can never be repeated + arrived on a page whose every link was dead. + """ + return _a_project_page( + session, + slug, + "settings", + "Settings", + "Everything this instance will do to this project when told to, and nothing it does on its " + "own.", + lambda one: _the_rest_of_its_life(one, acting) + + _sweeping(one, acting) + + _reading_the_repository(one, acting) + + what_this_can_do(session, one, settings), + acting=acting, + said=said, + refused=refused, + rotated=rotated, + ) + + def projects( session: Session, settings: Settings, @@ -1951,7 +3775,7 @@ def projects( acting: Acting = READING, just_made: object = None, refused: str | None = None, - rotated: tuple[str, str | None] | None = None, + said: str | None = None, ) -> str: """Every project this instance serves. Item 142, and the level the tree was missing. @@ -1966,24 +3790,34 @@ def projects( """ prices = spend.Prices.from_settings(settings) found = list(session.scalars(select(_Project).order_by(_Project.slug)).all()) - answered = _what_was_just_made(just_made) + _what_was_rotated(rotated) + _refusal(refused) + # **A rotated secret is not answered here** (item 250). It was, and this view is written for + # `/page//projects` — returned from `projects/` its every link resolved one level + # too deep. It is shown where the button is, on that project's settings. + answered = _what_was_just_made(just_made) + _refusal(refused) if not found: body = ( '

    Projects

    ' + _the_form(acting, answered=answered) + "

    No project is registered yet.

    " ) - return _document("Hullwork — projects", body, acting=acting, here="projects") + return _document( + "Hullwork — projects", body, acting=acting, here="projects", + projects=each_project(session), + ) - blocks: list[str] = [] + blocks: list[str] = [_outcome(said)] for project in found[:MAX_ITEMS]: - health, _ = _project_health(project) + health, _ = _project_health(project, settings) blocks.append( f'

    {_h(project.slug)}

    ' f'

    {_h(project.forge)} · {_h(project.repo)}' f"{'' if project.active else ' · not active'}

    " - f"
      {health}
    " - f"{_project_columns(session, project.id)}" + + (f"
      {health}
    " if health else "") + + f"{_project_columns(session, project.id)}" + # **A list is a list** (items 223 and 225). The controls went to the project's own + # view first; the feature block followed, because it was 85% of this page and rendered + # once per project. On a list a reader is not looking *at* a project — they are looking + # *for* one. ) bound = ( f"

    Showing {min(len(found), MAX_ITEMS)} of {len(found)}.

    " @@ -2000,11 +3834,24 @@ def projects( + "".join(blocks) ) del prices - return _document("Hullwork — projects", body, acting=acting, here="projects") + return _document( + "Hullwork — projects", body, acting=acting, here="projects", + projects=each_project(session), + ) + +def project( + session: Session, + settings: Settings, + slug: str, + *, + acting: Acting = READING, + said: str | None = None, +) -> str | None: + """A project's overview: what is wrong, where everything is, and what each feature holds. -def project(session: Session, settings: Settings, slug: str) -> str | None: - """One project: health, board, cost, items. `None` when there is no project with that slug. + **The overview, and not the whole project** (item 237). Every feature has its own page under + this one now; what is here is the answer to *how is this project* and a way into each of them. `None` rather than a message, so the route answers the same `404` an unknown path gets — a distinct body would let somebody enumerate the slugs an instance serves with a valid token. @@ -2013,45 +3860,67 @@ def project(session: Session, settings: Settings, slug: str) -> str | None: if found is None: return None - prices = spend.Prices.from_settings(settings) - health, _ = _project_health(found) - rows = list( - session.scalars( - select(_Item) - .where(_Item.project_id == found.id) - .order_by(_Item.id.desc()) - .limit(MAX_ITEMS) - ).all() + health, _ = _project_health(found, settings) + counts = how_much_of_each(session, found.id) + # **The rail says how much; this says what it is.** A reader who has just arrived on a project + # should not have to read a sidebar's badges to find out which of its five features has anything + # in it, and a link that says what it holds is one they can decide about before clicking. + where_to = "".join( + f'
  • ' + f'{how_many if how_many else "—"}' + f'{_h(name)}' + f'

    {says}

    • ' + for where, name, how_many, says in ( + ( + "errors", "Errors", counts.errors, + "the bugs its tracker sent, and what state each is in", + ), + ( + "fixes", "Fixes", counts.fixes, + "what this instance attempted, how far it got, and what it cost", + ), + ( + "dependencies", "Dependencies", counts.dependencies, + "what OSV publishes against what it pins, and what came of trying the fix", + ), + ( + "deliveries", "Deliveries", counts.deliveries, + "what its tracker actually sent, and whether it could be read", + ), + ) ) - listed = "".join( - f"{item_row.id}" - f"{_h(item_row.title)}" - f"{_h(item_row.state.value)}" - f"{_h(item_row.lane.value)}" - f"{_h(_ago(item_row.state_since))}" - for item_row in rows - ) - total = len(list(session.scalars(select(_Item).where(_Item.project_id == found.id)).all())) - bound = f"

    Showing {len(rows)} of {total}.

    " if total > len(rows) else "" - body = ( - f"

    {_h(found.slug)}

    " + _outcome(said) + + f'

    {_h(found.slug)} Overview

    ' f'

    {_h(found.forge)} · {_h(found.repo)}' - f"{'' if found.active else ' · not active'} · " - f'All projects · This instance

    ' - f"

    Health

      {health}
    " - f"

    Where everything is

    {_project_columns(session, found.id)}" - f"

    What its attempts cost

    {_project_cost(session, found.id, prices)}" - f"

    Items

    {bound}" - + ( - "" - f"{listed}
    idtitlestatelanesince
    " - if listed - else "

    No item has arrived for this project.

    " + f"{'' if found.active else ' · not active'}

    " + # **Nothing to say is the common case now** (item 228): a project whose credential is + # narrow and whose manifest reads is a project with no health section at all, rather than + # two green lines at the volume of a fault. + + (_section("What is wrong", "", f"
      {health}
    ") if health else "") + + _section( + "Where everything is", + "Every item this project has, by who it is waiting on.", + _project_columns(session, found.id), + ) + + _section( + "What it holds", + "Each of this project's features, and how much is in it.", + f'
      {where_to}
    ', ) ) - return _document(f"Hullwork — {found.slug}", body) - + return _document( + f"Hullwork — {found.slug}", + body, + acting=acting, + here="", + # **How far this view is from `/page//`** (item 227). Every URL on this page is + # relative on purpose — that is what keeps the token out of the HTML — so a view one level + # down that does not say so sends every rail link to `projects/`, and all of them 404. + up="../", + inside=found.slug, + counts=counts, + ) #: The lines `evidence` emits around a collapsible block, and around the captured output inside it. @@ -2175,6 +4044,182 @@ def artefact( ) +def front_door( + session: Session, + settings: Settings, + *, + acting: Acting = READING, + error_reporting: bool = False, + said: str | None = None, +) -> str: + """What needs you, and then one line per project. Item 237. + + **The door answers one question and lists one thing.** Before this it was every item on the + instance in one table, which is a list nobody wants first: with two projects it is already two + projects\' bugs interleaved, and *whose* is the column a reader has to scan for. + + The number against a project is **what is waiting on a person**, not how much exists. A count of + items reads the same on a project that is fine and one that is stuck, and the whole of a front + door is *which of these wants me*. + """ + from hullwork.models import DependencyReport + + answer, badge = does_this_need_you(session, settings, acting, error_reporting=error_reporting) + found = list(session.scalars(select(_Project).order_by(_Project.slug)).all()) + rows = "" + for one in found: + counts = how_much_of_each(session, one.id) + # **Two states, two sentences, because they are two different things** (item 247). This + # summed `waiting-approval` and `human-only` and called both *waiting on you*, two lines + # under a headline that counts only the first and had just said **Nothing needs you**. Both + # numbers were right; one name for them was not. A decision is owed on the first; the second + # is work no agent may attempt, and nothing is owed until somebody chooses to do it. + # Bound rather than closed over: `one` is the loop variable, and a closure reading it would + # answer for whichever project the loop was on when it ran (the shape of the bug item 233's + # `_read` carries a comment about). + def _in(*states: ItemState, project_id: int = one.id) -> int: + return int( + session.scalar( + select(func.count()) + .select_from(_Item) + .where(_Item.project_id == project_id, _Item.state.in_(states)) + ) + or 0 + ) + + waiting = _in(ItemState.WAITING_APPROVAL) + only_you = _in(ItemState.HUMAN_ONLY) + report = session.get(DependencyReport, one.id) + said_of: list[str] = [] + if waiting: + said_of.append(f"{waiting} awaiting your decision") + if only_you: + said_of.append(f"{only_you} only a person can do") + if counts.errors: + said_of.append(f"{counts.errors} item(s)") + # **Three states, and *could not ask* is one of them** (DR-0024). A project whose report + # failed must not read the same as one with nothing published against it. + if report is None: + said_of.append("dependencies not asked about yet") + elif not report.asked: + said_of.append('could not ask OSV') + elif report.findings: + packages = len({str(one.get("package") or "") for one in report.findings}) + said_of.append(f"{packages} package(s) with something published") + if not one.active: + said_of.append("not watched") + # **What is waiting on a person is the action**, so it sits where an action sits and leads + # to the items it counted (item 166). Everything else about the project is context. + owed = ( + f'{waiting} waiting' if waiting else "" + ) + rows += ( + '' + f'' + f'{_h(one.slug)}' + f'{" · ".join(said_of) or "nothing to report"}' + f'{owed}' + ) + body = ( + _outcome(said) + + answer + # **What it is doing, above what there is** (item 242): the operator watching a five-minute + # verification through `docker logs` had no other way to see it. + + _what_it_says_it_is_doing(session) + + _section( + "Projects", + # **The paragraph explaining the list is documentation** (DR-0028): it was re-read every + # day by somebody who had understood it the first time. + "One line each. Everything else about a project lives inside it.", + f'{rows}
    ' + if found + else "

    No project is connected yet.

    ", + ) + + what_it_has_been_doing(session) + ) + return _document( + "Hullwork", + body, + acting=acting, + here="./", + state=badge, + projects=each_project(session), + ) + + +#: What each of `_COLUMNS` means, for the heading that now carries the grouping. DR-0028. +#: +#: **The words are the reader's question, not the state machine's.** `pr-open` is not a state a +#: person cares about; *a draft pull request is waiting for somebody to read it* is. +_WHO_IS_BLOCKED: dict[str, str] = { + "waiting": "a decision from you, or work no agent may attempt", + "review": "a draft pull request is waiting for somebody to read it", + "arrived": "triaged and not queued yet", + "queued": "eligible, and the dispatcher takes one per turn", + "working": "an attempt is running now", + "closed": "merged, rejected, or answered — nothing owed", +} + + +def _items_by_who_is_blocked(rows: Sequence[_Item], pulls: Mapping[int, str]) -> str: + """Every item, grouped by who is blocked and ordered by whether that is you. DR-0028, item 247. + + **`_COLUMNS` is the grouping, and it already existed.** Inventing a second vocabulary for this + list is how it and the strip on the front page would come to disagree about what *waiting* means + — which is the drift DR-0027 spent an item undoing. So the six columns are the six bands, in an + order this view chooses: **what the reader owns first, `closed` last**. + + The chronological order the flat table used put the one item waiting for a person at the top by + luck — it happened to be the most recently seen. At two hundred items that is wherever the clock + left it. + """ + owned = [one for one in _COLUMNS if one[4]] + rest = [one for one in _COLUMNS if not one[4] and one[1] != "closed"] + closed = [one for one in _COLUMNS if one[1] == "closed"] + bands = [] + for title, key, colour, states, _ in [*owned, *rest, *closed]: + here = [row for row in rows if row.state in states] + if not here: + continue + lines = "".join(_an_item(row, colour, pulls) for row in here) + bands.append( + f'

    {_h(title)}' + f'{_h(_WHO_IS_BLOCKED.get(key, ""))}{len(here)}

    ' + f'{lines}
    ' + ) + return "".join(bands) + + +def _an_item(row: _Item, colour: str, pulls: Mapping[int, str]) -> str: + """One item, one row: which one, whose, what it says, when, and where it reached. + + **No state column** (DR-0028): it is the heading above this row, and printing it here is the + twenty-five identical words the decision is named after. The lane stays — it is what + distinguishes two rows inside one band, which is the test a column has to pass to exist. + """ + reached = ( + f'{_h(pulls[row.id])}' + if row.id in pulls + else (_h(row.forge_issue_ref) if row.forge_issue_ref else "") + ) + # **`never` is a fact about the item, not context about it** (item 166): this one can never be + # attempted. Rendered in the context cell it was clipped to `N…` by that column's ellipsis, and + # a truncated warning is worse than none — it reads as a rendering fault rather than a state. + stuck = ' never' if _stuck(row) else "" + title = _h(row.title.splitlines()[0] if row.title else "") + return ( + '' + f'' + f'#{row.id}' + f'{title}{stuck}' + f'{_h(row.project.slug)} · {_h(row.lane.value)}' + f'" + f'{reached}' + "" + ) + + def items( session: Session, *, @@ -2215,40 +4260,11 @@ def items( ): pulls[item_id] = ref - body_rows = [] - for row in rows: - reached = ( - _h(pulls[row.id]) - if row.id in pulls - else (_h(row.forge_issue_ref) if row.forge_issue_ref else "—") - ) - state = _h(row.state.value) + ( - ' never' if _stuck(row) else "" - ) - body_rows.append( - "" - f'#{row.id}' - f"{_h(row.project.slug)}" - f"{state}" - f"{_h(row.lane.value)}" - f"{_h(row.title.splitlines()[0] if row.title else '')}" - f"{_h(row.last_seen)}" - f"{reached}" - "" - ) + grouped = _items_by_who_is_blocked(rows, pulls) scope = "" if states is None else f" in {_h(only)}" if rows: - # **The widest table in the product, and the only one that was not allowed to scroll** - # (item 215). Seven columns on the view a person lands on: without this the body itself - # scrolls sideways on a narrow window, which moves the navigation while you read a title. - table = ( - '
    ' - "" - "" - + "".join(body_rows) - + "
    itemprojectstatelanetitlelast seenissue / pull
    " - ) + table = grouped # The bound, stated. Silently showing 200 of 4,000 is how a page teaches a reader that an # instance has done less than it has. shown = ( @@ -2261,7 +4277,7 @@ def items( shown = ( "Nothing here now" if states is not None - else "No items yet. Nothing has arrived from the error tracker on this instance" + else _why_it_is_empty(session) ) # **The emptiness has a cause and the cause has an action** (item 214). On an instance with # no projects the sentence above is true and useless: nothing arrived because nothing is @@ -2289,12 +4305,23 @@ def items( # **Only what is true of what is on screen.** *Most recently seen first* under an empty list # describes an order there is nothing to order, and the link to the instance was a second copy # of a noun the rail already carries. - order = " Most recently seen first." if rows else "" + # **It said *most recently seen first* and that stopped being true** (item 247): the list is + # grouped by who is blocked, and the clock only decides inside a group. + order = " Grouped by who is blocked; newest first inside each." if rows else "" + # **A link and the page it reaches have to be called the same thing** (item 235). The rail said + # *Items* and so did this heading, and neither is a word somebody arriving with a broken + # checkout would look for. *Errors* is what they are; *item* is what this product calls one, and + # that sentence is worth one line rather than a heading nobody can navigate by. body = ( - answer + "

    Items

    " + answer + "

    Errors

    " + '

    The bugs your error tracker sent. Hullwork calls each one an ' + "item.

    " f'

    {shown}{scope}.{order}{everything}

    ' + table ) - return _document("Hullwork — items", body, acting=acting, here=here, state=badge) + return _document( + "Hullwork — errors", body, acting=acting, here=here, state=badge, + projects=each_project(session), + ) def _above_the_fold(attempt: Attempt, prices: Prices | None) -> str: @@ -2374,6 +4401,17 @@ def _next_action(found: Item, acting: Acting, *, up: str) -> str: parts.append(f"

    Waiting for {_own_prose(waiting)}

    ") if stuck: parts.append(f'

    But {_h(stuck)}.

    ') + # **The one state `requeue` exists for** (item 093), and it had a route and no button until + # item 223: an item left `human-only` by a red baseline holds an attempt it never spent, and the + # only way to give it back was an `UPDATE` against a SQLite file inside a Docker volume. + if found.state is ItemState.HUMAN_ONLY and acting.csrf: + parts.append( + f'
    ' + f'' + '
    ' + '

    Only when what stopped it was the environment. Its attempt was never ' + "spent, so it still has one.

    " + ) if found.state is ItemState.WAITING_APPROVAL and not stuck: # The buttons when there is a session, and otherwise how to get them — here as well as on # the front page, because a reader can arrive straight at an item from a forge issue. @@ -2409,7 +4447,7 @@ def _decide(found: Item, acting: Acting, *, up: str) -> str: return "" -def just_the_login(acting: Acting) -> str: +def just_the_login(acting: Acting, *, going_to: str = "") -> str: """The login and nothing else, for the door that replaces the token (DR-0021, item 204). **Nothing about the instance is on it.** A page showing a name, a version or a count beside the @@ -2434,12 +4472,12 @@ def just_the_login(acting: Acting) -> str: f'' f"Sign in\n" '
    ' - f'

    Sign in

    {_login(acting, up="")}' + f'

    Sign in

    {_login(acting, up="", going_to=going_to)}' "
    \n" ) -def _login(acting: Acting, *, up: str) -> str: +def _login(acting: Acting, *, up: str, going_to: str = "") -> str: """The login, or what to run when there is nothing to log in to. Item 168. **`autocomplete="current-password"` and a real `
    ` are the whole feature.** A browser @@ -2453,14 +4491,47 @@ def _login(acting: Acting, *, up: str) -> str: f'

    Too many wrong passwords. This waits ' f"{_h(acting.locked_minutes)} more minute(s) before it will try again.

    " ) + # **Where they were going** (item 224). Signing in used to land on the front door whatever URL + # you had opened, which on an instance you reach by bookmark means finding the view again by + # hand every twelve hours. + onward = ( + f'' if going_to else "" + ) return ( - f'' + f'{onward}' '' '
    ' ) +#: The views a `going_to` may name, so a redirect after signing in cannot be pointed anywhere else. +#: A literal list rather than a pattern: `../` and `//host` and `%2e%2e` are all things a pattern +#: written in a hurry lets through, and there are eight of these. +WHERE_YOU_CAN_LAND: tuple[str, ...] = ( + "", "items", "instance", "projects", "doctor", "config", +) + + +def where_it_may_land(asked: str | None) -> str: + """The tail of a path this instance will send somebody to after they sign in, or `""`. + + **Anything it does not recognise becomes the front door**, silently: a login that argues with + you about where you were going is worse than one that takes you home, and an open redirect is + the classic way a sign-in form becomes somebody else's. + """ + if not asked: + return "" + tail = asked.removeprefix(f"{PREFIX}/{MINE}/").strip("/") + if tail in WHERE_YOU_CAN_LAND: + return tail + # One shape beyond the flat list, because it is where half the work is: a project of its own. + named = tail.removeprefix("projects/") + if tail.startswith("projects/") and "/" not in named and named.replace("-", "").isalnum(): + return tail + return "" + + def _signing_in(acting: Acting, *, up: str = "") -> str: """A way in that does not depend on there being something to decide. Item 168. @@ -2509,8 +4580,138 @@ def _how_to_decide(acting: Acting, found: Item | None = None, *, up: str = "") - ) +def the_error_itself(session: Session, item_id: int) -> str: + """The full error as the tracker recorded it. Item 232, item 036's table finally on the page. + + **The webhook cuts the title at 100 characters**, and for a `KeyError` or a `ValueError` the + half it cuts is often the input that reproduces the bug. The item's title is the cut one; this + is the whole one, and it is nowhere else on this page. + + **Nothing is scrubbed here.** The adapter does it on the way in, which is what that table exists + to say after an audit found a live DSN in one field and this product's own webhook token in + another, on real events. A second scrubber would be a second thing to keep correct. + """ + from hullwork.models import FetchedEvent + + seen = list( + session.scalars( + select(FetchedEvent) + .where(FetchedEvent.item_id == item_id) + .order_by(FetchedEvent.occurred_at.desc().nullslast(), FetchedEvent.id.desc()) + ).all() + ) + if not seen: + return "" + + newest = seen[0] + # **`prune` empties this row and keeps it** (item 231's neighbour): rendering *no frames* for a + # pruned event would report an error with no stack rather than one whose stack this instance + # chose to forget. Different sentences, and the second is the true one. + forgotten = not newest.frames and not newest.packages + said = [] + if newest.message: + said.append( + f'

    The whole message, untruncated — the webhook cuts it at 100 ' + f"characters and the half it cuts is often what reproduces the bug:

    " + f'
    {_h(newest.message)}
    ' + ) + facts = [ + (label, value) + for label, value in ( + ("type", newest.exception_type), + ("where", newest.culprit), + ("level", newest.level), + ("handled", None if newest.handled is None else ("yes" if newest.handled else "no")), + ("release", newest.release), + ("host", newest.server_name), + ("happened", _ago(newest.occurred_at) if newest.occurred_at else None), + ) + if value + ] + said.append( + "
      " + + "".join(f"
    • {_h(label)}: {_h(value)}
      • " for label, value in facts) + + "
    " + ) + if len(seen) > 1: + said.append( + f'

    {len(seen)} occurrences of this are stored. Two samples of one bug ' + "are worth more than one: what differs between them is usually the input that " + "triggers it, and the tracker notifies once per issue and never again.

    " + ) + if forgotten: + said.append( + '

    Its frames, its locals and its pinned versions were forgotten by ' + "hullwork prune. The error is not missing them — this instance stopped " + "keeping them.

    " + ) + else: + said.append(_the_frames(newest.frames)) + if newest.packages: + said.append( + _fold( + f"What was installed when it failed — {len(newest.packages)} version(s)", + '
    ' + "" + + "".join( + f'' + f'' + for name, version in sorted(newest.packages.items()) + ) + + "
    packageversion
    {_h(name)}{_h(version)}
    ", + ) + ) + return _fold("The error, as the tracker recorded it", "".join(said)) + + +def _the_frames(frames: list[dict[str, object]]) -> str: + """The stack, innermost last, with the line each one stopped on. + + The locals get their own disclosure: scrubbed, and still the thing a reader is least often + looking for and most likely to be surprised to find rendered. + """ + if not frames: + return '

    No frames were recorded for this occurrence.

    ' + said = [] + for frame in frames: + where = _h(str(frame.get("filename") or frame.get("module") or "?")) + line = frame.get("lineno") + held = frame.get("variables") + said.append( + f'
  • {where}' + + (f" line {_h(line)}" if line else "") + + (f" in {_h(frame.get('function'))}" if frame.get("function") else "") + + ( + f'
    {_h(frame.get("context_line"))}
    ' + if frame.get("context_line") + else "" + ) + + ( + _fold( + "What the code was holding here", + '
    ' + + "".join( + f'' + f'' + for name, value in dict(held).items() + ) + + "
    {_h(name)}{_h(value)}
    ", + ) + if isinstance(held, dict) and held + else "" + ) + + "
    • " + ) + return f"

    Where it stopped

      {''.join(said)}
    " + + def item( - session: Session, settings: Settings, item_id: int, *, acting: Acting = READING + session: Session, + settings: Settings, + item_id: int, + *, + acting: Acting = READING, + said: str | None = None, ) -> str | None: """One item and every attempt on it. `None` when there is no such item, which the route 404s. @@ -2580,10 +4781,12 @@ def item( title = found.title.splitlines()[0] if found.title else f"item {found.id}" body = ( - f"

    #{_h(found.id)} {_h(title)}

    " + _outcome(said) + + f"

    #{_h(found.id)} {_h(title)}

    " f'

    All items · Instance

    ' f'
    {table}
    ' + _next_action(found, acting, up="../") + + the_error_itself(session, found.id) + ( f'

    {_h(_NOT_STORED)}

    ' + "".join(blocks) if blocks @@ -2610,6 +4813,7 @@ def item( acting=acting, up="../", state=(found.state.value, tone), + projects=each_project(session), ) diff --git a/hullwork/resolve.py b/hullwork/resolve.py index 440cf1c..f6fb29c 100644 --- a/hullwork/resolve.py +++ b/hullwork/resolve.py @@ -20,13 +20,24 @@ import json import logging import tomllib -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager from dataclasses import dataclass from enum import StrEnum from pathlib import Path log = logging.getLogger(__name__) + +class ResolveError(RuntimeError): + """The socket refused something this needed: a volume, a carrier, a copy. + + Its own class rather than `SandboxError`, because this is not an attempt and nothing here holds + a sandbox — and it is turned into an ordinary non-zero exit by `in_a_container`, so a failure to + reach Docker is reported as *this upgrade could not be moved* rather than as a crash in a + dispatcher that has other work to do. + """ + #: How long a resolver may take. Generous: it is a registry round trip plus a graph solve, and a #: cold npm cache on a large tree is genuinely slow. RESOLVE_TIMEOUT_SECONDS = 600 @@ -165,14 +176,25 @@ def command_for(resolver: Resolver, package: str, version: str) -> str: return resolver.command.format(package=package, version=version) -def missing_from(resolver: Resolver, present: Sequence[str]) -> list[str]: - """Which of the files this resolver needs are not in the checkout. +def beside(path: str) -> str: + """The directory a path is in, as the repository writes it, and `""` at the root.""" + return path.rsplit("/", 1)[0] if "/" in path else "" + + +def missing_from(resolver: Resolver, present: Sequence[str], at: str = "") -> list[str]: + """Which of the files this resolver needs are not **beside the lock**. Item 239. Checked before the container starts: a `uv.lock` with no `pyproject.toml` beside it cannot be resolved by anything, and finding that out after pulling an image is a minute wasted on a fact that was on disk. + + **`at` is the whole of item 239.** This compared basenames anywhere in the checkout, so a + `pyproject.toml` in `backend/` satisfied a check about a lock in `frontend/` — and the honest + refusal it exists to produce was unreachable for any repository with more than one lock file. + Measured on `simplecheck`, which is a monorepo: it pulled an image to be told the file was not + where it was looking. """ - names = {path.rsplit("/", 1)[-1] for path in present} + names = {path.rsplit("/", 1)[-1] for path in present if beside(path) == at} return [needed for needed in resolver.needs if needed not in names] @@ -184,14 +206,22 @@ def upgrade( version: str, present: Sequence[str], run: Callable[[Resolver, Path, str], tuple[int, str]], + at: str = "", ) -> Result: """Move the graph, then check that it actually moved. Item 175. `run` takes the resolver, the directory to mount and the command, and returns an exit code and the tool's output. Injected for the reason every other boundary here is: this stays testable without a daemon, and nothing in this module knows Docker exists. + + **`at` is where the lock lives**, relative to the worktree, and it is item 239. This mounted the + root and ran the tool there, so a monorepo — `backend/uv.lock`, `frontend/package.json` — was + told *No `pyproject.toml` found in current directory or any parent directory* and recorded + `cannot-move`, which is a sentence about somebody else's repository that was our own working + directory. It defaults to the root, so a repository with one lock at the top behaves exactly as + it did. """ - absent = missing_from(resolver, present) + absent = missing_from(resolver, present, at) if absent: return Result( Outcome.MISSING, @@ -199,11 +229,12 @@ def upgrade( f"the manifest to know which versions are allowed, and there is none here.", ) - code, output = run(resolver, worktree, command_for(resolver, package, version)) + where = worktree / at if at else worktree + code, output = run(resolver, where, command_for(resolver, package, version)) if code != 0: return Result(Outcome.FAILED, output) - lock_path = worktree / resolver.lock + lock_path = where / resolver.lock landed = version_in_lock(lock_path.read_text(encoding="utf-8"), resolver.lock, package) if landed != version: # **The tool's exit code is not the verdict.** Every one of these resolves happily within @@ -218,21 +249,73 @@ def upgrade( return Result(Outcome.RESOLVED) +@contextmanager +def _carrying(context: Path, docker: str) -> Iterator[str]: + """A named volume holding a copy of `context`, seeded and read back over the socket. Item 240. + + **A bind mount cannot be used here and this is the third time this repository has learned it.** + `-v {path}:/w` is resolved by the *daemon*: the dispatcher runs in a container, so the path + exists in one filesystem and is looked up in another. The daemon finds nothing, mounts an empty + directory, and the resolver reports the project has no manifest — measured on atlas, where every + dependency verification this instance ever ran took this path. + + Item 055 moved the attempt's worktree off a bind mount for exactly this, and item 082 the + contract directory. This is that recipe with a different working directory, and it is imported + from `sandbox.run` rather than written a second time. + """ + import secrets + + from hullwork.sandbox.docker import run_docker + from hullwork.sandbox.inventory import label_args + from hullwork.sandbox.run import CARRIER_IMAGE + + name = f"hullwork-resolve-{secrets.token_hex(4)}" + made = run_docker([docker, "volume", "create", *label_args(), name], timeout=60) + if made.returncode != 0: + raise ResolveError(made.stdout + made.stderr) + + def carrier() -> str: + created = run_docker( + [docker, "create", "--volume", f"{name}:/w", CARRIER_IMAGE, "true"], timeout=120 + ) + if created.returncode != 0: + raise ResolveError(created.stdout + created.stderr) + return created.stdout.strip() + + try: + one = carrier() + try: + pushed = run_docker([docker, "cp", f"{context}/.", f"{one}:/w"], timeout=300) + finally: + run_docker([docker, "rm", "-f", "-v", one], timeout=60) + if pushed.returncode != 0: + raise ResolveError(pushed.stdout + pushed.stderr) + yield name + # **Back to the dispatcher's own filesystem**, because the regenerated lock is the entire + # point: everything downstream — `version_in_lock`, the rebuild, the guard that restores + # what this touched — reads files, and reads them here. + two = carrier() + try: + run_docker([docker, "cp", f"{two}:/w/.", str(context)], timeout=300) + finally: + run_docker([docker, "rm", "-f", "-v", two], timeout=60) + finally: + run_docker([docker, "volume", "rm", "-f", name], timeout=60) + + def in_a_container( resolver: Resolver, context: Path, command: str, *, docker: str = "docker" ) -> tuple[int, str]: - """Run one resolver's command in an ephemeral container. The only Docker in this module. + """Run one resolver's command against a copy of the checkout. The only Docker in this module. - **A bind mount rather than a volume**, unlike an attempt's worktree (item 055), and the - difference is worth stating so it does not later look like an oversight. An attempt's phases run - **the project's own untrusted code**, where a bind mount would let it write to the host as the - uid that started it. This runs one package manager's own command with no project code executing, - and the entire purpose is to get a regenerated file back — which a bind mount does and a volume - does not. + **On a volume rather than a bind mount** (item 240). The docstring here used to argue the + opposite — a bind mount gets the regenerated file back, a volume does not — and that was true + while the dispatcher ran on a host and false the moment it ran in a container, where the daemon + resolves the path in its own filesystem and mounts nothing. Neither the comment nor anything + else was re-read when the ground moved. - **`--user` is not a detail.** Without it `npm` leaves root-owned files in the operator's - checkout, and the next ordinary command they run fails with a permission error nothing connects - back to us. + **`--user` is not a detail.** Without it `npm` leaves root-owned files in the copy, and the + `docker cp` back hands the operator's checkout a file they cannot write. **And this one has a network, deliberately.** Resolving *is* asking the registry what exists. It is the trade `image.build` already makes, and it changes nothing about the phase that later runs @@ -241,27 +324,33 @@ def in_a_container( import os import subprocess - argv = [ - docker, "run", "--rm", - "--user", f"{os.getuid()}:{os.getgid()}", - # A resolver that hangs must not hold the run: these are network calls to a registry. - "--stop-timeout", "10", - "-v", f"{context}:/w", - "-w", "/w", - # `HOME` so the tools have somewhere to write their caches; `/w` is the only writable path - # and a cache in the checkout would be left behind for the operator to find. - "-e", "HOME=/tmp", - resolver.image, - "sh", "-lc", command, - ] - log.info("resolving", extra={"image": resolver.image, "command": command}) try: - done = subprocess.run( # noqa: S603 - argv, capture_output=True, text=True, check=False, timeout=RESOLVE_TIMEOUT_SECONDS - ) - except subprocess.TimeoutExpired: - return 1, f"the resolver did not finish within {RESOLVE_TIMEOUT_SECONDS}s" - return done.returncode, (done.stdout + done.stderr).strip() + with _carrying(context, docker) as volume: + argv = [ + docker, "run", "--rm", + "--user", f"{os.getuid()}:{os.getgid()}", + # A resolver that hangs must not hold the run: these are network calls to a + # registry. + "--stop-timeout", "10", + "-v", f"{volume}:/w", + "-w", "/w", + # `HOME` so the tools have somewhere to write their caches; `/w` is the only + # writable path and a cache in the checkout would be left behind for the operator. + "-e", "HOME=/tmp", + resolver.image, + "sh", "-lc", command, + ] + log.info("resolving", extra={"image": resolver.image, "command": command}) + try: + done = subprocess.run( # noqa: S603 + argv, capture_output=True, text=True, check=False, + timeout=RESOLVE_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + return 1, f"the resolver did not finish within {RESOLVE_TIMEOUT_SECONDS}s" + return done.returncode, (done.stdout + done.stderr).strip() + except ResolveError as failed: + return 1, str(failed) def touches(resolver: Resolver) -> tuple[str, ...]: diff --git a/hullwork/sandbox/net.py b/hullwork/sandbox/net.py index 2e127bf..9ff2780 100644 --- a/hullwork/sandbox/net.py +++ b/hullwork/sandbox/net.py @@ -216,7 +216,7 @@ def close(self) -> None: # Last chance at the journal: after the volume goes it is gone, and an attempt that died # before anything asked for its seal is exactly the one worth reading afterwards. self._pull_journal() - _quietly(self._docker, ["rm", "-f", self.container]) + _quietly(self._docker, ["rm", "-f", "-v", self.container]) _quietly(self._docker, ["network", "rm", self.network]) if self._cable_volume: _quietly(self._docker, ["volume", "rm", "-f", self._cable_volume]) @@ -556,7 +556,7 @@ def _carrier(self) -> Iterator[str]: try: yield carrier finally: - _quietly(self._docker, ["rm", "-f", carrier]) + _quietly(self._docker, ["rm", "-f", "-v", carrier]) def _wait_until_listening(self) -> None: """Wait for the gateway to say it has bound, or say what it said instead. diff --git a/hullwork/sandbox/run.py b/hullwork/sandbox/run.py index 4f6765d..348fd37 100644 --- a/hullwork/sandbox/run.py +++ b/hullwork/sandbox/run.py @@ -463,7 +463,7 @@ def _in_container( out_of_memory=bool(state.get("OOMKilled")), ) finally: - run_docker([self.docker, "rm", "-f", self._container], timeout=60) + run_docker([self.docker, "rm", "-f", "-v", self._container], timeout=60) self._container = None if self.volume: self._pull() @@ -560,7 +560,7 @@ def _contract_carrier(self) -> "Iterator[str]": try: yield carrier finally: - run_docker([self.docker, "rm", "-f", carrier], timeout=60) + run_docker([self.docker, "rm", "-f", "-v", carrier], timeout=60) # --- the worktree the container owns (item 055) -------------------------------------------- @@ -702,7 +702,7 @@ def _carrier(self) -> "Iterator[str]": try: yield carrier finally: - run_docker([self.docker, "rm", "-f", carrier], timeout=60) + run_docker([self.docker, "rm", "-f", "-v", carrier], timeout=60) def _wait(self, timeout: int) -> bool: """Poll until the container exits. False if it had to be killed.""" diff --git a/hullwork/sandbox/services.py b/hullwork/sandbox/services.py index 4086c91..758f4ef 100644 --- a/hullwork/sandbox/services.py +++ b/hullwork/sandbox/services.py @@ -232,9 +232,22 @@ def __exit__( self.close() def close(self) -> None: - """Remove every container and the network. Safe to call twice, and never raises.""" + """Remove every container, its databases and the network. Safe to call twice, never raises. + + **`-v`, and it is the whole of item 244.** `postgres:16` declares + `VOLUME /var/lib/postgresql/data` in its own Dockerfile, so every `docker run` of it creates + an anonymous volume — and a `docker rm` without `-v` leaves it behind. One per service, per + phase, on every attempt and every verification: sixty-nine volumes and 3.2GB on the + operator's own host, in a day, after the images had already been fixed. + + `-v` removes the container's **anonymous** volumes and leaves named ones alone, which is + the distinction that matters: `hullwork-worktree-*` and `hullwork-envcache-*` have names + and owners; this one has neither, and cannot be collected by the reaper for that reason — + an anonymous volume is a hex string that says nothing about who made it, and removing those + by pattern would delete everything else on the host (item 125). + """ for container in self._containers: - _quietly(self._docker, ["rm", "-f", container]) + _quietly(self._docker, ["rm", "-f", "-v", container]) self._containers = [] if self._names: _quietly(self._docker, ["network", "rm", self.network]) diff --git a/hullwork/upgrades.py b/hullwork/upgrades.py index f7f3356..e60e580 100644 --- a/hullwork/upgrades.py +++ b/hullwork/upgrades.py @@ -21,16 +21,33 @@ from __future__ import annotations +import json import logging +import os import re -from collections.abc import Mapping, Sequence - -from hullwork import bump, evidence +import shutil +from collections.abc import Callable, Iterable, Mapping, MutableMapping, Sequence +from contextlib import ExitStack +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, TextIO + +import sqlalchemy as sa +from sqlalchemy.orm import Session + +from hullwork import bump, dependencies, evidence, resolve +from hullwork import dispatch as dispatch_module +from hullwork.config import Settings from hullwork.forge import BranchExistsError, ForgeError +from hullwork.manifest import Manifest from hullwork.osv import Advisory log = logging.getLogger(__name__) +#: How long an opened pull request goes unasked about. The report's own clock (item 253), and the +#: same number `recurrence.RECHECK_SECONDS` uses for the identical question about an item. +RECHECK_SECONDS = 6 * 60 * 60 + #: Where these branches live. Namespaced under `hullwork/` like every other branch this product #: creates, and under `deps/` beneath that so an operator can tell an upgrade from an agent's fix #: without opening either. @@ -218,3 +235,767 @@ def _open_one( log.error("the forge did not mark it a draft", extra={"pull": pull.ref}) log.info("opened", extra={"package": answer.package, "to": answer.to, "pull": pull.ref}) return str(pull.html_url) + + +def verify_one( + checkout: Path, + paths: Sequence[str], + read: Callable[[str], str | None], + manifest: Manifest, + dep: dependencies.Dependency, + versions: list[str], + out: TextIO, +) -> bump.Report | None: + """One package, every candidate, each in its own sandbox. + + **Moved here from the CLI by item 233**, unchanged, because the dispatcher needs the same + function and a private name in `cli` is a function only one caller can have. It takes a + checkout and returns a report; it prints progress to a stream and knows nothing about who + asked — which is what makes it callable from a loop as well as from a terminal. + """ + from hullwork import trial + from hullwork.sandbox import image as image_module + from hullwork.sandbox.run import Sandbox + + runtime = manifest.runtime + assert runtime is not None # noqa: S101 - refused above, and mypy cannot see that + tests = manifest.tests or "" + source = dep.source + + # Which candidate `verify` is on, so a resolver-backed mover knows what to ask for. + _pending: dict[str, str] = {"version": ""} + + with ExitStack() as stack: + worktree = dispatch_module.prepare_worktree(checkout) + stack.callback(shutil.rmtree, worktree, ignore_errors=True) + + def files_now() -> dict[str, bytes]: + """The declared dependency files as they are in the worktree right now. + + Read per build rather than once: the rewrite happens between the two, and the second + build has to see it — `image.dependency_digest` then makes the tag differ by itself, + which is what turns the second build into a real rebuild. + """ + found: dict[str, bytes] = {} + for path in runtime.dependencies or [source]: + whole = worktree / path + if whole.exists(): + found[path] = whole.read_bytes() + return found + + built: dict[str, str] = {} + leaves_behind: set[str] = set() + stack.callback(lambda: _drop_images(leaves_behind)) + # **The commit the source is at, when the source goes into the build at all** (item 182). + # Read once: it is what `image_tag` hashes to decide whether an image can be reused, and the + # source does not move between candidates — only the dependency files do, and those are + # hashed separately by `dependency_digest`. + source_ref = trial.head_sha(checkout) if runtime.install_needs_source else None + + def build_now() -> str | None: + try: + image = image_module.build( + runtime, files_now(), None, + # **Item 113's fix, which this path never inherited** (found by item 182, on + # the first third-party tree it was pointed at). The build context holds the + # declared dependency files and never the source, and three ordinary installers + # read the source anyway: a `requirements.txt` beginning `-e .`, a `Gemfile` + # that says `gemspec`, and `mvn test`. Measured on `encode/httpx`, whose first + # requirement is `-e .[brotli,cli,http2,socks,zstd]`: + # + # ERROR: file:///work does not appear to be a Python project: + # neither 'setup.py' nor 'pyproject.toml' found. + # + # Reported as *your own environment does not build*, which was true of what we + # built and false of the project. Ruby, Java and PHP are on the roadmap as + # stacks whose attempts work; every one of them reaches this the same way. + source=worktree if runtime.install_needs_source else None, + source_ref=source_ref, + ) + except image_module.ImageBuildError as failed: + return str(failed) + built["tag"] = image.tag + # **Every image this verification builds, so every one of them can go** (item 241). + # A candidate's image is not a cache: the lock it was built from exists for one run, + # and the next candidate rewrites that lock and builds another. Measured on atlas — + # seven of them, 1.09GB each, one every six minutes, on a disk that then had 211MB. + leaves_behind.add(image.tag) + return None + + # The baseline image, before anything is rewritten. A failure here is the project's + # environment, not the upgrade's, so it is said as that. + problem = build_now() + if problem is not None: + print(f" {dep.name}: your own environment does not build — {problem}\n", file=out) + return None + + made = {"n": 0} + + def make_box(_version: str) -> bump.Box: + """A box on **whatever image `built` holds right now**. + + Called once per run rather than once per candidate, because the second run has to + happen on the rebuilt image — reusing the first box measures the upgraded project's + suite against the environment it replaced, and reports `clean` for a version that was + never installed. Found by a real Docker run; see item 174. + """ + made["n"] += 1 + # Built from the worktree **as it is now**, which is what makes each run happen in the + # environment its own tree describes. Cheap when nothing changed: the digest is the + # content, so `build` reuses the existing image rather than making another. + build_now() + # **The services the manifest declared** (item 238). `work.py` has passed these since + # item 052 and this path never did, so a project with a database reached + # `localhost:5432`, found nothing, and was reported `already-red` — the honest sentence + # about the wrong thing: the suite was not failing, it was never given what it asked + # for. Every project with a database, for ever, with no way for the report to say + # anything else. + box = Sandbox( + image=built["tag"], worktree=worktree, services=list(runtime.services) + ) + stack.callback(box.cleanup) + box.ensure_volume( + f"hullwork-deps-{os.getpid()}-{made['n']}", + # **Item 114's fix, which this path never inherited either** (item 182). Anything + # the build installed under `/work` is erased by the worktree volume unless the + # image goes down first — which is what `vendor/` is for PHP, and the reason that + # item exists. Off unless the project asks, so every other project takes the path + # it took yesterday. + seed_from_image=runtime.install_needs_source, + ) + return box # type: ignore[return-value] + + # How this file is moved, and everything moving it can touch (items 175 and 176). For a + # list the line is the pin; for a resolved graph only the ecosystem's own tool may move it, + # and `touches` is what stops one candidate leaving a widened range behind for the next. + resolver = resolve.resolver_for(source) + mover = None + guarded: tuple[str, ...] = (source,) + if resolver is not None: + guarded = files_touched_by(resolver, source) + here = [p for p in paths if p.rsplit("/", 1)[-1] in set(resolver.needs)] + mover = mover_for(resolver, source, dep.name, _pending, here) + + report = bump.verify( + tests=tests, source=source, package=dep.name, + was=dep.version, versions=versions, + make_box=make_box, rebuild=lambda _text: build_now(), + mover=mover, touches=guarded, pending=_pending, + ) + + for answer in report.answers: + print(f" {answer.says}", file=out) + if answer.detail: + for line in answer.detail.splitlines()[:8]: + print(f" {line}", file=out) + print("", file=out) + return report + + +def files_touched_by(resolver: resolve.Resolver, source: str) -> tuple[str, ...]: + """Every file this resolver may rewrite, **where the repository actually keeps them**. + + Item 239: `touches` names them relative to the lock, and on a monorepo the lock is not at the + root — so a guard listing `pyproject.toml` protected a file that does not exist while + `backend/pyproject.toml` was rewritten unwatched. + """ + at = resolve.beside(source) + return tuple(f"{at}/{one}" if at else one for one in resolve.touches(resolver)) + + +def mover_for( + resolver: resolve.Resolver, + source: str, + package: str, + pending: MutableMapping[str, str], + present: Sequence[str], +) -> Callable[[Path], str | None]: + """How this graph is moved, in the directory the finding says it lives in. Item 239. + + **A function rather than a closure inside `verify_one`**, because the one thing worth asserting + about it — that it runs where the lock is — was three levels of nesting deep and therefore + untested: `at` was wrong for every monorepo and the suite was green. + + `pending` is read at call time on purpose: `bump.verify` moves through candidates and the mover + is built once. + """ + at = resolve.beside(source) + + def move(worktree: Path) -> str | None: + outcome = resolve.upgrade( + resolver=resolver, worktree=worktree, package=package, version=pending["version"], + present=present, run=resolve.in_a_container, at=at, + ) + return None if outcome.ok else f"{outcome.outcome.value}: {outcome.detail}" + + return move + + +def _drop_images(tags: Iterable[str]) -> None: + """Remove the images a verification built, and the environment cache each one owns. Item 241. + + **Never raises, and never reported as a failure.** A host that could not delete an image is a + host with debris on it, which is worse than it was and is not a wrong verdict about somebody's + upgrade — the same rule every teardown in `sandbox` follows. + + The environment cache is named from a digest of the tag (`Sandbox._env_cache`), and it is + derived here rather than asked for because the sandbox that owned it is already gone by the + time this runs. It is the one place in this repository that computes that name twice, and the + test that covers it asserts they agree. + """ + import hashlib + + from hullwork.sandbox.docker import run_docker + + for tag in tags: + try: + run_docker(["docker", "image", "rm", tag], timeout=120) + digest = hashlib.sha256(tag.encode()).hexdigest()[:12] + run_docker(["docker", "volume", "rm", "-f", f"hullwork-envcache-{digest}"], timeout=60) + except Exception: # this runs while a stack unwinds; see below + # **Broad on purpose, and this is the one place it is right.** `run_docker` swallows + # what Docker answers; it does not swallow the socket being gone. This is a + # `stack.callback`, so anything raised here replaces whatever the verification was + # already reporting — a disk that will not let go would arrive as a crash in place of + # a verdict that had already been measured. + log.warning("could not remove what a verification built", extra={"image": tag}) + + +#: What the dispatcher may spend on this in one turn. **One**, because each is a clone, an image +#: build and a suite run — and a queue that empties itself as fast as it can is a queue nobody can +#: watch (DR-0026). +ONE_PER_TURN = 1 + + +def forget_stale( + session: Session, project_id: int, findings: Sequence[Mapping[str, Any]] +) -> int: + """Drop the artefact of every verdict about a version this project no longer pins. Item 245. + + Called where a new report is written, which is the only event that can make one stale. Returns + how many were forgotten. + + **Why the artefact cannot simply sit there.** A verdict about a version no longer pinned is + already hidden from the page for being stale, so nothing would ever open it — but a repository + that bumps a dependency by hand leaves that row behind for good, and the row is now carrying a + lock file. The report is the event that knows, so the report is where the forgetting goes. + + **The request goes with it.** A pending *open this* whose artefact has just been dropped would + reach the dispatcher with nothing to commit, and the honest state of that row is *never asked* + rather than *asked and failed*: nobody asked for a pull request against a version this project + stopped pinning. + """ + from hullwork.models import UpgradeVerdict + + pinned = { + (str(one.get("package") or ""), str(one.get("version") or "")) for one in findings + } + forgotten = 0 + carrying = ( + session.query(UpgradeVerdict) + .filter( + UpgradeVerdict.project_id == project_id, + UpgradeVerdict.artefact.is_not(None), + ) + .all() + ) + for verdict in carrying: + if (verdict.package, verdict.was) in pinned: + continue + verdict.artefact = None + verdict.asked_to_open_at = None + forgotten += 1 + if forgotten: + log.info( + "forgot what stale verdicts passed with", + extra={"project_id": project_id, "verdicts": forgotten}, + ) + return forgotten + + +def open_requested( + session: Session, + code_forge: object | None, + *, + secrets: list[str] | None = None, +) -> str | None: + """Open one pull request a person asked for, or `None`. Item 245, DR-0026's other half. + + **DR-0026 said this was a button and the button had nowhere to press.** The receiver renders the + page and cannot open anything — it refuses to start holding a credential that can push, and that + refusal is load-bearing (DR-0009, spec M2 §1) — so the page writes an intention on the verdict + and this reads it, in the process that holds the code token and binds no socket. + + **One per turn, oldest first**, for the same reason `open_them` opens one per package: a click + that produces thirty-one pull requests is not a convenience. + + **A missing credential is not the verdict's fault.** It leaves the request pending and says so, + because spending somebody's request on a dispatcher that was misconfigured for an afternoon + would make them press a button that can never work twice. + """ + from hullwork.models import DependencyReport, UpgradeVerdict + from hullwork.models import Project as ProjectRow + + asked = ( + session.query(UpgradeVerdict) + .filter( + UpgradeVerdict.asked_to_open_at.is_not(None), + UpgradeVerdict.opened_where.is_(None), + UpgradeVerdict.open_note.is_(None), + ) + .order_by(UpgradeVerdict.asked_to_open_at) + .first() + ) + if asked is None: + return None + if code_forge is None: + log.warning( + "somebody asked for an upgrade to be opened and this process cannot push", + extra={"package": asked.package, "to": asked.to}, + ) + return None + + project = session.get(ProjectRow, asked.project_id) + if project is None: # pragma: no cover - a foreign key says otherwise + return None + pair = f"{asked.package} {asked.was} → {asked.to}" + + def refuse(why: str) -> str: + asked.open_note = why + session.commit() + log.info("did not open", extra={"package": asked.package, "why": why}) + return f"{project.slug}: {pair} was not opened — {why}" + + permitted = False + if project.manifest: + from hullwork.manifest import parse_manifest + + permitted = parse_manifest(json.dumps(project.manifest)).autofix.open_upgrades + if not permitted: + # **The manifest outranks the button, and this is where a race lands** (DR-0019). The page + # does not offer the control without the permission, so arriving here means it was withdrawn + # between the click and the turn — which is the project changing its mind, and it wins. + return refuse( + "this project has not permitted opening upgrades: set " + "`autofix: {open_upgrades: true}` in its manifest" + ) + + report = session.get(DependencyReport, project.id) + still_pinned = report is not None and any( + one.get("package") == asked.package and one.get("version") == asked.was + for one in (report.findings or []) + ) + if not still_pinned: + return refuse( + f"{asked.package} {asked.was} is not what this project pins any more, so what was " + f"verified is not what would be opened" + ) + + answer = answer_from(asked) + if answer is None: + return refuse( + "the files this verdict passed with were not kept, so there is nothing to open" + ) + if not asked.base_sha: + return refuse( + "the commit this verdict was verified at was not kept, so a branch has no root" + ) + + advisories: dict[str, Sequence[Advisory]] = {} + if report is not None: + for one in report.findings or []: + if one.get("package") != asked.package: + continue + advisories[asked.package] = tuple( + Advisory( + id=str(each.get("id") or ""), + summary=str(each.get("summary") or ""), + fixed=tuple(str(version) for version in (each.get("fixed") or [])), + ) + for each in (one.get("advisories") or []) + ) + + where = _open_one( + code_forge, + repo=project.repo, + answer=answer, + advisories=tuple(advisories.get(asked.package, ())), + base_sha=asked.base_sha, + secrets=secrets, + ) + if where is None: + # Never silence (item 178's rule, one layer along): a request that produced nothing is + # either already open from an earlier run or something the forge refused, and both are facts + # the person who pressed the button needs to read without opening a log. + return refuse("already open from an earlier run, or the forge refused it") + + asked.opened_where = where + # **The artefact has done its job.** The pull request now holds those exact files, and keeping a + # second copy in the database is the unbounded half of this feature's cost. + asked.artefact = None + session.commit() + return f"{project.slug}: {pair} → {where}" + + +def watch_opened( + session: Session, forge: object, *, now: datetime | None = None +) -> str | None: + """Ask the forge what became of one opened pull request. Item 253. + + **`opened_where` was written once and never read back**, so the page said *a draft pull request + is waiting for a person* about two that had been merged days before — and would have said it for + ever about one somebody closed without merging, which displays their explicit "no" as work they + still owe. Measured on the live instance on 14 August: two rows *already open*, both `merged` at + the forge. + + **This is `recurrence._watch_one`, one noun along**, and item 138's split is the whole of it: + *not merged* is two facts wearing one answer, and a pull request nobody has looked at is not a + pull request somebody refused. + + One per turn, oldest unchecked first, and never while rendering — a forge request per render is + what item 142 forbids. A forge that will not answer changes nothing: the row keeps saying what + it last knew, because a verdict written for a bad afternoon is worse than a stale one. + + A read, so the read credential is enough; this asks about a pull request and writes nothing to + any repository. + """ + from hullwork.forge import ForgeError as ForgeFailure + from hullwork.models import Project as ProjectRow + from hullwork.models import UpgradeVerdict + from hullwork.outcomes import rejection_reason + + moment = now or datetime.now(UTC) + cutoff = moment - timedelta(seconds=RECHECK_SECONDS) + watching = ( + session.query(UpgradeVerdict) + .filter( + UpgradeVerdict.opened_where.is_not(None), + # Terminal states are never asked about again: merged is merged, and a person who closed + # one has answered. This is where the watch stops costing requests (item 121's lesson). + # + # **The `IS NULL` half is not decoration.** `NULL NOT IN (…)` is `NULL` in SQL, which is + # not true, so `notin_` alone excludes every row that has never been asked about — which + # is every row that exists when this ships, and the only ones with anything to learn. + # Written that way first; the watcher did nothing at all and said nothing about it. + sa.or_( + UpgradeVerdict.opened_state.is_(None), + UpgradeVerdict.opened_state.notin_(("merged", "closed")), + ), + sa.or_( + UpgradeVerdict.open_checked_at.is_(None), + UpgradeVerdict.open_checked_at < cutoff, + ), + ) + .order_by(UpgradeVerdict.open_checked_at.is_not(None), UpgradeVerdict.open_checked_at) + .first() + ) + if watching is None or forge is None: + return None + number = _pull_request_number(str(watching.opened_where)) + project = session.get(ProjectRow, watching.project_id) + if number is None or project is None: + # Permanent, so it is recorded rather than retried: a stored reference with no number in it + # cannot be asked about however many times this runs. `recurrence._settled`'s reasoning. + watching.opened_state = "unreadable" + watching.open_checked_at = moment + session.commit() + return None + + try: + state = forge.merge_state(project.repo, number) # type: ignore[attr-defined] + except ForgeFailure as exc: + log.warning( + "the forge could not be asked about an opened upgrade", extra={"error": str(exc)} + ) + return None + + watching.open_checked_at = moment + pair = f"{watching.package} {watching.was} → {watching.to}" + if state.merged: + watching.opened_state = "merged" + session.commit() + return f"{project.slug}: {pair} was merged" + if state.state == "closed": + watching.opened_state = "closed" + why = rejection_reason(state.labels) + # **Never silence** (item 178). A reviewer who gave no reason is a fact about the review, + # not a blank to fill in — `rejection_reason` answers `None` for exactly that and this says + # so rather than inventing one. + watching.open_note = ( + f"a person closed the pull request without merging — {why}" + if why + else "a person closed the pull request without merging, and gave no reason" + ) + session.commit() + return f"{project.slug}: {pair} was closed without merging" + watching.opened_state = "open" + session.commit() + return None + + +def _pull_request_number(where: str) -> int | None: + """The number out of a stored pull request URL, or `None` if there is not one in it.""" + found = re.search(r"(\d+)\s*$", where.strip().rstrip("/")) + return int(found.group(1)) if found else None + + +def next_to_try( + session: Session, project_id: int, findings: Sequence[Mapping[str, Any]] +) -> tuple[str, str, str] | None: + """The oldest `(package, was, to)` this instance has no current verdict for, or `None`. + + **A verdict is about a pair of versions, not about a package.** `cryptography 48.0.1 → 49.0.0` + and `48.0.1 → 50.0.0` are two questions with two answers, and OSV publishes both when an + advisory was fixed on two release branches. + """ + from hullwork.models import UpgradeVerdict + + for one in findings: + was = str(one.get("version") or "") + package = str(one.get("package") or "") + advisories = one.get("advisories") or [] + for to in dict.fromkeys( + str(version) + for advisory in advisories + for version in (advisory.get("fixed") or []) + ): + # **A published version older than the one you pin is not a fix you can take** (item + # 243). OSV publishes one per release branch — `brace-expansion` is fixed in 1.1.18, + # 2.1.4, 3.0.6 *and* 5.0.9 — and this tried every one of them against a project pinned + # at 5.0.6, at five minutes each, to be told the resolver will not go backwards. On a + # resolver that would accept it, taking it is a regression shipped as a security fix. + # + # `None` means neither version could be read as one, and then it is tried: OSV carries + # `1.2.3.RELEASE` and `2024-11-01` among the ordinary ones, and a rule that guessed + # would hide a real fix. + if dependencies.newer(to, was) is False: + continue + already = ( + session.query(UpgradeVerdict) + .filter( + UpgradeVerdict.project_id == project_id, + UpgradeVerdict.package == package, + UpgradeVerdict.was == was, + UpgradeVerdict.to == to, + ) + .one_or_none() + ) + if already is None: + return package, was, to + return None + + +def _its_baseline_was_red(session: Session, project_id: int, taken_at: datetime) -> bool: + """Whether this project's own suite was failing the last time anything was tried. Item 234. + + **The baseline is a property of the project at a commit, not of the upgrade.** `simplecheck`'s + suite cannot reach a database inside the sandbox, so item 233's first hour on atlas spent a + clone, an image build and a suite run per pair to print *your suite was already failing* fifty + times over. Measuring it once answers every question in that queue at the same time. + + The way back in is a **new report**, which this instance takes on its own clock every six hours: + a repository that fixes its suite is picked up again without anybody typing anything, and one + that does not costs four builds a day instead of one a minute. + """ + from hullwork.models import UpgradeVerdict + + latest = ( + session.query(UpgradeVerdict) + .filter(UpgradeVerdict.project_id == project_id) + .order_by(UpgradeVerdict.tried_at.desc()) + .first() + ) + if latest is None or latest.outcome != "already-red": + return False + tried_at = latest.tried_at + if tried_at.tzinfo is None: + tried_at = tried_at.replace(tzinfo=UTC) + asked_at = taken_at if taken_at.tzinfo is not None else taken_at.replace(tzinfo=UTC) + return tried_at >= asked_at + + +def verify_next( + session: Session, + settings: Settings, + *, + clone: Callable[..., Path], + say: Callable[[str | None], None] = lambda _: None, +) -> str | None: + """Try one published fix, in a clone, and keep what happened. DR-0026, item 233. + + **The read credential, not the code one.** A verification writes nothing to a repository — that + is the whole of what DR-0026 decided — so it clones with the token that cannot push, and the + property holds by construction rather than by care. + + One per turn, and only where a bug is not waiting: a production error outranks a dependency + upgrade, and the loop calls this after `work.run` found nothing. + """ + import io + import tempfile + + from hullwork import trial + from hullwork.manifest import parse_manifest + from hullwork.models import DependencyReport, UpgradeVerdict + from hullwork.models import Project as ProjectRow + + projects = ( + session.query(ProjectRow) + .filter(ProjectRow.active.is_(True)) + .order_by(ProjectRow.id) + .all() + ) + for project in projects: + report = session.get(DependencyReport, project.id) + if report is None or not report.asked or not report.findings or not project.manifest: + continue + if _its_baseline_was_red(session, project.id, report.taken_at): + continue + chosen = next_to_try(session, project.id, report.findings) + if chosen is None: + continue + package, was, to = chosen + manifest = parse_manifest(json.dumps(project.manifest)) + if manifest.runtime is None or not manifest.tests: + continue + + # **What it is doing, said as it happens** (item 242). Four to five minutes pass between + # here and a verdict — a clone, an image build and the project's own suite run twice — and + # the page called all of it *nothing in progress*. + pair = f"{package} {was} → {to}" + said = io.StringIO() + with tempfile.TemporaryDirectory() as where: + say(f"{project.slug}: cloning to try {pair}") + worktree = clone(settings, project, Path(where)) + paths = [ + str(one.relative_to(worktree)) + for one in worktree.rglob("*") + if one.is_file() and ".git/" not in str(one) + ] + found = [ + one + for one in report.findings + if one.get("package") == package and one.get("version") == was + ] + source = str(found[0].get("source")) if found else "" + # **Bound, because `worktree` is a loop variable.** A closure over it would read + # whichever project the loop was on when it ran, which is exactly the bug that is + # invisible until there are two projects. + def _read(path: str, tree: Path = worktree) -> str: + return (tree / path).read_text(encoding="utf-8", errors="replace") + + say(f"{project.slug}: verifying {pair}") + report_of = verify_one( + worktree, + paths, + _read, + manifest, + dependencies.Dependency("PyPI", package, was, source), + [to], + said, + ) + # **Read here or never**: the clone goes with the `with`, and this is the only commit a + # pull request opened later may be rooted at. `head_sha` answers `working tree` for a + # directory that is not a repository — impossible for a clone, and stored as *no sha* + # rather than as that string, because a branch cannot be rooted at a sentence. + verified_at: str | None = trial.head_sha(worktree) + if verified_at == "working tree": # pragma: no cover - a clone is always a repository + verified_at = None + + # **A `Report` holds one answer per candidate**, and one candidate was asked for. No + # answer at all is the build refusing before anything could be tried, which is + # `will-not-install` — a different fact from the suite failing, and DR-0026 says so. + answered = report_of.answers[0] if report_of is not None and report_of.answers else None + outcome = answered.verdict.value if answered is not None else "will-not-install" + session.merge( + UpgradeVerdict( + project_id=project.id, + package=package, + was=was, + to=to, + outcome=outcome, + detail=(answered.detail if answered is not None else said.getvalue())[:4000], + # **Kept only for a verdict somebody could act on** (item 245). A `breaks` has a + # finding and nothing to open; an `already-red` has neither. Storing files for + # those would be paying for a button that must never exist. + artefact=keepable(answered) if answered is not None else None, + base_sha=verified_at, + ) + ) + session.commit() + return f"{project.slug}: {package} {was} → {to} is {outcome}" + return None + + +def keepable(answer: bump.Answer) -> dict[str, Any] | None: + """What a clean verdict has to keep to be openable later, or `None`. Item 245. + + **Both halves or neither.** The files are what gets committed; the runs are the evidence the + pull request body is mostly made of. Keeping the files alone would produce a pull request that + is quietly thinner than the one `hullwork deps --open` produces from the same verdict, and two + surfaces disagreeing about what was measured is the failure this repository keeps finding. + + **Text and not bytes**, because every dependency file a resolver writes is text, and a base64 + blob in a database is a thing nobody can read when they are trying to work out what a pull + request would contain. A file that does not decode keeps **no** artefact rather than part of + one: a commit missing a file it needed is worse than a button that is not there, and the + verdict — which is the valuable half — stands either way. + """ + if answer.verdict is not bump.Verdict.CLEAN or not answer.files: + return None + files: dict[str, str] = {} + for path, blob in answer.files.items(): + try: + files[path] = blob.decode("utf-8") + except UnicodeDecodeError: + log.warning( + "keeping no artefact: a dependency file is not text", + extra={"package": answer.package, "path": path}, + ) + return None + runs = answer.runs + return { + "files": files, + "runs": None + if runs is None + else { + "command": runs.command, + "before_exit": runs.before_exit, + "after_exit": runs.after_exit, + "before_summary": runs.before_summary, + "after_summary": runs.after_summary, + }, + } + + +def answer_from(verdict: object) -> bump.Answer | None: + """Rebuild the answer a stored artefact describes, or `None` when it cannot be opened. + + **The inverse of `keepable`, and it refuses rather than improvises.** A row with no artefact, or + one whose artefact has no files, is not an answer with something missing — it is a verdict that + was never openable, and `_open_one` must never be reached with an empty file set: that would + branch, commit nothing and open a pull request claiming an upgrade it does not contain. + """ + kept = getattr(verdict, "artefact", None) + if not isinstance(kept, Mapping): + return None + files = kept.get("files") + if not isinstance(files, Mapping) or not files: + return None + said = kept.get("runs") + return bump.Answer( + verdict=bump.Verdict.CLEAN, + package=str(getattr(verdict, "package", "")), + was=str(getattr(verdict, "was", "")), + to=str(getattr(verdict, "to", "")), + detail=str(getattr(verdict, "detail", "") or ""), + files={str(path): str(text).encode("utf-8") for path, text in files.items()}, + runs=None + if not isinstance(said, Mapping) + else bump.Runs( + command=str(said.get("command") or ""), + before_exit=int(said.get("before_exit") or 0), + after_exit=int(said.get("after_exit") or 0), + before_summary=str(said.get("before_summary") or ""), + after_summary=str(said.get("after_summary") or ""), + ), + ) diff --git a/migrations/versions/a91c4e75d203_what_is_published_against_what_you_pin.py b/migrations/versions/a91c4e75d203_what_is_published_against_what_you_pin.py new file mode 100644 index 0000000..ad46de4 --- /dev/null +++ b/migrations/versions/a91c4e75d203_what_is_published_against_what_you_pin.py @@ -0,0 +1,41 @@ +"""What is published against what you pin, and when that was asked. + +DR-0024, accepted 2026-08-11: the receiver may fetch the lock files it can already read, ask OSV on +its own clock, and keep the answer — so the half of the product that needs no model, no write +credential and no Docker stops being invisible from a browser. + +One row per project, overwritten. **`asked` and `taken_at` are the operator's two conditions on +accepting it**: a report with no timestamp is a claim about a moment presented as a standing fact, +and an advisory list that silently reads empty when the network was down says *you are fine* on no +evidence at all. + +No application imports, so this revision keeps describing the schema as it was when it was written. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "a91c4e75d203" +down_revision: str | None = "b2e9f47a10c3" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "dependency_reports", + sa.Column("project_id", sa.Integer(), nullable=False), + sa.Column("taken_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("asked", sa.Boolean(), nullable=False), + sa.Column("note", sa.Text(), nullable=True), + sa.Column("pinned", sa.Integer(), nullable=False), + sa.Column("findings", sa.JSON(), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"]), + sa.PrimaryKeyConstraint("project_id"), + ) + + +def downgrade() -> None: + op.drop_table("dependency_reports") diff --git a/migrations/versions/a9f4d1c07b83_the_verdict_carries_what_it_passed_with.py b/migrations/versions/a9f4d1c07b83_the_verdict_carries_what_it_passed_with.py new file mode 100644 index 0000000..a6668a5 --- /dev/null +++ b/migrations/versions/a9f4d1c07b83_the_verdict_carries_what_it_passed_with.py @@ -0,0 +1,48 @@ +"""The verdict carries what it passed with, and whether somebody asked for it. Item 245. + +`verify_next` produced an `Answer` holding the dependency files as the passing run saw them, wrote +six columns and dropped the rest. So a clean verdict could be read and never acted on: opening a +pull request needs those exact files — a lock regenerated a second time can differ, and publishing +files the suite did not pass against is the defect item 045 is named after — and the sha they were +verified at, which is what roots the branch. + +Five columns rather than a table: it is the same row's business. Three of them are the request that +DR-0026 always intended (*open stays a button somebody presses*) and never had anywhere to live — +the receiver cannot open anything, so the page writes an intention here and the dispatcher, which +holds the code credential, reads it. + +`artefact` is text keyed by path, not bytes: every dependency file a resolver writes is text, and one +that is not stores no artefact rather than a base64 blob nobody can read in a database browser. + +Revision ID: a9f4d1c07b83 +Revises: f4b2e8d71a05 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "a9f4d1c07b83" +down_revision = "f4b2e8d71a05" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("upgrade_verdicts", sa.Column("artefact", sa.JSON(), nullable=True)) + op.add_column("upgrade_verdicts", sa.Column("base_sha", sa.String(length=64), nullable=True)) + op.add_column( + "upgrade_verdicts", + sa.Column("asked_to_open_at", sa.DateTime(timezone=True), nullable=True), + ) + op.add_column("upgrade_verdicts", sa.Column("opened_where", sa.Text(), nullable=True)) + op.add_column("upgrade_verdicts", sa.Column("open_note", sa.Text(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("upgrade_verdicts", "open_note") + op.drop_column("upgrade_verdicts", "opened_where") + op.drop_column("upgrade_verdicts", "asked_to_open_at") + op.drop_column("upgrade_verdicts", "base_sha") + op.drop_column("upgrade_verdicts", "artefact") diff --git a/migrations/versions/b1e7c3d94f26_the_emptied_artefacts_read_as_empty.py b/migrations/versions/b1e7c3d94f26_the_emptied_artefacts_read_as_empty.py new file mode 100644 index 0000000..8b12071 --- /dev/null +++ b/migrations/versions/b1e7c3d94f26_the_emptied_artefacts_read_as_empty.py @@ -0,0 +1,41 @@ +"""The emptied artefacts read as empty from SQL too. Item 245. + +`artefact` was a plain `JSON` column, so assigning `None` wrote the JSON text `null` rather than SQL +`NULL`. Four bytes — the 400 KB really is released, and the row reads `None` in Python — but +`WHERE artefact IS NOT NULL` counts it, and `forget_stale` filters on exactly that predicate. + +Measured the minute it mattered: the check run just after the first pull request was opened reported +**two** artefacts where the database held one, because the one that had just been handed to the forge +was still matching. A count that quietly disagrees with what is there is the failure this repository +spends its items on, so the column now declares `none_as_null` and the rows written before it are +normalised here. + +Rewritten rather than left: a predicate that is right for new rows and wrong for old ones is worse +than one that is wrong everywhere, because nothing tells you which half you are reading. + +Revision ID: b1e7c3d94f26 +Revises: a9f4d1c07b83 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "b1e7c3d94f26" +down_revision = "a9f4d1c07b83" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # `'null'` is the only JSON text that means *nothing kept*; a real artefact is an object. + op.execute( + sa.text("UPDATE upgrade_verdicts SET artefact = NULL WHERE artefact = 'null'") + ) + + +def downgrade() -> None: + # **Nothing.** The two states mean the same thing to every reader of this column, and writing + # `'null'` back would restore a distinction whose only effect was a wrong count. + pass diff --git a/migrations/versions/b2e9f47a10c3_the_token_is_the_fact_not_the_account.py b/migrations/versions/b2e9f47a10c3_the_token_is_the_fact_not_the_account.py new file mode 100644 index 0000000..5f37454 --- /dev/null +++ b/migrations/versions/b2e9f47a10c3_the_token_is_the_fact_not_the_account.py @@ -0,0 +1,39 @@ +"""The token is the fact, not the account. + +`d5a7c31f9e04` added `ingest_can_push`, and item 228 filled it from `PushCapability.can_push` — which +is **the account's** access to the repository, not the token's. `credentials.py` says so in a +docstring written after measuring exactly this: *a token scoped to reads and issues is refused +regardless — measured on the live instance, where this flag fired for both projects while +`POST /branches` came back `403 … scope(s): [write:repository]`*. + +So the first thing the new clock did was record `True` for a correctly configured project, and the +page was one deploy from painting a permanent red *its ingest credential CAN push, which DR-0009 +forbids* over an instance where the credential cannot. That is the permanently-on signal item 073 +deleted a whole check for, rebuilt by hand three items later. + +The column carries `token_can_push` now, and its name says which of the two questions it answers. +Every existing value is dropped rather than migrated: they answer the other question. + +No application imports, so this revision keeps describing the schema as it was when it was written. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "b2e9f47a10c3" +down_revision: str | None = "d5a7c31f9e04" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.drop_column("projects", "ingest_can_push") + op.add_column("projects", sa.Column("ingest_token_can_push", sa.Boolean(), nullable=True)) + op.execute(sa.text("UPDATE projects SET ingest_checked_at = NULL")) + + +def downgrade() -> None: + op.drop_column("projects", "ingest_token_can_push") + op.add_column("projects", sa.Column("ingest_can_push", sa.Boolean(), nullable=True)) diff --git a/migrations/versions/c4d81b6ea295_what_happened_when_it_tried_the_fix.py b/migrations/versions/c4d81b6ea295_what_happened_when_it_tried_the_fix.py new file mode 100644 index 0000000..107ddd1 --- /dev/null +++ b/migrations/versions/c4d81b6ea295_what_happened_when_it_tried_the_fix.py @@ -0,0 +1,49 @@ +"""What happened when it tried the fix. + +DR-0026, accepted 2026-08-12: the dispatcher may verify an upgrade on its own clock and may not open +one. This is where the verdict goes. + +One row per `(project, package, was, to)`, overwritten: *does this upgrade hold today* is one +question, and yesterday's answer about the same pair is the same fact gone stale rather than a +second one. + +`was` is stored because a verdict about a version that is no longer pinned reads as current and is +not — the page compares it against the report before showing anything. + +No application imports, so this revision keeps describing the schema as it was when it was written. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "c4d81b6ea295" +down_revision: str | None = "a91c4e75d203" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "upgrade_verdicts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("project_id", sa.Integer(), nullable=False), + sa.Column("package", sa.String(length=200), nullable=False), + sa.Column("was", sa.String(length=100), nullable=False), + sa.Column("to", sa.String(length=100), nullable=False), + sa.Column("outcome", sa.String(length=30), nullable=False), + sa.Column("detail", sa.Text(), nullable=True), + sa.Column("tried_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("project_id", "package", "was", "to", name="uq_upgrade_verdict"), + ) + op.create_index( + op.f("ix_upgrade_verdicts_project_id"), "upgrade_verdicts", ["project_id"], unique=False + ) + + +def downgrade() -> None: + op.drop_index(op.f("ix_upgrade_verdicts_project_id"), table_name="upgrade_verdicts") + op.drop_table("upgrade_verdicts") diff --git a/migrations/versions/c4d8a1f36b92_what_became_of_the_pull_request.py b/migrations/versions/c4d8a1f36b92_what_became_of_the_pull_request.py new file mode 100644 index 0000000..1dadbd5 --- /dev/null +++ b/migrations/versions/c4d8a1f36b92_what_became_of_the_pull_request.py @@ -0,0 +1,39 @@ +"""What became of the pull request, asked rather than assumed. Item 253. + +`opened_where` was written the moment the dispatcher opened one and never read back, so the page +said *a draft pull request is waiting for a person* about pull requests that had been merged days +before — and, worse, about ones a person had closed without merging, which displayed their explicit +"no" as work they still owed. + +Measured on the live instance before this ran: two rows reading *already open*, both `merged=True` +at the forge. + +`opened_state` is `NULL` for every existing row on purpose. **Not backfilled to `'open'`**: the +difference between *nobody has asked yet* and *the forge said it is open* is the whole point of the +column, and a backfill would erase it on exactly the rows that need asking first. + +Revision ID: c4d8a1f36b92 +Revises: b1e7c3d94f26 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "c4d8a1f36b92" +down_revision = "b1e7c3d94f26" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("upgrade_verdicts", sa.Column("opened_state", sa.String(10), nullable=True)) + op.add_column( + "upgrade_verdicts", sa.Column("open_checked_at", sa.DateTime(timezone=True), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("upgrade_verdicts", "open_checked_at") + op.drop_column("upgrade_verdicts", "opened_state") diff --git a/migrations/versions/d5a7c31f9e04_what_the_ingest_credential_may_do.py b/migrations/versions/d5a7c31f9e04_what_the_ingest_credential_may_do.py new file mode 100644 index 0000000..34665a4 --- /dev/null +++ b/migrations/versions/d5a7c31f9e04_what_the_ingest_credential_may_do.py @@ -0,0 +1,41 @@ +"""What the ingest credential may do, and when that was measured. + +Two nullable columns on `projects`, for the answer to the question the whole two-program split +exists to guarantee: **can the credential this instance ingests with also push code?** DR-0009 +forbids it, `credentials.audit` measures it, and until item 228 the page read the answer out of a +key inside the manifest JSON that **nothing ever wrote** — so it said *not asked yet* for the life +of every instance, and running the command it told you to run would not have changed that. + +A column rather than a key in the manifest, because the manifest is the project's own document +adopted verbatim (DR-0012) and instance-measured facts have no business inside it. + +**Nullable, and the reason is the same one it always is.** `NULL` is *not measured*, which is a +third answer and never a `False`: reporting *cannot push* for a project nobody has asked about is +the mistake this project has now made in three different places. + +`checked_at` alongside, because an answer with no timestamp is the permanently-on signal item 073 +deleted a whole check for. A verdict from three weeks ago is a different thing from one from ten +minutes ago, and only the row can say which it is. + +No application imports, so this revision keeps describing the schema as it was when it was written. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "d5a7c31f9e04" +down_revision: str | None = "c8f4a1d63b27" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("projects", sa.Column("ingest_can_push", sa.Boolean(), nullable=True)) + op.add_column("projects", sa.Column("ingest_checked_at", sa.DateTime(timezone=True), nullable=True)) + + +def downgrade() -> None: + op.drop_column("projects", "ingest_checked_at") + op.drop_column("projects", "ingest_can_push") diff --git a/migrations/versions/e0a7f31c88b2_a_verdict_taken_without_its_database.py b/migrations/versions/e0a7f31c88b2_a_verdict_taken_without_its_database.py new file mode 100644 index 0000000..2d9a844 --- /dev/null +++ b/migrations/versions/e0a7f31c88b2_a_verdict_taken_without_its_database.py @@ -0,0 +1,37 @@ +"""A verdict taken without the database the project declared. Item 238. + +`upgrades.verify_one` built its sandbox without the services the manifest asks for, so any project +declaring `postgres-16` ran its suite against nothing listening on 5432 and was recorded +`already-red`. That verdict is DR-0026's honest one — *your suite was failing before anything was +touched, so no claim can be made either way* — and here it was **the truth about the wrong thing**: +the suite was not failing, it was never given what it asked for. + +Eleven of them on the operator's own instance, and item 234 then stopped the queue on the most +recent one, so nothing would have re-asked them without this. + +**Deleted rather than corrected**, because there is nothing to correct them to: the question was +never asked. A row that is gone is re-asked on the next idle turn, which is exactly the behaviour +wanted. And an `already-red` that was genuine — a project whose suite really is failing — costs one +verification to say so again. + +Revision ID: e0a7f31c88b2 +Revises: c4d81b6ea295 +""" + +from __future__ import annotations + +from alembic import op + +revision = "e0a7f31c88b2" +down_revision = "c4d81b6ea295" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("DELETE FROM upgrade_verdicts WHERE outcome = 'already-red'") + + +def downgrade() -> None: + """Nothing to put back. The rows said a thing that was not measured, and re-taking them is + what the dispatcher does on its own clock.""" diff --git a/migrations/versions/f4b2e8d71a05_what_the_dispatcher_is_doing.py b/migrations/versions/f4b2e8d71a05_what_the_dispatcher_is_doing.py new file mode 100644 index 0000000..7f1be42 --- /dev/null +++ b/migrations/versions/f4b2e8d71a05_what_the_dispatcher_is_doing.py @@ -0,0 +1,35 @@ +"""What the dispatcher is doing, on the lease. Item 242. + +The instance report said *nothing in progress* while a verification built an image and ran somebody +else's suite twice, because it read `Item.state == IN_PROGRESS` and a dependency verification is not +an item. Nothing a page could infer covers the gap between two writes, and that gap is exactly the +four minutes an operator is trying to watch. + +On the lease rather than in a table of its own: it is the same fact as *who is dispatching now*, and +two rows could disagree about whether one exists. + +Revision ID: f4b2e8d71a05 +Revises: e0a7f31c88b2 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "f4b2e8d71a05" +down_revision = "e0a7f31c88b2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("dispatcher_lease", sa.Column("doing", sa.String(length=200), nullable=True)) + op.add_column( + "dispatcher_lease", sa.Column("doing_since", sa.DateTime(timezone=True), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("dispatcher_lease", "doing_since") + op.drop_column("dispatcher_lease", "doing") diff --git a/tests/test_a_fact_should_not_wait.py b/tests/test_a_fact_should_not_wait.py new file mode 100644 index 0000000..46be924 --- /dev/null +++ b/tests/test_a_fact_should_not_wait.py @@ -0,0 +1,291 @@ +"""The push audit runs on the instance's own clock, and what is fine says nothing. Item 228. + +The operator, reading a project's view: + +> *`not asked yet — hullwork status records this when it runs`, ¿esto porque depende de un comando? +> debería de ser automático* + +It did not depend on a command. **It depended on code that was never written**: the page read +`manifest["__ingest_can_push__"]`, a key that appears exactly twice in this repository and is read +both times. Running `hullwork status` would not have changed it either, and the docstring claiming +otherwise was wrong for two items. + +The line answers *can the credential this instance ingests with also push code* — DR-0009's whole +subject. A signal that waits for somebody to remember is not a signal, which is item 073's rule +arriving from the other side: it deleted a check that was permanently on, and this one was +permanently unknown. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, Project + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/fact.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + monkeypatch.setenv("HULLWORK_FORGE_URL", "https://forge.example") + monkeypatch.setenv("HULLWORK_FORGE_TOKEN", "t") + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + manifest={ + "project": "shop", + "git": {"provider": "forgejo", "repo": "acme/shop"}, + "errors": {"provider": "glitchtip"}, + }, + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _view(db: Session) -> str: + return page.project(db, Settings(), "shop", acting=page.Acting(csrf="c", offered=True)) or "" + + +# --- what the page says now ------------------------------------------------------------------- + + +def test_a_project_that_is_fine_says_nothing_about_it(db: Session) -> None: + """**`cached manifest validates` was the operator's second question**, and the answer is that + it should not have been on the page: three internal words describing a normal state, at the + volume of a fault. Item 203 — what is not fine, and a count of what is.""" + row = db.query(Project).one() + row.ingest_token_can_push = False + row.ingest_checked_at = dt.datetime.now(dt.UTC) + db.commit() + + shown = _view(db) + + assert "validates" not in shown + assert "What is wrong" not in shown, "a healthy project has a section headed *what is wrong*" + + +def test_a_credential_that_can_push_is_loud(db: Session) -> None: + """DR-0009's subject: the receiver must not hold a credential that can push. When the forge + says it can, that is the loudest thing this page has to say about a project.""" + row = db.query(Project).one() + row.ingest_token_can_push = True + row.ingest_checked_at = dt.datetime.now(dt.UTC) + db.commit() + + shown = _view(db) + + assert "can write code" in shown + assert "DR-0009" in shown + assert 'class="bad"' in shown + + +def test_an_unmeasured_project_says_the_instance_will_ask_itself(db: Session) -> None: + """**Not `hullwork status records this when it runs`.** That sentence sent a person to type a + command that would not have helped, and the honest one says who is going to answer it.""" + shown = _view(db) + + assert "hullwork status" not in shown + assert "not measured yet" in shown + assert "on its own clock" in shown + + +def test_a_manifest_that_no_longer_reads_is_still_loud(db: Session) -> None: + """The loudest thing that can be wrong with a project: every error from it lands red, silently + by design, and until item 142 the only way to know was to read that function.""" + row = db.query(Project).one() + row.manifest = {"project": "shop", "runtime": {"base": 5}} + row.ingest_token_can_push = False + row.ingest_checked_at = dt.datetime.now(dt.UTC) + db.commit() + + shown = _view(db) + + assert "no longer validates" in shown + assert "lands red" in shown + + +# --- and the clock measures it ---------------------------------------------------------------- + + +class _Reader: + """The forge, answering what the **account** may do — which is not the fact that is stored. + + `can_push` is the account's access. A token scoped to reads and issues is refused regardless, + and on this project's own instance that flag was `True` for both projects while `POST /branches` + answered `403 … scope(s): [write:repository]`. The probe below is what decides. + """ + + def __init__(self, can_push: bool) -> None: + self._can_push = can_push + self.asked: list[str] = [] + + def can_write_code(self, repo: str) -> bool: + self.asked.append(repo) + return self._can_push + + def close(self) -> None: + pass + + +def _sweep( + db: Session, monkeypatch: pytest.MonkeyPatch, reader: _Reader, *, probe: bool | None = False +) -> None: + """**Patched where they are defined.** The measurement imports both inside the function, so a + name bound on `hullwork.main` is a name nothing looks at — the second time that has cost a + round in this repository.""" + from hullwork import cli as cli_module + from hullwork import main as main_module + from hullwork.forge import factory + + monkeypatch.setattr(factory, "make_permission_reader", lambda settings: reader) + monkeypatch.setattr(cli_module, "_scope_probe", lambda settings: lambda repo: probe) + main_module._measure_what_the_ingest_credential_may_do( + lambda: _Scoped(db), # type: ignore[arg-type] + get_settings(), + ) + + +class _Scoped: + """The session factory's contract, over a session the test keeps open.""" + + def __init__(self, session: Session) -> None: + self._session = session + + def __enter__(self) -> Session: + return self._session + + def __exit__(self, *exc: object) -> None: + return None + + +def test_the_sweep_measures_it_with_no_command_typed( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """**The item.** The receiver already runs a clock; this is the fact that was waiting for a + person to remember.""" + reader = _Reader(can_push=False) + + _sweep(db, monkeypatch, reader) + + row = db.query(Project).one() + assert reader.asked == ["acme/shop"] + assert row.ingest_token_can_push is False + assert row.ingest_checked_at is not None, "a verdict with no timestamp is the old signal again" + + +def test_it_does_not_ask_again_before_the_interval( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two forge calls per active project per interval is the cost, and it is spent on the clock — + not once a minute because the sweep runs once a minute.""" + reader = _Reader(can_push=False) + _sweep(db, monkeypatch, reader) + _sweep(db, monkeypatch, reader) + + assert reader.asked == ["acme/shop"], "it asked again inside the interval" + + +def test_it_asks_again_once_the_answer_is_old( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """A verdict from three weeks ago is a different thing from one from ten minutes ago.""" + reader = _Reader(can_push=False) + _sweep(db, monkeypatch, reader) + row = db.query(Project).one() + row.ingest_checked_at = dt.datetime.now(dt.UTC) - dt.timedelta(days=21) + db.commit() + + _sweep(db, monkeypatch, reader) + + assert len(reader.asked) == 2 + + +def test_rendering_a_page_spends_no_forge_request( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """Item 142's rule, restated where the answer moved: the page reads a column, and a reader + refreshing must not cost somebody their forge quota.""" + from hullwork.forge import factory + + def _refuse(settings: object) -> object: + raise AssertionError("a page render asked the forge") + + monkeypatch.setattr(factory, "make_permission_reader", _refuse) + + assert re.search(r"

    ", _view(db)), "the view did not render at all" + + +def test_the_sweep_itself_calls_it(db: Session, monkeypatch: pytest.MonkeyPatch) -> None: + """**Written because a mutation escaped.** Every test above calls the measurement directly, so + deleting the one line that wires it into the sweep left them all green — a function that works + and is called by nothing, which is the same shape as a route with no button (item 223). + """ + from hullwork import cli as cli_module + from hullwork import main as main_module + from hullwork.forge import factory + from hullwork.ingest import SweepResult + + reader = _Reader(can_push=False) + monkeypatch.setattr(factory, "make_permission_reader", lambda settings: reader) + monkeypatch.setattr(cli_module, "_scope_probe", lambda settings: None) + monkeypatch.setattr(main_module, "make_forge", lambda settings: None) + monkeypatch.setattr(main_module, "make_tracker", lambda settings: None) + monkeypatch.setattr(main_module, "make_inventory", lambda settings: None) + nothing = SweepResult(deliveries=0, filed=0, resolved=0) + monkeypatch.setattr(main_module, "sweep", lambda *a, **k: nothing) + + main_module._sweep_once(lambda: _Scoped(db), get_settings()) # type: ignore[arg-type] + + assert reader.asked == ["acme/shop"], "the sweep does not measure it" + + +def test_the_accounts_access_is_not_what_is_stored( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """**The alarm I nearly shipped.** The first thing the new clock recorded on the operator's own + instance was `True` — from `can_push`, which is the *account's* access to the repository. Their + token is refused with `403 … scope(s): [write:repository]`, which is what the module's own + docstring describes measuring — so the page was one deploy from a permanent red *your + credential can push* over a correct configuration. + + That is item 073's permanently-on signal, rebuilt by hand three items after it was deleted. + """ + reader = _Reader(can_push=True) + + _sweep(db, monkeypatch, reader, probe=False) + + row = db.query(Project).one() + assert row.ingest_token_can_push is False, "the account's answer was stored as the token's" + assert "can write code" not in _view(db) + + +def test_a_token_that_really_can_write_code_is_loud( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """And the other direction, because a check that can only be quiet is not a check: when the + probe says a request only a code scope allows was accepted, that is the fiction DR-0009 exists + to prevent and it is measured rather than inferred.""" + reader = _Reader(can_push=True) + + _sweep(db, monkeypatch, reader, probe=True) + + assert db.query(Project).one().ingest_token_can_push is True + assert "can write code" in _view(db) diff --git a/tests/test_a_red_baseline_answers_once.py b/tests/test_a_red_baseline_answers_once.py new file mode 100644 index 0000000..e0daf20 --- /dev/null +++ b/tests/test_a_red_baseline_answers_once.py @@ -0,0 +1,203 @@ +"""A project whose own suite is already failing is asked once, not fifty times. Item 234. + +Item 233 shipped at 07:40 and the dispatcher did exactly what it was built to do: `simplecheck`'s +suite cannot reach a database inside the sandbox, so it is red before anything is touched, and every +pair in the queue got its own clone, image build and suite run to print *your suite was already +failing* again. + +`already-red` is the honest verdict — no claim can be made either way. **What is wrong is what it +costs to say it fifty times.** The baseline is a property of the project at a commit, not of the +upgrade, so measuring it once answers every question in that queue at the same time. + +The way back in is a new dependency report, taken on this instance's own clock every six hours: a +repository that fixes its suite is picked up again without anybody typing anything. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterator, Sequence +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import bump, upgrades +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, DependencyReport, Project, UpgradeVerdict + +MANIFEST = { + "project": "shop", + "git": {"provider": "forgejo", "repo": "acme/shop"}, + "errors": {"provider": "glitchtip"}, + "runtime": { + "base": "python:3.12", + "install": "pip install -r requirements.txt", + "dependencies": ["requirements.txt"], + }, + "tests": "pytest", +} + +FINDING = { + "package": "cryptography", + "version": "48.0.1", + "source": "requirements.txt", + "advisories": [{"id": "GHSA-g6cj", "summary": "one", "fixed": ["49.0.0", "50.0.0"]}], +} + +AN_HOUR_AGO = dt.datetime.now(dt.UTC) - dt.timedelta(hours=1) + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/baseline.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + monkeypatch.setenv("HULLWORK_FORGE_URL", "https://forge.example") + monkeypatch.setenv("HULLWORK_FORGE_TOKEN", "read-only") + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + project = Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + manifest=MANIFEST, + ) + session.add(project) + session.flush() + # **Taken an hour ago**, so a verdict can be placed on either side of it. A report taken *now* + # could only ever be older than the verdict, which is one of the two states under test. + session.merge( + DependencyReport( + project_id=project.id, taken_at=AN_HOUR_AGO, asked=True, pinned=50, findings=[FINDING], + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +class _Tried: + """A `verify_one` that answers as it is told and remembers being asked.""" + + def __init__(self, verdict: bump.Verdict = bump.Verdict.CLEAN) -> None: + self.verdict = verdict + self.asked: list[str] = [] + + def __call__( + self, + checkout: Path, + paths: Sequence[str], + read: object, + manifest: object, + dep: object, + versions: list[str], + out: object, + ) -> bump.Report: + self.asked.append(versions[0]) + name, was = dep.name, dep.version # type: ignore[attr-defined] + return bump.Report( + package=name, + was=was, + answers=( + bump.Answer( + verdict=self.verdict, package=name, was=was, to=versions[0], detail="1 failed" + ), + ), + ) + + +class _Cloned: + def __call__(self, settings: Settings, project: Project, into: Path) -> Path: + (into / "requirements.txt").write_text("cryptography==48.0.1\n") + return into + + +def _verdict(db: Session, outcome: str, *, when: dt.datetime, to: str = "49.0.0") -> None: + project = db.query(Project).one() + db.merge( + UpgradeVerdict( + project_id=project.id, package="cryptography", was="48.0.1", to=to, + outcome=outcome, detail="", tried_at=when, + ) + ) + db.commit() + + +def _turn(db: Session, tried: _Tried, monkeypatch: pytest.MonkeyPatch) -> str | None: + monkeypatch.setattr(upgrades, "verify_one", tried) + return upgrades.verify_next(db, get_settings(), clone=_Cloned()) + + +# --- when it stops ----------------------------------------------------------------------------- + + +def test_a_red_baseline_stops_the_queue_for_that_project( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """**The whole item.** 25 findings and ~50 published versions between them, each costing a + clone, an image build and a suite run, to print the same sentence fifty times.""" + _verdict(db, "already-red", when=dt.datetime.now(dt.UTC)) + tried = _Tried() + + said = _turn(db, tried, monkeypatch) + + assert said is None + assert tried.asked == [], "it built an image to ask a question already answered" + + +def test_a_report_taken_since_puts_it_back_in_the_queue( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """**The way back in, and it needs nobody to type anything.** A repository that fixes its suite + is picked up on this instance's own six-hourly clock — a stop with no way out of it would be a + project silently dropped for ever.""" + _verdict(db, "already-red", when=AN_HOUR_AGO - dt.timedelta(minutes=5)) + tried = _Tried() + + said = _turn(db, tried, monkeypatch) + + assert said is not None + assert tried.asked == ["50.0.0"] + + +def test_a_green_baseline_is_not_stopped_by_a_broken_upgrade( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """`breaks` is a fact about one upgrade and says nothing about the next one. Stopping on it + would turn the most valuable verdict this product produces into a reason to stop working.""" + _verdict(db, "breaks", when=dt.datetime.now(dt.UTC)) + tried = _Tried() + + said = _turn(db, tried, monkeypatch) + + assert said is not None + assert tried.asked == ["50.0.0"] + + +def test_a_baseline_that_came_back_green_is_not_held_by_the_old_red( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """Only the most recent verdict is the baseline. An older `already-red` still in the table is + history, and reading the whole table for one would stop a project that had recovered.""" + _verdict(db, "already-red", when=AN_HOUR_AGO, to="49.0.0") + _verdict(db, "clean", when=dt.datetime.now(dt.UTC), to="50.0.0") + tried = _Tried() + + monkeypatch.setattr(upgrades, "verify_one", tried) + db.merge( + DependencyReport( + project_id=db.query(Project).one().id, taken_at=AN_HOUR_AGO, asked=True, + pinned=50, + findings=[ + {**FINDING, "advisories": [{"id": "x", "summary": "s", "fixed": ["51.0.0"]}]} + ], + ) + ) + db.commit() + + assert upgrades.verify_next(db, get_settings(), clone=_Cloned()) is not None + assert tried.asked == ["51.0.0"] diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index cc91deb..754c4ad 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -429,7 +429,7 @@ def test_verify_hands_the_source_to_a_build_that_reads_it( """ import pytest - from hullwork import cli + from hullwork import upgrades from hullwork.manifest import parse_manifest asked: list[dict[str, object]] = [] @@ -455,7 +455,7 @@ def watch(runtime: object, files: object, engine: object, **kwargs: object) -> B # The first build is the baseline and is the only one this needs: it either carries the source # or it does not, and everything after it is the same call. with pytest.raises(Exception): # noqa: B017 - it stops at the sandbox, after the build - cli._verify_one( + upgrades.verify_one( tmp_path, ["requirements.txt"], lambda p: (tmp_path / p).read_text(), manifest, dependencies.Dependency("PyPI", "jinja2", "2.4.1", "requirements.txt"), @@ -475,7 +475,7 @@ def test_a_project_whose_install_does_not_read_the_source_still_gets_none( """The cheap path stays cheap: the tag does not move and the image is reused between runs.""" import pytest - from hullwork import cli + from hullwork import upgrades from hullwork.manifest import parse_manifest from hullwork.sandbox import image as image_module @@ -496,7 +496,7 @@ def watch(runtime: object, files: object, engine: object, **kwargs: object) -> B ) monkeypatch.setattr(image_module, "build", watch) # type: ignore[attr-defined] with pytest.raises(Exception): # noqa: B017 - cli._verify_one( + upgrades.verify_one( tmp_path, ["requirements.txt"], lambda p: (tmp_path / p).read_text(), manifest, dependencies.Dependency("PyPI", "jinja2", "2.4.1", "requirements.txt"), @@ -625,3 +625,201 @@ def test_a_refusal_is_counted_rather_than_only_printed() -> None: assert bump.needs_of(reports[0]) is bump.Needs.BLOCKED assert bump.summary(reports)[bump.Needs.BLOCKED] == 1 assert "not one of the files your image is built from" in printed.getvalue() + + +# --- the environment the project asked for (item 238) ------------------------------------------ + + +def _box_built_for(tmp_path: Path, declared: str, monkeypatch: object) -> dict[str, object]: + """Run `verify_one` far enough to see the sandbox it constructs, and report its arguments.""" + import io + + import pytest + + from hullwork import dependencies, upgrades + from hullwork.manifest import parse_manifest + from hullwork.sandbox import image as image_module + from hullwork.sandbox import run as run_module + + made: dict[str, object] = {} + + class Built: + tag = "img:1" + + class Box: + def __init__(self, **kwargs: object) -> None: + made.update(kwargs) + raise RuntimeError("far enough: the box exists and this is what it was given") + + monkeypatch.setattr( # type: ignore[attr-defined] + image_module, "build", lambda *a, **k: Built() + ) + monkeypatch.setattr(run_module, "Sandbox", Box) # type: ignore[attr-defined] + (tmp_path / "requirements.txt").write_text("jinja2==2.4.1\n") + manifest = parse_manifest( + "project: p\ngit: {provider: github, repo: o/r}\n" + 'tests: "pytest"\ntest_path: tests\n' + "runtime: {base: python-3.12, install: pip, " + f"dependencies: [requirements.txt]{declared}}}\n" + ) + # **`pytest.raises(Exception)` would swallow a `NameError`** and report the wrong thing passing, + # which this file has already been bitten by once. The message is checked. + with pytest.raises(RuntimeError, match="far enough"): + upgrades.verify_one( + tmp_path, ["requirements.txt"], lambda p: (tmp_path / p).read_text(), + manifest, + dependencies.Dependency("PyPI", "jinja2", "2.4.1", "requirements.txt"), + ["2.10.1"], io.StringIO(), + ) + return made + + +def test_the_sandbox_gets_the_services_the_manifest_declared( + tmp_path: Path, monkeypatch: object +) -> None: + """**Item 238, and eleven wrong verdicts on the operator's own instance.** `work.py` has passed + these since item 052; this path never did, so a project declaring `postgres-16` ran its suite + against nothing listening on 5432 and was reported `already-red`. + + That verdict is DR-0026's honest one — *no claim can be made either way* — and it was the truth + about the wrong thing: the suite was not failing, it was never given the database it asked for. + """ + made = _box_built_for(tmp_path, ", services: [postgres-16]", monkeypatch) + + assert made.get("services") == ["postgres-16"], "the box runs without the declared database" + + +def test_a_project_that_declares_none_gets_none(tmp_path: Path, monkeypatch: object) -> None: + """The other half, and the cheaper one: a project with no services must not pay for a container + it never asked for — this path is the one built to be cheap.""" + made = _box_built_for(tmp_path, "", monkeypatch) + + assert made.get("services") == [] + + +def test_the_mover_runs_where_the_finding_says_the_lock_is(monkeypatch: object) -> None: + """**Item 239, and the assertion that could not be written before it.** This lived three levels + of nesting inside `verify_one`, so *does it run beside the lock* was untestable — and it was + wrong for every monorepo while the suite stayed green. + + `simplecheck` is one: `backend/uv.lock`, `backend/pyproject.toml`, `frontend/package.json`. It + mounted the worktree root, `uv` answered *No `pyproject.toml` found in current directory or any + parent directory*, and that was recorded as `cannot-move` — a sentence about our own working + directory, stored as a fact about somebody else's repository. + """ + from pathlib import Path + + from hullwork import resolve, upgrades + + uv = resolve.resolver_for("backend/uv.lock") + assert uv is not None + asked: dict[str, object] = {} + + def upgrade(**kwargs: object) -> resolve.Result: + asked.update(kwargs) + return resolve.Result(resolve.Outcome.RESOLVED) + + monkeypatch.setattr(resolve, "upgrade", upgrade) # type: ignore[attr-defined] + move = upgrades.mover_for(uv, "backend/uv.lock", "cryptography", {"version": "50.0.0"}, []) + + assert move(Path("/w")) is None + assert asked["at"] == "backend", "the resolver runs somewhere other than beside the lock" + assert asked["version"] == "50.0.0", "the candidate is read when the mover is built, not run" + + +def test_what_is_guarded_is_where_the_repository_keeps_it(monkeypatch: object) -> None: + """`touches` names the files relative to the lock. On a monorepo a guard listing + `pyproject.toml` protects a path that does not exist, while `backend/pyproject.toml` — which the + resolver does rewrite — goes unwatched, and the next candidate's baseline describes the previous + one. That is item 174's defect, one directory over.""" + del monkeypatch + from hullwork import resolve, upgrades + + uv = resolve.resolver_for("backend/uv.lock") + assert uv is not None + + guarded = upgrades.files_touched_by(uv, "backend/uv.lock") + + assert "backend/uv.lock" in guarded + assert "backend/pyproject.toml" in guarded + assert "pyproject.toml" not in guarded + + +def test_the_images_a_verification_built_are_removed(monkeypatch: object) -> None: + """**Item 241, found by reading an hour of the dispatcher's own execution.** No errors in the + log at all — 224 lines, every one INFO — and the disk at 100%: seven sandbox images of 1.09GB, + one every six minutes, which is the verification queue's cadence. + + Each candidate builds its own image, correctly: the lock changed, so the environment is a + different environment. Nothing removed them, and eighteen candidates were still to come. + """ + from hullwork import upgrades + from hullwork.sandbox import run as run_module + + asked: list[list[str]] = [] + monkeypatch.setattr( # type: ignore[attr-defined] + "hullwork.sandbox.docker.run_docker", + lambda argv, **k: asked.append(argv), + ) + + upgrades._drop_images(["hullwork-sandbox:abc123"]) + + assert ["docker", "image", "rm", "hullwork-sandbox:abc123"] in asked + # The cache's name is derived from the tag in two places, and this is what keeps them agreeing. + owned = run_module.Sandbox(image="hullwork-sandbox:abc123", worktree=Path("/w"))._env_cache() + assert ["docker", "volume", "rm", "-f", owned] in asked + + +def test_a_disk_that_will_not_let_go_is_not_a_wrong_verdict(monkeypatch: object) -> None: + """A host that could not delete an image is a host with debris on it, which is worse than it + was and is **not** a claim about somebody's upgrade. + + This runs from a `stack.callback`, so anything raised here replaces what the verification was + already reporting: the verdict was measured, and it would arrive as a crash instead. + """ + from hullwork import upgrades + + def refuse(argv: list[str], **kwargs: object) -> None: + raise OSError("no space left on device") + + monkeypatch.setattr("hullwork.sandbox.docker.run_docker", refuse) # type: ignore[attr-defined] + + upgrades._drop_images(["x"]) # does not raise + + +def test_the_verification_removes_what_it_built_when_it_ends( + tmp_path: Path, monkeypatch: object +) -> None: + """**The half a unit test of `_drop_images` cannot reach**: that the tags are collected as they + are built and handed over when the stack unwinds. Registered on the way in rather than returned, + because a candidate that fails half way through has still built an image.""" + asked: list[list[str]] = [] + monkeypatch.setattr( # type: ignore[attr-defined] + "hullwork.sandbox.docker.run_docker", lambda argv, **k: asked.append(argv) + ) + + _box_built_for(tmp_path, "", monkeypatch) + + removed = [argv for argv in asked if argv[:3] == ["docker", "image", "rm"]] + assert removed == [["docker", "image", "rm", "img:1"]], "the image it built is still there" + + +def test_versions_are_compared_as_versions() -> None: + """`5.0.10` is newer than `5.0.9`, and sorts before it as a string — a rule built on strings + would skip the one upgrade that mattered. Item 243.""" + from hullwork.dependencies import newer + + assert newer("5.0.10", "5.0.9") is True + assert newer("5.0.9", "5.0.10") is False + assert newer("2.1.3", "5.0.6") is False + assert newer("5.0.7", "5.0.6") is True + # Different depths, and a leading `v`, which npm advisories carry. Both directions, because + # only one of them separates "pad the shorter one" from "compare what is there": `1.2.0` and + # `1.2` are the same version, and without padding the longer one reads as newer. + assert newer("v2", "1.9.9") is True + assert newer("1.2", "1.2.0") is False + assert newer("1.2.0", "1.2") is False + # Unreadable on either side is not a comparison, and the caller is told so rather than guessed + # at: `None` means *try it*. + assert newer("RELEASE-10", "RELEASE-9") is None + assert newer("1.0.0", "not-a-version") is None diff --git a/tests/test_every_feature_has_a_place.py b/tests/test_every_feature_has_a_place.py new file mode 100644 index 0000000..9a36d3f --- /dev/null +++ b/tests/test_every_feature_has_a_place.py @@ -0,0 +1,294 @@ +"""Every command is on the page, or says in writing why it is not. Item 218. + +The operator, told the dependency half worked: *pero no hay ninguna ejecución del informe de +dependencias y verificación. Al menos en la página web.* There was not, and there is no view, no +route and no table for it — `_cmd_deps` opens no session and cannot run inside the container at all. + +So this file is the guard that makes *every feature has a place* a fact rather than an intention: +a command added tomorrow fails here until somebody has decided where it lives, or written down why +it does not. **A reason is as good as a route** — `work` will never be on a page served by the +receiver, and saying so is the answer rather than an omission. + +Three lists and not two, because *never* and *not yet* are different claims and collapsing them is +how a to-do becomes a design. `NOT_YET` is the work remaining, and it is meant to empty. + +Read off the parser rather than the published surface on purpose: the surface records the last +release, so a command added today would not appear in it until after it shipped undocumented, which +is the failure item 209 is about arriving through a second door. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterator + +import pytest +from sqlalchemy.orm import Session + +from hullwork import page +from hullwork.cli import build_parser +from hullwork.models import Item as _Item +from hullwork.page import Acting + +#: Where each command lives, as the path under `/page/{token}/` **and what is rendered there**. +#: +#: **The second half is item 223.** Until then this named a path and checked the application's route +#: table, and five commands passed that check with no button anywhere: `refresh`, `disable`, +#: `rotate-secret`, `set-tracker` and `requeue` were reachable by `curl` and by nothing a person +#: could press. Their tests posted straight at the route, which is a fair test of a route and no +#: test at all of a page. +#: +#: So a placement now names a string that has to appear in the rendered view — `value="disable"` for +#: an action, a heading or a link for a reading. A route is not a control. +#: +#: **Item 235 moved half of these** (DR-0027). Dependencies, deliveries and fixes each stopped being +#: a fold inside one project and became a page across every project, and this table is what proves +#: the move lost nothing: a command whose page vanished fails here, before anybody looks at a +#: screen. Three placements changed and every other one had to keep working, which is the whole +#: reason this file was written before the redesign rather than after it. +ON_THE_PAGE: dict[str, tuple[str, str]] = { + "status": ("instance", "

    This instance

    "), + "doctor": ("doctor", "

    Diagnostics

    "), + "config": ("config", "

    What it received

    "), + "approve": ("items/{item_id}", 'items/{id}/approve"'), + "requeue": ("items/{item_id}", 'value="requeue"'), + "republish": ("instance", 'value="republish"'), + "lease": ("instance", "Releasing it means the next dispatcher"), + "lease release": ("instance", 'value="lease-release"'), + "prune": ("instance", 'value="prune-preview"'), + "page-token": ("instance", 'value="page-token"'), + "deps": ("projects/{slug}/dependencies", "Dependencies"), + "sweep": ("projects/{slug}/settings", 'value="sweep"'), + "features": ("projects/{slug}/settings", "What Hullwork can do for"), + "propose": ("projects/{slug}/settings", 'value="propose"'), + "projects lanes": ("projects/{slug}/settings", 'value="lanes"'), + "projects": ("projects", "

    Projects

    "), + "projects add": ("projects", "Connect a project"), + "projects list": ("projects", "

    Projects

    "), + "projects refresh": ("projects/{slug}/settings", 'value="refresh"'), + "projects disable": ("projects/{slug}/settings", 'value="disable-preview"'), + "projects enable": ("projects/{slug}/settings", 'value="enable"'), + "projects rotate-secret": ("projects/{slug}/settings", 'value="rotate-secret"'), + "projects set-tracker": ("projects/{slug}/settings", 'value="set-tracker"'), +} + +#: Never, and each with the reason it is never. **Prose here is the point**: a command missing from +#: all three lists is one nobody decided about, and that is what this file catches. +NEVER_ON_THE_PAGE: dict[str, str] = { + "work": ( + "the receiver holds no Docker socket and no credential that can push, and refuses to start " + "if it finds one (DR-0005). A page served by it that could attempt a fix would undo the " + "split this product is sold on." + ), + "try": "needs the Docker socket, which the receiver does not have (DR-0005).", + "gateway": "needs the Docker socket, which the receiver does not have (DR-0005).", + "init": ( + "writes the two files a deployment needs, before an instance exists. There is no page to " + "put it on, because there is nothing running yet." + ), + "password": ( + "sets the credential that decides who may hold the session in future. DR-0025, accepted " + "2026-08-11: a session obtained once — a borrowed laptop, an unlocked screen — would " + "become permanent, silently, and unlike every other control here it could not be undone " + "from the page. Rotating a read link revokes; setting a password grants." + ), +} + +#: Not yet, and the item that will place it. This list is the work remaining and it is meant to +#: empty; an entry here is a promise with a number on it rather than a decision. +#: **Empty, and it is meant to stay that way.** DR-0024 was the last thing in it: the receiver may +#: ask OSV now, so `deps` has a place rather than a promise. An entry here is a promise with a +#: number on it; the list existing is what stops one becoming a design. +NOT_YET: dict[str, str] = {} + + +@pytest.fixture +def an_operator() -> Acting: + return Acting(csrf="c", offered=True) + + +@pytest.fixture +def an_instance(tmp_path: object, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + """One project and two items, because half these controls appear only when there is something + for them to act on — which is why a route-table check could not see them.""" + import datetime as dt + + from sqlalchemy.orm import sessionmaker + + from hullwork.config import get_settings + from hullwork.db import make_engine + from hullwork.models import ( + Attempt, + AttemptOutcome, + AttemptPhase, + Base, + ItemState, + Lane, + Project, + ) + + url = f"sqlite:///{tmp_path}/place.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + project = Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + tracker_project="shop", + ) + # **And one that is not watched**, because stopping and starting are each offered only in the + # state the other one leaves you in — the same reason there are two items below. + session.add( + Project( + slug="stopped", forge="forgejo", repo="acme/stopped", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + active=False, + ) + ) + session.add(project) + session.flush() + # **Two items, because the two item controls live in different states** and neither is wrong to + # hide in the other: `requeue` is for one a red baseline left with a human, `approve` for one + # waiting on a decision. A fixture with one of them measures whichever it happens to be. + stopped = _Item( + project_id=project.id, fingerprint="a", title="KeyError", state=ItemState.HUMAN_ONLY, + lane=Lane.GREEN, last_seen=dt.datetime.now(dt.UTC), + ) + waiting = _Item( + project_id=project.id, fingerprint="b", title="ValueError", + state=ItemState.WAITING_APPROVAL, lane=Lane.AMBER, last_seen=dt.datetime.now(dt.UTC), + state_since=dt.datetime.now(dt.UTC), + ) + session.add_all([stopped, waiting]) + session.flush() + session.add( + Attempt( + item_id=stopped.id, phase_reached=AttemptPhase.BASELINE, + outcome=AttemptOutcome.BASELINE_RED, consumed=False, + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _every_command() -> Iterator[str]: + """Every command this build offers, from the parser that offers them.""" + + def walk(parser: argparse.ArgumentParser, prefix: tuple[str, ...] = ()) -> Iterator[str]: + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + for name, sub in action.choices.items(): + yield " ".join((*prefix, name)) + yield from walk(sub, (*prefix, name)) + + yield from walk(build_parser()) + + +def test_no_command_is_undecided() -> None: + """**The whole item.** A command in neither dictionary is one nobody has placed, and the way + that reaches a release is exactly how the dependency half ended up invisible: nothing was ever + wrong, so nothing ever failed.""" + placed = set(ON_THE_PAGE) | set(NEVER_ON_THE_PAGE) | set(NOT_YET) + commands = set(_every_command()) + + assert commands - placed == set(), f"no place and no reason: {sorted(commands - placed)}" + gone = sorted(placed - commands) + assert gone == [], f"placed, but no longer a command: {gone}" + + +def test_every_reason_is_a_sentence() -> None: + """A reason of `""` or `"n/a"` would satisfy the test above and answer nobody. This is the + cheapest way to keep the second dictionary from becoming a list of names.""" + for command, why in NEVER_ON_THE_PAGE.items(): + assert len(why) > 25, f"{command} has no real reason: {why!r}" + assert why.endswith("."), f"{command}'s reason is not a sentence: {why!r}" + + +def test_nothing_is_both_placed_and_pending() -> None: + """A command in two lists is a command whose status nobody can read off this file — and the one + that would go unnoticed is `NOT_YET` left behind after the work landed.""" + assert set(ON_THE_PAGE) & set(NOT_YET) == set() + assert set(ON_THE_PAGE) & set(NEVER_ON_THE_PAGE) == set() + assert set(NEVER_ON_THE_PAGE) & set(NOT_YET) == set() + + +def test_every_promise_names_an_item() -> None: + """`NOT_YET` is the work remaining, and a promise with no work item behind it is a wish. The + file it names has to exist, which is what stops this list from outliving its plan.""" + from pathlib import Path + + work = Path(__file__).resolve().parent.parent / "work" + if not work.is_dir(): + pytest.skip("the work items are withheld from the published tree, and this reads them") + + for command, item in NOT_YET.items(): + assert list(work.glob(f"{item}-*.md")), f"{command} promises item {item}, which is not one" + + +def test_every_placement_names_a_route_that_exists() -> None: + """**Asserted against the app's own route table**, because a placement is a claim about where a + person goes, and a claim naming a path that 404s is worse than no claim: it closes the question + without answering it.""" + from hullwork.main import app + + routes = {getattr(one, "path", "") for one in app.routes} + + for command, (where, _) in ON_THE_PAGE.items(): + full = f"{page.PREFIX}/{{token}}/{where}" if where else f"{page.PREFIX}/{{token}}/" + assert full in routes, f"{command} is placed at {where}, which is not a route" + + +def test_every_placement_is_something_a_person_can_press( + an_instance: Session, an_operator: Acting +) -> None: + """**The check that would have caught five of these** (item 223). A route in the table is not a + place on a page: `refresh`, `disable`, `rotate-secret`, `set-tracker` and `requeue` all passed + the test above with no button rendered anywhere, and on an item stuck `human-only` — the one + state `requeue` is for — the only button on the page was *Sign out*. + + Rendered with an operator and with data in it, because half of these appear only for a session + and half only for a project or an item that exists. + """ + from hullwork.config import Settings + + settings = Settings() + items = an_instance.query(_Item).order_by(_Item.id).all() + views = { + "instance": page.instance( + an_instance, settings, error_reporting=False, acting=an_operator + ), + "doctor": page.why_it_will_not_work(an_instance, settings, acting=an_operator), + "config": page.what_it_received(settings, acting=an_operator), + "projects": page.projects(an_instance, settings, acting=an_operator), + # Item 237: a feature lives inside the project it is about. + "projects/{slug}/dependencies": "".join( + page.dependencies(an_instance, settings, slug, acting=an_operator) or "" + for slug in ("shop", "stopped") + ), + "projects/{slug}/settings": "".join( + page.settings_for(an_instance, settings, slug, acting=an_operator) or "" + for slug in ("shop", "stopped") + ), + # Both states, joined: a control offered only when it applies is not a control missing. + "projects/{slug}": "".join( + page.project(an_instance, settings, slug, acting=an_operator) or "" + for slug in ("shop", "stopped") + ), + # Both states, joined: what is asserted is that the control exists in the state it is for. + "items/{item_id}": "".join( + page.item(an_instance, settings, one.id, acting=an_operator) or "" for one in items + ), + } + + for command, (where, rendered) in ON_THE_PAGE.items(): + shown = views.get(where) + assert shown is not None, f"{command} names {where}, which this test does not render" + wanted = rendered.replace("{id}", str(items[1].id)) if items else rendered + assert wanted in shown, ( + f"{command} claims {where} and nothing there renders {wanted!r}" + ) diff --git a/tests/test_every_link_lands.py b/tests/test_every_link_lands.py new file mode 100644 index 0000000..1095a9b --- /dev/null +++ b/tests/test_every_link_lands.py @@ -0,0 +1,267 @@ +"""Every link on every view, followed. Item 227. + +The operator, one URL: `/page/me/projects/projects` → `{"detail":"Not Found"}`. + +They had clicked **Projects** in the rail, from a project's own view. Every URL on this page is +relative on purpose — that is what keeps the token out of the HTML, so a saved page or a screenshot +of the source carries no key — and `_document` takes `up` for exactly that reason. Item 223 gave the +project view a rail and did not tell it how deep it sits, so all five nouns resolved one level too +far — `projects/projects`, `projects/instance`, `projects/doctor`, `projects/config`, `projects/`. + +**Five broken links, and every existing test passed**, because a test that renders a view by +calling a function never resolves an `href`. `test_the_evidence_a_reviewer_came_for` walks links +from the front door for this reason, and had no reason to walk them from anywhere else. + +So this file walks them from **everywhere**: for each view, at the URL it is really served from, +every link is resolved the way a browser resolves it and followed. A rail that 404s is a page whose +navigation moves under the reader, and it is invisible to anything short of clicking. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Iterator +from pathlib import Path +from urllib.parse import urljoin + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import operator +from hullwork.config import get_settings +from hullwork.db import make_engine +from hullwork.models import ( + Attempt, + AttemptOutcome, + Base, + DependencyReport, + Item, + ItemState, + Lane, + Project, +) + + +def _where_you_can_be() -> tuple[str, ...]: + """Every GET the page serves, derived from the application rather than listed by hand. + + **Item 249, and the reason it exists.** This was seven URLs written out, and DR-0027 then gave a + project five views of its own — `errors`, `fixes`, `dependencies`, `deliveries`, `settings` — + none of which was ever added here. The operator found the gap by clicking: an item on a + project's *fixes* view had a relative `items/27`, which resolves three levels deep to + `projects/shop/items/27` and answers `{"detail":"Not Found"}`. + + A hand-written list of *where you can be* is a list that goes stale on the next route, silently, + and its silence reads as coverage. Derived, a new page is crawled the day it exists. + + The trailing shape matters and is preserved: a browser resolves `projects` differently from + `/page/me/projects` and from `projects/shop`. + """ + from hullwork import page as page_module + from hullwork.main import app + + prefix = f"{page_module.PREFIX}/{{token}}" + found = [] + for route in app.routes: + path = getattr(route, "path", "") + if not path.startswith(prefix) or "GET" not in getattr(route, "methods", set()): + continue + here = path.replace(prefix, "/page/me") + if "{" in here.replace("{token}", ""): + here = here.replace("{slug}", "shop").replace("{item_id}", "1") + if "{" in here: + continue + # **The slashless variant redirects and serves nothing**, so resolving a relative href + # against it measures a URL no reader is ever on: `/page/me` + `projects` is + # `/page/projects`, which is the redirect's job to prevent rather than a broken link. + if here == "/page/me": + continue + found.append(here or "/page/me/") + return tuple(sorted(set(found))) + + +#: Every view a person can reach, at the path it is served from. +WHERE_YOU_CAN_BE = _where_you_can_be() + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/links.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + project = Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + project.tracker_project = "shop" + session.add(project) + session.flush() + session.add( + Item( + project_id=project.id, fingerprint="f", title="KeyError", + state=ItemState.NEW, lane=Lane.GREEN, last_seen=dt.datetime.now(dt.UTC), + ) + ) + session.flush() + # **Enough in it that every view emits its links** (item 249). Three of that item's four + # reintroductions escaped a crawl that visited the right pages and found them empty: `fixes` + # renders no row without an attempt, the sweep form does not exist without a tracker name, and + # the dependency view has nothing to link to without a report. A crawl over empty pages reports + # coverage it does not have, which is the same failure as the hand-written list it replaced. + now = dt.datetime.now(dt.UTC) + session.add(Attempt( + item_id=1, outcome=AttemptOutcome.PR_OPEN, pull_request_ref="#4", + started_at=now, finished_at=now, + )) + session.merge(DependencyReport( + project_id=project.id, taken_at=now, asked=True, pinned=9, + findings=[{ + "package": "thing", "version": "1.0", "source": "uv.lock", + "advisories": [{"id": "GHSA-x", "summary": "something", "fixed": ["2.0"]}], + }], + )) + operator.set_password(session, "correct horse") + session.commit() + yield session + get_settings.cache_clear() + + +@pytest.fixture +def client(db: Session) -> TestClient: + from hullwork.main import app + + made = TestClient(app) + made.post("/page/me/login", data={"password": "correct horse"}) + return made + + +def _links(html: str, *, only: str | None = None) -> list[str]: + within = html + if only: + found = re.search(rf"<{only}[^>]*>.*?", html, re.S) + within = found.group(0) if found else "" + return re.findall(r' None: + """**The whole item.** From a project's own view every noun in the rail resolved one level too + deep and answered `404`, which is what the operator opened and what nothing here could see.""" + shown = client.get(here) + + assert shown.status_code == 200, here + rail = _links(shown.text, only='nav class="rail"') + + assert rail, f"{here} has no rail" + for href in rail: + landed = urljoin(f"http://testserver{here}", href) + assert client.get(landed).status_code == 200, f"{here} → {href} → {landed}" + + +@pytest.mark.parametrize("here", WHERE_YOU_CAN_BE) +def test_every_link_on_every_view_lands(client: TestClient, here: str) -> None: + """Not only the rail. A view's own links — *All projects*, *This instance*, an item's id — are + written by hand in nine functions, and each of them can be one `../` out.""" + shown = client.get(here) + + for href in _links(shown.text): + if href.startswith(("http://", "https://", "mailto:", "#", "data:")): + continue + landed = urljoin(f"http://testserver{here}", href) + assert client.get(landed).status_code == 200, f"{here} → {href} → {landed}" + + +@pytest.mark.parametrize("here", WHERE_YOU_CAN_BE) +def test_every_form_posts_somewhere_that_exists(client: TestClient, here: str) -> None: + """**A form is a URL like any other**, and `_document`'s own docstring says so: from `items/28` + the sign-out has to post to `../logout`, and hardcoding `logout` would have posted to + `items/logout`. Asserted by resolving, not by reading. + + **Asked of the router rather than shaped by hand** (item 250). This used to turn a resolved URL + back into a route template with a ladder of `re.sub` — `/projects/` to + `/projects/{slug}`, and so on — which is the hand-written list of item 249 wearing a different + hat: it went stale the moment a route had a shape the ladder did not know, and said *not a + route* about a route that was there. The router already answers this question. + """ + from starlette.routing import Match + + from hullwork.main import app + + shown = client.get(here) + + for action in re.findall(r']*action="([^"]+)"', shown.text): + landed = urljoin(f"http://testserver{here}", action) + path = landed.removeprefix("http://testserver") + scope = {"type": "http", "method": "POST", "path": path, "root_path": "", "headers": []} + matched = any(one.matches(scope)[0] is Match.FULL for one in app.routes) + + assert matched, f"{here} posts to {action} → {path}, which is not a route" + + +#: What each label promises, as the path it has to reach. A link that lands is not a link that is +#: honest: `This instance` pointed at the front door for as long as the front door was the instance +#: report, and item 212 moved that without moving the label. +LABELS_PROMISE = { + "This instance": "/page/me/instance", + "All projects": "/page/me/projects", + "All items": "/page/me/items", + "Projects": "/page/me/projects", + "Items": "/page/me/", + "Why it will not work": "/page/me/doctor", + "What it received": "/page/me/config", +} + + +@pytest.mark.parametrize("here", WHERE_YOU_CAN_BE) +def test_a_label_goes_where_it_says(client: TestClient, here: str) -> None: + """**Landing is not the same as being honest.** Every test in this file follows links and checks + they answer `200`; a label pointing at the wrong working page passes all of them, and is worse + than a broken one — a `404` tells you something is wrong, and this quietly does not.""" + shown = client.get(here).text + + for href, label in re.findall(r']*>([^<]+)', shown): + promised = LABELS_PROMISE.get(label.strip()) + if promised is None: + continue + landed = urljoin(f"http://testserver{here}", href).removeprefix("http://testserver") + + assert landed == promised, f"{here}: {label.strip()!r} goes to {landed}, not {promised}" + + +def test_the_crawl_covers_every_page_the_application_serves() -> None: + """**The guard's own scope, asserted** (item 249). + + Reintroducing the hand-written list of seven changes nothing while the code is correct, so the + property has to be measured directly: what this file walks is what the application serves, and + a route added tomorrow is walked without anybody editing this file. + + That is the defect item 249 is about. DR-0027 gave a project five views and none of them was + added here; the crawl kept passing over the seven it knew, and its silence read as coverage. + """ + from hullwork import page as page_module + from hullwork.main import app + + prefix = f"{page_module.PREFIX}/{{token}}" + served = { + getattr(one, "path", "") + for one in app.routes + if getattr(one, "path", "").startswith(prefix) + and "GET" in getattr(one, "methods", set()) + } + walked = { + one.replace("/page/me", prefix).replace("/shop", "/{slug}").replace("/1", "/{item_id}") + for one in WHERE_YOU_CAN_BE + } + # The slashless variant redirects and is excluded above, with its reason. + missing = served - walked - {prefix} + + assert not missing, f"the crawl does not visit: {sorted(missing)}" + for feature in ("errors", "fixes", "dependencies", "deliveries", "settings"): + assert f"/page/me/projects/shop/{feature}" in WHERE_YOU_CAN_BE diff --git a/tests/test_nothing_outlives_the_container.py b/tests/test_nothing_outlives_the_container.py new file mode 100644 index 0000000..960b774 --- /dev/null +++ b/tests/test_nothing_outlives_the_container.py @@ -0,0 +1,84 @@ +"""Every container this product removes takes its anonymous volumes with it. Item 244. + +Item 241 stopped the verification queue leaving a gigabyte of image per candidate, and the disk kept +climbing — 45% to 54% in an hour, with **zero** sandbox images on the host: + +``` +Local Volumes 69 ACTIVE 5 3.217GB 96% reclaimable +``` + +Ten of those carried a hullwork name. The rest were anonymous, and they were databases. +`postgres:16` declares `VOLUME /var/lib/postgresql/data` in its Dockerfile, so every `docker run` +of it makes one, and `docker rm -f` **without `-v`** leaves it behind. One per service, per phase +— item 052 starts them fresh around each one on purpose — on every attempt and verification. + +**The reaper cannot collect these.** `inventory` matches by name and an anonymous volume has none: +a 64-character hex string that says nothing about who made it. Removing those by pattern would +delete everything else on the host, which is what item 125 exists to prevent. They have to go with +whoever created them, at the moment that one knows it is done. + +So this is a rule rather than seven fixes, and this file is the rule. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +SOURCE = Path(__file__).resolve().parent.parent / "hullwork" + +#: `docker rm` of a container, in both the spellings this repository uses: `run_docker([docker, +#: "rm", …])` and `_quietly(self._docker, ["rm", …])`. A volume is removed with +#: `["volume", "rm", …]` and is a different call under a different rule: those are **named**, they +#: have owners, and `-v` never touches them. +#: +#: **The whitespace lives inside the lookahead**, and that is not a detail: written the other way +#: — `\s*,\s*(?!"-v")` — the engine backtracks that `\s*` to zero, the lookahead sees ` "-v"` rather +#: than `"-v"`, and the pattern matches the very lines that are correct. It reported all seven fixed +#: call sites as offenders, which is how it was found. +_REMOVES_A_CONTAINER = re.compile(r'(? None: + """**The whole item, as a rule instead of seven patches.** `-v` removes a container's anonymous + volumes and leaves named ones alone, which is exactly the distinction that matters here: + `hullwork-worktree-*` and `hullwork-envcache-*` have names and owners; a database started for + one phase has neither. + """ + offenders: list[str] = [] + for path in sorted(SOURCE.rglob("*.py")): + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if _REMOVES_A_CONTAINER.search(line): + offenders.append(f"{path.relative_to(SOURCE.parent)}:{number}: {line.strip()}") + + assert offenders == [], ( + "these remove a container without `-v`, so anything the image declared as a VOLUME " + "outlives it:\n " + "\n ".join(offenders) + ) + + +def test_the_rule_can_tell_a_container_from_a_volume() -> None: + """A guard that also matched `["volume", "rm", "-f", name]` would demand `-v` on a call that + does not take it, and the fix would be to weaken the guard — which is how a rule stops being + one. Asserted so the pattern itself is under test rather than only its verdict.""" + assert _REMOVES_A_CONTAINER.search('_quietly(self._docker, ["rm", "-f", container])') + assert _REMOVES_A_CONTAINER.search('run_docker([docker, "rm", "-f", one], timeout=60)') + assert not _REMOVES_A_CONTAINER.search('_quietly(self._docker, ["rm", "-f", "-v", container])') + assert not _REMOVES_A_CONTAINER.search( + 'run_docker([docker, "rm", "-f", "-v", one], timeout=60)' + ) + assert not _REMOVES_A_CONTAINER.search( + 'run_docker([self.docker, "volume", "rm", "-f", name], timeout=60)' + ) + + +def test_the_services_that_hold_a_database_are_the_ones_this_is_about() -> None: + """Named rather than inferred: `postgres` is the service whose image declares a `VOLUME`, and + it is why 69 volumes existed on a host that had been cleaned four hours earlier.""" + from hullwork.sandbox.services import SERVICES + + assert any(name.startswith("postgres") for name in SERVICES), ( + "the service this item was found on is gone; check whether the rule still has a subject" + ) diff --git a/tests/test_one_feature_one_section.py b/tests/test_one_feature_one_section.py new file mode 100644 index 0000000..6e97f16 --- /dev/null +++ b/tests/test_one_feature_one_section.py @@ -0,0 +1,291 @@ +"""A project is the map, and its features are what is on it. Items 235 and 237, DR-0027. + +Third time the operator said the page was difficult, and the first two answers were reductions — +1,932 words to 172, five `curl`-only routes given a button. Both were true and neither was it. + +What was wrong was one decision nobody made on purpose: **the page was laid out along the database +tables**, so finding a feature required knowing which table it hung off. *Where are my +dependencies?* had the answer *inside a project, inside a closed fold*, and no page said so. + +Item 235 named the features and gave each a page holding **every project's**, and the operator +corrected the axis rather than the naming: *¿no será mejor plantear esto mismo, pero a nivel de +proyecto? Así no mezclamos cosas.* He is right — a page called *Dependencies* listing two projects' +advisories one after another is a wall at two and unusable at ten. Nobody works by feature across +clients; they work on a client. + +So this file holds the properties that make a project a map: every one of its features is a word in +its rail, every one of those words leads somewhere that exists, none of them is behind a disclosure, +and **nothing inside a project is ever about another project**. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import ( + Attempt, + AttemptOutcome, + AttemptPhase, + Base, + Delivery, + DependencyReport, + Item, + ItemState, + Lane, + Project, +) + +SIGNED_IN = page.Acting(csrf="c", offered=True) +READING = page.Acting(csrf=None, offered=False) + +#: Every feature, the page it lives on, and what that page must render. **The whole item in one +#: table**: a feature missing from the rail is one a reader cannot find, and a rail entry whose page +#: renders nothing is a name that leads nowhere — worse than the fold it replaced, because it closes +#: the question without answering it. +#: The fourth column is **the feature itself**, and it is what makes this table more than a list of +#: headings: a page could keep its `

    ` in the open and fold everything under it, which is exactly +#: the state this item exists to leave. So each row names a string that has to survive deleting +#: every `
    ` block on the page. +FEATURES: tuple[tuple[str, str, str, str], ...] = ( + ("Overview", "", "Overview

    ", "Where everything is"), + ("Errors", "errors", "Errors", "KeyError"), + ("Fixes", "fixes", "Fixes", "green-gate"), + ("Dependencies", "dependencies", "Dependencies", "cryptography"), + ("Deliveries", "deliveries", "Deliveries", "understood"), + ("Settings", "settings", "Settings", 'value="refresh"'), +) + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + """An instance with something in every feature, because a page that is empty everywhere renders + the same eight sentences whatever it is asked for.""" + url = f"sqlite:///{tmp_path}/map.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + project = Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + session.add(project) + session.flush() + bug = Item( + project_id=project.id, fingerprint="f", title="KeyError: 'total'", + state=ItemState.WAITING_APPROVAL, lane=Lane.AMBER, last_seen=dt.datetime.now(dt.UTC), + ) + session.add(bug) + session.flush() + session.add( + Attempt( + item_id=bug.id, phase_reached=AttemptPhase.GREEN_GATE, + outcome=AttemptOutcome.PR_OPEN, consumed=True, + ) + ) + session.add( + Delivery( + project_id=project.id, provider_delivery_id="d1", payload_hash="h", + payload_json="{}", received_at=dt.datetime.now(dt.UTC), + processed_at=dt.datetime.now(dt.UTC), attempts=1, + ) + ) + session.merge( + DependencyReport( + project_id=project.id, taken_at=dt.datetime.now(dt.UTC), asked=True, pinned=50, + findings=[ + { + "package": "cryptography", "version": "48.0.1", "source": "requirements.txt", + "advisories": [{"id": "GHSA-x", "summary": "s", "fixed": ["49.0.0"]}], + } + ], + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _render(db: Session, where: str, acting: page.Acting) -> str: + settings = Settings() + if where == "": + return page.project(db, settings, "shop", acting=acting) or "" + view = { + "errors": page.errors, + "fixes": page.fixes, + "dependencies": page.dependencies, + "deliveries": page.deliveries, + "settings": page.settings_for, + }[where] + return view(db, settings, "shop", acting=acting) or "" + + +def _rail_of(shown: str) -> str: + found = re.search(r'', shown, re.S) + assert found is not None, "the page has no rail at all" + return found.group(1) + + +# --- the rail is the map ---------------------------------------------------------------------- + + +def test_every_feature_is_a_word_in_the_rail(db: Session) -> None: + """**The whole item.** *Items · Projects · This instance · Why it will not work · What it + received* contained no word a reader looking for their dependencies could have clicked.""" + rail = _rail_of(_render(db, "", SIGNED_IN)) + + for name, _, _, _ in FEATURES: + assert f">{name}<" in rail, f"{name} is not in the navigation" + + +@pytest.mark.parametrize(("name", "where", "renders", "_open"), FEATURES) +def test_every_name_in_the_rail_reaches_a_page_that_exists( + db: Session, name: str, where: str, renders: str, _open: str +) -> None: + """A name leading nowhere is worse than the fold it replaced: it closes the question without + answering it. The link's `href` is checked against the view it claims to reach.""" + rail = _rail_of(_render(db, "", SIGNED_IN)) + # From `projects/` the rail is written `../projects//`, which is the + # arithmetic item 227 got wrong by one and 404'd every link on the page. + wanted = f'href="../projects/shop{"/" + where if where else ""}"' + + assert wanted in rail, f"{name} points somewhere else" + assert renders in _render(db, where, SIGNED_IN) + + +def test_a_reader_is_shown_no_control_they_cannot_use(db: Session) -> None: + """DR-0021: a read link gets the instance and nothing that administers it. The rail changed + shape twice in two items and the operator-only rule has to survive both.""" + rail = _rail_of(_render(db, "", READING)) + + assert "Settings" not in rail, "a reader is offered the controls" + assert ">Dependencies<" in rail, "a reading is still allowed to see what is published" + + +# --- and no feature is behind a disclosure ------------------------------------------------------ + + +@pytest.mark.parametrize(("_name", "where", "_renders", "in_the_open"), FEATURES) +def test_a_features_page_opens_with_the_feature_on_it( + db: Session, _name: str, where: str, _renders: str, in_the_open: str +) -> None: + """**The failure this item is named after.** Four of these were `
    ` on a project's view, + so the answer to *what is published against what I pin* was one click and one guess away. + + `
    ` keeps what item 167 built it for — a stack trace, a payload, a table of pinned + versions — and stops being where a feature lives. + + **Measured by deleting every fold on the page** and looking for the feature in what is left. + Asserting that the `

    ` is unfolded would pass over a page whose headline stands alone above + everything it is the headline of, which is the shape being replaced. + """ + body = _render(db, where, SIGNED_IN).split('
    ')[1] + unfolded = re.sub(r"]*>.*?

    ", "", body, flags=re.S) + + assert in_the_open in unfolded, f"{where} holds its own feature behind a disclosure" + + +def test_the_counts_say_whether_there_is_anything_in_there(db: Session) -> None: + """A number in the navigation is what makes it a map rather than a list of words: it answers + *is there anything in there* before the click.""" + rail = _rail_of(_render(db, "", SIGNED_IN)) + + assert '1' in rail + + +def test_a_count_of_zero_is_not_rendered(db: Session, tmp_path: Path) -> None: + """Item 073's rule one turn further: a badge reading `0` on every row on every page is furniture + pretending to be information.""" + empty = make_engine(f"sqlite:///{tmp_path}/empty.db") + Base.metadata.create_all(empty) + with sessionmaker(bind=empty)() as blank: + blank.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + ) + blank.commit() + rail = _rail_of(_render(blank, "", SIGNED_IN)) + + assert 'class="count"' not in rail + + +# --- and the headings are labels ---------------------------------------------------------------- + + +def test_a_heading_is_a_label_and_its_sentence_is_under_it(db: Session) -> None: + """*What is published against what it pins* is accurate and unscannable, and an eye moving down + a page cannot use it. The sentence is not deleted — it moves one line down and into grey.""" + shown = _render(db, "dependencies", SIGNED_IN) + + assert "Dependencies" in shown + assert '

    ' in shown + assert "What OSV publishes against the versions this project pins" in shown + + +def test_no_heading_on_any_feature_page_is_a_sentence(db: Session) -> None: + """**Measured rather than promised.** Nine of them were sentences; the guard is a word count, + because that is what separates a label from prose and it cannot be argued with.""" + for _, where, _, _ in FEATURES: + body = _render(db, where, SIGNED_IN).split('

    ')[1] + for heading in re.findall(r"]*>(.*?)", body, re.S): + words = len(re.sub(r"<[^>]+>", "", heading).split()) + assert words <= 4, f"{where} has a heading that is a sentence: {heading!r}" + + +def test_a_section_without_its_sentence_is_not_a_section() -> None: + """`_section` is the shape DR-0027 decided on — label, sentence, thing — and a version that + silently dropped the middle one would leave a page of bare labels that passes every other test + in this file: the lede on a feature page is written inline, so nothing else covers this. + """ + made = page._section("Dependencies", "What OSV publishes against what you pin.", "

    x

    ") + + assert "

    Dependencies

    " in made + assert '

    What OSV publishes against what you pin.

    ' in made + assert made.index("

    ") < made.index('class="says"') < made.index("

    x

    ") + + +def test_a_project_says_what_each_of_its_sections_is(db: Session) -> None: + """The view with the most sections, and the one an operator uses to act. A label with no + sentence under it is the terse half of the redesign without the half that explains.""" + shown = page.project(db, Settings(), "shop", acting=SIGNED_IN) or "" + labels = re.findall(r'

    ([^<]+)

    (

    )?', shown) + + assert labels, "the project view has no sections at all" + for label, says in labels: + if label != "What is wrong": + assert says, f"{label} is a label with nothing under it" + + +def test_nothing_inside_a_project_is_about_another_project(db: Session) -> None: + """**The operator's whole correction.** Item 235 put every project's advisories on one page, + which is a wall at two projects and unusable at ten. Inside a project, the only slug on screen + is this one — and the way back out is a link rather than a heading, because a rail that replaces + itself has to say what it replaced. + """ + db.add( + Project( + slug="other", forge="forgejo", repo="acme/other", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + ) + db.commit() + + for _, where, _, _ in FEATURES: + shown = _render(db, where, SIGNED_IN) + assert "other" not in _rail_of(shown), f"{where} lists another project in its rail" + assert "acme/other" not in shown, f"{where} renders another project" + assert "All projects" in shown, f"{where} has no way back out" diff --git a/tests/test_one_package_one_row.py b/tests/test_one_package_one_row.py new file mode 100644 index 0000000..7090622 --- /dev/null +++ b/tests/test_one_package_one_row.py @@ -0,0 +1,253 @@ +"""The dependency view is a table of packages. DR-0028, item 246. + +The view it replaces was 11,330px — 12.6 screens — for twenty-six findings, because each fact was +a paragraph and the outcome of a package lived in a second list six screens below its advisory. +What is under test is the shape that fixed it, and the three faults that shape's prototype had: + +* one row per **package**, carrying every version of it that is pinned; +* the state's sentence in the **heading**, said once, never once per row; +* **no action column** — the control renders in the row that has one and nowhere else. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import get_settings +from hullwork.db import make_engine +from hullwork.models import Base, DependencyReport, Project, UpgradeVerdict + +SIGNED_IN = page.Acting(csrf="c", offered=True) + + +def _finding( + package: str, version: str, fixed: list[str], source: str = "uv.lock" +) -> dict[str, object]: + return { + "package": package, "version": version, "source": source, + "advisories": [{"id": f"GHSA-{package}", "summary": "something", "fixed": fixed}], + } + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/rows.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add(Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + manifest={"project": "shop", "autofix": {"open_upgrades": True}}, + )) + session.commit() + yield session + get_settings.cache_clear() + + +def _report(db: Session, findings: list[dict[str, object]]) -> None: + db.merge(DependencyReport( + project_id=1, taken_at=dt.datetime.now(dt.UTC), asked=True, + pinned=99, findings=findings, + )) + db.commit() + + +def _verdict(db: Session, package: str, was: str, to: str, outcome: str, **kept: object) -> None: + db.add(UpgradeVerdict(project_id=1, package=package, was=was, to=to, outcome=outcome, **kept)) + db.commit() + + +def _view(db: Session, *, acting: page.Acting = SIGNED_IN) -> str: + project = db.query(Project).one() + return page.what_is_published_against_it(db, project, acting=acting) + + +class TestThePackageIsTheRow: + def test_a_package_pinned_three_times_is_one_row(self, db: Session) -> None: + """**The fault the first prototype had.** `brace-expansion` is pinned at three versions in + one lock file, and rendering a row per pinned version made it three rows that share a name. + """ + _report(db, [ + _finding("brace-expansion", "1.1.15", ["5.0.7"], "package-lock.json"), + _finding("brace-expansion", "2.1.1", ["5.0.7"], "package-lock.json"), + _finding("brace-expansion", "5.0.6", ["5.0.7"], "package-lock.json"), + ]) + + shown = _view(db) + + assert shown.count('') == 1 + assert "1.1.15" in shown and "2.1.1" in shown + + def test_the_report_handing_the_same_pair_twice_is_one_row(self, db: Session) -> None: + """Measured on the operator's own instance: 26 findings, 25 distinct pairs, 21 packages — + `brace-expansion 5.0.6` arrives twice with the same source. The page does not multiply + it.""" + _report(db, [ + _finding("brace-expansion", "5.0.6", ["5.0.7"]), + _finding("brace-expansion", "5.0.6", ["5.0.7"]), + ]) + + shown = _view(db) + row = shown.split('')[1].split("")[0] + + assert shown.count('') == 1 + # **And the version is not printed twice inside it.** Counting rows alone let a defect that + # appended the same pinned version per finding pass, which is what the operator's own report + # hands over: 26 findings, 25 distinct pairs. + assert row.count("5.0.6") == 1 + + def test_neither_side_of_the_move_is_a_cartesian_product(self, db: Session) -> None: + """**Found by rendering it**: pairing every pinned version with every published destination + printed `5.0.6` thirty times and stretched the table to 7,208px.""" + _report(db, [ + _finding("thing", "1.0", ["2.0", "3.0", "4.0"]), + _finding("thing", "1.1", ["2.0", "3.0", "4.0"]), + ]) + + row = _view(db).split('')[1].split("")[0] + + assert row.count("1.0") == 1 + assert row.count("2.0") == 1 + + def test_a_package_with_nowhere_to_go_still_says_what_it_is(self, db: Session) -> None: + """The one row a reader can do nothing about was also the one rendering an empty span where + its version should be.""" + _report(db, [_finding("left-pad", "1.0.0", [])]) + + row = _view(db).split('')[1].split("")[0] + + assert "left-pad" in row + assert "1.0.0" in row + + +class TestTheStateThatMostNeedsAPerson: + def test_a_package_takes_the_state_that_needs_a_person(self, db: Session) -> None: + """One version ready to open and another stuck is a row you can act on. A row is a place to + act, so the quietest version must not hide the loudest.""" + # **Two pinned versions in two states**, which is the only shape where the rule fires: + # within one pinned version the order inside `_state_of` decides, and a test that used one + # finding measured that instead — it passed with the rule inverted. + _report(db, [ + _finding("thing", "1.0", ["2.0"]), + _finding("thing", "9.0", ["9.9"]), + ]) + _verdict(db, "thing", "1.0", "2.0", "clean", artefact={"files": {"a": "b"}}) + _verdict(db, "thing", "9.0", "9.9", "cannot-move") + + shown = _view(db) + + assert "Ready to open" in shown + assert "The pin would not move" not in shown + + def test_an_outcome_this_page_does_not_know_is_still_shown(self, db: Session) -> None: + _report(db, [_finding("thing", "1.0", ["2.0"])]) + _verdict(db, "thing", "1.0", "2.0", "cannot-parse") + + assert "it ended as cannot-parse" in _view(db) + + def test_a_refusal_is_the_one_sentence_a_row_still_says(self, db: Session) -> None: + """Each refusal differs — *already open from an earlier run* and *the forge refused it* are + not the same sentence — so this one cannot move to a heading.""" + _report(db, [_finding("thing", "1.0", ["2.0"])]) + _verdict( + db, "thing", "1.0", "2.0", "clean", + asked_to_open_at=dt.datetime.now(dt.UTC), open_note="the forge refused it", + ) + + shown = _view(db) + + assert "Asked for, and not opened" in shown + assert "the forge refused it" in shown + + +class TestTheSentenceIsSaidOnce: + def test_the_state_is_explained_in_the_heading_not_in_the_rows(self, db: Session) -> None: + """**The fault the first prototype had**: seventeen rows reading *passed, but nothing kept + to open it from*. A column repeating one sentence is a column that should not exist.""" + _report(db, [_finding(f"pkg{n}", "1.0", ["2.0"]) for n in range(4)]) + for n in range(4): + _verdict(db, f"pkg{n}", "1.0", "2.0", "clean") + + shown = _view(db) + + assert shown.count("passed your suite before the change and after it") == 1 + assert shown.count('') == 4 + + def test_the_band_names_every_package_it_covers(self, db: Session) -> None: + """Saying it once must not mean saying it about nobody.""" + _report(db, [_finding(f"pkg{n}", "1.0", ["2.0"]) for n in range(3)]) + for n in range(3): + _verdict(db, f"pkg{n}", "1.0", "2.0", "clean") + + shown = _view(db) + + for n in range(3): + assert f"pkg{n}" in shown + + +class TestTheActionHasNoColumn: + def test_the_control_renders_only_where_it_exists(self, db: Session) -> None: + """It was empty in twenty-five rows of twenty-six, paying width to say nothing.""" + _report(db, [ + _finding("openable", "1.0", ["2.0"]), + _finding("not-openable", "1.0", ["2.0"]), + ]) + _verdict(db, "openable", "1.0", "2.0", "clean", artefact={"files": {"a": "b"}}) + _verdict(db, "not-openable", "1.0", "2.0", "cannot-move") + + shown = _view(db) + + assert shown.count('value="open-upgrade"') == 1 + + def test_a_reader_who_cannot_act_is_offered_nothing(self, db: Session) -> None: + _report(db, [_finding("thing", "1.0", ["2.0"])]) + _verdict(db, "thing", "1.0", "2.0", "clean", artefact={"files": {"a": "b"}}) + + shown = _view(db, acting=page.READING) + + assert "open-upgrade" not in shown + assert "Signing in is what offers the control" in shown + + def test_an_open_one_is_a_link_and_not_a_button(self, db: Session) -> None: + _report(db, [_finding("thing", "1.0", ["2.0"])]) + _verdict( + db, "thing", "1.0", "2.0", "clean", + artefact={"files": {"a": "b"}}, opened_where="https://forge/pull/10", + ) + + shown = _view(db) + + assert 'href="https://forge/pull/10"' in shown + assert "open-upgrade" not in shown + + +class TestWhatTheViewMustNotBecome: + def test_there_is_no_second_list(self, db: Session) -> None: + """**The fault DR-0028 exists for.** What OSV publishes about a package and what this + instance did about it were two sections six screens apart.""" + _report(db, [_finding("thing", "1.0", ["2.0"])]) + _verdict(db, "thing", "1.0", "2.0", "clean") + + shown = _view(db) + + assert "What happened when it was tried" not in shown + assert shown.count('') == 1 + + def test_the_advisory_texts_stay_behind_the_disclosure(self, db: Session) -> None: + _report(db, [_finding("thing", "1.0", ["2.0"])]) + + row = _view(db).split('')[1].split("")[0] + + assert "something" not in row.split(" Iterator[Session]: + url = f"sqlite:///{tmp_path}/repo.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + monkeypatch.setenv("HULLWORK_FORGE_URL", "https://forge.example") + monkeypatch.setenv("HULLWORK_FORGE_TOKEN", "t") + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + manifest={ + "project": "shop", + "git": {"provider": "forgejo", "repo": "acme/shop"}, + "errors": {"provider": "glitchtip"}, + }, + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +@pytest.fixture +def client() -> TestClient: + from hullwork.main import app + + return TestClient(app) + + +def _signed_in(db: Session, client: TestClient) -> None: + operator.set_password(db, "correct horse") + db.commit() + client.post("/page/me/login", data={"password": "correct horse"}) + + +def _csrf(client: TestClient) -> str: + found = re.search(r'name="csrf" value="([^"]+)"', client.get("/page/me/instance").text) + assert found is not None, "no CSRF field on the instance report" + return found.group(1) + + +#: **Patched where they are defined, not where they are used.** `_propose_one` and `_lanes_of` +#: import from `hullwork.cli` inside the function, so a name bound on `hullwork.main` is a name +#: nothing looks at — the first version of these tests patched that and every one errored. +class _Tree: + """What a forge answers when asked for a tree. Named fields, because a hand-built double that + drifts from its protocol is a mistake this project has now made four times.""" + + def __init__(self, paths: tuple[str, ...], *, truncated: bool = False) -> None: + self.paths = paths + self.ref = "0123456789abcdef" + self.truncated = truncated + + +class _Forge: + def __init__(self, tree: _Tree | None = None) -> None: + self._tree = tree + self.closed = False + + def tree(self, repo: str) -> _Tree: + assert self._tree is not None + return self._tree + + def close(self) -> None: + self.closed = True + + +# --- the lane policy, applied to their code --------------------------------------------------- + + +def test_the_lane_policy_is_shown_over_the_projects_own_tree( + db: Session, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """**An operator who cannot see the policy applied to their code is being asked to trust a + paragraph**, and this product's first principle is that trust is the product.""" + from hullwork import cli as cli_module + + forge = _Forge(_Tree(("app/checkout.py", "app/payments/charge.py", "README.md"))) + monkeypatch.setattr(cli_module, "_forge_for", lambda settings, kind: forge) + _signed_in(db, client) + + shown = client.post( + "/page/me/projects/shop/settings", data={"action": "lanes", "csrf": _csrf(client)} + ) + + assert shown.status_code == 200 + assert "3 file(s)" in shown.text + assert "keeps a human on" in shown.text + assert forge.closed, "the forge connection was left open" + + +def test_a_truncated_tree_says_so_rather_than_reading_as_clean( + db: Session, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """**What is missing is unclassified, not classified as ordinary.** A partial listing rendered + without that sentence is a page claiming a repository has no sensitive files because the forge + stopped talking halfway.""" + from hullwork import cli as cli_module + + forge = _Forge(_Tree(("app/checkout.py",), truncated=True)) + monkeypatch.setattr(cli_module, "_forge_for", lambda settings, kind: forge) + _signed_in(db, client) + + shown = client.post( + "/page/me/projects/shop/settings", data={"action": "lanes", "csrf": _csrf(client)} + ) + + assert "did not serve the whole tree" in shown.text + assert "not classified as ordinary" in shown.text + + +def test_the_lane_policy_is_never_stored( + db: Session, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """**A derived policy kept on disk is a snapshot of which code is dangerous**, and + `territory.py` explains why that fails in the direction that matters: the tree moves, the + snapshot does not, and the file that became sensitive last week reads as ordinary. + + So this is an action and never a cache — asserted, because *read-only and stores nothing* was + a sentence in a docstring and nothing was checking it on this path.""" + from hullwork import cli as cli_module + + monkeypatch.setattr( + cli_module, "_forge_for", + lambda settings, kind: _Forge(_Tree(("app/payments/charge.py", "README.md"))), + ) + _signed_in(db, client) + before = db.query(Project).one().manifest + + client.post("/page/me/projects/shop/settings", data={"action": "lanes", "csrf": _csrf(client)}) + + db.rollback() + assert db.query(Project).one().manifest == before, "the derived policy was written down" + + +def test_a_forge_that_will_not_answer_says_which_repository( + db: Session, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """A refusal naming nothing sends somebody to check every project they have.""" + from hullwork import cli as cli_module + + class _Refuses(_Forge): + def tree(self, repo: str) -> _Tree: + raise RuntimeError("404 Not Found") + + monkeypatch.setattr(cli_module, "_forge_for", lambda settings, kind: _Refuses()) + _signed_in(db, client) + + shown = client.post( + "/page/me/projects/shop/settings", data={"action": "lanes", "csrf": _csrf(client)} + ) + + assert "acme/shop" in shown.text + assert "404 Not Found" in shown.text + + +# --- the manifest read from their CI ---------------------------------------------------------- + + +def test_a_manifest_is_proposed_and_not_written( + db: Session, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """**It prints and does not write**, here as in the terminal. A manifest belongs in the + project's repository, committed by somebody who read it — and DR-0006's rule that what was + inferred stays commented only means anything if a person uncomments it.""" + from hullwork import cli as cli_module + + forge = _Forge() + monkeypatch.setattr(cli_module, "_forge_for", lambda settings, kind: forge) + monkeypatch.setattr( + cli_module, "propose_from_ci", lambda forge_, repo: "project: shop\nruntime:\n base: x" + ) + _signed_in(db, client) + before = db.query(Project).one().manifest + + shown = client.post( + "/page/me/projects/shop/settings", data={"action": "propose", "csrf": _csrf(client)} + ) + + assert "runtime:" in shown.text + assert forge.closed, "the forge connection was left open" + # **`rollback` and not `expire_all`.** This session opened a transaction reading the row above, + # and SQLite serves it that snapshot until the transaction ends — so an expiry re-reads the + # same view and a write by the application looks like no write at all. A mutation that stored + # the proposal on the project escaped exactly here. + db.rollback() + assert db.query(Project).one().manifest == before, "it wrote the proposal to the project" + + +def test_a_repository_with_no_ci_says_what_to_do_instead( + db: Session, client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """*Nothing proposes a manifest* is not a refusal to connect the project, and the page has to + say the field that decides whether anything can be built at all.""" + from hullwork import cli as cli_module + + monkeypatch.setattr(cli_module, "_forge_for", lambda settings, kind: _Forge()) + monkeypatch.setattr(cli_module, "propose_from_ci", lambda forge, repo: None) + _signed_in(db, client) + + shown = client.post( + "/page/me/projects/shop/settings", data={"action": "propose", "csrf": _csrf(client)} + ) + + assert "written by hand" in shown.text + assert "runtime.base" in shown.text + + +# --- and the line every action on this page holds ---------------------------------------------- + + +@pytest.mark.parametrize("what", ["lanes", "propose"]) +def test_a_read_link_is_offered_neither(db: Session, client: TestClient, what: str) -> None: + """DR-0021. Each of these spends a forge request; a control that does that for anybody holding + a saved URL is a control that spends somebody else's rate limit.""" + minted = generate_token() + page.issue(db, hash_token(minted)) + db.commit() + + seen = client.get(f"/page/{minted}/projects").text + assert "Which files keep a human on" not in seen + assert "Read a manifest from its CI" not in seen + + refused = client.post(f"/page/{minted}/projects/shop/settings", data={"action": what}) + assert refused.status_code == 404 diff --git a/tests/test_resolve.py b/tests/test_resolve.py index 26dc10f..b988833 100644 --- a/tests/test_resolve.py +++ b/tests/test_resolve.py @@ -63,7 +63,81 @@ def test_the_manifest_is_required_and_named_when_absent() -> None: assert resolve.missing_from(uv, ["uv.lock"]) == ["pyproject.toml"] assert resolve.missing_from(uv, ["pyproject.toml", "uv.lock"]) == [] # And from a subdirectory, because a monorepo pins per package. - assert resolve.missing_from(uv, ["svc/pyproject.toml", "svc/uv.lock"]) == [] + assert resolve.missing_from(uv, ["svc/pyproject.toml", "svc/uv.lock"], "svc") == [] + + +def test_a_manifest_in_another_directory_does_not_count(monkeypatch: object) -> None: + """**Item 239, and the reason item 238's fix surfaced a second bug rather than a verdict.** + + This compared basenames anywhere in the checkout, so `backend/pyproject.toml` satisfied a check + about `frontend/`'s lock — and the honest refusal it exists to produce was unreachable for any + repository with more than one lock file. Measured on `simplecheck`, which is a monorepo: it + pulled a 200MB image to be told the file was not where it was looking. + """ + del monkeypatch + uv = resolve.resolver_for("uv.lock") + assert uv is not None + + tree = ["backend/pyproject.toml", "backend/uv.lock", "frontend/uv.lock"] + + assert resolve.missing_from(uv, tree, "backend") == [] + assert resolve.missing_from(uv, tree, "frontend") == ["pyproject.toml"] + + +def test_the_resolver_runs_where_the_lock_is() -> None: + """**The defect itself**: `uv lock` in the worktree root, on a repository whose `pyproject.toml` + is in `backend/`, answers *No `pyproject.toml` found in current directory or any parent + directory* — a sentence about our own working directory, recorded as a fact about somebody + else's repository (`cannot-move`). + """ + import tempfile + from pathlib import Path + + uv = resolve.resolver_for("uv.lock") + assert uv is not None + root = Path(tempfile.mkdtemp()) + (root / "backend").mkdir() + (root / "backend" / "pyproject.toml").write_text("[project]\nname='x'\n") + (root / "backend" / "uv.lock").write_text(UV_LOCK % "3.1.6") + mounted: list[Path] = [] + + def run(_r: object, context: Path, _c: str) -> tuple[int, str]: + mounted.append(context) + return 0, "" + + outcome = resolve.upgrade( + resolver=uv, worktree=root, package="jinja2", version="3.1.6", + present=["backend/pyproject.toml", "backend/uv.lock"], run=run, at="backend", + ) + + assert mounted == [root / "backend"], "the tool ran somewhere other than beside the lock" + assert outcome.ok, outcome.detail + + +def test_a_lock_at_the_root_is_unchanged() -> None: + """Every project that worked yesterday takes the path it took yesterday: `at` defaults to the + root, and this is what says so rather than the default's existence.""" + import tempfile + from pathlib import Path + + uv = resolve.resolver_for("uv.lock") + assert uv is not None + root = Path(tempfile.mkdtemp()) + (root / "pyproject.toml").write_text("[project]\nname='x'\n") + (root / "uv.lock").write_text(UV_LOCK % "3.1.6") + mounted: list[Path] = [] + + def run(_r: object, context: Path, _c: str) -> tuple[int, str]: + mounted.append(context) + return 0, "" + + outcome = resolve.upgrade( + resolver=uv, worktree=root, package="jinja2", version="3.1.6", + present=["pyproject.toml", "uv.lock"], run=run, + ) + + assert mounted == [root] + assert outcome.ok, outcome.detail def test_the_version_is_read_back_out_of_each_lock_shape() -> None: @@ -167,3 +241,82 @@ def test_a_resolver_may_rewrite_every_file_it_needs_not_only_the_lock() -> None: assert resolver.lock in resolve.touches(resolver) # The manifest is in there too, which is the whole point of this test. assert len(resolve.touches(resolver)) >= 2 + + +# --- the checkout the daemon can actually see (item 240) --------------------------------------- + + +def test_the_resolver_never_bind_mounts_the_checkout(monkeypatch: object) -> None: + """**The third time this repository has learned it**, and the first time it is asserted. + + `-v {path}:/w` is resolved by the *daemon*. The dispatcher runs in a container, so that path + exists in one filesystem and is looked up in another: the daemon finds nothing, mounts an empty + directory, and `uv` reports the project has no manifest. Measured from inside the deployed + dispatcher — a file written there, and `.`/`..` seen by the daemon. + + Item 055 moved the attempt's worktree off a bind mount for exactly this and item 082 the + contract directory; this path kept one, with a docstring arguing a bind mount was *better* + here — true on a host, false in a container, and never re-read when the ground moved. + """ + import subprocess + from pathlib import Path + + ran: list[list[str]] = [] + + class Done: + returncode = 0 + stdout = "carrier\n" + stderr = "" + + def watch(argv: list[str], **kwargs: object) -> Done: + ran.append(argv) + return Done() + + monkeypatch.setattr(subprocess, "run", watch) # type: ignore[attr-defined] + monkeypatch.setattr( # type: ignore[attr-defined] + "hullwork.sandbox.docker.run_docker", lambda argv, **k: watch(argv) + ) + uv = resolve.resolver_for("uv.lock") + assert uv is not None + + resolve.in_a_container(uv, Path("/inside/the/dispatcher"), "uv lock") + + resolving = [argv for argv in ran if "sh" in argv and "-lc" in argv] + assert resolving, "the resolver never ran" + mounts = [argv[i + 1] for argv in resolving for i, a in enumerate(argv) if a == "-v"] + assert mounts, "the resolver runs with nothing mounted at all" + for mount in mounts: + assert not mount.startswith("/"), f"a host path is bind-mounted: {mount}" + assert mount.startswith("hullwork-resolve-"), mount + + +def test_the_regenerated_lock_is_copied_back(monkeypatch: object) -> None: + """The whole point of running it: `version_in_lock`, the rebuild and the guard that restores + what this touched all read files, and they read them in the dispatcher's own filesystem.""" + import subprocess + from pathlib import Path + + ran: list[list[str]] = [] + + class Done: + returncode = 0 + stdout = "carrier\n" + stderr = "" + + def watch(argv: list[str], **kwargs: object) -> Done: + ran.append(argv) + return Done() + + monkeypatch.setattr(subprocess, "run", watch) # type: ignore[attr-defined] + monkeypatch.setattr( # type: ignore[attr-defined] + "hullwork.sandbox.docker.run_docker", lambda argv, **k: watch(argv) + ) + uv = resolve.resolver_for("uv.lock") + assert uv is not None + + resolve.in_a_container(uv, Path("/w/backend"), "uv lock") + + copies = [argv for argv in ran if len(argv) > 1 and argv[1] == "cp"] + assert any(one[-1] == "/w/backend" for one in copies), "nothing is copied back out" + assert any(one[2] == "/w/backend/." for one in copies), "nothing is copied in" + assert any(one[1] == "volume" and one[2] == "rm" for one in ran), "the volume is left behind" diff --git a/tests/test_sandbox_net.py b/tests/test_sandbox_net.py index eb7afa0..eed3c4f 100644 --- a/tests/test_sandbox_net.py +++ b/tests/test_sandbox_net.py @@ -276,7 +276,8 @@ def test_everything_is_torn_down_even_when_construction_fails( pass # pragma: no cover - the cable never comes up lines = _log(tmp_path) - assert any(line.startswith("rm -f hullwork-cable-t8") for line in lines) + # `-v` since item 244: what the gateway's image declared as a VOLUME goes with the container. + assert any(line.startswith("rm -f -v hullwork-cable-t8") for line in lines) assert any(line.startswith("network rm hullwork-attempt-t8") for line in lines) @@ -481,7 +482,9 @@ def test_teardown_completes_when_the_journal_cannot_be_fetched( lines = _log(tmp_path) # Teardown ran to the end: all three removals happened despite the journal being unreadable. - assert any(line.startswith("rm -f hullwork-cable-t15") for line in lines), ( + # `-v` since item 244: the gateway's own anonymous volumes go with it, and a removal without + # it left 69 of them on the operator's host in a day. + assert any(line.startswith("rm -f -v hullwork-cable-t15") for line in lines), ( f"the container was not removed: {lines}" ) assert any(line == "network rm hullwork-attempt-t15" for line in lines), ( diff --git a/tests/test_the_desk_it_cleared.py b/tests/test_the_desk_it_cleared.py index 9b6f628..711ecca 100644 --- a/tests/test_the_desk_it_cleared.py +++ b/tests/test_the_desk_it_cleared.py @@ -314,5 +314,7 @@ def test_the_number_is_on_the_page_and_not_only_in_the_terminal( body = page.instance(session, Settings(), error_reporting=False) assert "went onto your desk rather than off it" in body - assert "2 claim(s) have arrived" in body - assert body.index("how much left your desk") < body.index("What its attempts came to") + assert "claims arrived" in body + # The two sections were folds titled with sentences until item 235; the order is the property + # and it survived the rename, which is why this asserts on the labels rather than on the prose. + assert body.index("What left your desk") < body.index("What attempts came to") diff --git a/tests/test_the_door_and_the_errors.py b/tests/test_the_door_and_the_errors.py new file mode 100644 index 0000000..ff5b3b6 --- /dev/null +++ b/tests/test_the_door_and_the_errors.py @@ -0,0 +1,219 @@ +"""The daily views are tables of subjects too. DR-0028, item 247. + +Errors was already a table and still broke the decision in two places: `state` printed itself +twenty-five times down a column, and the order was the clock's rather than *who is blocked* — which +put the one item waiting for a person at the top by luck. + +The door said two things about one number in consecutive lines: **Nothing needs you** in the +headline, and *2 waiting on you* in the project row underneath. Both were right; one name for them +was not. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, DependencyReport, Item, ItemState, Lane, Project + +SIGNED_IN = page.Acting(csrf="c", offered=True) + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/door.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add(Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + )) + session.commit() + yield session + get_settings.cache_clear() + + +def _item(db: Session, state: ItemState, title: str = "boom", lane: Lane = Lane.GREEN) -> Item: + row = Item( + project_id=1, fingerprint=f"{title}-{state.value}", state=state, lane=lane, + title=title, last_seen=dt.datetime.now(dt.UTC), first_seen=dt.datetime.now(dt.UTC), + ) + db.add(row) + db.commit() + return row + + +class TestErrorsIsGroupedByWhoIsBlocked: + def test_the_state_is_the_heading_and_not_a_column(self, db: Session) -> None: + """Twenty-five rows reading `done` is the column DR-0028 is named after.""" + for n in range(3): + _item(db, ItemState.DONE, title=f"closed {n}") + + shown = page.items(db, acting=SIGNED_IN) + + assert "Closed" in shown + assert shown.count("merged, rejected, or answered") == 1 + assert "state" not in shown + + def test_what_the_reader_owns_comes_first(self, db: Session) -> None: + """**The order was the clock's.** `last_seen desc` put the one item waiting for a person at + the top by luck; at two hundred items it is wherever the clock left it.""" + _item(db, ItemState.WAITING_APPROVAL, title="decide me") + for n in range(3): + _item(db, ItemState.DONE, title=f"closed {n}") + + shown = page.items(db, acting=SIGNED_IN) + + assert shown.index("Waiting on you") < shown.index("Closed") + + def test_closed_is_last_whatever_else_is_there(self, db: Session) -> None: + _item(db, ItemState.DONE, title="closed") + _item(db, ItemState.READY, title="queued") + + shown = page.items(db, acting=SIGNED_IN) + + assert shown.index("Queued") < shown.index("Closed") + + def test_the_grouping_is_the_one_the_front_page_uses(self) -> None: + """**Not a second vocabulary.** Two lists that group the same states under different names + drift the first time either changes, which is what DR-0027 spent an item undoing.""" + assert {key for _, key, _, _, _ in page._COLUMNS} == set(page._WHO_IS_BLOCKED) + + def test_an_empty_group_is_not_rendered(self, db: Session) -> None: + _item(db, ItemState.DONE, title="closed") + + shown = page.items(db, acting=SIGNED_IN) + + assert "Waiting on you" not in shown + + def test_the_sentence_above_the_list_describes_the_order_it_has(self, db: Session) -> None: + """It said *most recently seen first*, which stopped being true the moment it grouped.""" + _item(db, ItemState.DONE, title="closed") + + shown = page.items(db, acting=SIGNED_IN) + + assert "Grouped by who is blocked" in shown + assert "Most recently seen first" not in shown + + def test_an_item_that_can_never_be_attempted_says_so_where_it_fits(self, db: Session) -> None: + """`never` is a fact about the item. In the context cell it was clipped to `N…`, and a + truncated warning reads as a rendering fault rather than as a state.""" + db.query(Project).one().active = False + db.commit() + _item(db, ItemState.READY, title="unreachable") + + row = page.items(db, acting=SIGNED_IN).split('')[1] + + assert "never" in row.split('')[0] + + +class TestTheDoorSaysOneThingAboutOneNumber: + def test_a_decision_and_work_only_a_person_can_do_are_two_sentences( + self, db: Session + ) -> None: + """**The contradiction this item found.** The headline counts `waiting-approval` and said + *Nothing needs you*; the project row summed `human-only` into the same words two lines + below. Both numbers were right; one name for them was not.""" + _item(db, ItemState.HUMAN_ONLY, title="only a person") + + shown = page.front_door(db, Settings(), acting=SIGNED_IN) + + assert "Nothing needs you" in shown + assert "1 only a person can do" in shown + assert "waiting on you" not in shown + + def test_a_decision_owed_is_counted_as_one(self, db: Session) -> None: + _item(db, ItemState.WAITING_APPROVAL, title="decide me") + + shown = page.front_door(db, Settings(), acting=SIGNED_IN) + + assert "1 awaiting your decision" in shown + assert "Nothing needs you" not in shown + + def test_a_project_is_a_row_of_the_same_table(self, db: Session) -> None: + shown = page.front_door(db, Settings(), acting=SIGNED_IN) + + assert '' in shown + assert 'shop' in shown + + def test_the_paragraph_explaining_the_list_is_gone(self, db: Session) -> None: + """It was re-read every day by somebody who understood it the first time.""" + shown = page.front_door(db, Settings(), acting=SIGNED_IN) + + assert "the number is what is waiting on a person rather than how much" not in shown + + +class TestTheRailCountsWhatItsViewCounts: + def test_a_package_pinned_twice_is_one_of_each(self, db: Session) -> None: + """`Dependencies 25` beside a view saying `20 packages`: two true numbers of two different + things, in one eye-line.""" + db.merge(DependencyReport( + project_id=1, taken_at=dt.datetime.now(dt.UTC), asked=True, pinned=9, + findings=[ + {"package": "thing", "version": "1.0", "source": "a", "advisories": []}, + {"package": "thing", "version": "2.0", "source": "a", "advisories": []}, + {"package": "other", "version": "1.0", "source": "a", "advisories": []}, + ], + )) + db.commit() + + assert page.how_much_of_each(db, 1).dependencies == 2 + + def test_a_report_that_could_not_ask_counts_nothing(self, db: Session) -> None: + """**Carrying findings, which is the only shape where the guard fires.** With an empty list + the count is zero either way, and the first version of this test passed with `asked` + deleted from the condition — an advisory list that silently reads as current when the + question never reached OSV is the failure DR-0024 exists to prevent.""" + db.merge(DependencyReport( + project_id=1, taken_at=dt.datetime.now(dt.UTC), asked=False, pinned=9, + findings=[{"package": "stale", "version": "1.0", "source": "a", "advisories": []}], + )) + db.commit() + + assert page.how_much_of_each(db, 1).dependencies == 0 + + +class TestWhatTheDeployedPageSaid: + """Two defects visible on the door the moment it was deployed, and neither was on the list.""" + + def test_a_phrase_that_is_not_a_duration_does_not_get_ago(self) -> None: + """`_ago` answers `just now` under a minute, and every caller appending *ago* to it + rendered **just now ago** — on the door, on an idle instance, which is most of the time.""" + assert page._since(None) == "not recorded" + assert page._since(dt.datetime.now(dt.UTC)) == "just now" + assert page._since(dt.datetime.now(dt.UTC) - dt.timedelta(hours=3)) == "3h ago" + + def test_no_caller_appends_ago_to_it_by_hand(self) -> None: + """**Asserted on the source**, because the four that did were found by reading the rendered + page rather than by a test — and a fifth would be written the same way.""" + import inspect + + assert ")} ago" not in inspect.getsource(page), "a caller composes the phrase by hand" + + def test_the_project_line_counts_packages_like_its_view(self, db: Session) -> None: + """`25 published against what it pins` beside a view that says `20 packages`: the rail's + fault, in the door's own words.""" + db.merge(DependencyReport( + project_id=1, taken_at=dt.datetime.now(dt.UTC), asked=True, pinned=9, + findings=[ + {"package": "thing", "version": "1.0", "source": "a", "advisories": []}, + {"package": "thing", "version": "2.0", "source": "a", "advisories": []}, + ], + )) + db.commit() + + shown = page.front_door(db, Settings(), acting=SIGNED_IN) + + assert "1 package(s) with something published" in shown diff --git a/tests/test_the_door_answers_one_question.py b/tests/test_the_door_answers_one_question.py new file mode 100644 index 0000000..b0f4330 --- /dev/null +++ b/tests/test_the_door_answers_one_question.py @@ -0,0 +1,184 @@ +"""The front door: what needs you, then one line per project. Item 237. + +It was every item on the instance in one table. With one project that is a list; with two it is two +projects' bugs interleaved and **whose** is the column a reader has to scan for — which is the +mixing the operator asked to stop: + +> ¿No será mejor plantear esto mismo, pero a nivel de proyecto? Así no mezclamos cosas. + +So the door answers one question and lists one thing, and the number against a project is **what is +waiting on a person** rather than how much exists. A count of items reads the same on a project that +is fine and one that is stuck, and the whole of a front door is *which of these wants me*. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, DependencyReport, Item, ItemState, Lane, Project + +SIGNED_IN = page.Acting(csrf="c", offered=True) + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/door.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + for slug in ("shop", "warehouse"): + session.add( + Project( + slug=slug, forge="forgejo", repo=f"acme/{slug}", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _an_item(db: Session, slug: str, state: ItemState) -> Item: + project = db.query(Project).filter(Project.slug == slug).one() + seen = db.query(Item).count() + one = Item( + project_id=project.id, fingerprint=f"f{seen}", title="KeyError: 'total'", + state=state, lane=Lane.AMBER, last_seen=dt.datetime.now(dt.UTC), + ) + db.add(one) + db.commit() + return one + + +def _report(db: Session, slug: str, **fields: object) -> None: + project = db.query(Project).filter(Project.slug == slug).one() + db.merge( + DependencyReport( + project_id=project.id, taken_at=dt.datetime.now(dt.UTC), **fields + ) + ) + db.commit() + + +def _door(db: Session) -> str: + return page.front_door(db, Settings(), acting=SIGNED_IN) + + +def _line_for(db: Session, slug: str) -> str: + """The project's own row, and **not** its entry in the rail, which carries the same name and + the same href one element up. Slicing on the name alone found the rail first. + + A row of the subject table since item 247, so the slice is ``-shaped; the rail's copy of the + name is not inside one, which is what keeps this pointing at the right element. + """ + shown = _door(db) + marker = f'{slug}' + assert marker in shown, f"{slug} is not listed on the door" + at = shown.index(marker) + return shown[shown.rindex("", at)] + + +# --- what it says ------------------------------------------------------------------------------- + + +def test_it_answers_whether_anything_needs_you(db: Session) -> None: + """The one question a person opens this to ask, and the reason the door stopped being a table + of every item on the instance.""" + shown = _door(db) + + assert "Nothing needs you" in shown + + +def test_every_project_is_one_line(db: Session) -> None: + _an_item(db, "shop", ItemState.NEW) + + shown = _door(db) + + assert ">shop" in shown + assert ">warehouse" in shown + + +def test_the_number_is_what_waits_on_a_person(db: Session) -> None: + """**Not how much exists.** Four items nobody is blocked on and one waiting for a decision are + different situations, and a count of items renders them identically.""" + _an_item(db, "shop", ItemState.NEW) + _an_item(db, "shop", ItemState.READY) + _an_item(db, "shop", ItemState.WAITING_APPROVAL) + + line = _line_for(db, "shop") + + # **Two states, two sentences** (item 247): this line summed `waiting-approval` and `human-only` + # and called both *waiting on you*, two lines under a headline that counts only the first — and + # which had just said *Nothing needs you*. A decision is owed on one; the other is work no agent + # may attempt, and nothing is owed until somebody chooses to do it. + assert "1 awaiting your decision" in line + assert "3 item(s)" in line, "how much there is is still said, after what needs doing" + + +def test_a_project_with_nothing_waiting_says_so_quietly(db: Session) -> None: + """Nothing rather than a zero: item 073's rule, which is that a signal on every row all the time + is not a signal. The dash it used to print was that signal in punctuation — the row now says it + by having nothing in the column where an action goes.""" + line = _line_for(db, "warehouse") + + assert "awaiting your decision" not in line + assert 'class="do">' in line + + +# --- and what it must not flatten --------------------------------------------------------------- + + +def test_the_three_dependency_states_survive_the_summary(db: Session) -> None: + """**DR-0024's condition, one level up.** A project whose report failed must never read like one + with nothing published against it — that is the failure this whole half must not have, and a + one-line summary is exactly where it would be lost.""" + _report(db, "shop", asked=False, pinned=9, note="could not reach OSV", findings=[]) + _report( + db, "warehouse", asked=True, pinned=50, + findings=[{"package": "x", "version": "1", "source": "s", "advisories": []}], + ) + + could_not = _line_for(db, "shop") + counted = _line_for(db, "warehouse") + + assert "could not ask OSV" in could_not + assert "published against" not in could_not, "a failed request reads as a clean report" + assert "1 package(s) with something published" in counted + + +def test_a_project_nobody_has_asked_about_is_a_fourth_state(db: Session) -> None: + """*Not asked yet* is not *nothing published*, and the six-hour clock is what makes the first + one temporary rather than a thing to go and do.""" + line = _line_for(db, "shop") + + assert "not asked about yet" in line + + +def test_a_project_that_is_not_watched_says_it_on_the_door(db: Session) -> None: + """The state an operator forgets they left something in — and the one where every other number + on the line is about to stop moving.""" + db.query(Project).filter(Project.slug == "shop").one().active = False + db.commit() + + assert "not watched" in _line_for(db, "shop") + + +def test_no_project_lines_are_about_another_project(db: Session) -> None: + """**The mixing this item removed.** Every number on a line is that project's; the door was one + table of every item on the instance, with the project as a column to scan.""" + _an_item(db, "shop", ItemState.WAITING_APPROVAL) + + assert "awaiting your decision" in _line_for(db, "shop") + assert "awaiting your decision" not in _line_for(db, "warehouse") diff --git a/tests/test_the_door_you_arrive_at.py b/tests/test_the_door_you_arrive_at.py new file mode 100644 index 0000000..197af6f --- /dev/null +++ b/tests/test_the_door_you_arrive_at.py @@ -0,0 +1,181 @@ +"""Every URL on the operator's path offers the way in. Item 224. + +The operator: *la web no es reactiva, es como que sólo se sirve `page/me/`, entro a cualquier otra +ruta, y me da error en el navegador.* + +Reproduced exactly. With no session, `/page/me/` answered `200` with a login and **every other path +answered `{"detail":"Not Found"}`** — so a bookmark, a second browser, or a session twelve hours old +turned every view into a wall. + +**The `404` is right where it is right, and it was in the wrong place.** DR-0021's reason is that +a distinct refusal tells somebody holding a *read link* which doors exist behind it. `me` is not a +read link: it is a literal anybody can type, and the front door already answers it with a login. +Refusing the rest protected nothing and cost everything. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import operator, page +from hullwork.config import get_settings +from hullwork.db import make_engine +from hullwork.models import Base, Project +from hullwork.security import generate_token, hash_token + +#: Every view on the operator's path. The point of the item is that none of them is a wall. +VIEWS = ("", "items", "instance", "projects", "doctor", "config", "projects/shop") + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/door.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +@pytest.fixture +def client() -> TestClient: + from hullwork.main import app + + return TestClient(app, follow_redirects=False) + + +# --- no view is a wall --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("view", VIEWS) +def test_every_view_offers_the_way_in(db: Session, client: TestClient, view: str) -> None: + """**The whole item.** A person arriving at any of these with no session gets the login, not a + JSON `404` — which is what a browser shows as a bare error page.""" + operator.set_password(db, "correct horse") + db.commit() + + shown = client.get(f"/page/me/{view}") + + assert shown.status_code == 200, view + assert "current-password" in shown.text, f"/page/me/{view} is a wall" + + +def test_the_login_it_offers_still_says_nothing_about_the_instance( + db: Session, client: TestClient +) -> None: + """Item 204's rule, which this must not undo: the one page a prober can reach discloses that + this host has a login and nothing else — not a name, not a version, not a count. A version tells + somebody which advisories to go and read.""" + operator.set_password(db, "correct horse") + db.commit() + + # **Without the stylesheet**, which is inlined and whose comments cite WCAG 2.5.8 and CSS + # escapes — the first version of this failed on a rule number in a comment and called it a + # version. What is being asserted is what a reader sees, not what the bytes contain. + shown = re.sub(r"", "", client.get("/page/me/config").text, flags=re.S) + + assert "hullwork" not in shown.lower() + assert not re.search(r"\d+\.\d+\.\d+", shown), "a version reached the door" + + +def test_an_instance_with_no_password_still_refuses(db: Session, client: TestClient) -> None: + """**Offering a login where there is none would be a lie**, and a door that says *sign in* on an + instance with no password sends somebody looking for a credential that does not exist. With + nothing to sign in to, the refusal is the honest answer and stays a `404`.""" + for view in ("instance", "projects", "doctor"): + assert client.get(f"/page/me/{view}").status_code == 404, view + + +def test_a_read_link_is_still_told_nothing(db: Session, client: TestClient) -> None: + """DR-0021 intact. The `404` is right where it is right: a token is a secret, and a distinct + refusal on one path and not another tells its holder which doors exist behind it.""" + operator.set_password(db, "correct horse") + minted = generate_token() + page.issue(db, hash_token(minted)) + db.commit() + + assert client.get(f"/page/{minted}/doctor").status_code == 404 + assert client.get(f"/page/{minted}/config").status_code == 404 + + +def test_a_link_that_stopped_working_is_not_an_invitation( + db: Session, client: TestClient +) -> None: + """**A rotated link must not look like a door.** The colleague holding yesterday's URL is not + somebody who should sign in — they have no password and never did — and answering them with a + login sends them to ask for one. It stays a `404`, which is what a wrong path is. + + Written because a mutation escaped: the read-link test above uses a *valid* token, so it never + reaches this decision at all.""" + operator.set_password(db, "correct horse") + stale = generate_token() + page.issue(db, hash_token(generate_token())) # somebody rotated it + db.commit() + + for view in ("", "instance", "doctor"): + shown = client.get(f"/page/{stale}/{view}") + + assert shown.status_code == 404, view + assert "current-password" not in shown.text, f"a stale link is offered a login at {view!r}" + + +# --- and it takes you where you were going ------------------------------------------------------- + + +def test_signing_in_lands_where_you_were_going(db: Session, client: TestClient) -> None: + """Twelve hours is the session, and an instance reached by bookmark is reached at a view. Being + returned to the front door every time means finding it again by hand.""" + operator.set_password(db, "correct horse") + db.commit() + + offered = client.get("/page/me/doctor").text + field = re.search(r'name="going_to" value="([^"]*)"', offered) + + assert field is not None, "the login forgot where it was asked from" + + arrived = client.post( + "/page/me/login", data={"password": "correct horse", "going_to": field.group(1)} + ) + + assert arrived.headers["location"] == "/page/me/doctor" + + +@pytest.mark.parametrize( + "asked", + [ + "https://evil.example/x", + "//evil.example", + "/page/me/../../etc/passwd", + "/page/me/config/../../elsewhere", + "instance;rm -rf /", + "/page/OTHER-TOKEN/instance", + ], +) +def test_it_will_not_be_pointed_anywhere_else(asked: str) -> None: + """**An open redirect is how a sign-in form becomes somebody else's.** A literal list rather + than a pattern, because `../`, `//host` and `%2e%2e` are each something a pattern written in a + hurry lets through.""" + assert page.where_it_may_land(asked) == "" + + +def test_it_does_take_you_to_a_project_of_its_own(db: Session, client: TestClient) -> None: + """The one shape beyond the flat list, because it is where half the work is.""" + assert page.where_it_may_land("/page/me/projects/shop") == "projects/shop" + assert page.where_it_may_land("/page/me/projects/shop/../../x") == "" diff --git a/tests/test_the_error_as_the_tracker_recorded_it.py b/tests/test_the_error_as_the_tracker_recorded_it.py new file mode 100644 index 0000000..ccfde39 --- /dev/null +++ b/tests/test_the_error_as_the_tracker_recorded_it.py @@ -0,0 +1,178 @@ +"""The full error, on the item's own view. Item 232. + +`FetchedEvent` is the largest object in the system — the untruncated message, the frames with their +source context, the locals, and the 33 to 71 dependency versions pinned at the moment it failed. It +is what item 036 built the tracker reader for, it is what an attempt is constructed from, and it +appeared on the page zero times. + +**The webhook cuts the title at 100 characters**, and the model says why that is not a detail: for a +`KeyError` or a `ValueError` the half it cuts is often the input that reproduces the bug. The item's +title is the cut version; the whole one lives here and nowhere else on this page. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, FetchedEvent, Item, ItemState, Lane, Project + +SIGNED_IN = page.Acting(csrf="c", offered=True) + +#: The half a webhook cuts. The item's title stops at *checkout*; the input is after it. +WHOLE = "KeyError: 'total' in checkout — the input that reproduces it: {'lines': []}" + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/error.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + project = Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + session.add(project) + session.flush() + session.add( + Item( + project_id=project.id, fingerprint="f", title="KeyError: 'total' in checkout", + state=ItemState.NEW, lane=Lane.GREEN, last_seen=dt.datetime.now(dt.UTC), + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _recorded(db: Session, **fields: object) -> FetchedEvent: + item = db.query(Item).one() + seen = db.query(FetchedEvent).count() + fetched = FetchedEvent( + item_id=item.id, + provider_event_id=f"ev-{seen}", + exception_type="KeyError", + message=WHOLE, + culprit="app/checkout.py in total", + level="error", + handled=False, + occurred_at=dt.datetime.now(dt.UTC) - dt.timedelta(hours=3), + frames=[ + { + "filename": "app/checkout.py", + "function": "total", + "lineno": 42, + "context_line": "return sum(line['total'] for line in lines)", + "variables": {"lines": "[]", "user_id": "9"}, + } + ], + packages={"requests": "2.31.0", "fastapi": "0.115.0"}, + ) + for name, value in fields.items(): + setattr(fetched, name, value) + db.add(fetched) + db.commit() + return fetched + + +def _view(db: Session) -> str: + item = db.query(Item).one() + return page.item(db, Settings(), item.id, acting=SIGNED_IN) or "" + + +# --- what it shows ------------------------------------------------------------------------------- + + +def test_the_untruncated_message_is_there_and_says_it_is(db: Session) -> None: + """**The item's title is the cut one.** A page that showed only it would be showing the half + the provider kept rather than the half that reproduces the bug.""" + _recorded(db) + + shown = _view(db) + + assert "the input that reproduces it" in shown + assert "untruncated" in shown + assert "cuts it at 100" in shown + + +def test_the_frames_carry_the_line_they_stopped_on(db: Session) -> None: + _recorded(db) + + shown = _view(db) + + assert "app/checkout.py" in shown + assert "line 42" in shown + assert "sum(line[" in shown, "the source line itself is not there" + + +def test_the_locals_are_behind_their_own_disclosure(db: Session) -> None: + """**The sharpest thing on this page.** Scrubbed on the way in, and still what a reader is + least often looking for and most likely to be surprised to find rendered.""" + _recorded(db) + + shown = _view(db) + fold = re.search(r"]*>\s*What the code was holding here", shown) + + assert fold is not None, "the locals are not folded" + assert " open" not in fold.group(0), "they are open, so they are read whether or not you asked" + assert "user_id" in shown + + +def test_the_pinned_versions_are_counted_not_poured_out(db: Session) -> None: + """33 to 71 of them. A wall of versions above the stack would bury the stack.""" + _recorded(db) + + shown = _view(db) + + assert "2 version(s)" in shown + fold = re.search(r"]*>\s*What was installed when it failed", shown) + assert fold is not None and " open" not in fold.group(0) + + +def test_more_than_one_occurrence_says_why_that_is_worth_having(db: Session) -> None: + """Several rows per item are allowed on purpose: what differs between two samples is usually + the input that triggers it, and it is the only route to occurrences 2..N.""" + _recorded(db) + _recorded(db) + + shown = _view(db) + + assert "2 occurrences" in shown + assert "once per issue and never again" in shown + + +# --- and what it must not claim ------------------------------------------------------------------- + + +def test_a_pruned_event_says_it_was_forgotten(db: Session) -> None: + """**`prune` empties the row and keeps it.** Rendering *no frames* would report an error with no + stack rather than one whose stack this instance chose to forget — different sentences, and the + second is the true one.""" + _recorded(db, frames=[], packages={}) + + shown = _view(db) + + assert "forgotten by" in shown + assert "stopped keeping them" in shown + assert "No frames were recorded" not in shown, "it reports absence as if it were the error's" + + +def test_an_item_nothing_was_fetched_for_shows_nothing_at_all(db: Session) -> None: + """No fold, not an empty one: an item whose tracker was never read has nothing to say here, and + a disclosure promising an error and holding none is worse than no disclosure.""" + shown = _view(db) + + assert "The error, as the tracker recorded it" not in shown diff --git a/tests/test_the_evidence_a_reviewer_came_for.py b/tests/test_the_evidence_a_reviewer_came_for.py index 39b0b58..ef1c708 100644 --- a/tests/test_the_evidence_a_reviewer_came_for.py +++ b/tests/test_the_evidence_a_reviewer_came_for.py @@ -158,13 +158,20 @@ def test_the_list_says_which_attempt_reached_a_pull_request(db: Session, project rendered = page.items(db) - row = re.search(r"(?:(?!).)*fixed one.*?", rendered, re.DOTALL) + # Rows carry a class since item 247; the reach of each one is still its last cell. + row = re.search(r'(?:(?!).)*fixed one.*?', rendered, re.DOTALL) assert row is not None and "#6" in row.group(0) - filed_row = re.search(r"(?:(?!).)*filed only.*?", rendered, re.DOTALL) + filed_row = re.search( + r'(?:(?!).)*filed only.*?', rendered, re.DOTALL + ) assert filed_row is not None and "#10" in filed_row.group(0) assert filed.forge_issue_ref == "#10" - never = re.search(r"(?:(?!).)*never filed.*?", rendered, re.DOTALL) - assert never is not None and "—" in never.group(0) + never = re.search( + r'(?:(?!).)*never filed.*?', rendered, re.DOTALL + ) + # **Nothing rather than a dash**: the column where a reference goes is empty, which is what the + # other views do with an action nobody can take. + assert never is not None and '' in never.group(0) def test_the_bound_is_on_the_page_and_not_only_in_the_query( @@ -180,7 +187,8 @@ def test_the_bound_is_on_the_page_and_not_only_in_the_query( rendered = page.items(db) assert "2 most recently seen of 5 item(s)" in rendered - assert rendered.count("") == 3, "two rows and the header" + # Two rows and no header: the grouping's heading is what names them now (DR-0028). + assert rendered.count('') == 2 def test_an_instance_with_no_items_says_so(db: Session) -> None: @@ -487,26 +495,44 @@ def test_the_relative_links_actually_reach_the_other_views( door = client.get(f"/page/{token}") assert str(door.url).endswith(f"/page/{token}/"), "the slash is what makes the rest relative" - # Item 212 made the door the items themselves, so the first hop of this walk is gone and the - # rail — which is on every page and therefore has five chances to resolve wrongly — is what the - # rest of it follows. - assert "

    Items

    " in door.text + # Item 212 made the door the work rather than the arithmetic; item 237 made it the projects and + # what is waiting in each. The rail is on every page and therefore has as many chances to + # resolve wrongly as it has entries, which is what the rest of this walk follows. + assert "

    Hullwork

    " in door.text or "Projects" in door.text report = client.get(urljoin(str(door.url), _href(door.text, "This instance"))) assert report.status_code == 200, "the noun the arithmetic moved behind" - detail = client.get(urljoin(str(door.url), _href(door.text, f"#{item.id}"))) - assert detail.status_code == 200 + # **Three depths of relative link, which is where item 227 broke** — one level for `items/`, + # two for `projects/`, three for `projects//`. Each is written once, in + # `_document`, and each is followed here rather than asserted about. + walked = client.get(urljoin(str(door.url), _href(door.text, project.slug))) + assert walked.status_code == 200, "a project named on the door does not resolve" + + seen: dict[str, str] = {} + for feature in ("Errors", "Fixes", "Dependencies", "Deliveries"): + inside = client.get(urljoin(str(walked.url), _href(walked.text, feature))) + assert inside.status_code == 200, f"{feature} does not resolve from the project" + assert f"{feature}" in inside.text + seen[feature] = inside.text + + from_errors = _href(seen["Errors"], str(item.id)) + detail = client.get(urljoin(f"/page/{token}/projects/{project.slug}/errors", from_errors)) + assert detail.status_code == 200, "an item does not resolve from its project's errors" assert "Attempt 1" in detail.text - back = client.get(urljoin(str(detail.url), _href(detail.text, "Items"))) + back = client.get(urljoin(str(detail.url), _href(detail.text, "What needs you"))) assert back.status_code == 200 - assert "

    Items

    " in back.text + assert "Projects" in back.text def _href(html_text: str, label: str) -> str: """The `href` of the link whose text is `label`. A reader clicks these; so does this test.""" - found = re.search(rf']*>{re.escape(label)}', html_text) + # The rail carries a count inside the link since item 235, so a label is followed by the end of + # the anchor **or** by that badge. Anchored on the label rather than on the whole anchor. + found = re.search(rf']*>{re.escape(label)}(?:)', html_text) + if found is None: # a name rendered inside its own element, as the door renders a project + found = re.search(rf']*>\s*{re.escape(label)}\s*<', html_text) assert found is not None, f"no link labelled {label!r}" return found.group(1) diff --git a/tests/test_the_figures_are_a_table.py b/tests/test_the_figures_are_a_table.py new file mode 100644 index 0000000..99baf3a --- /dev/null +++ b/tests/test_the_figures_are_a_table.py @@ -0,0 +1,118 @@ +"""The instance report's counts are figures in a column. DR-0028, item 248. + +Six sections of prose bullets were 500 of this view's 779 words, and every bullet was **a number +with a sentence wrapped around it**. A reader comparing this week to last had to parse eight of them +to find two figures — and the one section that was already a two-column table is the one that reads +at a glance. + +What is under test is that nothing was lost on the way: every count and every caveat the sentences +carried is still on the page, and the skin that renders them agrees with the one the terminal +prints. Two skins of one structure is the item 050 pattern; two *computations* is what comes apart. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import pytest + +from hullwork import outcomes, page +from hullwork.models import AttemptOutcome + + +def _desk(**fields: int) -> outcomes.Desk: + return outcomes.Desk(**fields) + + +class TestAFigureAndWhatItCounts: + def test_the_count_is_its_own_column(self) -> None: + shown = page._figures([(28, "claims arrived", "")]) + + assert '    ' in shown + assert "claims arrived" in shown + + def test_the_caveat_is_under_the_meaning_not_inside_it(self) -> None: + """*3 baseline-red, 2 abandoned* used to be the second half of the same sentence, which is + where a reader looking for the number has to read past it.""" + shown = page._figures([(5, "never counted against an item", "3 baseline-red")]) + + assert '3 baseline-red' in shown + + def test_no_rows_is_no_table(self) -> None: + """Silence rather than an empty frame: `desk_lines`'s own rule, and the two skins have to + agree on it or one of them reports a beginning as a failure.""" + assert page._figures([]) == "" + + def test_it_sits_in_a_container_that_scrolls_on_its_own(self) -> None: + """Item 215's rule about every table on this page, and a two-column table being unable to + overflow is a reason to believe it rather than to exempt it.""" + assert '
    ' in page._figures([(1, "a thing", "")]) + + +class TestNothingTheSentencesCarriedIsLost: + def test_every_figure_the_desk_reports_is_on_the_page(self) -> None: + """**The two skins are checked against each other**, which is what keeps them from drifting + without duplicating the arithmetic: every number the terminal prints has to be a figure.""" + counted = _desk( + arrived=28, left_with_evidence=6, with_a_change=4, with_a_refusal=2, + still_waiting=20, handed_over=2, + ) + + shown = page._desk_figures(counted) + spoken = " ".join(outcomes.desk_lines(counted)) + + for number in ("28", "20", "6", "4", "2"): + assert number in shown, f"{number} is in the sentences and not in the figures" + assert "28" in spoken + + def test_the_figure_that_can_embarrass_it_keeps_its_words(self) -> None: + """*Put on* a desk rather than taken off it. Rounding this into good news is exactly how a + report stops being one.""" + shown = page._desk_figures(_desk(arrived=3, handed_over=2)) + + assert "went onto your desk rather than off it" in shown + assert "red lane, or a pull request somebody read and refused" in shown + + def test_nothing_arrived_is_nothing_rendered(self) -> None: + assert page._desk_figures(_desk()) == "" + + def test_the_funnel_keeps_its_denominator_and_its_exclusions(self) -> None: + counted = outcomes.Funnel( + fair_try=6, pull_requests=4, merged=4, not_reproducible=1, failed=1, + rehearsals=10, + never_counted={ + AttemptOutcome.BASELINE_RED: 3, AttemptOutcome.ABANDONED: 2, + }, + ) + + shown = page._funnel_figures(counted) + + assert "attempts got a fair try" in shown + assert "of those 4 pull request(s) were merged" in shown + assert "baseline-red" in shown and "abandoned" in shown + # Rehearsals publish nothing, and an instance that has only rehearsed has done work that + # produced no forge state to count. Dropping them reads as an instance that did nothing. + assert "rehearsals" in shown + + def test_a_percentage_is_never_printed(self) -> None: + """Item 119: four attempts are not a rate, and a percentage invites comparing instances + running different code over different repositories.""" + shown = page._funnel_figures( + outcomes.Funnel(fair_try=6, pull_requests=4, merged=4) + ) + + assert "%" not in shown + + +class TestWhatStaysProse: + @pytest.mark.parametrize( + "line", ["median time from first error to decision: 2h 1m"] + ) + def test_a_duration_is_not_forced_into_a_column(self, line: str) -> None: + """A figure goes in the column; prose that is genuinely prose stays prose. A median in a + column of counts is a number that cannot be compared with the ones above it.""" + shown = page._review_figures(outcomes.Reviewed(merged=4), [line]) + + assert '
    ' in shown + assert line in shown + assert f'' in shown + assert "") == 1, "one package is one row" + assert "49.0.0" in shown + assert "50.0.0" in shown + + +def test_nothing_tried_yet_says_who_will_try_it(db: Session) -> None: + """The old sentence handed the question back to the reader — *only the half that holds a Docker + socket can answer it* — which is true and useless to somebody reading a page.""" + shown = _view(db) + + assert "What happened when it was tried" not in shown, "no second list (DR-0028)" + assert "one per idle turn" in shown + assert "Not tried yet" in shown + + +# --- and what it must never claim ------------------------------------------------------------ + + +def test_a_verdict_about_a_version_no_longer_pinned_is_not_shown(db: Session) -> None: + """**The stale one.** A row saying `47.0.0 → 48.0.0 is clean` about a version this repository + stopped pinning wears no mark of its age, so it reads as a current statement about a current + pin. Dropping it is the only honest render.""" + _verdict(db, "clean", was="47.0.0", to="48.0.0") + + shown = _view(db) + + assert "47.0.0" not in shown + assert "What happened when it was tried" not in shown + + +def test_a_build_that_refused_is_not_painted_as_a_broken_suite(db: Session) -> None: + """`will-not-install` is the build failing. Rendering it red would be this instance telling + somebody their tests fail on an upgrade whose tests never ran.""" + _verdict(db, "will-not-install") + + shown = _view(db) + + assert "the build refused it, so your suite never ran" in shown + assert "your suite fails on it" not in shown + + +def test_a_suite_already_failing_claims_nothing_either_way(db: Session) -> None: + """The one state where the honest answer is *I could not tell you* — and it needs a person, not + a colour that reads as bad news about the upgrade.""" + _verdict(db, "already-red") + + shown = _view(db) + + assert "own test suite was already failing" in shown + assert "your suite fails on it" not in shown + + +def test_a_red_baseline_is_said_once_rather_than_once_per_pair(db: Session) -> None: + """**Item 234, as the page renders it.** `simplecheck` produced 50 identical rows in an hour: + the fact is about the project, not about any of the upgrades, and repeating it per pair buries + whatever else the list has to say.""" + for to in ("49.0.0", "50.0.0"): + _verdict(db, "already-red", to=to) + + shown = _view(db) + + assert shown.count("own test suite was already failing") == 1, "once, in the band's heading" + assert "That covers 2 upgrade(s)" in shown + assert "cryptography" in shown, "the packages it covers are not named" + + +def test_a_red_baseline_does_not_bury_a_real_verdict(db: Session) -> None: + """The one that matters is the one that is not `already-red`. A project part-way through a + recovery has both, and the summary must not swallow the row.""" + _verdict(db, "already-red", to="49.0.0") + _verdict(db, "breaks", to="50.0.0") + + shown = _view(db) + + # **The real verdict wins the row** (DR-0028): a package takes the state that most needs a + # person, so one pair being unclaimable never hides the one that broke. + assert "Breaks your suite" in shown + assert "50.0.0" in shown + assert "Nothing could be claimed" not in shown, "the red pair is not this package's state" + + +def test_an_outcome_this_page_does_not_know_is_shown_rather_than_swallowed(db: Session) -> None: + """A verdict `bump` adds tomorrow must not vanish from the page because this table was not + updated: an unknown state is still a state, and rendering nothing would report *not tried*.""" + _verdict(db, "cannot-parse") + + shown = _view(db) + + assert "it ended as cannot-parse" in shown diff --git a/tests/test_what_is_published_against_you.py b/tests/test_what_is_published_against_you.py new file mode 100644 index 0000000..4161b36 --- /dev/null +++ b/tests/test_what_is_published_against_you.py @@ -0,0 +1,467 @@ +"""The dependency report, on the instance and on the page. DR-0024, item 230. + +Accepted 2026-08-11 with two conditions, and they are the interesting half: + +> *el informe se guarda con cuándo se tomó, y "no pude preguntar a OSV" es una respuesta de primera +> clase, nunca una lista vacía.* + +They are the same sentence twice. A report rendered without its timestamp is a claim about a moment +presented as a standing fact; an advisory list that silently reads empty when the request failed +says *you are fine* on no evidence at all. Either one turns the half of the product an evaluator +can use on their first day into the kind of green tick this product exists to distrust. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import advisories, page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, DependencyReport, Project +from hullwork.osv import Advisory, Finding + +LOCK = """[[package]] +name = "requests" +version = "2.31.0" +""" + + +class _Tree: + def __init__(self, paths: tuple[str, ...]) -> None: + self.paths = paths + self.truncated = False + + +class _Forge: + """A forge that lists a tree and reads files, which is all this needs.""" + + def __init__(self, paths: tuple[str, ...] = ("uv.lock",), text: str | None = LOCK) -> None: + self._paths = paths + self._text = text + self.read: list[str] = [] + + def tree(self, repo: str) -> _Tree: + return _Tree(self._paths) + + def read_file(self, repo: str, path: str) -> str | None: + self.read.append(path) + return self._text + + def close(self) -> None: + pass + + +class _Refuses(_Forge): + def tree(self, repo: str) -> _Tree: + raise RuntimeError("403 Forbidden") + + +def _one_advisory(deps: Sequence[Any]) -> list[Finding]: + return [ + Finding( + dependency=deps[0], + advisories=( + Advisory(id="GHSA-xxxx", summary="a real one", fixed=("2.32.0", "2.31.1")), + ), + ) + ] + + +def _nothing(deps: Sequence[Any]) -> list[Finding]: + return [] + + +def _unreachable(deps: Sequence[Any]) -> list[Finding]: + raise TimeoutError("api.osv.dev did not answer") + + +# --- reading and asking ------------------------------------------------------------------------ + + +def test_it_reads_what_is_pinned_and_asks_about_it() -> None: + """One forge listing, one file per lock, one batch to OSV. Nothing here needs a credential OSV + does not take or a socket the receiver does not have.""" + forge = _Forge() + + report = advisories.about("acme/shop", forge, _one_advisory) + + assert forge.read == ["uv.lock"] + assert report.asked is True + assert report.pinned == 1 + assert report.findings[0]["package"] == "requests" + assert report.findings[0]["advisories"][0]["fixed"] == ["2.32.0", "2.31.1"] + + +def test_a_forge_that_will_not_list_is_not_a_clean_report() -> None: + """**The condition, half one.** Two halves can fail and they are different problems: this one + names which.""" + report = advisories.about("acme/shop", _Refuses(), _one_advisory) + + assert report.asked is False + assert "could not list" in (report.note or "") + assert report.findings == [] + + +def test_osv_being_unreachable_is_not_an_empty_report() -> None: + """**The condition, half two, and the one that matters.** `asked=False` with a note is the + answer; `findings=[]` with `asked=True` would be a page saying *you are fine* because the + network was down.""" + report = advisories.about("acme/shop", _Forge(), _unreachable) + + assert report.asked is False + assert report.pinned == 1, "it read the lock file before it failed, and says so" + assert "could not reach OSV" in (report.note or "") + + +def test_nothing_pinned_is_its_own_sentence() -> None: + """*Nothing published* and *nothing pinned* are different facts about different problems, and a + report that blurred them would tell a project with no lock file that it is clean.""" + report = advisories.about("acme/shop", _Forge(paths=("README.md",)), _one_advisory) + + assert report.asked is True + assert report.pinned == 0 + assert "nothing here pins a version" in (report.note or "") + + +def test_a_file_that_will_not_read_costs_only_itself() -> None: + """A `package-lock.json` the forge refuses while `uv.lock` comes back fine is one file saying + nothing, not a failed report.""" + forge = _Forge(paths=("package-lock.json", "uv.lock"), text=None) + + report = advisories.about("acme/shop", forge, _nothing) + + assert forge.read == ["package-lock.json", "uv.lock"] + assert report.asked is True + + +# --- and what the page says about it ------------------------------------------------------------ + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/adv.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _stored(db: Session, **fields: object) -> None: + row = db.query(Project).one() + db.merge( + DependencyReport( + project_id=row.id, + taken_at=dt.datetime.now(dt.UTC) - dt.timedelta(hours=2), + **fields, + ) + ) + db.commit() + + +def _view(db: Session) -> str: + """**Its own page since item 235.** This was a closed `
    ` on a project's view, and the + operator said three times that he could not find things: a feature that spans projects is a + page that spans projects.""" + signed_in = page.Acting(csrf="c", offered=True) + return page.dependencies(db, Settings(), "shop", acting=signed_in) or "" + + +def test_the_page_says_when_it_was_asked(db: Session) -> None: + """**The operator's first condition.** A dependency report is a claim about a moment; one + rendered without its timestamp is the permanently-on signal item 073 deleted a check for.""" + _stored(db, asked=True, pinned=12, findings=[]) + + shown = _view(db) + + assert "Asked 2h" in shown or "Asked 2 h" in shown + + +def test_could_not_ask_never_reads_as_clean(db: Session) -> None: + """**The second condition, on the page.** The words have to say that nothing was learned — + a fold headed *none published* over a failed request is the lie this guards against.""" + _stored(db, asked=False, pinned=9, note="read 9 pinned version(s) and could not reach OSV: x") + + shown = _view(db) + + # **Lowercased, because the sentence moved out of a `
    ` summary and into the body** + # (item 235). What is asserted is the claim, not the sentence case it happens to carry. + assert "could not ask" in shown.lower() + assert "not an empty report" in shown + # **Asserted on the claim, not on a substring of the document.** The old form sliced the whole + # page around two phrases and looked for `none`, which the stylesheet supplies twenty times over + # (`text-decoration: none`) — it passed for four months because the slice happened to miss them. + assert "nothing published against" not in shown.lower(), "it reads clean over a failed request" + + +def test_nothing_published_says_how_many_it_asked_about(db: Session) -> None: + """*None of twelve* and *none of nothing* are different answers, and only one of them is good + news.""" + _stored(db, asked=True, pinned=12, findings=[]) + + shown = _view(db) + + # The count was in the fold's summary — *none, of 12* — until item 235 unfolded the feature. + # The claim is unchanged and it is the one that matters: **of how many**. + assert "nothing published against any of the 12" in shown + assert "12 pinned version(s)" in shown + + +def test_a_finding_carries_its_fix(db: Session) -> None: + """An advisory with nothing to upgrade to is a different situation from one with two, and both + happen. The versions come from OSV and a person picks.""" + _stored( + db, + asked=True, + pinned=3, + findings=[ + { + "package": "requests", + "version": "2.31.0", + "source": "uv.lock", + "advisories": [ + {"id": "GHSA-xxxx", "summary": "a real one", "fixed": ["2.32.0"]}, + {"id": "GHSA-yyyy", "summary": "no fix yet", "fixed": []}, + ], + } + ], + ) + + shown = _view(db) + + assert "requests" in shown and "2.31.0" in shown + assert "2.32.0" in shown + # One advisory of the two publishes nothing to move to; the package still has somewhere to go, + # so it is not in the band for packages that do not. + assert "Nothing published to upgrade to" not in shown + + +def test_a_project_nobody_has_asked_about_says_who_will(db: Session) -> None: + """Not *run this command*: item 228's lesson, applied to the feature next to it.""" + shown = _view(db) + + assert "Not asked yet" in shown + assert "on its own clock" in shown + + +def test_the_page_never_asks_osv_itself(db: Session, monkeypatch: pytest.MonkeyPatch) -> None: + """Item 142's rule, again: a render spends no request — not to a forge and not to OSV.""" + from hullwork import osv as osv_module + + def _refuse(*args: object, **kwargs: object) -> object: + raise AssertionError("a page render asked OSV") + + monkeypatch.setattr(osv_module, "Osv", _refuse) + _stored(db, asked=True, pinned=1, findings=[]) + + assert "

    " in _view(db) + + +# --- and the clock asks it ---------------------------------------------------------------------- + + +class _Scoped: + def __init__(self, session: Session) -> None: + self._session = session + + def __enter__(self) -> Session: + return self._session + + def __exit__(self, *exc: object) -> None: + return None + + +def test_the_sweep_asks_and_stores_it(db: Session, monkeypatch: pytest.MonkeyPatch) -> None: + """**The item, wired.** Every test above calls `advisories.about` directly; deleting the line + that hangs it on the clock would leave them green and the instance silent — which is exactly + what escaped in item 228, one item ago, in the function next door.""" + from hullwork import advisories as advisories_module + from hullwork import main as main_module + + monkeypatch.setenv("HULLWORK_FORGE_URL", "https://forge.example") + monkeypatch.setenv("HULLWORK_FORGE_TOKEN", "t") + get_settings.cache_clear() + monkeypatch.setattr(main_module, "make_forge", lambda settings: _Forge()) + monkeypatch.setattr(advisories_module, "asking", lambda timeout=20.0: _one_advisory) + + main_module._ask_what_is_published_against_what_they_pin( + lambda: _Scoped(db), # type: ignore[arg-type] + get_settings(), + ) + + stored = db.query(DependencyReport).one() + assert stored.asked is True + assert stored.pinned == 1 + assert stored.findings[0]["package"] == "requests" + assert stored.taken_at is not None + + +def test_it_does_not_ask_again_inside_the_interval( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + """Six hours, not sixty seconds: advisories are published on a human's schedule, and asking a + public database every minute would be spending somebody else's API to learn nothing.""" + from hullwork import advisories as advisories_module + from hullwork import main as main_module + + monkeypatch.setenv("HULLWORK_FORGE_URL", "https://forge.example") + monkeypatch.setenv("HULLWORK_FORGE_TOKEN", "t") + get_settings.cache_clear() + forge = _Forge() + monkeypatch.setattr(main_module, "make_forge", lambda settings: forge) + monkeypatch.setattr(advisories_module, "asking", lambda timeout=20.0: _one_advisory) + + for _ in range(2): + main_module._ask_what_is_published_against_what_they_pin( + lambda: _Scoped(db), # type: ignore[arg-type] + get_settings(), + ) + + assert forge.read == ["uv.lock"], "it asked again inside the interval" + + +def test_the_sweep_itself_calls_it(db: Session, monkeypatch: pytest.MonkeyPatch) -> None: + """**And the escape happened anyway.** The test above says deleting the wiring would leave + everything green, and then nothing checked it — so the mutation walked through, in the item + right after the one where the identical thing happened. + + Predicting a hole is not covering it.""" + from hullwork import advisories as advisories_module + from hullwork import main as main_module + from hullwork.ingest import SweepResult + + monkeypatch.setenv("HULLWORK_FORGE_URL", "https://forge.example") + monkeypatch.setenv("HULLWORK_FORGE_TOKEN", "t") + get_settings.cache_clear() + forge = _Forge() + monkeypatch.setattr(main_module, "make_forge", lambda settings: forge) + monkeypatch.setattr(advisories_module, "asking", lambda timeout=20.0: _one_advisory) + monkeypatch.setattr(main_module, "make_tracker", lambda settings: None) + monkeypatch.setattr(main_module, "make_inventory", lambda settings: None) + nothing = SweepResult(deliveries=0, filed=0, resolved=0) + monkeypatch.setattr(main_module, "sweep", lambda *a, **k: nothing) + monkeypatch.setattr( + main_module, "_measure_what_the_ingest_credential_may_do", lambda *a, **k: None + ) + + main_module._sweep_once(lambda: _Scoped(db), get_settings()) # type: ignore[arg-type] + + assert db.query(DependencyReport).count() == 1, "the sweep does not ask" + + +def test_a_finding_is_three_facts_and_not_its_prose(db: Session) -> None: + """**Item 236, measured on the operator's own instance.** Every advisory's summary was joined + inline with semicolons: ten lines for one row, twenty-five rows — and OSV carries a GHSA *and* + a PYSEC identifier for the same advisory, so half of it was the other half word for word. + + The row is which package, at which version, and what to move to. The summaries are why somebody + cares once they have decided to look, which is evidence, and evidence is what a fold is for. + """ + _stored( + db, + asked=True, + pinned=3, + findings=[ + { + "package": "cryptography", + "version": "48.0.1", + "source": "backend/uv.lock", + "advisories": [ + {"id": "GHSA-g6cj", "summary": "a Bleichenbacher oracle", "fixed": ["50.0.0"]}, + {"id": "PYSEC-3552", "summary": "a Bleichenbacher oracle", "fixed": ["50.0.0"]}, + {"id": "GHSA-jwv3", "summary": "path-building", "fixed": ["49.0.0"]}, + ], + } + ], + ) + + shown = _view(db) + row = shown.split('

    ')[1].split("")[0] + + assert "backend/uv.lock" in row + # **Two identifiers, one advisory, one version to move to.** `50.0.0, 50.0.0, 49.0.0` is a list + # that has been counted wrong, and it is what a reader would have had to de-duplicate by eye. + assert "50.0.0 · 49.0.0" in row + # The fold lives inside the row now, so *not in the row* is measured where it means something: + # nothing before the disclosure opens is a summary. + assert "Bleichenbacher" not in row.split(" None: + """A fold whose summary does not say what is inside it is a mystery box (item 167), and the + count is the thing that tells a reader whether opening it is worth it.""" + _stored( + db, + asked=True, + pinned=3, + findings=[ + { + "package": "requests", "version": "2.31.0", "source": "uv.lock", + "advisories": [{"id": "GHSA-x", "summary": "a real one", "fixed": ["2.32.0"]}], + } + ], + ) + + shown = _view(db) + + # The count is the summary now (DR-0028): a disclosure whose label is a sentence spends a line + # of the row saying what every other row also says. + assert "1" in shown + assert "GHSA-x" in shown and "a real one" in shown + + +def test_nothing_to_upgrade_to_is_said_in_the_row(db: Session) -> None: + """**The worse case, and it stays out of the fold.** An upgrade nobody can make is the one thing + on this page a reader cannot act on, and it must not need a click to find out.""" + _stored( + db, + asked=True, + pinned=3, + findings=[ + { + "package": "left-pad", "version": "1.0.0", "source": "package-lock.json", + "advisories": [{"id": "GHSA-z", "summary": "unfixed", "fixed": []}], + } + ], + ) + + shown = _view(db) + row = shown.split('')[1].split("")[0] + + # **In the band's heading now** (DR-0028), which is where the sentence is said once — and the + # row is still there, still named, still carrying its version. + assert "Nothing published to upgrade to" in shown + assert "left-pad" in row and "1.0.0" in row + + +def test_the_disclosure_spans_its_row_rather_than_the_pills_column() -> None: + """**Seen in a browser, on atlas.** `.standing li` is a grid whose first column is 6.2rem wide + for the count, and a `
    ` left in it set *What these 6 advisory(s) say* one word per + line, five lines deep, on every row. + + Asserted on the rule because nothing in this repository draws anything — the browser is the only + thing that could have caught it, and did. + """ + from hullwork import page + + assert ".standing li > details" in page._STYLE + assert "grid-column: 1 / -1" in page._STYLE.split(".standing li > details")[1][:60] diff --git a/tests/test_what_it_is_doing_right_now.py b/tests/test_what_it_is_doing_right_now.py new file mode 100644 index 0000000..d9629b5 --- /dev/null +++ b/tests/test_what_it_is_doing_right_now.py @@ -0,0 +1,251 @@ +"""What the page may say about a dispatcher that is working. Item 242. + +The operator, watching a verification queue he could only see through `docker logs`: *sería +interesante ver en la página web qué está pasando. Ahora mismo no tenemos trazabilidad ninguna.* + +It was worse than missing. The instance report has a band for *the attempt in flight* and it read +**nothing running** while the dispatcher spent five minutes building an image and running somebody +else's suite twice — because it looked at `Item.state == IN_PROGRESS`, and a verification is not an +item. A page that reports calm during four minutes of work is not missing a feature; it is answering +wrongly. + +So the dispatcher says what it is doing and the page reads that. Two claims are worth guarding: that +it is **the dispatcher's own word**, and that a **stale heartbeat is not busy** — a process killed +mid-sentence leaves its last one behind, and rendering that as *now* is this page's own version of +the defect it fixes. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import lease, page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, DependencyReport, DispatcherLease, Project, UpgradeVerdict + +SIGNED_IN = page.Acting(csrf="c", offered=True) + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/doing.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _holding(db: Session) -> str: + holder = lease.new_holder() + assert lease.acquire(db, holder) + return holder + + +def _door(db: Session) -> str: + return page.front_door(db, Settings(), acting=SIGNED_IN) + + +# --- what it is doing --------------------------------------------------------------------------- + + +def test_the_page_says_what_the_dispatcher_said(db: Session) -> None: + """**The whole item.** Not deduced: everything a page could infer has the same hole in the + middle, and that hole is the four minutes somebody is trying to watch.""" + holder = _holding(db) + lease.doing(db, holder, "shop: verifying cryptography 48.0.1 → 49.0.0") + + shown = _door(db) + + assert "shop: verifying cryptography 48.0.1 → 49.0.0" in shown + assert "working" in shown + + +def test_it_says_how_long_that_has_been_going_on(db: Session) -> None: + """A step that has taken nine minutes is the interesting one, and *what* alone cannot say so.""" + holder = _holding(db) + lease.doing(db, holder, "shop: building the image") + row = db.get(DispatcherLease, 1) + assert row is not None + row.doing_since = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=9) + db.commit() + + assert "for 9m" in _door(db) + + +def test_the_clock_does_not_restart_while_the_step_is_the_same(db: Session) -> None: + """The loop writes this every turn. If each write moved the timestamp, a nine-minute step would + read as *just now* for ever — the permanently-reset version of item 073's rule.""" + holder = _holding(db) + lease.doing(db, holder, "shop: verifying x") + row = db.get(DispatcherLease, 1) + assert row is not None + row.doing_since = dt.datetime.now(dt.UTC) - dt.timedelta(minutes=6) + db.commit() + + lease.doing(db, holder, "shop: verifying x") + + db.refresh(row) + assert row.doing_since is not None + assert (dt.datetime.now(dt.UTC) - row.doing_since).total_seconds() > 300 + + +def test_an_idle_dispatcher_says_so_and_says_it_is_there(db: Session) -> None: + """**Idle and unreachable are different facts**, and a door that renders nothing in both cases + answers *is it running?* with silence — which is the question that was opened to ask.""" + holder = _holding(db) + lease.doing(db, holder, "shop: verifying x") + lease.doing(db, holder, None) + + shown = _door(db) + + assert "nothing running" in shown + assert "The dispatcher answered" in shown + assert "verifying x" not in shown + + +def test_an_instance_with_no_dispatcher_at_all_says_what_that_costs(db: Session) -> None: + """The state an evaluator meets first, and the one where every queue silently stops.""" + shown = _door(db) + + assert "no dispatcher" in shown + assert "Nothing will be attempted or verified until one does" in shown + + +# --- and what it must never claim ---------------------------------------------------------------- + + +def test_a_stale_heartbeat_is_not_busy(db: Session) -> None: + """**The defect this page would otherwise inherit.** A dispatcher killed mid-sentence leaves its + last one behind, and rendering it as *now* claims work that stopped — which is the same shape as + the band that read *nothing running* through five minutes of it.""" + holder = _holding(db) + lease.doing(db, holder, "shop: verifying cryptography 48.0.1 → 49.0.0") + row = db.get(DispatcherLease, 1) + assert row is not None + row.renewed_at = dt.datetime.now(dt.UTC) - dt.timedelta(hours=4) + db.commit() + + shown = _door(db) + + assert "not answering" in shown + assert "what it was doing rather than what it is doing" in shown + assert "working" not in shown + + +def test_a_dispatcher_that_was_released_leaves_nothing_behind(db: Session) -> None: + """A stopped dispatcher is not still doing the last thing it was doing.""" + holder = _holding(db) + lease.doing(db, holder, "shop: verifying x") + lease.release(db, holder) + + assert "verifying x" not in _door(db) + + +def test_only_the_dispatcher_that_holds_the_lease_may_say_what_it_is_doing(db: Session) -> None: + """**A second process must not narrate over the one that is working.** Losing a lease mid-run is + the state `renew` exists to catch; a stale process still writing here would put its sentence on + the page under the live one's name, which is worse than saying nothing. + + The fixture holds the lease first, because a write that finds no lease at all returns either + way — and a test that measured that would pass over the defect. + """ + holder = _holding(db) + lease.doing(db, holder, "shop: verifying the real one") + + lease.doing(db, "somebody-else", "shop: verifying x") + + shown = _door(db) + assert "verifying the real one" in shown + assert "verifying x" not in shown + + +# --- and what it has been doing ------------------------------------------------------------------- + + +def test_the_recent_history_merges_what_is_already_stored(db: Session) -> None: + """**No log table.** A second record of the same events could disagree with the first, and then + a reader has to decide which to believe.""" + project = db.query(Project).one() + db.merge( + UpgradeVerdict( + project_id=project.id, package="cryptography", was="48.0.1", to="49.0.0", + outcome="clean", detail="", tried_at=dt.datetime.now(dt.UTC) - dt.timedelta(minutes=2), + ) + ) + db.merge( + DependencyReport( + project_id=project.id, taken_at=dt.datetime.now(dt.UTC) - dt.timedelta(hours=1), + asked=True, pinned=50, findings=[], + ) + ) + db.commit() + + shown = _door(db) + + assert "What it has been doing" in shown + assert "tried cryptography 48.0.1 → 49.0.0" in shown + assert "asked OSV about 50 pinned version(s)" in shown + # Newest first: a history in the other order buries what just happened under what happened + # yesterday, which is the reading nobody wants. + assert shown.index("tried cryptography") < shown.index("asked OSV") + + +def test_a_report_that_could_not_be_taken_is_not_dressed_as_one_that_was(db: Session) -> None: + """DR-0024's condition, carried into the history: *could not ask* is not a report.""" + project = db.query(Project).one() + db.merge( + DependencyReport( + project_id=project.id, taken_at=dt.datetime.now(dt.UTC), asked=False, pinned=0, + note="OSV timed out", findings=[], + ) + ) + db.commit() + + shown = _door(db) + + assert "could not ask OSV" in shown + assert "with something published" not in shown + + +def test_an_instance_that_has_done_nothing_shows_no_history(db: Session) -> None: + """An empty list titled *what it has been doing* is furniture claiming to be information.""" + assert "What it has been doing" not in _door(db) + + +def test_the_history_is_set_as_sentences_and_not_as_labels(db: Session) -> None: + """**Seen in a browser.** `.standing .name` is small caps because it holds a thing's name — + `CRYPTOGRAPHY 48.0.1` — and the history holds whole sentences, which small caps makes slower to + read and louder than the thing they describe.""" + project = db.query(Project).one() + db.merge( + UpgradeVerdict( + project_id=project.id, package="x", was="1", to="2", outcome="clean", detail="", + tried_at=dt.datetime.now(dt.UTC), + ) + ) + db.commit() + + shown = _door(db) + history = shown[shown.index("What it has been doing") :] + + assert '' in history + assert '' not in history, "the history is set as labels" + assert ".standing .said" in page._STYLE diff --git a/tests/test_what_this_can_do_for_you.py b/tests/test_what_this_can_do_for_you.py new file mode 100644 index 0000000..a319a03 --- /dev/null +++ b/tests/test_what_this_can_do_for_you.py @@ -0,0 +1,208 @@ +"""`hullwork features`, per project, on the page. Item 220, item 218 §2. + +The command answers for the checkout it is run in. A page serves an instance that may watch somebody +else's repositories entirely, so the answer has to be per project — and the instance holds the +manifest (DR-0012) and its own variable names, and nothing else. + +**Unmet means three different things here and only one of them is a defect**, which is the whole +item and what `Need.reads` was added to name: + +* a *manifest* requirement the instance can answer, so unmet is a fact about the project; +* a *checkout* requirement nothing on the instance can answer — item 142 forbids a forge request per + render — so unmet reads *not asked yet*, never as a pass and never as a no; +* an *instance* credential, which on the receiver is often the dispatcher's by design (DR-0005), + so it is downgraded exactly as `doctor.not_from_here` downgrades it. Reporting the model key + missing where the half that uses it holds it sends somebody to repair a working machine. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import features as features_module +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, Project +from hullwork.page import Acting + +SIGNED_IN = Acting(csrf="c", offered=True) + +#: Enough manifest for the two features that only need one. Anything the page cannot answer has to +#: read as *not asked yet*, and a fixture that supplies everything would never exercise that. +A_MANIFEST = { + "project": "shop", + "git": {"provider": "forgejo", "repo": "acme/shop"}, + "errors": {"provider": "glitchtip"}, + "runtime": {"base": "python:3.12", "install": "pip install -r requirements.txt"}, + "tests": {"command": "pytest"}, + "autofix": {"agent": "none", "lanes": {"green": ["keyerror"], "red": ["payment"]}}, +} + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/can.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + session.add( + Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + manifest=A_MANIFEST, + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _shown(db: Session, settings: Settings | None = None) -> str: + """**The project's own view, not the list** (item 225). The block was 85% of the list and was + rendered once per project there: on a list a reader is not looking *at* a project, they are + looking *for* one.""" + # **On the project's Settings page since item 237.** What this instance can do *for* a project + # is the same question as what it can be told to do, and both are that project's, not a + # feature the reader browses across clients. + return page.settings_for(db, settings or Settings(), "shop", acting=SIGNED_IN) or "" + + +def _block(html: str) -> str: + found = re.search(r"What Hullwork can do for shop.*?", html, re.S) + assert found is not None, "the feature block is not on the project's view" + return found.group(0) + + +# --- every feature, with its limits ------------------------------------------------------------- + + +def test_the_list_does_not_carry_it(db: Session) -> None: + """**871 of the list's 1,021 words were this block**, rendered once per project (item 225). On + a list a reader is not looking *at* a project — they are looking *for* one, and two projects + made that page 1,932 words.""" + listed = page.projects(db, Settings(), acting=SIGNED_IN) + + assert "What Hullwork can do for" not in listed + assert "It reads what you pinned" not in listed + + +def test_it_is_folded_where_it_lives(db: Session) -> None: + """Reference rather than news: the settings page opens on what you can tell it, and this is one + click below that. **A fold is for an evaluator's questions** (item 167, DR-0027) and this is + exactly one: what could this do for me, asked once and not on every visit.""" + shown = _shown(db) + + fold = re.search(r"]*>\s*What Hullwork can do for", shown) + + assert fold is not None, "the block is not folded on the settings page" + assert " open" not in fold.group(0), "it is open, so it is the first thing read" + + +def test_every_feature_is_on_the_page(db: Session) -> None: + """**All five, not the four item 203 counts.** Those are the instance-shaped ones — filing an + issue, the page, notifications, the recurrence watch. These are the product: what it can do for + a repository.""" + block = _block(_shown(db)) + + for feature in features_module.FEATURES: + assert feature.name in block, feature.name + + +def test_the_limits_are_there_for_what_is_available_too(db: Session) -> None: + """**A limit is true whether or not the feature is available**, which is what makes it a + description and not an excuse. *What is measured is your suite* is worth more to an evaluator + than any green tick, and it is written already — this item moves it.""" + block = _block(_shown(db)) + + assert "It reads what you pinned" in block + assert "does not exercise the dependency" in block + + +# --- the three meanings of unmet ---------------------------------------------------------------- + + +def test_a_checkout_requirement_reads_not_asked_yet(db: Session) -> None: + """Nothing on the instance reads your tree and a page render does not spend a forge request to + find out (item 142). So *do you pin your dependencies* has no answer here — and **the answer to + a question nobody asked is not `no`**, which is the same `None != False` this project has got + wrong three times.""" + block = _block(_shown(db)) + + assert "not asked yet" in block + assert "hullwork features --checkout ." in block + + +def test_a_dispatchers_credential_is_not_reported_missing_here(db: Session) -> None: + """**The false alarm item 208's gate names.** The receiver holds no model key by design; the + half that uses it does. With a dispatcher alive, saying *missing* would send somebody to repair + a working machine, so it is downgraded exactly as `doctor.not_from_here` downgrades it — the + same rule, not a second one.""" + from hullwork import lease + + lease.acquire(db, lease.new_holder()) + db.commit() + + block = _block(_shown(db)) + + assert "not from here" in block + assert "a dispatcher is running" in block + + +def test_with_no_dispatcher_alive_the_credential_is_reported_missing(db: Session) -> None: + """**The other half of the same rule, and the reason it is not a blanket exemption.** With no + dispatcher running, the absence of a model credential is exactly what somebody needs to know — + `doctor` downgrades nothing in that state either.""" + block = _block(_shown(db)) + + assert "not from here" not in block + assert "a model credential" in block + + +def test_a_manifest_requirement_the_instance_can_answer_is_answered(db: Session) -> None: + """The instance holds the manifest (DR-0012), so this one is a fact about the project rather + than a question nobody asked — and it must not be softened into *not asked yet* along with the + others.""" + from hullwork.models import Project as Row + + row = db.query(Row).one() + row.manifest = { + "project": "shop", + "git": {"provider": "forgejo", "repo": "acme/shop"}, + "errors": {"provider": "glitchtip"}, + } + db.commit() + + block = _block(_shown(db)) + + assert "hullwork.yml naming an image" in block + assert "hullwork propose" in block, "it does not say what to do about it" + + +# --- what it must not become -------------------------------------------------------------------- + + +def test_it_answers_for_the_project_and_not_for_this_checkout(db: Session) -> None: + """**The reason this is not just a rendering of the command.** `hullwork features` run on this + instance answers for `hullwork` itself, from the tree the receiver happens to be installed in. + The page serves an instance that may watch somebody else's repositories entirely.""" + from hullwork.models import Project as Row + + row = db.query(Row).one() + row.manifest = None + db.commit() + + block = _block(_shown(db)) + + assert "hullwork.yml naming an image" in block, ( + "a project with no manifest reads as satisfied, which is this checkout's answer" + )
    284{line}' not in shown diff --git a/tests/test_the_item_list_on_a_narrow_screen.py b/tests/test_the_item_list_on_a_narrow_screen.py new file mode 100644 index 0000000..95b6ab7 --- /dev/null +++ b/tests/test_the_item_list_on_a_narrow_screen.py @@ -0,0 +1,244 @@ +"""The item list where a table stops working. Item 221. + +Item 215 left the narrow viewport *written and unverified*, because the browser I had renders at a +fixed layout width and the page correctly refuses both framing and `fetch` from its own context. +This is what a throwaway Chromium said when I stopped accepting that: no view scrolls the body +sideways at 390, 768 or 1440 — and the seven-column item list was unreadable anyway. Titles wrapped +to five lines, the timestamp was cut mid-value, and *issue / pull* sat behind a horizontal scroll +nobody discovers. + +**Not breaking the page and being readable are different bars**, and item 215 cleared the first. + +The browser tests here are skipped unless `playwright` is installed, because it is not a dependency +of this product and a page that needs a test harness to be looked at is not the thing being tested. +What is asserted without it is the markup those rules act on — the labels, the relative time, and +the target — since a stylesheet that reflows nothing is caught by the browser tests and a stylesheet +acting on labels that are not there is caught by these. + +Every test here was verified by reintroducing the defect it covers. +""" + +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Iterator +from pathlib import Path +from typing import Protocol + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from hullwork import page +from hullwork.config import Settings, get_settings +from hullwork.db import make_engine +from hullwork.models import Base, Item, ItemState, Lane, Project +from hullwork.page import Acting + + +class _Page(Protocol): + """The four things these tests ask of a browser page, and nothing else. + + A protocol rather than playwright's own types, because it is **not a dependency of this + product**: a suite that type-checks differently depending on what somebody happens to have + installed is worse than one that names the surface it uses. `Any` is banned here, and rightly. + """ + + def goto(self, url: str) -> object: ... + def evaluate(self, expression: str) -> object: ... + + +def _read(page: _Page, expression: str) -> list[dict[str, object]]: + """What the browser answered, as the shape every caller here expects. + + The protocol says `object` because a page can return anything; the casts live in one place + rather than at four call sites, which is the same reason `_rows_for_standing` exists.""" + got = page.evaluate(expression) + assert isinstance(got, list), f"the browser answered {type(got).__name__}, not a list" + return got + + +class _Context(Protocol): + def new_page(self) -> _Page: ... + + +class _Browser(Protocol): + def new_context(self, *, viewport: dict[str, int]) -> _Context: ... + def close(self) -> None: ... + + +SIGNED_IN = Acting(csrf="c", offered=True) + +#: The width the measurement was taken at, and the one the rules are written for. +NARROW = 390 + + +@pytest.fixture +def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Session]: + url = f"sqlite:///{tmp_path}/narrow.db" + engine = make_engine(url) + Base.metadata.create_all(engine) + monkeypatch.setenv("HULLWORK_DATABASE_URL", url) + get_settings.cache_clear() + session = sessionmaker(bind=engine)() + project = Project( + slug="shop", forge="forgejo", repo="acme/shop", + webhook_secret_hash="x", # noqa: S106 - a fixture, not a credential + ) + session.add(project) + session.flush() + for n in range(3): + session.add( + Item( + project_id=project.id, + fingerprint=f"f{n}", + title=f"KeyError: 'total' in checkout handler {n}", + state=ItemState.NEW, + lane=Lane.GREEN, + last_seen=dt.datetime.now(dt.UTC) - dt.timedelta(hours=n * 9), + ) + ) + session.commit() + yield session + get_settings.cache_clear() + + +def _front_door(db: Session) -> str: + return page.items(db, acting=SIGNED_IN, here="./", settings=Settings(), front=True) + + +# --- the markup the rules act on ------------------------------------------------------------ + + +def test_one_markup_serves_both_shapes(db: Session) -> None: + """**One markup, two shapes**, which is the property; `data-label` was one way of having it. + + Since DR-0028 the header and the labels are both gone: the grouping's heading names what the + rows are, and each field is legible without one — an id, a title, a slug and a lane, a time. + What has to stay true is that the narrow layout is a **stylesheet**, so this asserts there is + exactly one table markup and that the rules acting on it exist. + """ + shown = _front_door(db) + + assert '
    " not in shown, "a header row is a second thing to keep in step" + assert "@media (max-width: 46rem)" in shown, "nothing reflows the row" + + +def test_the_time_is_relative_and_the_exact_value_survives(db: Session) -> None: + """`2026-08-11 10:45:13.847329+00:00` in a 90px column, with microseconds. `_ago` has existed + since item 141 and renders *9h*; the exact value is what somebody needs when comparing against a + log, and a `title` is where that belongs rather than in the cell.""" + shown = _front_door(db) + + assert "