trust-factor W re-measurement: contention-safe sweep harness

benchmarks/lib/disk_space_snapshot.py

@@ -0,0 +1,67 @@
+"""Disk-space snapshot for a benchmark cell (issue #368 follow-up).
+
+2026-08-10 incident (reported by a neighboring session sharing this
+machine): leaked throwaway SQLite test databases (226-357 MB per run)
+filled the host disk to 100% on 2026-08-09 — PostgreSQL went down and even
+the Bash tool could no longer write its own output file. A cell can finish
+cleanly and still return degraded numbers because the volume filled under
+it, and that leaves no more trace in the artifact than CPU contention does
+(see `machine_load_snapshot.py`'s docstring — this is the same failure
+shape, a different resource). Recorded the same way and for the same
+reason: best-effort, taken at cell start and cell end, never aborts
+manifest generation.
+
+Two paths are measured, both best-effort:
+  - the repo root's filesystem — where benchmark datasets, throwaway test
+    databases, and HuggingFace cache actually land;
+  - Docker's storage root (`docker info`'s `DockerRootDir`) — under a VM
+    backend (colima, Docker Desktop) this is a guest-internal path and
+    `shutil.disk_usage` on it measures the HOST filesystem backing the VM
+    image file, not a second independent volume; when it resolves to the
+    same filesystem as the repo root this is redundant but harmless, and
+    when Docker is unreachable it is simply `None`.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+
+
+def _usage(path: str) -> dict | None:
+    try:
+        usage = shutil.disk_usage(path)
+    except OSError:
+        return None
+    return {
+        "path": path,
+        "free_bytes": usage.free,
+        "total_bytes": usage.total,
+    }
+
+
+def _docker_root_dir() -> str | None:
+    try:
+        out = subprocess.run(
+            ["docker", "info", "--format", "{{.DockerRootDir}}"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=False,
+        ).stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return out or None
+
+
+def disk_space_snapshot() -> dict:
+    """Free/total bytes on the volume(s) backing benchmark data + Docker
+    storage, as this run saw them. See this module's docstring for why."""
+    docker_root = _docker_root_dir()
+    return {
+        "repo_root": _usage(str(_REPO_ROOT)),
+        "docker_root": _usage(docker_root) if docker_root else None,
+    }

benchmarks/lib/machine_load_snapshot.py

@@ -0,0 +1,90 @@
+"""Machine-load snapshot for a benchmark cell (issue #368 follow-up).
+
+2026-08-10 incident: a 5-cell trust-factor sweep ran while three other
+agents' full pytest suites were active on the same machine (load average
+~11-14 on a 10-core box); one cell crashed on a native fatal error, and the
+crash was the ONLY visible signal — cells that merely finished under the
+same contention could have returned degraded numbers (saturated connection
+pool, cold cache, GC pressure) with nothing in the artifact to show it. The
+whole grid was discarded and re-run rather than salvaged, per this
+project's own rule: a measurement from a harness with a known defect is
+invalid and is redone, not patched after the fact — and contention is
+exactly such a defect. This snapshot is recorded so that rule can be
+applied by inspection later, instead of by asking whoever happened to be
+watching at the time.
+
+Taken TWICE per cell (same-day follow-up, same incident): once at cell
+START (`write_manifest.write_start_snapshot`, called before `start_db` so
+it predates the container/DB overhead too) and once at cell END (inside
+`write_manifest.build_manifest`). A crash is the visible failure mode; a
+cell that merely FINISHES under contention is the invisible one, and a
+single end-of-run snapshot cannot distinguish "ran under load throughout"
+from "load spiked right at the end". Two points at least bound the window.
+
+Every probe here is best-effort: a failure records `None` rather than
+aborting manifest generation, matching every other field `write_manifest.py`
+records.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+
+
+def _run(cmd: list[str], *, env: dict[str, str] | None = None) -> str | None:
+    try:
+        return subprocess.run(
+            cmd, capture_output=True, text=True, timeout=10, check=False, env=env
+        ).stdout
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def count_pytest_processes() -> int | None:
+    """Concurrent `pytest` processes system-wide, or None if unreadable.
+
+    Filtered in Python, not via a shell `grep -c "[p]ytest"` idiom: a
+    subprocess.run argv has no shell to bracket-escape a self-match, so the
+    filter runs here instead, over the same process list that idiom reads.
+
+    The `COLUMNS` override fixes a real undercount, not just a test flake
+    (caught by tests_py/benchmarks/test_write_manifest_machine_load.py's
+    own self-referential assertion failing on GitHub's Linux CI runner,
+    2026-08-10): both BSD ps (macOS) and GNU procps (Linux) truncate the
+    COMMAND column to `$COLUMNS` when stdout is not a terminal and COLUMNS
+    is unset, and `ps aux`'s fixed-width USER/PID/... columns alone can
+    exceed a default 80-column budget before COMMAND even starts — cutting
+    off the "pytest" substring entirely on a long interpreter path.
+    """
+    ps_out = _run(["ps", "aux"], env={**os.environ, "COLUMNS": "1000"})
+    if ps_out is None:
+        return None
+    return sum(
+        1 for line in ps_out.splitlines() if "pytest" in line and "grep" not in line
+    )
+
+
+def count_docker_containers() -> int | None:
+    """Concurrent running Docker containers, or None if unreadable."""
+    docker_out = _run(["docker", "ps", "-q"])
+    if docker_out is None:
+        return None
+    return len([line for line in docker_out.splitlines() if line.strip()])
+
+
+def machine_load_snapshot() -> dict:
+    """Load average + concurrent pytest/container counts, as this run saw
+    them. See this module's docstring for why."""
+    try:
+        load1, load5, load15 = os.getloadavg()
+    except OSError:  # not available on this platform (e.g. Windows)
+        load1 = load5 = load15 = None
+    return {
+        "load_average_1m": load1,
+        "load_average_5m": load5,
+        "load_average_15m": load15,
+        "cpu_count": os.cpu_count(),
+        "concurrent_pytest_processes": count_pytest_processes(),
+        "concurrent_docker_containers": count_docker_containers(),
+    }

benchmarks/lib/sweep_progress.py

@@ -0,0 +1,104 @@
+"""Per-cell checkpoint/resume state for the trust-factor sweep (issue #368).
+
+2026-08-10 incident: five sweep campaigns died over one session, each with a
+different attributed cause (CPU contention, a native crash, a missing
+dataset file, a session gap, a noisy neighbor). Five explanations for five
+occurrences of the same symptom was itself the signal missed at the time:
+those four "causes" described the state found on wake-up, not the actual
+failure mechanism. The mechanism is that a long-running job driven from a
+sub-agent's own foreground loop does not survive whatever ends that
+sub-agent's turn -- confirmed by a neighboring session that measured the
+identical failure shape on five unrelated tasks, and that succeeded only
+once it stopped depending on that survival.
+
+The fix is not to prevent the death (it cannot be prevented from here) but
+to make it cost one cell, not the whole grid: `trust_factor_sweep.sh` now
+runs exactly ONE pending cell per invocation and returns control immediately
+after, recording that cell's completion (grid point, result location, code
+sha, and machine-load/disk-space snapshots at both cell start and cell end)
+to a fixed-location PROGRESS.json. A killed or crashed run leaves no entry
+for the cell it was running, so the next invocation finds it still pending
+and retries exactly that one -- never the cells already recorded.
+
+The load/disk snapshots are NOT a defence against this failure mode (a kill
+happens regardless of what they read) -- they remain what they were built
+for: metadata that lets a later reader requalify a cell's number without
+trusting the runner's word for it. Load and disk are deliberately two
+separate signals, not redundant: load average integrates I/O wait as well
+as CPU, so it rises under a saturated disk with no CPU contention at all.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+
+PROGRESS_FILENAME = "PROGRESS.json"
+
+
+def _git_sha() -> str:
+    try:
+        out = subprocess.run(
+            ["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10
+        )
+    except (OSError, subprocess.SubprocessError):
+        return "unknown"
+    return out.stdout.strip() if out.returncode == 0 else "unknown"
+
+
+def read_progress(sweep_dir: str) -> dict:
+    """Read PROGRESS.json, or an empty structure if absent/unreadable --
+    never raises, matching this project's other best-effort snapshot code."""
+    path = Path(sweep_dir) / PROGRESS_FILENAME
+    try:
+        return json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return {"cells": []}
+
+
+def completed_w_values(sweep_dir: str) -> set[float]:
+    progress = read_progress(sweep_dir)
+    return {c["w"] for c in progress.get("cells", []) if c.get("status") == "complete"}
+
+
+def next_pending_w(grid: list[float], sweep_dir: str) -> float | None:
+    """First grid value with no `status: complete` entry, in grid order, or
+    None once every cell has one -- the resume point after a kill/crash."""
+    done = completed_w_values(sweep_dir)
+    for w in grid:
+        if w not in done:
+            return w
+    return None
+
+
+def record_cell_result(
+    sweep_dir: str,
+    w: float,
+    *,
+    status: str,
+    repro_dir: str | None,
+    snapshots: dict,
+) -> None:
+    """Append (or replace) one cell's completion record in PROGRESS.json.
+
+    `snapshots` carries the four points this incident asks for:
+    machine_load_at_start/_at_end, disk_space_at_start/_at_end -- passed as
+    one dict rather than four parameters to stay under this project's
+    4-argument-max convention (coding-standards.md §3.2) without losing any
+    of the four points to a size-cap-driven abbreviation.
+    """
+    progress = read_progress(sweep_dir)
+    progress.setdefault("cells", [])
+    progress["cells"] = [c for c in progress["cells"] if c.get("w") != w]
+    progress["cells"].append(
+        {
+            "w": w,
+            "status": status,
+            "git_sha": _git_sha(),
+            "repro_dir": repro_dir,
+            **snapshots,
+        }
+    )
+    path = Path(sweep_dir) / PROGRESS_FILENAME
+    path.write_text(json.dumps(progress, indent=2))