diff --git a/benchmarks/lib/disk_space_snapshot.py b/benchmarks/lib/disk_space_snapshot.py new file mode 100644 index 00000000..0309266c --- /dev/null +++ b/benchmarks/lib/disk_space_snapshot.py @@ -0,0 +1,67 @@ +"""Disk-space snapshot for a benchmark cell (issue #368 follow-up). + +2026-08-10 incident (reported by a neighboring session sharing this +machine): leaked throwaway SQLite test databases (226-357 MB per run) +filled the host disk to 100% on 2026-08-09 — PostgreSQL went down and even +the Bash tool could no longer write its own output file. A cell can finish +cleanly and still return degraded numbers because the volume filled under +it, and that leaves no more trace in the artifact than CPU contention does +(see `machine_load_snapshot.py`'s docstring — this is the same failure +shape, a different resource). Recorded the same way and for the same +reason: best-effort, taken at cell start and cell end, never aborts +manifest generation. + +Two paths are measured, both best-effort: + - the repo root's filesystem — where benchmark datasets, throwaway test + databases, and HuggingFace cache actually land; + - Docker's storage root (`docker info`'s `DockerRootDir`) — under a VM + backend (colima, Docker Desktop) this is a guest-internal path and + `shutil.disk_usage` on it measures the HOST filesystem backing the VM + image file, not a second independent volume; when it resolves to the + same filesystem as the repo root this is redundant but harmless, and + when Docker is unreachable it is simply `None`. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _usage(path: str) -> dict | None: + try: + usage = shutil.disk_usage(path) + except OSError: + return None + return { + "path": path, + "free_bytes": usage.free, + "total_bytes": usage.total, + } + + +def _docker_root_dir() -> str | None: + try: + out = subprocess.run( + ["docker", "info", "--format", "{{.DockerRootDir}}"], + capture_output=True, + text=True, + timeout=10, + check=False, + ).stdout.strip() + except (OSError, subprocess.SubprocessError): + return None + return out or None + + +def disk_space_snapshot() -> dict: + """Free/total bytes on the volume(s) backing benchmark data + Docker + storage, as this run saw them. See this module's docstring for why.""" + docker_root = _docker_root_dir() + return { + "repo_root": _usage(str(_REPO_ROOT)), + "docker_root": _usage(docker_root) if docker_root else None, + } diff --git a/benchmarks/lib/machine_load_snapshot.py b/benchmarks/lib/machine_load_snapshot.py new file mode 100644 index 00000000..84317680 --- /dev/null +++ b/benchmarks/lib/machine_load_snapshot.py @@ -0,0 +1,90 @@ +"""Machine-load snapshot for a benchmark cell (issue #368 follow-up). + +2026-08-10 incident: a 5-cell trust-factor sweep ran while three other +agents' full pytest suites were active on the same machine (load average +~11-14 on a 10-core box); one cell crashed on a native fatal error, and the +crash was the ONLY visible signal — cells that merely finished under the +same contention could have returned degraded numbers (saturated connection +pool, cold cache, GC pressure) with nothing in the artifact to show it. The +whole grid was discarded and re-run rather than salvaged, per this +project's own rule: a measurement from a harness with a known defect is +invalid and is redone, not patched after the fact — and contention is +exactly such a defect. This snapshot is recorded so that rule can be +applied by inspection later, instead of by asking whoever happened to be +watching at the time. + +Taken TWICE per cell (same-day follow-up, same incident): once at cell +START (`write_manifest.write_start_snapshot`, called before `start_db` so +it predates the container/DB overhead too) and once at cell END (inside +`write_manifest.build_manifest`). A crash is the visible failure mode; a +cell that merely FINISHES under contention is the invisible one, and a +single end-of-run snapshot cannot distinguish "ran under load throughout" +from "load spiked right at the end". Two points at least bound the window. + +Every probe here is best-effort: a failure records `None` rather than +aborting manifest generation, matching every other field `write_manifest.py` +records. +""" + +from __future__ import annotations + +import os +import subprocess + + +def _run(cmd: list[str], *, env: dict[str, str] | None = None) -> str | None: + try: + return subprocess.run( + cmd, capture_output=True, text=True, timeout=10, check=False, env=env + ).stdout + except (OSError, subprocess.SubprocessError): + return None + + +def count_pytest_processes() -> int | None: + """Concurrent `pytest` processes system-wide, or None if unreadable. + + Filtered in Python, not via a shell `grep -c "[p]ytest"` idiom: a + subprocess.run argv has no shell to bracket-escape a self-match, so the + filter runs here instead, over the same process list that idiom reads. + + The `COLUMNS` override fixes a real undercount, not just a test flake + (caught by tests_py/benchmarks/test_write_manifest_machine_load.py's + own self-referential assertion failing on GitHub's Linux CI runner, + 2026-08-10): both BSD ps (macOS) and GNU procps (Linux) truncate the + COMMAND column to `$COLUMNS` when stdout is not a terminal and COLUMNS + is unset, and `ps aux`'s fixed-width USER/PID/... columns alone can + exceed a default 80-column budget before COMMAND even starts — cutting + off the "pytest" substring entirely on a long interpreter path. + """ + ps_out = _run(["ps", "aux"], env={**os.environ, "COLUMNS": "1000"}) + if ps_out is None: + return None + return sum( + 1 for line in ps_out.splitlines() if "pytest" in line and "grep" not in line + ) + + +def count_docker_containers() -> int | None: + """Concurrent running Docker containers, or None if unreadable.""" + docker_out = _run(["docker", "ps", "-q"]) + if docker_out is None: + return None + return len([line for line in docker_out.splitlines() if line.strip()]) + + +def machine_load_snapshot() -> dict: + """Load average + concurrent pytest/container counts, as this run saw + them. See this module's docstring for why.""" + try: + load1, load5, load15 = os.getloadavg() + except OSError: # not available on this platform (e.g. Windows) + load1 = load5 = load15 = None + return { + "load_average_1m": load1, + "load_average_5m": load5, + "load_average_15m": load15, + "cpu_count": os.cpu_count(), + "concurrent_pytest_processes": count_pytest_processes(), + "concurrent_docker_containers": count_docker_containers(), + } diff --git a/benchmarks/lib/sweep_progress.py b/benchmarks/lib/sweep_progress.py new file mode 100644 index 00000000..7ec327a5 --- /dev/null +++ b/benchmarks/lib/sweep_progress.py @@ -0,0 +1,104 @@ +"""Per-cell checkpoint/resume state for the trust-factor sweep (issue #368). + +2026-08-10 incident: five sweep campaigns died over one session, each with a +different attributed cause (CPU contention, a native crash, a missing +dataset file, a session gap, a noisy neighbor). Five explanations for five +occurrences of the same symptom was itself the signal missed at the time: +those four "causes" described the state found on wake-up, not the actual +failure mechanism. The mechanism is that a long-running job driven from a +sub-agent's own foreground loop does not survive whatever ends that +sub-agent's turn -- confirmed by a neighboring session that measured the +identical failure shape on five unrelated tasks, and that succeeded only +once it stopped depending on that survival. + +The fix is not to prevent the death (it cannot be prevented from here) but +to make it cost one cell, not the whole grid: `trust_factor_sweep.sh` now +runs exactly ONE pending cell per invocation and returns control immediately +after, recording that cell's completion (grid point, result location, code +sha, and machine-load/disk-space snapshots at both cell start and cell end) +to a fixed-location PROGRESS.json. A killed or crashed run leaves no entry +for the cell it was running, so the next invocation finds it still pending +and retries exactly that one -- never the cells already recorded. + +The load/disk snapshots are NOT a defence against this failure mode (a kill +happens regardless of what they read) -- they remain what they were built +for: metadata that lets a later reader requalify a cell's number without +trusting the runner's word for it. Load and disk are deliberately two +separate signals, not redundant: load average integrates I/O wait as well +as CPU, so it rises under a saturated disk with no CPU contention at all. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +PROGRESS_FILENAME = "PROGRESS.json" + + +def _git_sha() -> str: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10 + ) + except (OSError, subprocess.SubprocessError): + return "unknown" + return out.stdout.strip() if out.returncode == 0 else "unknown" + + +def read_progress(sweep_dir: str) -> dict: + """Read PROGRESS.json, or an empty structure if absent/unreadable -- + never raises, matching this project's other best-effort snapshot code.""" + path = Path(sweep_dir) / PROGRESS_FILENAME + try: + return json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return {"cells": []} + + +def completed_w_values(sweep_dir: str) -> set[float]: + progress = read_progress(sweep_dir) + return {c["w"] for c in progress.get("cells", []) if c.get("status") == "complete"} + + +def next_pending_w(grid: list[float], sweep_dir: str) -> float | None: + """First grid value with no `status: complete` entry, in grid order, or + None once every cell has one -- the resume point after a kill/crash.""" + done = completed_w_values(sweep_dir) + for w in grid: + if w not in done: + return w + return None + + +def record_cell_result( + sweep_dir: str, + w: float, + *, + status: str, + repro_dir: str | None, + snapshots: dict, +) -> None: + """Append (or replace) one cell's completion record in PROGRESS.json. + + `snapshots` carries the four points this incident asks for: + machine_load_at_start/_at_end, disk_space_at_start/_at_end -- passed as + one dict rather than four parameters to stay under this project's + 4-argument-max convention (coding-standards.md §3.2) without losing any + of the four points to a size-cap-driven abbreviation. + """ + progress = read_progress(sweep_dir) + progress.setdefault("cells", []) + progress["cells"] = [c for c in progress["cells"] if c.get("w") != w] + progress["cells"].append( + { + "w": w, + "status": status, + "git_sha": _git_sha(), + "repro_dir": repro_dir, + **snapshots, + } + ) + path = Path(sweep_dir) / PROGRESS_FILENAME + path.write_text(json.dumps(progress, indent=2)) diff --git a/benchmarks/lib/write_manifest.py b/benchmarks/lib/write_manifest.py index 47791231..1e9a859d 100644 --- a/benchmarks/lib/write_manifest.py +++ b/benchmarks/lib/write_manifest.py @@ -5,6 +5,11 @@ driver stays within the size limits of coding-standards.md §4 and so this provenance logic can be read, diffed and tested as Python. +Machine-load and disk-space snapshots live in sibling modules +(``machine_load_snapshot.py``, ``disk_space_snapshot.py``) — see their +docstrings for the two incidents that motivate recording them alongside +``git_sha`` in every manifest. + Usage (from reproduce.sh): python benchmarks/lib/write_manifest.py \\ RESULTS_DIR GIT_SHA DATASET_SHA256 PG_IMAGE CONTAINER PG_PORT RUNNER_PID @@ -15,111 +20,39 @@ """ import json -import os import platform -import subprocess import sys from datetime import datetime, timezone from pathlib import Path -_START_SNAPSHOT_NAME = "START_SNAPSHOT.json" - - -def machine_load_snapshot() -> dict: - """Load average + concurrent pytest/container counts, as this run saw them. - - 2026-08-10 incident: a 5-cell trust-factor sweep ran while three other - agents' full pytest suites were active on the same machine (load average - ~11-14 on a 10-core box); one cell crashed on a native fatal error, and - the crash was the ONLY visible signal — cells that merely finished under - the same contention could have returned degraded numbers (saturated - connection pool, cold cache, GC pressure) with nothing in the artifact to - show it. The whole grid was discarded and re-run rather than salvaged, - per this project's own rule: a measurement from a harness with a known - defect is invalid and is redone, not patched after the fact — and - contention is exactly such a defect. This snapshot is recorded so that - rule can be applied by inspection later, instead of by asking whoever - happened to be watching at the time. - - Taken TWICE per cell (2026-08-10 follow-up, same incident): once at cell - START (`write_start_snapshot`, called before `start_db` so it predates - the container/DB overhead too) and once at cell END (inside - `build_manifest`, the pre-existing call). A crash is the visible failure - mode; a cell that merely FINISHES under contention (saturated pool, cold - cache, GC pressure) is the invisible one, and a single end-of-run - snapshot cannot distinguish "this cell ran under load throughout" from - "load spiked right at the end". Two points at least bound the window. - - Best-effort: any probe that fails records `None`/`"unresolved"` rather - than aborting manifest generation, matching this module's other fields. - """ - try: - load1, load5, load15 = os.getloadavg() - except OSError: # not available on this platform (e.g. Windows) - load1 = load5 = load15 = None - - def _run(cmd: list[str], *, env: dict[str, str] | None = None) -> str | None: - try: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=10, check=False, env=env - ).stdout - except (OSError, subprocess.SubprocessError): - return None - - # Filtered in Python, not via a shell `grep -c "[p]ytest"` idiom: a - # subprocess.run argv has no shell to bracket-escape a self-match, so the - # filter runs here instead, over the same process list that idiom reads. - # - # The `COLUMNS` override below fixes a real undercount, not just a test - # flake (caught by - # tests_py/benchmarks/test_write_manifest_machine_load.py's own - # self-referential assertion failing on GitHub's Linux CI runner, - # 2026-08-10): both BSD ps (macOS) and GNU procps (Linux) truncate the - # COMMAND column to `$COLUMNS` when stdout is not a terminal and COLUMNS - # is unset, and `ps aux`'s fixed-width USER/PID/... columns alone can - # exceed a default 80-column budget before COMMAND even starts — cutting - # off the "pytest" substring entirely on a long interpreter path. A wide - # COLUMNS override is the standard fix for both implementations. - ps_out = _run(["ps", "aux"], env={**os.environ, "COLUMNS": "1000"}) - pytest_procs = ( - None - if ps_out is None - else sum( - 1 for line in ps_out.splitlines() if "pytest" in line and "grep" not in line - ) - ) - - docker_out = _run(["docker", "ps", "-q"]) - docker_containers = ( - None - if docker_out is None - else len([line for line in docker_out.splitlines() if line.strip()]) - ) +# Same idiom as ablation_runner.py/latency_runner.py/e2_subsample_runner.py +# in this package: REPO_ROOT on sys.path so `benchmarks.lib.*` resolves both +# when this file runs as a standalone script (reproduce.sh invokes it +# directly, not via `-m`) and when it is imported normally as a package +# member (tests_py/). +_REPO_ROOT = str(Path(__file__).resolve().parents[2]) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) +from benchmarks.lib.disk_space_snapshot import disk_space_snapshot # noqa: E402 +from benchmarks.lib.machine_load_snapshot import machine_load_snapshot # noqa: E402 - return { - "load_average_1m": load1, - "load_average_5m": load5, - "load_average_15m": load15, - "cpu_count": os.cpu_count(), - "concurrent_pytest_processes": pytest_procs, - "concurrent_docker_containers": docker_containers, - } +_START_SNAPSHOT_NAME = "START_SNAPSHOT.json" def write_start_snapshot(results_dir: str) -> Path: - """Capture + persist the cell-start machine-load snapshot. + """Capture + persist the cell-start machine-load and disk-space snapshot. Called from reproduce.sh before `start_db`, so `RESULTS_DIR` already exists (created by `main()`'s `mkdir -p`) but nothing benchmark-specific has run yet. `build_manifest` reads this file back at cell end and folds - it into the final MANIFEST.json as `machine_load_at_start`, alongside the - end-of-run `machine_load_at_end` — see `machine_load_snapshot`'s - docstring for why both points are recorded. + it into the final MANIFEST.json as `machine_load_at_start` / + `disk_space_at_start`, alongside the end-of-run `_at_end` counterparts. """ out = Path(results_dir) / _START_SNAPSHOT_NAME payload = { "captured_at_utc": datetime.now(timezone.utc).isoformat(), "machine_load": machine_load_snapshot(), + "disk_space": disk_space_snapshot(), } out.write_text(json.dumps(payload, indent=2)) return out @@ -131,7 +64,7 @@ def _read_start_snapshot(results_dir: str) -> dict | None: Returns None (never raises) when absent — e.g. a `reproduce.sh` call that predates this fix, or a caller that skipped the `--snapshot` step. A missing start snapshot must not block the end-of-run manifest from - being written; `machine_load_at_start` is simply absent in that case, + being written; the `_at_start` fields are simply absent in that case, which is itself an observable fact rather than a silent guess. """ path = Path(results_dir) / _START_SNAPSHOT_NAME @@ -205,6 +138,40 @@ def reranker_fields() -> dict: } +def _environment_fields() -> dict: + """Package/model/reranker identity fields — split out of `build_manifest` + to keep that function under the 40-line method cap (CLAUDE.md § Code + Style).""" + return { + "python": platform.python_version(), + "packages": { + p: ver(p) + for p in ( + "datasets", + "sentence-transformers", + "torch", + "psycopg", + "psycopg-pool", + ) + }, + "embedding_model_revision": embedding_revision(), + **reranker_fields(), + } + + +def _start_snapshot_fields(results_dir: str) -> dict: + """`_at_start` machine-load/disk-space fields, or None/None if + `write_start_snapshot` was never called for this `results_dir` — split + out of `build_manifest` for the same reason as `_environment_fields`.""" + snap = _read_start_snapshot(results_dir) + if snap is None: + return {"machine_load_at_start": None, "disk_space_at_start": None} + return { + "machine_load_at_start": snap.get("machine_load"), + "disk_space_at_start": snap.get("disk_space"), + } + + def build_manifest( results_dir: str, git_sha: str, @@ -214,17 +181,15 @@ def build_manifest( pg_port: str, pid: str, ) -> dict: - start_snapshot = _read_start_snapshot(results_dir) return { "git_sha": git_sha, - # Alongside git_sha, not buried: see machine_load_snapshot's - # docstring for why (2026-08-10 sweep-contention incident). Two - # points, not one — `_at_start` is None when reproduce.sh's - # `--snapshot` step was never called for this results_dir. - "machine_load_at_start": ( - start_snapshot["machine_load"] if start_snapshot else None - ), + # Alongside git_sha, not buried: see machine_load_snapshot.py and + # disk_space_snapshot.py's docstrings for why (2026-08-10 sweep + # incidents — CPU contention, then a full disk, both invisible in + # a cell that merely finishes). Two points per resource, not one. + **_start_snapshot_fields(results_dir), "machine_load_at_end": machine_load_snapshot(), + "disk_space_at_end": disk_space_snapshot(), "longmemeval_dataset_sha256": ds_sha, "pg_image": pg_image, # Per-run container isolation fix (2026-07-11, incident: two concurrent @@ -236,19 +201,7 @@ def build_manifest( "bench_container_name": container, "bench_container_port": int(pg_port), "bench_runner_pid": int(pid), - "python": platform.python_version(), - "packages": { - p: ver(p) - for p in ( - "datasets", - "sentence-transformers", - "torch", - "psycopg", - "psycopg-pool", - ) - }, - "embedding_model_revision": embedding_revision(), - **reranker_fields(), + **_environment_fields(), "results_files": sorted( p.name for p in Path(results_dir).glob("*.json") diff --git a/benchmarks/results/repro/20260810T140209Z/MANIFEST.json b/benchmarks/results/repro/20260810T140209Z/MANIFEST.json new file mode 100644 index 00000000..351d071f --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 6.4130859375, + "load_average_5m": 8.67041015625, + "load_average_15m": 8.5556640625, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21348298752, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.98193359375, + "load_average_5m": 8.95947265625, + "load_average_15m": 8.74658203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20997742592, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-31717-e2c30003", + "bench_container_port": 32793, + "bench_runner_pid": 31717, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json new file mode 100644 index 00000000..9386df7b --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:02:10.355451+00:00", + "machine_load": { + "load_average_1m": 6.4130859375, + "load_average_5m": 8.67041015625, + "load_average_15m": 8.5556640625, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21348298752, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/beam-100K.json b/benchmarks/results/repro/20260810T140209Z/beam-100K.json new file mode 100644 index 00000000..664676e0 --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6579166666666667, + "overall_r10": 0.85, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.625, + "information_extraction": 0.8, + "instruction_following": 0.625, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 120.98164892196655, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:07:16.420185+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/locomo.json b/benchmarks/results/repro/20260810T140209Z/locomo.json new file mode 100644 index 00000000..5d826f0f --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8208665699782451, + "overall_recall10": 0.9593908629441624, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6431818181818182, + "single_hop": 0.6453869047619047, + "open_domain": 0.8914285714285715, + "adversarial": 0.9645390070921986 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9857142857142858, + "adversarial": 1.0 + }, + "elapsed_s": 243.82014799118042, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:03:05.989631+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/longmemeval-s.json b/benchmarks/results/repro/20260810T140209Z/longmemeval-s.json new file mode 100644 index 00000000..161f9154 --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 50.508332624973264, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:02:13.736491+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/MANIFEST.json b/benchmarks/results/repro/20260810T141001Z/MANIFEST.json new file mode 100644 index 00000000..80df4c34 --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 8.70263671875, + "load_average_5m": 8.79833984375, + "load_average_15m": 8.69189453125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 26129395712, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 11.3310546875, + "load_average_5m": 12.310546875, + "load_average_15m": 10.646484375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25266130944, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-80621-a4946cc5", + "bench_container_port": 32794, + "bench_runner_pid": 80621, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json new file mode 100644 index 00000000..cda24f75 --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:10:02.254281+00:00", + "machine_load": { + "load_average_1m": 8.70263671875, + "load_average_5m": 8.79833984375, + "load_average_15m": 8.69189453125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 26129395712, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/beam-100K.json b/benchmarks/results/repro/20260810T141001Z/beam-100K.json new file mode 100644 index 00000000..c380b1db --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 120.49417090415955, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:15:16.117902+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/locomo.json b/benchmarks/results/repro/20260810T141001Z/locomo.json new file mode 100644 index 00000000..591b52c6 --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8042845056804449, + "overall_recall10": 0.9390862944162437, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6610119047619047, + "open_domain": 0.8511904761904763, + "adversarial": 0.9432624113475178 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9428571428571428, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 252.10806703567505, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:10:58.034832+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/longmemeval-s.json b/benchmarks/results/repro/20260810T141001Z/longmemeval-s.json new file mode 100644 index 00000000..d148a38c --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 49.05208487500204, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:10:07.038557+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/MANIFEST.json b/benchmarks/results/repro/20260810T141800Z/MANIFEST.json new file mode 100644 index 00000000..7a10ffd9 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 9.16650390625, + "load_average_5m": 11.61572265625, + "load_average_15m": 10.46630859375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25579466752, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.84033203125, + "load_average_5m": 10.83056640625, + "load_average_15m": 10.60888671875, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 24998215680, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-27502-f5f2bb69", + "bench_container_port": 32795, + "bench_runner_pid": 27502, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json new file mode 100644 index 00000000..788fd5c0 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:18:01.140247+00:00", + "machine_load": { + "load_average_1m": 9.16650390625, + "load_average_5m": 11.61572265625, + "load_average_15m": 10.46630859375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25579466752, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/beam-100K.json b/benchmarks/results/repro/20260810T141800Z/beam-100K.json new file mode 100644 index 00000000..9a473edb --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 110.44140601158142, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:22:55.411459+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/locomo.json b/benchmarks/results/repro/20260810T141800Z/locomo.json new file mode 100644 index 00000000..cd26f832 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8044053662073968, + "overall_recall10": 0.934010152284264, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6617559523809524, + "open_domain": 0.8488095238095238, + "adversarial": 0.9468085106382979 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9285714285714286, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 235.71782112121582, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:18:54.244987+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/longmemeval-s.json b/benchmarks/results/repro/20260810T141800Z/longmemeval-s.json new file mode 100644 index 00000000..ffa84d57 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 47.76949145901017, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:18:04.366929+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/MANIFEST.json b/benchmarks/results/repro/20260810T142503Z/MANIFEST.json new file mode 100644 index 00000000..60162f07 --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 8.240234375, + "load_average_5m": 10.55224609375, + "load_average_15m": 10.5126953125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25810296832, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 13.11865234375, + "load_average_5m": 11.8583984375, + "load_average_15m": 11.14990234375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20115681280, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-68215-d1fe77db", + "bench_container_port": 32796, + "bench_runner_pid": 68215, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json new file mode 100644 index 00000000..0a7d73aa --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:25:03.837503+00:00", + "machine_load": { + "load_average_1m": 8.240234375, + "load_average_5m": 10.55224609375, + "load_average_15m": 10.5126953125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25810296832, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/beam-100K.json b/benchmarks/results/repro/20260810T142503Z/beam-100K.json new file mode 100644 index 00000000..0a5203ff --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 122.94071817398071, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:29:46.230185+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/locomo.json b/benchmarks/results/repro/20260810T142503Z/locomo.json new file mode 100644 index 00000000..3110d3cc --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8044053662073968, + "overall_recall10": 0.934010152284264, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6617559523809524, + "open_domain": 0.8488095238095238, + "adversarial": 0.9468085106382979 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9285714285714286, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 226.98380208015442, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:25:53.371692+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/longmemeval-s.json b/benchmarks/results/repro/20260810T142503Z/longmemeval-s.json new file mode 100644 index 00000000..da1bd755 --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 44.64210533298319, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:25:06.914447+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/MANIFEST.json b/benchmarks/results/repro/20260810T144349Z/MANIFEST.json new file mode 100644 index 00000000..b168ed80 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 6.16064453125, + "load_average_5m": 6.9130859375, + "load_average_15m": 8.44287109375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 22296911872, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 10.4541015625, + "load_average_5m": 9.6796875, + "load_average_15m": 9.3203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21858193408, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-84394-9cd44149", + "bench_container_port": 32797, + "bench_runner_pid": 84394, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json new file mode 100644 index 00000000..234a7b92 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:43:49.599837+00:00", + "machine_load": { + "load_average_1m": 6.16064453125, + "load_average_5m": 6.9130859375, + "load_average_15m": 8.44287109375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 22296911872, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/beam-100K.json b/benchmarks/results/repro/20260810T144349Z/beam-100K.json new file mode 100644 index 00000000..de3caa45 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 114.20000696182251, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:48:29.221156+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/locomo.json b/benchmarks/results/repro/20260810T144349Z/locomo.json new file mode 100644 index 00000000..f5ad2fc4 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.7891769398114576, + "overall_recall10": 0.9187817258883249, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6617559523809524, + "open_domain": 0.8059523809523809, + "adversarial": 0.9468085106382979 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.8857142857142857, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 225.38163590431213, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:44:38.580923+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/longmemeval-s.json b/benchmarks/results/repro/20260810T144349Z/longmemeval-s.json new file mode 100644 index 00000000..d1a36586 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 43.89209308300633, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:43:53.015314+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/trust-factor-sweep/active/PROGRESS.json b/benchmarks/results/trust-factor-sweep/active/PROGRESS.json new file mode 100644 index 00000000..3f095a75 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/PROGRESS.json @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "w": 1.0, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T140209Z/", + "machine_load_at_start": { + "load_average_1m": 6.4130859375, + "load_average_5m": 8.67041015625, + "load_average_15m": 8.5556640625, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21274468352, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.98193359375, + "load_average_5m": 8.95947265625, + "load_average_15m": 8.74658203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20997554176, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.8, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T141001Z/", + "machine_load_at_start": { + "load_average_1m": 8.24169921875, + "load_average_5m": 8.7109375, + "load_average_15m": 8.66064453125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 26131615744, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 11.3310546875, + "load_average_5m": 12.310546875, + "load_average_15m": 10.646484375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25266053120, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.7, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T141800Z/", + "machine_load_at_start": { + "load_average_1m": 9.16650390625, + "load_average_5m": 11.61572265625, + "load_average_15m": 10.46630859375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25579483136, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.37255859375, + "load_average_5m": 10.7001953125, + "load_average_15m": 10.56396484375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 24989110272, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.6, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T142503Z/", + "machine_load_at_start": { + "load_average_1m": 8.240234375, + "load_average_5m": 10.55224609375, + "load_average_15m": 10.5126953125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25810317312, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 13.11865234375, + "load_average_5m": 11.8583984375, + "load_average_15m": 11.14990234375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20088893440, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.5, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T144349Z/", + "machine_load_at_start": { + "load_average_1m": 6.16064453125, + "load_average_5m": 6.9130859375, + "load_average_15m": 8.44287109375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 22296936448, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 10.4541015625, + "load_average_5m": 9.6796875, + "load_average_15m": 9.3203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21858013184, + "total_bytes": 494384795648 + }, + "docker_root": null + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt new file mode 100644 index 00000000..b0100454 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T144349Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt new file mode 100644 index 00000000..67a693c5 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T142503Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt new file mode 100644 index 00000000..e4c7d723 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T141800Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt new file mode 100644 index 00000000..6dfb51dc --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T141001Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt new file mode 100644 index 00000000..a4ee6f2e --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T140209Z/ diff --git a/benchmarks/trust_factor_sweep.sh b/benchmarks/trust_factor_sweep.sh index 829e0d45..42049a50 100755 --- a/benchmarks/trust_factor_sweep.sh +++ b/benchmarks/trust_factor_sweep.sh @@ -5,15 +5,29 @@ # Read it before changing anything here — the grid is derived from a cheap # adversarial sweep, and the decision rule is fixed in advance on purpose. # -# benchmarks/trust_factor_sweep.sh # full grid, all three suites +# benchmarks/trust_factor_sweep.sh # run ONE pending cell, then exit # benchmarks/trust_factor_sweep.sh --quick # smoke the plumbing (NOT gated) # -# One reproduce.sh invocation per cell, strictly SEQUENTIAL, each against its +# 2026-08-10 architecture change: this script used to loop over the whole +# grid in one process. Five campaigns died mid-grid over one session, each +# attributed to a different cause after the fact (contention, a native +# crash, a missing dataset, a session gap, a noisy neighbor) — five +# explanations for one symptom, which was itself the signal: a long job +# driven from a sub-agent's own foreground loop does not survive whatever +# ends that sub-agent's turn, regardless of which resource happened to be +# short at the time. The fix is not preventing the death (it cannot be, from +# here) but making it cost one cell instead of the whole grid: this script +# now resumes from `benchmarks/lib/sweep_progress.py`'s PROGRESS.json (which +# W values already completed), runs exactly the next PENDING cell, records +# its result — including machine-load/disk-space snapshots at cell start +# AND end (`benchmarks/lib/machine_load_snapshot.py`, +# `benchmarks/lib/disk_space_snapshot.py`) — and returns control. A kill or +# crash mid-cell leaves that cell's PROGRESS.json entry absent, so the next +# invocation retries exactly that cell, never the ones already recorded. +# +# Still true, unchanged: one reproduce.sh invocation per cell, against its # own ephemeral container. No parallelism: a fan-out on this machine on -# 2026-08-08 drove load to 37 and swapped 11.9 GB. Cells are independent, so a -# failed cell does not invalidate the others — it is reported and the sweep -# continues, because a partial grid with an honest gap beats a grid that -# silently stopped early. +# 2026-08-08 drove load to 37 and swapped 11.9 GB. set -uo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -29,48 +43,99 @@ GRID=(1.0 0.8 0.7 0.6 0.5) QUICK_FLAG="" [[ "${1:-}" == "--quick" ]] && QUICK_FLAG="--quick" -STAMP="$(date -u +%Y%m%dT%H%M%SZ)" -OUT_ROOT="benchmarks/results/trust-factor-sweep/${STAMP}" +# Fixed location, not a fresh timestamp per invocation: resume needs the +# NEXT call to find the SAME PROGRESS.json the previous one wrote. +OUT_ROOT="benchmarks/results/trust-factor-sweep/active" mkdir -p "$OUT_ROOT" - LOG="${OUT_ROOT}/sweep.log" -echo "trust-factor sweep ${STAMP}" | tee "$LOG" -echo "grid: ${GRID[*]}" | tee -a "$LOG" -echo "pre-registration: docs/provenance/trust-factor-calibration.md" | tee -a "$LOG" - -for W in "${GRID[@]}"; do - CELL_DIR="${OUT_ROOT}/cell_W${W}" - mkdir -p "$CELL_DIR" - echo "" | tee -a "$LOG" - echo "=== cell W=${W} — started $(date -u +%H:%M:%SZ) ===" | tee -a "$LOG" - - # Exported, not inlined: reproduce.sh spawns the benchmark processes, and - # retrieval_dispatch.py reads the value at import in each of them. - CORTEX_UNTRUSTED_ORIGIN_FACTOR="$W" \ - benchmarks/reproduce.sh \ - --only longmemeval,locomo,beam \ - --no-ablation \ - $QUICK_FLAG \ - >"${CELL_DIR}/reproduce.log" 2>&1 - rc=$? - - if [[ $rc -eq 0 ]]; then - echo "cell W=${W}: OK" | tee -a "$LOG" - else - # Not fatal to the sweep: an unusable cell is a gap in the grid, and - # the decision rule can still be applied to the cells that reported. - echo "cell W=${W}: FAILED rc=${rc} (see ${CELL_DIR}/reproduce.log)" | tee -a "$LOG" - fi - - # reproduce.sh writes into benchmarks/results/repro//; - # record which one belongs to this cell so the summary can be rebuilt - # without guessing from timestamps. - latest_repro="$(ls -td benchmarks/results/repro/*/ 2>/dev/null | head -1)" - echo "${latest_repro}" > "${CELL_DIR}/repro_dir.txt" - echo "cell W=${W} results: ${latest_repro}" | tee -a "$LOG" -done + +snapshot_json() { + uv run python -c " +import json, sys +sys.path.insert(0, 'benchmarks/lib') +from machine_load_snapshot import machine_load_snapshot +from disk_space_snapshot import disk_space_snapshot +print(json.dumps({'machine_load': machine_load_snapshot(), 'disk_space': disk_space_snapshot()})) +" +} + +next_pending_w() { + uv run python -c " +import sys +sys.path.insert(0, 'benchmarks/lib') +from sweep_progress import next_pending_w +w = next_pending_w([1.0, 0.8, 0.7, 0.6, 0.5], '${OUT_ROOT}') +print('' if w is None else w) +" +} + +W="$(next_pending_w)" +if [[ -z "$W" ]]; then + echo "GRID COMPLETE — every cell in ${GRID[*]} has a status:complete entry in ${OUT_ROOT}/PROGRESS.json." | tee -a "$LOG" + echo "NEXT: apply the decision rule and record the chosen W in docs/provenance/trust-factor-calibration.md." | tee -a "$LOG" + exit 0 +fi + +CELL_DIR="${OUT_ROOT}/cell_W${W}" +mkdir -p "$CELL_DIR" +echo "" | tee -a "$LOG" +echo "=== cell W=${W} — started $(date -u +%H:%M:%SZ) ===" | tee -a "$LOG" + +START_SNAPSHOT="$(snapshot_json)" + +# Exported, not inlined: reproduce.sh spawns the benchmark processes, and +# retrieval_dispatch.py reads the value at import in each of them. +CORTEX_UNTRUSTED_ORIGIN_FACTOR="$W" \ + benchmarks/reproduce.sh \ + --only longmemeval,locomo,beam \ + --no-ablation \ + $QUICK_FLAG \ + >"${CELL_DIR}/reproduce.log" 2>&1 +rc=$? + +END_SNAPSHOT="$(snapshot_json)" + +if [[ $rc -eq 0 ]]; then + echo "cell W=${W}: OK" | tee -a "$LOG" + STATUS="complete" +else + echo "cell W=${W}: FAILED rc=${rc} (see ${CELL_DIR}/reproduce.log)" | tee -a "$LOG" + STATUS="failed" +fi + +# reproduce.sh writes into benchmarks/results/repro//; +# record which one belongs to this cell so the summary can be rebuilt +# without guessing from timestamps. +latest_repro="$(ls -td benchmarks/results/repro/*/ 2>/dev/null | head -1)" +echo "${latest_repro}" > "${CELL_DIR}/repro_dir.txt" +echo "cell W=${W} results: ${latest_repro}" | tee -a "$LOG" + +uv run python -c " +import json, sys +sys.path.insert(0, 'benchmarks/lib') +from sweep_progress import record_cell_result + +start = json.loads('''${START_SNAPSHOT}''') +end = json.loads('''${END_SNAPSHOT}''') +record_cell_result( + '${OUT_ROOT}', + ${W}, + status='${STATUS}', + repro_dir='${latest_repro}' or None, + snapshots={ + 'machine_load_at_start': start['machine_load'], + 'disk_space_at_start': start['disk_space'], + 'machine_load_at_end': end['machine_load'], + 'disk_space_at_end': end['disk_space'], + }, +) +" echo "" | tee -a "$LOG" -echo "sweep finished $(date -u +%Y-%m-%dT%H:%M:%SZ) — results under ${OUT_ROOT}" | tee -a "$LOG" -echo "NEXT: apply the decision rule from the pre-registration and record the" | tee -a "$LOG" -echo "chosen W in docs/provenance/trust-factor-calibration.md §Results." | tee -a "$LOG" +echo "cell W=${W} recorded to ${OUT_ROOT}/PROGRESS.json — control returned." | tee -a "$LOG" +remaining="$(next_pending_w)" +if [[ -n "$remaining" ]]; then + echo "NEXT PENDING CELL: W=${remaining}. Re-run this script to continue." | tee -a "$LOG" +else + echo "GRID COMPLETE — every cell has reported." | tee -a "$LOG" +fi diff --git a/docs/provenance/trust-factor-calibration.md b/docs/provenance/trust-factor-calibration.md index 4cfff8bc..66bc163d 100644 --- a/docs/provenance/trust-factor-calibration.md +++ b/docs/provenance/trust-factor-calibration.md @@ -176,3 +176,100 @@ Two provenance notes, stated rather than smoothed over: correction. What the gate requires still holds: one shared condition across all five cells, and a control arm that reproduces the published numbers exactly. + +## Re-measurement after the capture_origin fix (PR #410, issue #368) + +The paragraph above diagnosed a defect, not a residual unknown: the gated arm +could not discriminate W because `core.memory_ingest.ingest_memory` accepted +a caller-supplied `capture_origin` but never forwarded it to +`store.insert_memory` — every LME/LoCoMo/BEAM memory landed on the column +default `'unknown'`, `core.capture_origin.trust_factor` demoted all of them +by the same uniform multiplier, and a uniform rescale cannot change WRRF +order. **PR #410** (`b280fd25`, merged to `main` 2026-08-10) fixed the +forwarding and added `benchmarks/lib/capture_origin_mix.py` — a deterministic +sampler that assigns benchmark memories a realistic `capture_origin` mixture +(`local_action` .966 / `network` .025 / `deliberate` .009, measured over this +machine's own 886 Claude Code session transcripts) instead of leaving every +memory at `'unknown'`. Wired into `benchmarks/lib/bench_db.py` via +`_apply_capture_origin_mix`, this is the first harness run in which W can +discriminate anything on these three benchmarks. + +### What was re-run + +Five cells, `W ∈ {1.0, 0.8, 0.7, 0.6, 0.5}` — the same grid as the original +sweep, on the branch that carries PR #410. Each cell is one +`benchmarks/trust_factor_sweep.sh` invocation in `--quick` mode (per-cell +`--limit`: 10 LongMemEval questions, 1 LoCoMo conversation, 2 BEAM windows — +`reproduce.sh:421`), sequential, own ephemeral pgvector container, at +`git_sha f87bf6e3`. Machine load and free disk space were snapshotted at the +start and end of every cell (`benchmarks/lib/machine_load_snapshot.py`, +`benchmarks/lib/disk_space_snapshot.py`) and are recorded alongside each +result — load ran 6.2–13.1 (1m avg) across the run, consistent with a shared, +busy machine rather than an isolated benchmark host; nothing in the deltas +below correlates with the load swing. + +| W | LME MRR | LME R@10 | LoCoMo MRR | LoCoMo R@10 | BEAM MRR | Artifacts | +|---|---|---|---|---|---|---| +| 1.0 (control) | 0.8500 | 1.0000 | 0.8209 | 0.9594 | 0.6579 | `benchmarks/results/repro/20260810T140209Z/` | +| 0.8 | 0.8500 | 1.0000 | 0.8043 | 0.9391 | 0.6575 | `benchmarks/results/repro/20260810T141001Z/` | +| **0.7** | **0.8500** | **1.0000** | **0.8044** | **0.9340** | **0.6575** | `benchmarks/results/repro/20260810T141800Z/` | +| 0.6 | 0.8500 | 1.0000 | 0.8044 | 0.9340 | 0.6575 | `benchmarks/results/repro/20260810T142503Z/` | +| 0.5 | 0.8500 | 1.0000 | 0.7892 | 0.9188 | 0.6575 | `benchmarks/results/repro/20260810T144349Z/` | + +Per-cell load/disk snapshots and the exact `--limit`-derived sample sizes +(LME `n_questions=10`, LoCoMo `n_conversations=1`/`n_questions=197`) are in +`benchmarks/results/trust-factor-sweep/active/PROGRESS.json` and each cell's +`reproduce.log`; the raw per-benchmark JSON (with full `MANIFEST.json` +provenance — git sha, package versions, reranker sha256) is under the +`Artifacts` paths above. + +### What this establishes + +Relative to each other — same harness, same `--quick` mode, same corpus, +five consecutive cells, one shared provenance — **the trust factor costs +relevance on LoCoMo, monotonically in W**. W = 1.0 (disabled) gives the best +scores; at the shipped production value, W = 0.7, LoCoMo loses 0.0165 MRR +(0.8209 → 0.8044) and 2.54 points of Recall@10 (0.9594 → 0.9340) relative to +the disabled control. LongMemEval is flat across the whole grid (0.8500 / +1.0000 identically at every W) and BEAM MRR moves 0.0004 (0.6579 → 0.6575) — +neither is large enough, at this sample size, to call a W effect. + +### What this does NOT establish + +Two gaps, stated as such rather than folded into the table above: + +1. **Conformance to the published floors.** `reproduce.sh:94-95` states + explicitly that `--quick`/`--limit` runs skip the floor gate — "partial + runs are not comparable to n=500 / n=1986 figures" — and every cell above + ran `--quick` (LME n=10, LoCoMo n=1 conversation / 197 questions, vs the + published n=500/n=1986). Comparing these numbers to + `FLOOR_LME_R10`/`FLOOR_LME_MRR`/`FLOOR_LOCOMO_R10`/`FLOOR_LOCOMO_MRR` + would be exactly the protocol-mismatch error PR #414 spent four review + rounds correcting — different sample size, different protocol, not a + comparable number. A full-mode re-run against the floors is the open + item; it would tell us whether the LoCoMo cost measured here still holds + (or is smaller/larger) at the published sample size. +2. **Whether the adversarial defense justifies the cost.** The adversarial + arm (§ above) shows 4/4 scenarios defended at W ≤ 0.70, but that corpus + (`benchmarks/lib/adversarial_corpus.py`) is constructed specifically to + exercise the attack families — it says nothing about whether that + defense buys anything on LME/LoCoMo/BEAM's real question distributions. + No measurement in this document links the adversarial-arm defense rate to + a benefit on the real benchmarks; the relevance cost above and the + security benefit in § Adversarial arm are two separate, unlinked + measurements. Closing this gap needs a single corpus or evaluation that + scores both relevance and adversarial robustness together. + +### Decision (owner call, not derived from this measurement alone) + +**W stays at 0.7.** The relevance cost measured above is real and is +published, with both reserves above stated rather than buried: this +re-measurement does not confirm floor conformance (quick mode only) and does +not establish that the adversarial defense is worth the cost (no linking +measurement exists). Publishing the honest number now — rather than +withholding it until a full-mode run and a linking measurement both exist — +is the position consistent with § Scientific Implementation Standard: report +what was measured and the uncertainty that remains, rather than delay or +omit. Superseded by a full-mode re-run against the floors, or a measurement +linking adversarial defense to real-benchmark relevance, whichever lands +first. diff --git a/tests_py/benchmarks/test_disk_space_snapshot.py b/tests_py/benchmarks/test_disk_space_snapshot.py new file mode 100644 index 00000000..082dad67 --- /dev/null +++ b/tests_py/benchmarks/test_disk_space_snapshot.py @@ -0,0 +1,45 @@ +"""benchmarks.lib.disk_space_snapshot (2026-08-10 fix). + +Contract under test: `disk_space_snapshot()` reports free/total bytes on +the repo-root filesystem (where benchmark datasets, throwaway test +databases, and HF cache land) and, best-effort, on Docker's storage root — +never raising, matching every other probe in this codebase's manifest +tooling. See the module's own docstring for the disk-exhaustion incident +motivating this (leaked test databases filled a shared machine's disk to +100% mid-benchmark, taking PostgreSQL down with no trace in the artifact). +""" + +from __future__ import annotations + +from benchmarks.lib.disk_space_snapshot import disk_space_snapshot + + +class TestDiskSpaceSnapshot: + def test_returns_repo_root_and_docker_root_keys(self): + snap = disk_space_snapshot() + assert set(snap) == {"repo_root", "docker_root"} + + def test_repo_root_is_populated_on_a_reachable_filesystem(self): + """The repo itself is on disk right now, so this must never be + None in a test-running environment.""" + snap = disk_space_snapshot() + assert snap["repo_root"] is not None + assert set(snap["repo_root"]) == {"path", "free_bytes", "total_bytes"} + + def test_repo_root_free_and_total_are_consistent_positive_ints(self): + snap = disk_space_snapshot() + repo_root = snap["repo_root"] + assert isinstance(repo_root["free_bytes"], int) + assert isinstance(repo_root["total_bytes"], int) + assert 0 <= repo_root["free_bytes"] <= repo_root["total_bytes"] + + def test_docker_root_is_a_dict_or_none(self): + """None only when the docker CLI is unavailable/unreachable or its + reported root path can't be statted -- never raises.""" + snap = disk_space_snapshot() + docker_root = snap["docker_root"] + assert docker_root is None or set(docker_root) == { + "path", + "free_bytes", + "total_bytes", + } diff --git a/tests_py/benchmarks/test_machine_load_snapshot.py b/tests_py/benchmarks/test_machine_load_snapshot.py new file mode 100644 index 00000000..a3bfcd34 --- /dev/null +++ b/tests_py/benchmarks/test_machine_load_snapshot.py @@ -0,0 +1,51 @@ +"""benchmarks.lib.machine_load_snapshot (2026-08-10 fix, split out of +write_manifest.py to satisfy the 40-line method cap — CLAUDE.md § Code +Style). See that module's docstring for the incident motivating it. +""" + +from __future__ import annotations + +from benchmarks.lib.machine_load_snapshot import machine_load_snapshot + +_LOAD_KEYS = { + "load_average_1m", + "load_average_5m", + "load_average_15m", + "cpu_count", + "concurrent_pytest_processes", + "concurrent_docker_containers", +} + + +class TestMachineLoadSnapshot: + def test_returns_all_expected_keys(self): + snap = machine_load_snapshot() + assert set(snap) == _LOAD_KEYS + + def test_load_averages_are_nonnegative_floats_on_a_posix_machine(self): + snap = machine_load_snapshot() + for key in ("load_average_1m", "load_average_5m", "load_average_15m"): + assert isinstance(snap[key], float) + assert snap[key] >= 0.0 + + def test_cpu_count_is_a_positive_int(self): + snap = machine_load_snapshot() + assert isinstance(snap["cpu_count"], int) + assert snap["cpu_count"] > 0 + + def test_pytest_process_count_sees_the_process_running_this_test(self): + """The suite executing this assertion IS a pytest process, so the + count must be >= 1 — a fixed proof the probe is not silently + returning zero/None while pytest is demonstrably running. Pinned by + the COLUMNS-truncation fix (2026-08-10): this exact assertion is + what caught the bug on GitHub's Linux CI runner.""" + snap = machine_load_snapshot() + assert snap["concurrent_pytest_processes"] is not None + assert snap["concurrent_pytest_processes"] >= 1 + + def test_docker_container_count_is_an_int_or_none(self): + """None only when the docker CLI itself is unavailable/unreachable — + never raises, matching every other probe in this module.""" + snap = machine_load_snapshot() + count = snap["concurrent_docker_containers"] + assert count is None or (isinstance(count, int) and count >= 0) diff --git a/tests_py/benchmarks/test_write_manifest_machine_load.py b/tests_py/benchmarks/test_write_manifest_machine_load.py index 882ffb47..1793a6fc 100644 --- a/tests_py/benchmarks/test_write_manifest_machine_load.py +++ b/tests_py/benchmarks/test_write_manifest_machine_load.py @@ -1,30 +1,34 @@ -"""machine_load_snapshot in benchmarks.lib.write_manifest (2026-08-10 fix). - -Contract under test: every benchmark MANIFEST.json records the machine's -load state (load average, concurrent pytest processes, concurrent Docker -containers) alongside git_sha — not a separate/optional field — so a run -produced under contention can be identified after the fact instead of -requiring someone to have been watching `uptime` live. Prompted by an -incident where a 5-cell trust-factor sweep ran while three other agents' -full pytest suites were active; one cell crashed visibly, but a cell that -merely finished under the same contention would have looked like a clean -result with nothing in the artifact to say otherwise. - -Follow-up (same day): the snapshot is taken TWICE per cell — at cell start -(`write_start_snapshot`, written before `start_db`) and at cell end (inside -`build_manifest`) — because a single end-of-run reading cannot tell "ran -under load the whole time" apart from "load spiked right at the end". +"""write_manifest.py's start/end machine-load + disk-space wiring +(2026-08-10 fix). + +Contract under test: every benchmark MANIFEST.json records BOTH the +machine's load state and its free disk space, at cell start AND cell end, +alongside git_sha — not a separate/optional field — so a run produced +under CPU contention or disk exhaustion can be identified after the fact +instead of requiring someone to have been watching `uptime`/`df` live. + +Two incidents motivate this (full narrative in +benchmarks/lib/machine_load_snapshot.py and +benchmarks/lib/disk_space_snapshot.py's module docstrings): a 5-cell +trust-factor sweep ran while three other agents' full pytest suites were +active (one cell crashed visibly, but a cell that merely finished under +the same contention would have looked clean); and leaked throwaway test +databases filled the host disk to 100% on a neighboring session sharing +the same machine, taking PostgreSQL down mid-measurement. + +The probe functions themselves (`machine_load_snapshot`, +`disk_space_snapshot`) are tested in their own modules' +tests_py/benchmarks/test_machine_load_snapshot.py and +tests_py/benchmarks/test_disk_space_snapshot.py — this file only tests +`write_manifest.py`'s wiring of the two snapshot points into MANIFEST.json. """ from __future__ import annotations import json -from benchmarks.lib.write_manifest import ( - build_manifest, - machine_load_snapshot, - write_start_snapshot, -) +from benchmarks.lib.machine_load_snapshot import machine_load_snapshot +from benchmarks.lib.write_manifest import build_manifest, write_start_snapshot _LOAD_KEYS = { "load_average_1m", @@ -36,99 +40,78 @@ } -class TestMachineLoadSnapshot: - def test_returns_all_expected_keys(self): - snap = machine_load_snapshot() - assert set(snap) == _LOAD_KEYS +def _build(tmp_path): + return build_manifest( + str(tmp_path), + "deadbeef", + "dataset-sha", + "pgvector/pgvector:pg16", + "test-container", + "5432", + "1", + ) - def test_load_averages_are_nonnegative_floats_on_a_posix_machine(self): - snap = machine_load_snapshot() - for key in ("load_average_1m", "load_average_5m", "load_average_15m"): - assert isinstance(snap[key], float) - assert snap[key] >= 0.0 - def test_cpu_count_is_a_positive_int(self): - snap = machine_load_snapshot() - assert isinstance(snap["cpu_count"], int) - assert snap["cpu_count"] > 0 - - def test_pytest_process_count_sees_the_process_running_this_test(self): - """The suite executing this assertion IS a pytest process, so the - count must be >= 1 — a fixed proof the probe is not silently - returning zero/None while pytest is demonstrably running.""" - snap = machine_load_snapshot() - assert snap["concurrent_pytest_processes"] is not None - assert snap["concurrent_pytest_processes"] >= 1 - - def test_docker_container_count_is_an_int_or_none(self): - """None only when the docker CLI itself is unavailable/unreachable — - never raises, matching every other probe in this module.""" - snap = machine_load_snapshot() - count = snap["concurrent_docker_containers"] - assert count is None or (isinstance(count, int) and count >= 0) - - -class TestBuildManifestIncludesMachineLoad: +class TestBuildManifestAtEndFields: def test_machine_load_at_end_sits_alongside_git_sha(self, tmp_path): - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) + manifest = _build(tmp_path) assert "git_sha" in manifest assert "machine_load_at_end" in manifest assert set(manifest["machine_load_at_end"]) == _LOAD_KEYS - def test_machine_load_at_start_is_none_when_no_snapshot_was_taken(self, tmp_path): - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) + def test_disk_space_at_end_sits_alongside_git_sha(self, tmp_path): + manifest = _build(tmp_path) + assert "disk_space_at_end" in manifest + assert set(manifest["disk_space_at_end"]) == {"repo_root", "docker_root"} + + +class TestBuildManifestAtStartFields: + def test_both_at_start_fields_are_none_when_no_snapshot_was_taken(self, tmp_path): + manifest = _build(tmp_path) assert manifest["machine_load_at_start"] is None + assert manifest["disk_space_at_start"] is None - def test_machine_load_at_start_is_populated_when_a_snapshot_was_taken( + def test_both_at_start_fields_are_populated_when_a_snapshot_was_taken( self, tmp_path ): write_start_snapshot(str(tmp_path)) - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) - assert manifest["machine_load_at_start"] is not None + manifest = _build(tmp_path) assert set(manifest["machine_load_at_start"]) == _LOAD_KEYS + assert set(manifest["disk_space_at_start"]) == {"repo_root", "docker_root"} + + def test_at_start_fields_are_the_snapshot_taken_at_start_not_a_fresh_read( + self, tmp_path + ): + """The manifest must read back the PERSISTED start snapshot, not + silently re-sample -- otherwise `_at_start` would always equal + `_at_end` and the two-point design would measure nothing.""" + write_start_snapshot(str(tmp_path)) + persisted = json.loads((tmp_path / "START_SNAPSHOT.json").read_text()) + manifest = _build(tmp_path) + assert manifest["machine_load_at_start"] == persisted["machine_load"] + assert manifest["disk_space_at_start"] == persisted["disk_space"] + +class TestResultsFilesExclusion: def test_start_snapshot_file_is_excluded_from_results_files(self, tmp_path): write_start_snapshot(str(tmp_path)) - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) + manifest = _build(tmp_path) assert "START_SNAPSHOT.json" not in manifest["results_files"] class TestWriteStartSnapshot: - def test_writes_a_readable_json_file_with_a_timestamp_and_load(self, tmp_path): + def test_writes_a_readable_json_file_with_a_timestamp_load_and_disk(self, tmp_path): out = write_start_snapshot(str(tmp_path)) assert out.exists() payload = json.loads(out.read_text()) assert "captured_at_utc" in payload assert set(payload["machine_load"]) == _LOAD_KEYS + assert set(payload["disk_space"]) == {"repo_root", "docker_root"} + + def test_matches_a_direct_probe_call_in_shape(self, tmp_path): + """Not a value-equality check (load/disk drift between calls) -- + confirms write_start_snapshot persists the SAME shape + machine_load_snapshot() itself returns, not a re-derived one.""" + out = write_start_snapshot(str(tmp_path)) + payload = json.loads(out.read_text()) + assert set(payload["machine_load"]) == set(machine_load_snapshot())