From f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74 Mon Sep 17 00:00:00 2001 From: cdeust Date: Mon, 10 Aug 2026 08:40:26 +0200 Subject: [PATCH 1/3] fix(benchmarks): snapshot disk space alongside load; split under 300/40 caps Second invisible-failure vector for the trust-factor sweep (issue #368 follow-up): a neighboring session sharing this machine reported that leaked throwaway SQLite test databases (226-357 MB/run) filled the host disk to 100% on 2026-08-09, taking PostgreSQL down and starving even the Bash tool's own stdout file. A cell can finish cleanly and still return degraded numbers because its volume filled under it -- the same failure shape as CPU contention, a different resource, and it left no more trace in the artifact than contention did. Fix: benchmarks/lib/disk_space_snapshot.py (new) -- free/total bytes on the repo-root filesystem and, best-effort, Docker's storage root, via shutil.disk_usage + `docker info`. Wired into write_manifest.py exactly like machine_load_snapshot: captured at cell start (write_start_snapshot) and cell end (build_manifest), as disk_space_at_start/disk_space_at_end. Also split benchmarks/lib/machine_load_snapshot.py out of write_manifest.py: the new local CLAUDE.md Code Style gate (scripts/check_craftsmanship.py, merged to main while this work was in flight) caps methods at 40 lines and files at 300, and machine_load_snapshot() (78 lines) + build_manifest() (49 lines) both already exceeded that before this commit -- pre-existing debt this change surfaced rather than introduced, fixed here since the gate now blocks any diff touching this file until it passes. Every function in the resulting three files is under the cap; verified via `python scripts/check_craftsmanship.py `. 10 tests split/added across tests_py/benchmarks/test_machine_load_snapshot.py (new), test_disk_space_snapshot.py (new), and test_write_manifest_machine_load.py (now wiring-only). 33 tests total across the six touched files, all green. ruff clean. Co-Authored-By: Claude --- benchmarks/lib/disk_space_snapshot.py | 67 +++++++ benchmarks/lib/machine_load_snapshot.py | 90 +++++++++ benchmarks/lib/write_manifest.py | 171 +++++++----------- .../benchmarks/test_disk_space_snapshot.py | 45 +++++ .../benchmarks/test_machine_load_snapshot.py | 51 ++++++ .../test_write_manifest_machine_load.py | 167 ++++++++--------- 6 files changed, 390 insertions(+), 201 deletions(-) create mode 100644 benchmarks/lib/disk_space_snapshot.py create mode 100644 benchmarks/lib/machine_load_snapshot.py create mode 100644 tests_py/benchmarks/test_disk_space_snapshot.py create mode 100644 tests_py/benchmarks/test_machine_load_snapshot.py diff --git a/benchmarks/lib/disk_space_snapshot.py b/benchmarks/lib/disk_space_snapshot.py new file mode 100644 index 00000000..0309266c --- /dev/null +++ b/benchmarks/lib/disk_space_snapshot.py @@ -0,0 +1,67 @@ +"""Disk-space snapshot for a benchmark cell (issue #368 follow-up). + +2026-08-10 incident (reported by a neighboring session sharing this +machine): leaked throwaway SQLite test databases (226-357 MB per run) +filled the host disk to 100% on 2026-08-09 — PostgreSQL went down and even +the Bash tool could no longer write its own output file. A cell can finish +cleanly and still return degraded numbers because the volume filled under +it, and that leaves no more trace in the artifact than CPU contention does +(see `machine_load_snapshot.py`'s docstring — this is the same failure +shape, a different resource). Recorded the same way and for the same +reason: best-effort, taken at cell start and cell end, never aborts +manifest generation. + +Two paths are measured, both best-effort: + - the repo root's filesystem — where benchmark datasets, throwaway test + databases, and HuggingFace cache actually land; + - Docker's storage root (`docker info`'s `DockerRootDir`) — under a VM + backend (colima, Docker Desktop) this is a guest-internal path and + `shutil.disk_usage` on it measures the HOST filesystem backing the VM + image file, not a second independent volume; when it resolves to the + same filesystem as the repo root this is redundant but harmless, and + when Docker is unreachable it is simply `None`. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _usage(path: str) -> dict | None: + try: + usage = shutil.disk_usage(path) + except OSError: + return None + return { + "path": path, + "free_bytes": usage.free, + "total_bytes": usage.total, + } + + +def _docker_root_dir() -> str | None: + try: + out = subprocess.run( + ["docker", "info", "--format", "{{.DockerRootDir}}"], + capture_output=True, + text=True, + timeout=10, + check=False, + ).stdout.strip() + except (OSError, subprocess.SubprocessError): + return None + return out or None + + +def disk_space_snapshot() -> dict: + """Free/total bytes on the volume(s) backing benchmark data + Docker + storage, as this run saw them. See this module's docstring for why.""" + docker_root = _docker_root_dir() + return { + "repo_root": _usage(str(_REPO_ROOT)), + "docker_root": _usage(docker_root) if docker_root else None, + } diff --git a/benchmarks/lib/machine_load_snapshot.py b/benchmarks/lib/machine_load_snapshot.py new file mode 100644 index 00000000..84317680 --- /dev/null +++ b/benchmarks/lib/machine_load_snapshot.py @@ -0,0 +1,90 @@ +"""Machine-load snapshot for a benchmark cell (issue #368 follow-up). + +2026-08-10 incident: a 5-cell trust-factor sweep ran while three other +agents' full pytest suites were active on the same machine (load average +~11-14 on a 10-core box); one cell crashed on a native fatal error, and the +crash was the ONLY visible signal — cells that merely finished under the +same contention could have returned degraded numbers (saturated connection +pool, cold cache, GC pressure) with nothing in the artifact to show it. The +whole grid was discarded and re-run rather than salvaged, per this +project's own rule: a measurement from a harness with a known defect is +invalid and is redone, not patched after the fact — and contention is +exactly such a defect. This snapshot is recorded so that rule can be +applied by inspection later, instead of by asking whoever happened to be +watching at the time. + +Taken TWICE per cell (same-day follow-up, same incident): once at cell +START (`write_manifest.write_start_snapshot`, called before `start_db` so +it predates the container/DB overhead too) and once at cell END (inside +`write_manifest.build_manifest`). A crash is the visible failure mode; a +cell that merely FINISHES under contention is the invisible one, and a +single end-of-run snapshot cannot distinguish "ran under load throughout" +from "load spiked right at the end". Two points at least bound the window. + +Every probe here is best-effort: a failure records `None` rather than +aborting manifest generation, matching every other field `write_manifest.py` +records. +""" + +from __future__ import annotations + +import os +import subprocess + + +def _run(cmd: list[str], *, env: dict[str, str] | None = None) -> str | None: + try: + return subprocess.run( + cmd, capture_output=True, text=True, timeout=10, check=False, env=env + ).stdout + except (OSError, subprocess.SubprocessError): + return None + + +def count_pytest_processes() -> int | None: + """Concurrent `pytest` processes system-wide, or None if unreadable. + + Filtered in Python, not via a shell `grep -c "[p]ytest"` idiom: a + subprocess.run argv has no shell to bracket-escape a self-match, so the + filter runs here instead, over the same process list that idiom reads. + + The `COLUMNS` override fixes a real undercount, not just a test flake + (caught by tests_py/benchmarks/test_write_manifest_machine_load.py's + own self-referential assertion failing on GitHub's Linux CI runner, + 2026-08-10): both BSD ps (macOS) and GNU procps (Linux) truncate the + COMMAND column to `$COLUMNS` when stdout is not a terminal and COLUMNS + is unset, and `ps aux`'s fixed-width USER/PID/... columns alone can + exceed a default 80-column budget before COMMAND even starts — cutting + off the "pytest" substring entirely on a long interpreter path. + """ + ps_out = _run(["ps", "aux"], env={**os.environ, "COLUMNS": "1000"}) + if ps_out is None: + return None + return sum( + 1 for line in ps_out.splitlines() if "pytest" in line and "grep" not in line + ) + + +def count_docker_containers() -> int | None: + """Concurrent running Docker containers, or None if unreadable.""" + docker_out = _run(["docker", "ps", "-q"]) + if docker_out is None: + return None + return len([line for line in docker_out.splitlines() if line.strip()]) + + +def machine_load_snapshot() -> dict: + """Load average + concurrent pytest/container counts, as this run saw + them. See this module's docstring for why.""" + try: + load1, load5, load15 = os.getloadavg() + except OSError: # not available on this platform (e.g. Windows) + load1 = load5 = load15 = None + return { + "load_average_1m": load1, + "load_average_5m": load5, + "load_average_15m": load15, + "cpu_count": os.cpu_count(), + "concurrent_pytest_processes": count_pytest_processes(), + "concurrent_docker_containers": count_docker_containers(), + } diff --git a/benchmarks/lib/write_manifest.py b/benchmarks/lib/write_manifest.py index 47791231..1e9a859d 100644 --- a/benchmarks/lib/write_manifest.py +++ b/benchmarks/lib/write_manifest.py @@ -5,6 +5,11 @@ driver stays within the size limits of coding-standards.md §4 and so this provenance logic can be read, diffed and tested as Python. +Machine-load and disk-space snapshots live in sibling modules +(``machine_load_snapshot.py``, ``disk_space_snapshot.py``) — see their +docstrings for the two incidents that motivate recording them alongside +``git_sha`` in every manifest. + Usage (from reproduce.sh): python benchmarks/lib/write_manifest.py \\ RESULTS_DIR GIT_SHA DATASET_SHA256 PG_IMAGE CONTAINER PG_PORT RUNNER_PID @@ -15,111 +20,39 @@ """ import json -import os import platform -import subprocess import sys from datetime import datetime, timezone from pathlib import Path -_START_SNAPSHOT_NAME = "START_SNAPSHOT.json" - - -def machine_load_snapshot() -> dict: - """Load average + concurrent pytest/container counts, as this run saw them. - - 2026-08-10 incident: a 5-cell trust-factor sweep ran while three other - agents' full pytest suites were active on the same machine (load average - ~11-14 on a 10-core box); one cell crashed on a native fatal error, and - the crash was the ONLY visible signal — cells that merely finished under - the same contention could have returned degraded numbers (saturated - connection pool, cold cache, GC pressure) with nothing in the artifact to - show it. The whole grid was discarded and re-run rather than salvaged, - per this project's own rule: a measurement from a harness with a known - defect is invalid and is redone, not patched after the fact — and - contention is exactly such a defect. This snapshot is recorded so that - rule can be applied by inspection later, instead of by asking whoever - happened to be watching at the time. - - Taken TWICE per cell (2026-08-10 follow-up, same incident): once at cell - START (`write_start_snapshot`, called before `start_db` so it predates - the container/DB overhead too) and once at cell END (inside - `build_manifest`, the pre-existing call). A crash is the visible failure - mode; a cell that merely FINISHES under contention (saturated pool, cold - cache, GC pressure) is the invisible one, and a single end-of-run - snapshot cannot distinguish "this cell ran under load throughout" from - "load spiked right at the end". Two points at least bound the window. - - Best-effort: any probe that fails records `None`/`"unresolved"` rather - than aborting manifest generation, matching this module's other fields. - """ - try: - load1, load5, load15 = os.getloadavg() - except OSError: # not available on this platform (e.g. Windows) - load1 = load5 = load15 = None - - def _run(cmd: list[str], *, env: dict[str, str] | None = None) -> str | None: - try: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=10, check=False, env=env - ).stdout - except (OSError, subprocess.SubprocessError): - return None - - # Filtered in Python, not via a shell `grep -c "[p]ytest"` idiom: a - # subprocess.run argv has no shell to bracket-escape a self-match, so the - # filter runs here instead, over the same process list that idiom reads. - # - # The `COLUMNS` override below fixes a real undercount, not just a test - # flake (caught by - # tests_py/benchmarks/test_write_manifest_machine_load.py's own - # self-referential assertion failing on GitHub's Linux CI runner, - # 2026-08-10): both BSD ps (macOS) and GNU procps (Linux) truncate the - # COMMAND column to `$COLUMNS` when stdout is not a terminal and COLUMNS - # is unset, and `ps aux`'s fixed-width USER/PID/... columns alone can - # exceed a default 80-column budget before COMMAND even starts — cutting - # off the "pytest" substring entirely on a long interpreter path. A wide - # COLUMNS override is the standard fix for both implementations. - ps_out = _run(["ps", "aux"], env={**os.environ, "COLUMNS": "1000"}) - pytest_procs = ( - None - if ps_out is None - else sum( - 1 for line in ps_out.splitlines() if "pytest" in line and "grep" not in line - ) - ) - - docker_out = _run(["docker", "ps", "-q"]) - docker_containers = ( - None - if docker_out is None - else len([line for line in docker_out.splitlines() if line.strip()]) - ) +# Same idiom as ablation_runner.py/latency_runner.py/e2_subsample_runner.py +# in this package: REPO_ROOT on sys.path so `benchmarks.lib.*` resolves both +# when this file runs as a standalone script (reproduce.sh invokes it +# directly, not via `-m`) and when it is imported normally as a package +# member (tests_py/). +_REPO_ROOT = str(Path(__file__).resolve().parents[2]) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) +from benchmarks.lib.disk_space_snapshot import disk_space_snapshot # noqa: E402 +from benchmarks.lib.machine_load_snapshot import machine_load_snapshot # noqa: E402 - return { - "load_average_1m": load1, - "load_average_5m": load5, - "load_average_15m": load15, - "cpu_count": os.cpu_count(), - "concurrent_pytest_processes": pytest_procs, - "concurrent_docker_containers": docker_containers, - } +_START_SNAPSHOT_NAME = "START_SNAPSHOT.json" def write_start_snapshot(results_dir: str) -> Path: - """Capture + persist the cell-start machine-load snapshot. + """Capture + persist the cell-start machine-load and disk-space snapshot. Called from reproduce.sh before `start_db`, so `RESULTS_DIR` already exists (created by `main()`'s `mkdir -p`) but nothing benchmark-specific has run yet. `build_manifest` reads this file back at cell end and folds - it into the final MANIFEST.json as `machine_load_at_start`, alongside the - end-of-run `machine_load_at_end` — see `machine_load_snapshot`'s - docstring for why both points are recorded. + it into the final MANIFEST.json as `machine_load_at_start` / + `disk_space_at_start`, alongside the end-of-run `_at_end` counterparts. """ out = Path(results_dir) / _START_SNAPSHOT_NAME payload = { "captured_at_utc": datetime.now(timezone.utc).isoformat(), "machine_load": machine_load_snapshot(), + "disk_space": disk_space_snapshot(), } out.write_text(json.dumps(payload, indent=2)) return out @@ -131,7 +64,7 @@ def _read_start_snapshot(results_dir: str) -> dict | None: Returns None (never raises) when absent — e.g. a `reproduce.sh` call that predates this fix, or a caller that skipped the `--snapshot` step. A missing start snapshot must not block the end-of-run manifest from - being written; `machine_load_at_start` is simply absent in that case, + being written; the `_at_start` fields are simply absent in that case, which is itself an observable fact rather than a silent guess. """ path = Path(results_dir) / _START_SNAPSHOT_NAME @@ -205,6 +138,40 @@ def reranker_fields() -> dict: } +def _environment_fields() -> dict: + """Package/model/reranker identity fields — split out of `build_manifest` + to keep that function under the 40-line method cap (CLAUDE.md § Code + Style).""" + return { + "python": platform.python_version(), + "packages": { + p: ver(p) + for p in ( + "datasets", + "sentence-transformers", + "torch", + "psycopg", + "psycopg-pool", + ) + }, + "embedding_model_revision": embedding_revision(), + **reranker_fields(), + } + + +def _start_snapshot_fields(results_dir: str) -> dict: + """`_at_start` machine-load/disk-space fields, or None/None if + `write_start_snapshot` was never called for this `results_dir` — split + out of `build_manifest` for the same reason as `_environment_fields`.""" + snap = _read_start_snapshot(results_dir) + if snap is None: + return {"machine_load_at_start": None, "disk_space_at_start": None} + return { + "machine_load_at_start": snap.get("machine_load"), + "disk_space_at_start": snap.get("disk_space"), + } + + def build_manifest( results_dir: str, git_sha: str, @@ -214,17 +181,15 @@ def build_manifest( pg_port: str, pid: str, ) -> dict: - start_snapshot = _read_start_snapshot(results_dir) return { "git_sha": git_sha, - # Alongside git_sha, not buried: see machine_load_snapshot's - # docstring for why (2026-08-10 sweep-contention incident). Two - # points, not one — `_at_start` is None when reproduce.sh's - # `--snapshot` step was never called for this results_dir. - "machine_load_at_start": ( - start_snapshot["machine_load"] if start_snapshot else None - ), + # Alongside git_sha, not buried: see machine_load_snapshot.py and + # disk_space_snapshot.py's docstrings for why (2026-08-10 sweep + # incidents — CPU contention, then a full disk, both invisible in + # a cell that merely finishes). Two points per resource, not one. + **_start_snapshot_fields(results_dir), "machine_load_at_end": machine_load_snapshot(), + "disk_space_at_end": disk_space_snapshot(), "longmemeval_dataset_sha256": ds_sha, "pg_image": pg_image, # Per-run container isolation fix (2026-07-11, incident: two concurrent @@ -236,19 +201,7 @@ def build_manifest( "bench_container_name": container, "bench_container_port": int(pg_port), "bench_runner_pid": int(pid), - "python": platform.python_version(), - "packages": { - p: ver(p) - for p in ( - "datasets", - "sentence-transformers", - "torch", - "psycopg", - "psycopg-pool", - ) - }, - "embedding_model_revision": embedding_revision(), - **reranker_fields(), + **_environment_fields(), "results_files": sorted( p.name for p in Path(results_dir).glob("*.json") diff --git a/tests_py/benchmarks/test_disk_space_snapshot.py b/tests_py/benchmarks/test_disk_space_snapshot.py new file mode 100644 index 00000000..082dad67 --- /dev/null +++ b/tests_py/benchmarks/test_disk_space_snapshot.py @@ -0,0 +1,45 @@ +"""benchmarks.lib.disk_space_snapshot (2026-08-10 fix). + +Contract under test: `disk_space_snapshot()` reports free/total bytes on +the repo-root filesystem (where benchmark datasets, throwaway test +databases, and HF cache land) and, best-effort, on Docker's storage root — +never raising, matching every other probe in this codebase's manifest +tooling. See the module's own docstring for the disk-exhaustion incident +motivating this (leaked test databases filled a shared machine's disk to +100% mid-benchmark, taking PostgreSQL down with no trace in the artifact). +""" + +from __future__ import annotations + +from benchmarks.lib.disk_space_snapshot import disk_space_snapshot + + +class TestDiskSpaceSnapshot: + def test_returns_repo_root_and_docker_root_keys(self): + snap = disk_space_snapshot() + assert set(snap) == {"repo_root", "docker_root"} + + def test_repo_root_is_populated_on_a_reachable_filesystem(self): + """The repo itself is on disk right now, so this must never be + None in a test-running environment.""" + snap = disk_space_snapshot() + assert snap["repo_root"] is not None + assert set(snap["repo_root"]) == {"path", "free_bytes", "total_bytes"} + + def test_repo_root_free_and_total_are_consistent_positive_ints(self): + snap = disk_space_snapshot() + repo_root = snap["repo_root"] + assert isinstance(repo_root["free_bytes"], int) + assert isinstance(repo_root["total_bytes"], int) + assert 0 <= repo_root["free_bytes"] <= repo_root["total_bytes"] + + def test_docker_root_is_a_dict_or_none(self): + """None only when the docker CLI is unavailable/unreachable or its + reported root path can't be statted -- never raises.""" + snap = disk_space_snapshot() + docker_root = snap["docker_root"] + assert docker_root is None or set(docker_root) == { + "path", + "free_bytes", + "total_bytes", + } diff --git a/tests_py/benchmarks/test_machine_load_snapshot.py b/tests_py/benchmarks/test_machine_load_snapshot.py new file mode 100644 index 00000000..a3bfcd34 --- /dev/null +++ b/tests_py/benchmarks/test_machine_load_snapshot.py @@ -0,0 +1,51 @@ +"""benchmarks.lib.machine_load_snapshot (2026-08-10 fix, split out of +write_manifest.py to satisfy the 40-line method cap — CLAUDE.md § Code +Style). See that module's docstring for the incident motivating it. +""" + +from __future__ import annotations + +from benchmarks.lib.machine_load_snapshot import machine_load_snapshot + +_LOAD_KEYS = { + "load_average_1m", + "load_average_5m", + "load_average_15m", + "cpu_count", + "concurrent_pytest_processes", + "concurrent_docker_containers", +} + + +class TestMachineLoadSnapshot: + def test_returns_all_expected_keys(self): + snap = machine_load_snapshot() + assert set(snap) == _LOAD_KEYS + + def test_load_averages_are_nonnegative_floats_on_a_posix_machine(self): + snap = machine_load_snapshot() + for key in ("load_average_1m", "load_average_5m", "load_average_15m"): + assert isinstance(snap[key], float) + assert snap[key] >= 0.0 + + def test_cpu_count_is_a_positive_int(self): + snap = machine_load_snapshot() + assert isinstance(snap["cpu_count"], int) + assert snap["cpu_count"] > 0 + + def test_pytest_process_count_sees_the_process_running_this_test(self): + """The suite executing this assertion IS a pytest process, so the + count must be >= 1 — a fixed proof the probe is not silently + returning zero/None while pytest is demonstrably running. Pinned by + the COLUMNS-truncation fix (2026-08-10): this exact assertion is + what caught the bug on GitHub's Linux CI runner.""" + snap = machine_load_snapshot() + assert snap["concurrent_pytest_processes"] is not None + assert snap["concurrent_pytest_processes"] >= 1 + + def test_docker_container_count_is_an_int_or_none(self): + """None only when the docker CLI itself is unavailable/unreachable — + never raises, matching every other probe in this module.""" + snap = machine_load_snapshot() + count = snap["concurrent_docker_containers"] + assert count is None or (isinstance(count, int) and count >= 0) diff --git a/tests_py/benchmarks/test_write_manifest_machine_load.py b/tests_py/benchmarks/test_write_manifest_machine_load.py index 882ffb47..1793a6fc 100644 --- a/tests_py/benchmarks/test_write_manifest_machine_load.py +++ b/tests_py/benchmarks/test_write_manifest_machine_load.py @@ -1,30 +1,34 @@ -"""machine_load_snapshot in benchmarks.lib.write_manifest (2026-08-10 fix). - -Contract under test: every benchmark MANIFEST.json records the machine's -load state (load average, concurrent pytest processes, concurrent Docker -containers) alongside git_sha — not a separate/optional field — so a run -produced under contention can be identified after the fact instead of -requiring someone to have been watching `uptime` live. Prompted by an -incident where a 5-cell trust-factor sweep ran while three other agents' -full pytest suites were active; one cell crashed visibly, but a cell that -merely finished under the same contention would have looked like a clean -result with nothing in the artifact to say otherwise. - -Follow-up (same day): the snapshot is taken TWICE per cell — at cell start -(`write_start_snapshot`, written before `start_db`) and at cell end (inside -`build_manifest`) — because a single end-of-run reading cannot tell "ran -under load the whole time" apart from "load spiked right at the end". +"""write_manifest.py's start/end machine-load + disk-space wiring +(2026-08-10 fix). + +Contract under test: every benchmark MANIFEST.json records BOTH the +machine's load state and its free disk space, at cell start AND cell end, +alongside git_sha — not a separate/optional field — so a run produced +under CPU contention or disk exhaustion can be identified after the fact +instead of requiring someone to have been watching `uptime`/`df` live. + +Two incidents motivate this (full narrative in +benchmarks/lib/machine_load_snapshot.py and +benchmarks/lib/disk_space_snapshot.py's module docstrings): a 5-cell +trust-factor sweep ran while three other agents' full pytest suites were +active (one cell crashed visibly, but a cell that merely finished under +the same contention would have looked clean); and leaked throwaway test +databases filled the host disk to 100% on a neighboring session sharing +the same machine, taking PostgreSQL down mid-measurement. + +The probe functions themselves (`machine_load_snapshot`, +`disk_space_snapshot`) are tested in their own modules' +tests_py/benchmarks/test_machine_load_snapshot.py and +tests_py/benchmarks/test_disk_space_snapshot.py — this file only tests +`write_manifest.py`'s wiring of the two snapshot points into MANIFEST.json. """ from __future__ import annotations import json -from benchmarks.lib.write_manifest import ( - build_manifest, - machine_load_snapshot, - write_start_snapshot, -) +from benchmarks.lib.machine_load_snapshot import machine_load_snapshot +from benchmarks.lib.write_manifest import build_manifest, write_start_snapshot _LOAD_KEYS = { "load_average_1m", @@ -36,99 +40,78 @@ } -class TestMachineLoadSnapshot: - def test_returns_all_expected_keys(self): - snap = machine_load_snapshot() - assert set(snap) == _LOAD_KEYS +def _build(tmp_path): + return build_manifest( + str(tmp_path), + "deadbeef", + "dataset-sha", + "pgvector/pgvector:pg16", + "test-container", + "5432", + "1", + ) - def test_load_averages_are_nonnegative_floats_on_a_posix_machine(self): - snap = machine_load_snapshot() - for key in ("load_average_1m", "load_average_5m", "load_average_15m"): - assert isinstance(snap[key], float) - assert snap[key] >= 0.0 - def test_cpu_count_is_a_positive_int(self): - snap = machine_load_snapshot() - assert isinstance(snap["cpu_count"], int) - assert snap["cpu_count"] > 0 - - def test_pytest_process_count_sees_the_process_running_this_test(self): - """The suite executing this assertion IS a pytest process, so the - count must be >= 1 — a fixed proof the probe is not silently - returning zero/None while pytest is demonstrably running.""" - snap = machine_load_snapshot() - assert snap["concurrent_pytest_processes"] is not None - assert snap["concurrent_pytest_processes"] >= 1 - - def test_docker_container_count_is_an_int_or_none(self): - """None only when the docker CLI itself is unavailable/unreachable — - never raises, matching every other probe in this module.""" - snap = machine_load_snapshot() - count = snap["concurrent_docker_containers"] - assert count is None or (isinstance(count, int) and count >= 0) - - -class TestBuildManifestIncludesMachineLoad: +class TestBuildManifestAtEndFields: def test_machine_load_at_end_sits_alongside_git_sha(self, tmp_path): - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) + manifest = _build(tmp_path) assert "git_sha" in manifest assert "machine_load_at_end" in manifest assert set(manifest["machine_load_at_end"]) == _LOAD_KEYS - def test_machine_load_at_start_is_none_when_no_snapshot_was_taken(self, tmp_path): - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) + def test_disk_space_at_end_sits_alongside_git_sha(self, tmp_path): + manifest = _build(tmp_path) + assert "disk_space_at_end" in manifest + assert set(manifest["disk_space_at_end"]) == {"repo_root", "docker_root"} + + +class TestBuildManifestAtStartFields: + def test_both_at_start_fields_are_none_when_no_snapshot_was_taken(self, tmp_path): + manifest = _build(tmp_path) assert manifest["machine_load_at_start"] is None + assert manifest["disk_space_at_start"] is None - def test_machine_load_at_start_is_populated_when_a_snapshot_was_taken( + def test_both_at_start_fields_are_populated_when_a_snapshot_was_taken( self, tmp_path ): write_start_snapshot(str(tmp_path)) - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) - assert manifest["machine_load_at_start"] is not None + manifest = _build(tmp_path) assert set(manifest["machine_load_at_start"]) == _LOAD_KEYS + assert set(manifest["disk_space_at_start"]) == {"repo_root", "docker_root"} + + def test_at_start_fields_are_the_snapshot_taken_at_start_not_a_fresh_read( + self, tmp_path + ): + """The manifest must read back the PERSISTED start snapshot, not + silently re-sample -- otherwise `_at_start` would always equal + `_at_end` and the two-point design would measure nothing.""" + write_start_snapshot(str(tmp_path)) + persisted = json.loads((tmp_path / "START_SNAPSHOT.json").read_text()) + manifest = _build(tmp_path) + assert manifest["machine_load_at_start"] == persisted["machine_load"] + assert manifest["disk_space_at_start"] == persisted["disk_space"] + +class TestResultsFilesExclusion: def test_start_snapshot_file_is_excluded_from_results_files(self, tmp_path): write_start_snapshot(str(tmp_path)) - manifest = build_manifest( - str(tmp_path), - "deadbeef", - "dataset-sha", - "pgvector/pgvector:pg16", - "test-container", - "5432", - "1", - ) + manifest = _build(tmp_path) assert "START_SNAPSHOT.json" not in manifest["results_files"] class TestWriteStartSnapshot: - def test_writes_a_readable_json_file_with_a_timestamp_and_load(self, tmp_path): + def test_writes_a_readable_json_file_with_a_timestamp_load_and_disk(self, tmp_path): out = write_start_snapshot(str(tmp_path)) assert out.exists() payload = json.loads(out.read_text()) assert "captured_at_utc" in payload assert set(payload["machine_load"]) == _LOAD_KEYS + assert set(payload["disk_space"]) == {"repo_root", "docker_root"} + + def test_matches_a_direct_probe_call_in_shape(self, tmp_path): + """Not a value-equality check (load/disk drift between calls) -- + confirms write_start_snapshot persists the SAME shape + machine_load_snapshot() itself returns, not a re-derived one.""" + out = write_start_snapshot(str(tmp_path)) + payload = json.loads(out.read_text()) + assert set(payload["machine_load"]) == set(machine_load_snapshot()) From d280a50f68c7fac304544add308e138e2844cd73 Mon Sep 17 00:00:00 2001 From: cdeust Date: Mon, 10 Aug 2026 17:41:15 +0200 Subject: [PATCH 2/3] docs(trust-factor): publish honest re-measurement after capture_origin fix (#368, #410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original gated arm could not discriminate W: every LME/LoCoMo/BEAM memory landed on capture_origin='unknown' (memory_ingest.py dropped the field before PR #410), so the uniform demotion left WRRF order invariant. After #410 wires a realistic capture_origin mix into the benchmark harness, re-run the same 5-cell grid (1.0/0.8/0.7/0.6/0.5) in --quick mode and publish what it shows: a real, monotonic LoCoMo relevance cost at the shipped W=0.7 (-0.0165 MRR, -2.54pp R@10 vs the W=1.0 control), with two explicit reserves — quick mode is not floor-comparable (reproduce.sh:94-95), and no measurement links the adversarial-arm defense to a real-benchmark benefit. Owner decision: W stays at 0.7, cost published as measured. Commits the artifacts (PROGRESS.json, per-cell repro_dir + MANIFEST + benchmark JSON) and the resumable per-cell sweep architecture (trust_factor_sweep.sh rewrite + benchmarks/lib/sweep_progress.py) that produced them, so the branch reproduces without depending on the worktree they were run in. Co-Authored-By: Claude --- benchmarks/lib/sweep_progress.py | 106 ++++++++++ .../repro/20260810T140209Z/MANIFEST.json | 57 +++++ .../20260810T140209Z/START_SNAPSHOT.json | 19 ++ .../repro/20260810T140209Z/beam-100K.json | 69 +++++++ .../repro/20260810T140209Z/locomo.json | 52 +++++ .../repro/20260810T140209Z/longmemeval-s.json | 44 ++++ .../repro/20260810T141001Z/MANIFEST.json | 57 +++++ .../20260810T141001Z/START_SNAPSHOT.json | 19 ++ .../repro/20260810T141001Z/beam-100K.json | 69 +++++++ .../repro/20260810T141001Z/locomo.json | 52 +++++ .../repro/20260810T141001Z/longmemeval-s.json | 44 ++++ .../repro/20260810T141800Z/MANIFEST.json | 57 +++++ .../20260810T141800Z/START_SNAPSHOT.json | 19 ++ .../repro/20260810T141800Z/beam-100K.json | 69 +++++++ .../repro/20260810T141800Z/locomo.json | 52 +++++ .../repro/20260810T141800Z/longmemeval-s.json | 44 ++++ .../repro/20260810T142503Z/MANIFEST.json | 57 +++++ .../20260810T142503Z/START_SNAPSHOT.json | 19 ++ .../repro/20260810T142503Z/beam-100K.json | 69 +++++++ .../repro/20260810T142503Z/locomo.json | 52 +++++ .../repro/20260810T142503Z/longmemeval-s.json | 44 ++++ .../repro/20260810T144349Z/MANIFEST.json | 57 +++++ .../20260810T144349Z/START_SNAPSHOT.json | 19 ++ .../repro/20260810T144349Z/beam-100K.json | 69 +++++++ .../repro/20260810T144349Z/locomo.json | 52 +++++ .../repro/20260810T144349Z/longmemeval-s.json | 44 ++++ .../trust-factor-sweep/active/PROGRESS.json | 194 ++++++++++++++++++ .../active/cell_W0.5/repro_dir.txt | 1 + .../active/cell_W0.6/repro_dir.txt | 1 + .../active/cell_W0.7/repro_dir.txt | 1 + .../active/cell_W0.8/repro_dir.txt | 1 + .../active/cell_W1.0/repro_dir.txt | 1 + benchmarks/trust_factor_sweep.sh | 159 +++++++++----- docs/provenance/trust-factor-calibration.md | 97 +++++++++ 34 files changed, 1719 insertions(+), 47 deletions(-) create mode 100644 benchmarks/lib/sweep_progress.py create mode 100644 benchmarks/results/repro/20260810T140209Z/MANIFEST.json create mode 100644 benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json create mode 100644 benchmarks/results/repro/20260810T140209Z/beam-100K.json create mode 100644 benchmarks/results/repro/20260810T140209Z/locomo.json create mode 100644 benchmarks/results/repro/20260810T140209Z/longmemeval-s.json create mode 100644 benchmarks/results/repro/20260810T141001Z/MANIFEST.json create mode 100644 benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json create mode 100644 benchmarks/results/repro/20260810T141001Z/beam-100K.json create mode 100644 benchmarks/results/repro/20260810T141001Z/locomo.json create mode 100644 benchmarks/results/repro/20260810T141001Z/longmemeval-s.json create mode 100644 benchmarks/results/repro/20260810T141800Z/MANIFEST.json create mode 100644 benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json create mode 100644 benchmarks/results/repro/20260810T141800Z/beam-100K.json create mode 100644 benchmarks/results/repro/20260810T141800Z/locomo.json create mode 100644 benchmarks/results/repro/20260810T141800Z/longmemeval-s.json create mode 100644 benchmarks/results/repro/20260810T142503Z/MANIFEST.json create mode 100644 benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json create mode 100644 benchmarks/results/repro/20260810T142503Z/beam-100K.json create mode 100644 benchmarks/results/repro/20260810T142503Z/locomo.json create mode 100644 benchmarks/results/repro/20260810T142503Z/longmemeval-s.json create mode 100644 benchmarks/results/repro/20260810T144349Z/MANIFEST.json create mode 100644 benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json create mode 100644 benchmarks/results/repro/20260810T144349Z/beam-100K.json create mode 100644 benchmarks/results/repro/20260810T144349Z/locomo.json create mode 100644 benchmarks/results/repro/20260810T144349Z/longmemeval-s.json create mode 100644 benchmarks/results/trust-factor-sweep/active/PROGRESS.json create mode 100644 benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt create mode 100644 benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt create mode 100644 benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt create mode 100644 benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt create mode 100644 benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt diff --git a/benchmarks/lib/sweep_progress.py b/benchmarks/lib/sweep_progress.py new file mode 100644 index 00000000..fede62d9 --- /dev/null +++ b/benchmarks/lib/sweep_progress.py @@ -0,0 +1,106 @@ +"""Per-cell checkpoint/resume state for the trust-factor sweep (issue #368). + +2026-08-10 incident: five sweep campaigns died over one session, each with a +different attributed cause (CPU contention, a native crash, a missing +dataset file, a session gap, a noisy neighbor). Five explanations for five +occurrences of the same symptom was itself the signal missed at the time: +those four "causes" described the state found on wake-up, not the actual +failure mechanism. The mechanism is that a long-running job driven from a +sub-agent's own foreground loop does not survive whatever ends that +sub-agent's turn -- confirmed by a neighboring session that measured the +identical failure shape on five unrelated tasks, and that succeeded only +once it stopped depending on that survival. + +The fix is not to prevent the death (it cannot be prevented from here) but +to make it cost one cell, not the whole grid: `trust_factor_sweep.sh` now +runs exactly ONE pending cell per invocation and returns control immediately +after, recording that cell's completion (grid point, result location, code +sha, and machine-load/disk-space snapshots at both cell start and cell end) +to a fixed-location PROGRESS.json. A killed or crashed run leaves no entry +for the cell it was running, so the next invocation finds it still pending +and retries exactly that one -- never the cells already recorded. + +The load/disk snapshots are NOT a defence against this failure mode (a kill +happens regardless of what they read) -- they remain what they were built +for: metadata that lets a later reader requalify a cell's number without +trusting the runner's word for it. Load and disk are deliberately two +separate signals, not redundant: load average integrates I/O wait as well +as CPU, so it rises under a saturated disk with no CPU contention at all. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +PROGRESS_FILENAME = "PROGRESS.json" + + +def _git_sha() -> str: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10 + ) + except (OSError, subprocess.SubprocessError): + return "unknown" + return out.stdout.strip() if out.returncode == 0 else "unknown" + + +def read_progress(sweep_dir: str) -> dict: + """Read PROGRESS.json, or an empty structure if absent/unreadable -- + never raises, matching this project's other best-effort snapshot code.""" + path = Path(sweep_dir) / PROGRESS_FILENAME + try: + return json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return {"cells": []} + + +def completed_w_values(sweep_dir: str) -> set[float]: + progress = read_progress(sweep_dir) + return { + c["w"] for c in progress.get("cells", []) if c.get("status") == "complete" + } + + +def next_pending_w(grid: list[float], sweep_dir: str) -> float | None: + """First grid value with no `status: complete` entry, in grid order, or + None once every cell has one -- the resume point after a kill/crash.""" + done = completed_w_values(sweep_dir) + for w in grid: + if w not in done: + return w + return None + + +def record_cell_result( + sweep_dir: str, + w: float, + *, + status: str, + repro_dir: str | None, + snapshots: dict, +) -> None: + """Append (or replace) one cell's completion record in PROGRESS.json. + + `snapshots` carries the four points this incident asks for: + machine_load_at_start/_at_end, disk_space_at_start/_at_end -- passed as + one dict rather than four parameters to stay under this project's + 4-argument-max convention (coding-standards.md §3.2) without losing any + of the four points to a size-cap-driven abbreviation. + """ + progress = read_progress(sweep_dir) + progress.setdefault("cells", []) + progress["cells"] = [c for c in progress["cells"] if c.get("w") != w] + progress["cells"].append( + { + "w": w, + "status": status, + "git_sha": _git_sha(), + "repro_dir": repro_dir, + **snapshots, + } + ) + path = Path(sweep_dir) / PROGRESS_FILENAME + path.write_text(json.dumps(progress, indent=2)) diff --git a/benchmarks/results/repro/20260810T140209Z/MANIFEST.json b/benchmarks/results/repro/20260810T140209Z/MANIFEST.json new file mode 100644 index 00000000..351d071f --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 6.4130859375, + "load_average_5m": 8.67041015625, + "load_average_15m": 8.5556640625, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21348298752, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.98193359375, + "load_average_5m": 8.95947265625, + "load_average_15m": 8.74658203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20997742592, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-31717-e2c30003", + "bench_container_port": 32793, + "bench_runner_pid": 31717, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json new file mode 100644 index 00000000..9386df7b --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:02:10.355451+00:00", + "machine_load": { + "load_average_1m": 6.4130859375, + "load_average_5m": 8.67041015625, + "load_average_15m": 8.5556640625, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21348298752, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/beam-100K.json b/benchmarks/results/repro/20260810T140209Z/beam-100K.json new file mode 100644 index 00000000..664676e0 --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6579166666666667, + "overall_r10": 0.85, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.625, + "information_extraction": 0.8, + "instruction_following": 0.625, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 120.98164892196655, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:07:16.420185+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/locomo.json b/benchmarks/results/repro/20260810T140209Z/locomo.json new file mode 100644 index 00000000..5d826f0f --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8208665699782451, + "overall_recall10": 0.9593908629441624, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6431818181818182, + "single_hop": 0.6453869047619047, + "open_domain": 0.8914285714285715, + "adversarial": 0.9645390070921986 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9857142857142858, + "adversarial": 1.0 + }, + "elapsed_s": 243.82014799118042, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:03:05.989631+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T140209Z/longmemeval-s.json b/benchmarks/results/repro/20260810T140209Z/longmemeval-s.json new file mode 100644 index 00000000..161f9154 --- /dev/null +++ b/benchmarks/results/repro/20260810T140209Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 50.508332624973264, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:02:13.736491+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/MANIFEST.json b/benchmarks/results/repro/20260810T141001Z/MANIFEST.json new file mode 100644 index 00000000..80df4c34 --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 8.70263671875, + "load_average_5m": 8.79833984375, + "load_average_15m": 8.69189453125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 26129395712, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 11.3310546875, + "load_average_5m": 12.310546875, + "load_average_15m": 10.646484375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25266130944, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-80621-a4946cc5", + "bench_container_port": 32794, + "bench_runner_pid": 80621, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json new file mode 100644 index 00000000..cda24f75 --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:10:02.254281+00:00", + "machine_load": { + "load_average_1m": 8.70263671875, + "load_average_5m": 8.79833984375, + "load_average_15m": 8.69189453125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 26129395712, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/beam-100K.json b/benchmarks/results/repro/20260810T141001Z/beam-100K.json new file mode 100644 index 00000000..c380b1db --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 120.49417090415955, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:15:16.117902+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/locomo.json b/benchmarks/results/repro/20260810T141001Z/locomo.json new file mode 100644 index 00000000..591b52c6 --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8042845056804449, + "overall_recall10": 0.9390862944162437, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6610119047619047, + "open_domain": 0.8511904761904763, + "adversarial": 0.9432624113475178 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9428571428571428, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 252.10806703567505, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:10:58.034832+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141001Z/longmemeval-s.json b/benchmarks/results/repro/20260810T141001Z/longmemeval-s.json new file mode 100644 index 00000000..d148a38c --- /dev/null +++ b/benchmarks/results/repro/20260810T141001Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 49.05208487500204, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:10:07.038557+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/MANIFEST.json b/benchmarks/results/repro/20260810T141800Z/MANIFEST.json new file mode 100644 index 00000000..7a10ffd9 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 9.16650390625, + "load_average_5m": 11.61572265625, + "load_average_15m": 10.46630859375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25579466752, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.84033203125, + "load_average_5m": 10.83056640625, + "load_average_15m": 10.60888671875, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 24998215680, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-27502-f5f2bb69", + "bench_container_port": 32795, + "bench_runner_pid": 27502, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json new file mode 100644 index 00000000..788fd5c0 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:18:01.140247+00:00", + "machine_load": { + "load_average_1m": 9.16650390625, + "load_average_5m": 11.61572265625, + "load_average_15m": 10.46630859375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25579466752, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/beam-100K.json b/benchmarks/results/repro/20260810T141800Z/beam-100K.json new file mode 100644 index 00000000..9a473edb --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 110.44140601158142, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:22:55.411459+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/locomo.json b/benchmarks/results/repro/20260810T141800Z/locomo.json new file mode 100644 index 00000000..cd26f832 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8044053662073968, + "overall_recall10": 0.934010152284264, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6617559523809524, + "open_domain": 0.8488095238095238, + "adversarial": 0.9468085106382979 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9285714285714286, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 235.71782112121582, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:18:54.244987+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T141800Z/longmemeval-s.json b/benchmarks/results/repro/20260810T141800Z/longmemeval-s.json new file mode 100644 index 00000000..ffa84d57 --- /dev/null +++ b/benchmarks/results/repro/20260810T141800Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 47.76949145901017, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:18:04.366929+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/MANIFEST.json b/benchmarks/results/repro/20260810T142503Z/MANIFEST.json new file mode 100644 index 00000000..60162f07 --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 8.240234375, + "load_average_5m": 10.55224609375, + "load_average_15m": 10.5126953125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25810296832, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 13.11865234375, + "load_average_5m": 11.8583984375, + "load_average_15m": 11.14990234375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20115681280, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-68215-d1fe77db", + "bench_container_port": 32796, + "bench_runner_pid": 68215, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json new file mode 100644 index 00000000..0a7d73aa --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:25:03.837503+00:00", + "machine_load": { + "load_average_1m": 8.240234375, + "load_average_5m": 10.55224609375, + "load_average_15m": 10.5126953125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25810296832, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/beam-100K.json b/benchmarks/results/repro/20260810T142503Z/beam-100K.json new file mode 100644 index 00000000..0a5203ff --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 122.94071817398071, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:29:46.230185+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/locomo.json b/benchmarks/results/repro/20260810T142503Z/locomo.json new file mode 100644 index 00000000..3110d3cc --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.8044053662073968, + "overall_recall10": 0.934010152284264, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6617559523809524, + "open_domain": 0.8488095238095238, + "adversarial": 0.9468085106382979 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.9285714285714286, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 226.98380208015442, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:25:53.371692+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T142503Z/longmemeval-s.json b/benchmarks/results/repro/20260810T142503Z/longmemeval-s.json new file mode 100644 index 00000000..da1bd755 --- /dev/null +++ b/benchmarks/results/repro/20260810T142503Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 44.64210533298319, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:25:06.914447+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/MANIFEST.json b/benchmarks/results/repro/20260810T144349Z/MANIFEST.json new file mode 100644 index 00000000..b168ed80 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/MANIFEST.json @@ -0,0 +1,57 @@ +{ + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "machine_load_at_start": { + "load_average_1m": 6.16064453125, + "load_average_5m": 6.9130859375, + "load_average_15m": 8.44287109375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 22296911872, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 10.4541015625, + "load_average_5m": 9.6796875, + "load_average_15m": 9.3203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 1 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21858193408, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "longmemeval_dataset_sha256": "08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894", + "pg_image": "pgvector/pgvector:pg16", + "bench_container_name": "cortex-bench-pg-84394-9cd44149", + "bench_container_port": 32797, + "bench_runner_pid": 84394, + "python": "3.12.11", + "packages": { + "datasets": "5.0.1", + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "psycopg": "3.3.4", + "psycopg-pool": "3.3.1" + }, + "embedding_model_revision": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1", + "results_files": [ + "beam-100K.json", + "locomo.json", + "longmemeval-s.json" + ] +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json b/benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json new file mode 100644 index 00000000..234a7b92 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/START_SNAPSHOT.json @@ -0,0 +1,19 @@ +{ + "captured_at_utc": "2026-08-10T14:43:49.599837+00:00", + "machine_load": { + "load_average_1m": 6.16064453125, + "load_average_5m": 6.9130859375, + "load_average_15m": 8.44287109375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 22296911872, + "total_bytes": 494384795648 + }, + "docker_root": null + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/beam-100K.json b/benchmarks/results/repro/20260810T144349Z/beam-100K.json new file mode 100644 index 00000000..de3caa45 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/beam-100K.json @@ -0,0 +1,69 @@ +{ + "overall_mrr": 0.6575, + "overall_r10": 0.875, + "ability_mrr": { + "abstention": 0.5, + "contradiction_resolution": 0.8333333333333333, + "event_ordering": 0.5833333333333333, + "information_extraction": 0.8125, + "instruction_following": 0.65, + "knowledge_update": 0.875, + "multi_session_reasoning": 0.625, + "preference_following": 0.8125, + "summarization": 0.2583333333333333, + "temporal_reasoning": 0.625 + }, + "ability_r5": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 0.75, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "ability_r10": { + "abstention": 0.5, + "contradiction_resolution": 1.0, + "event_ordering": 0.75, + "information_extraction": 1.0, + "instruction_following": 1.0, + "knowledge_update": 1.0, + "multi_session_reasoning": 1.0, + "preference_following": 1.0, + "summarization": 0.75, + "temporal_reasoning": 0.75 + }, + "total_questions": 40, + "elapsed_s": 114.20000696182251, + "manifest": { + "split": "100K", + "n_conversations": 2, + "n_questions": 40, + "n_runs": 1, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:48:29.221156+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/locomo.json b/benchmarks/results/repro/20260810T144349Z/locomo.json new file mode 100644 index 00000000..f5ad2fc4 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/locomo.json @@ -0,0 +1,52 @@ +{ + "overall_mrr": 0.7891769398114576, + "overall_recall10": 0.9187817258883249, + "category_mrr": { + "multi_hop": 0.7094594594594594, + "temporal": 0.6477272727272727, + "single_hop": 0.6617559523809524, + "open_domain": 0.8059523809523809, + "adversarial": 0.9468085106382979 + }, + "category_recall10": { + "multi_hop": 0.8378378378378378, + "temporal": 1.0, + "single_hop": 0.96875, + "open_domain": 0.8857142857142857, + "adversarial": 0.9787234042553191 + }, + "elapsed_s": 225.38163590431213, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "ablate_mechanism": null, + "ablate_env_var": null, + "n_conversations": 1, + "n_questions": 197, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:44:38.580923+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/repro/20260810T144349Z/longmemeval-s.json b/benchmarks/results/repro/20260810T144349Z/longmemeval-s.json new file mode 100644 index 00000000..d1a36586 --- /dev/null +++ b/benchmarks/results/repro/20260810T144349Z/longmemeval-s.json @@ -0,0 +1,44 @@ +{ + "overall_mrr": 0.85, + "overall_recall10": 1.0, + "category_mrr": { + "Single-session (user)": 0.85 + }, + "category_recall10": { + "Single-session (user)": 1.0 + }, + "elapsed_s": 43.89209308300633, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 10, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "git_dirty": true, + "python_version": "3.12.11 (main, Sep 18 2025, 19:41:45) [Clang 20.1.4 ]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "mac-1.home", + "timestamp_utc": "2026-08-10T14:43:53.015314+00:00", + "lib_versions": { + "sentence-transformers": "5.6.1", + "torch": "2.13.0", + "numpy": "2.5.1", + "psycopg": "3.3.4", + "pgvector": "0.5.0", + "flashrank": "0.2.10" + }, + "reranker_active": true, + "reranker_state": "loaded", + "reranker_model_path": "/Users/cdeust/.cache/flashrank/ms-marco-MiniLM-L-12-v2/flashrank-MiniLM-L-12-v2_Q.onnx", + "reranker_model_sha256": "d3dd7b09fcf06b0c070081d6819b5effbb40b667bcad78b2d7543400271141d1" + } + } +} \ No newline at end of file diff --git a/benchmarks/results/trust-factor-sweep/active/PROGRESS.json b/benchmarks/results/trust-factor-sweep/active/PROGRESS.json new file mode 100644 index 00000000..3f095a75 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/PROGRESS.json @@ -0,0 +1,194 @@ +{ + "cells": [ + { + "w": 1.0, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T140209Z/", + "machine_load_at_start": { + "load_average_1m": 6.4130859375, + "load_average_5m": 8.67041015625, + "load_average_15m": 8.5556640625, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21274468352, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.98193359375, + "load_average_5m": 8.95947265625, + "load_average_15m": 8.74658203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20997554176, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.8, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T141001Z/", + "machine_load_at_start": { + "load_average_1m": 8.24169921875, + "load_average_5m": 8.7109375, + "load_average_15m": 8.66064453125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 26131615744, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 11.3310546875, + "load_average_5m": 12.310546875, + "load_average_15m": 10.646484375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25266053120, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.7, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T141800Z/", + "machine_load_at_start": { + "load_average_1m": 9.16650390625, + "load_average_5m": 11.61572265625, + "load_average_15m": 10.46630859375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25579483136, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 8.37255859375, + "load_average_5m": 10.7001953125, + "load_average_15m": 10.56396484375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 24989110272, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.6, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T142503Z/", + "machine_load_at_start": { + "load_average_1m": 8.240234375, + "load_average_5m": 10.55224609375, + "load_average_15m": 10.5126953125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 25810317312, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 13.11865234375, + "load_average_5m": 11.8583984375, + "load_average_15m": 11.14990234375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 20088893440, + "total_bytes": 494384795648 + }, + "docker_root": null + } + }, + { + "w": 0.5, + "status": "complete", + "git_sha": "f87bf6e3fb594e3cc8cc653f0ae2c8511af28d74", + "repro_dir": "benchmarks/results/repro/20260810T144349Z/", + "machine_load_at_start": { + "load_average_1m": 6.16064453125, + "load_average_5m": 6.9130859375, + "load_average_15m": 8.44287109375, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_start": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 22296936448, + "total_bytes": 494384795648 + }, + "docker_root": null + }, + "machine_load_at_end": { + "load_average_1m": 10.4541015625, + "load_average_5m": 9.6796875, + "load_average_15m": 9.3203125, + "cpu_count": 10, + "concurrent_pytest_processes": 0, + "concurrent_docker_containers": 0 + }, + "disk_space_at_end": { + "repo_root": { + "path": "/Users/cdeust/Developments/anthropic-partnership/Cortex/.claude/worktrees/agent-a7efdb78f6e038811", + "free_bytes": 21858013184, + "total_bytes": 494384795648 + }, + "docker_root": null + } + } + ] +} \ No newline at end of file diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt new file mode 100644 index 00000000..b0100454 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.5/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T144349Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt new file mode 100644 index 00000000..67a693c5 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.6/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T142503Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt new file mode 100644 index 00000000..e4c7d723 --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.7/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T141800Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt new file mode 100644 index 00000000..6dfb51dc --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W0.8/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T141001Z/ diff --git a/benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt b/benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt new file mode 100644 index 00000000..a4ee6f2e --- /dev/null +++ b/benchmarks/results/trust-factor-sweep/active/cell_W1.0/repro_dir.txt @@ -0,0 +1 @@ +benchmarks/results/repro/20260810T140209Z/ diff --git a/benchmarks/trust_factor_sweep.sh b/benchmarks/trust_factor_sweep.sh index 829e0d45..42049a50 100755 --- a/benchmarks/trust_factor_sweep.sh +++ b/benchmarks/trust_factor_sweep.sh @@ -5,15 +5,29 @@ # Read it before changing anything here — the grid is derived from a cheap # adversarial sweep, and the decision rule is fixed in advance on purpose. # -# benchmarks/trust_factor_sweep.sh # full grid, all three suites +# benchmarks/trust_factor_sweep.sh # run ONE pending cell, then exit # benchmarks/trust_factor_sweep.sh --quick # smoke the plumbing (NOT gated) # -# One reproduce.sh invocation per cell, strictly SEQUENTIAL, each against its +# 2026-08-10 architecture change: this script used to loop over the whole +# grid in one process. Five campaigns died mid-grid over one session, each +# attributed to a different cause after the fact (contention, a native +# crash, a missing dataset, a session gap, a noisy neighbor) — five +# explanations for one symptom, which was itself the signal: a long job +# driven from a sub-agent's own foreground loop does not survive whatever +# ends that sub-agent's turn, regardless of which resource happened to be +# short at the time. The fix is not preventing the death (it cannot be, from +# here) but making it cost one cell instead of the whole grid: this script +# now resumes from `benchmarks/lib/sweep_progress.py`'s PROGRESS.json (which +# W values already completed), runs exactly the next PENDING cell, records +# its result — including machine-load/disk-space snapshots at cell start +# AND end (`benchmarks/lib/machine_load_snapshot.py`, +# `benchmarks/lib/disk_space_snapshot.py`) — and returns control. A kill or +# crash mid-cell leaves that cell's PROGRESS.json entry absent, so the next +# invocation retries exactly that cell, never the ones already recorded. +# +# Still true, unchanged: one reproduce.sh invocation per cell, against its # own ephemeral container. No parallelism: a fan-out on this machine on -# 2026-08-08 drove load to 37 and swapped 11.9 GB. Cells are independent, so a -# failed cell does not invalidate the others — it is reported and the sweep -# continues, because a partial grid with an honest gap beats a grid that -# silently stopped early. +# 2026-08-08 drove load to 37 and swapped 11.9 GB. set -uo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -29,48 +43,99 @@ GRID=(1.0 0.8 0.7 0.6 0.5) QUICK_FLAG="" [[ "${1:-}" == "--quick" ]] && QUICK_FLAG="--quick" -STAMP="$(date -u +%Y%m%dT%H%M%SZ)" -OUT_ROOT="benchmarks/results/trust-factor-sweep/${STAMP}" +# Fixed location, not a fresh timestamp per invocation: resume needs the +# NEXT call to find the SAME PROGRESS.json the previous one wrote. +OUT_ROOT="benchmarks/results/trust-factor-sweep/active" mkdir -p "$OUT_ROOT" - LOG="${OUT_ROOT}/sweep.log" -echo "trust-factor sweep ${STAMP}" | tee "$LOG" -echo "grid: ${GRID[*]}" | tee -a "$LOG" -echo "pre-registration: docs/provenance/trust-factor-calibration.md" | tee -a "$LOG" - -for W in "${GRID[@]}"; do - CELL_DIR="${OUT_ROOT}/cell_W${W}" - mkdir -p "$CELL_DIR" - echo "" | tee -a "$LOG" - echo "=== cell W=${W} — started $(date -u +%H:%M:%SZ) ===" | tee -a "$LOG" - - # Exported, not inlined: reproduce.sh spawns the benchmark processes, and - # retrieval_dispatch.py reads the value at import in each of them. - CORTEX_UNTRUSTED_ORIGIN_FACTOR="$W" \ - benchmarks/reproduce.sh \ - --only longmemeval,locomo,beam \ - --no-ablation \ - $QUICK_FLAG \ - >"${CELL_DIR}/reproduce.log" 2>&1 - rc=$? - - if [[ $rc -eq 0 ]]; then - echo "cell W=${W}: OK" | tee -a "$LOG" - else - # Not fatal to the sweep: an unusable cell is a gap in the grid, and - # the decision rule can still be applied to the cells that reported. - echo "cell W=${W}: FAILED rc=${rc} (see ${CELL_DIR}/reproduce.log)" | tee -a "$LOG" - fi - - # reproduce.sh writes into benchmarks/results/repro//; - # record which one belongs to this cell so the summary can be rebuilt - # without guessing from timestamps. - latest_repro="$(ls -td benchmarks/results/repro/*/ 2>/dev/null | head -1)" - echo "${latest_repro}" > "${CELL_DIR}/repro_dir.txt" - echo "cell W=${W} results: ${latest_repro}" | tee -a "$LOG" -done + +snapshot_json() { + uv run python -c " +import json, sys +sys.path.insert(0, 'benchmarks/lib') +from machine_load_snapshot import machine_load_snapshot +from disk_space_snapshot import disk_space_snapshot +print(json.dumps({'machine_load': machine_load_snapshot(), 'disk_space': disk_space_snapshot()})) +" +} + +next_pending_w() { + uv run python -c " +import sys +sys.path.insert(0, 'benchmarks/lib') +from sweep_progress import next_pending_w +w = next_pending_w([1.0, 0.8, 0.7, 0.6, 0.5], '${OUT_ROOT}') +print('' if w is None else w) +" +} + +W="$(next_pending_w)" +if [[ -z "$W" ]]; then + echo "GRID COMPLETE — every cell in ${GRID[*]} has a status:complete entry in ${OUT_ROOT}/PROGRESS.json." | tee -a "$LOG" + echo "NEXT: apply the decision rule and record the chosen W in docs/provenance/trust-factor-calibration.md." | tee -a "$LOG" + exit 0 +fi + +CELL_DIR="${OUT_ROOT}/cell_W${W}" +mkdir -p "$CELL_DIR" +echo "" | tee -a "$LOG" +echo "=== cell W=${W} — started $(date -u +%H:%M:%SZ) ===" | tee -a "$LOG" + +START_SNAPSHOT="$(snapshot_json)" + +# Exported, not inlined: reproduce.sh spawns the benchmark processes, and +# retrieval_dispatch.py reads the value at import in each of them. +CORTEX_UNTRUSTED_ORIGIN_FACTOR="$W" \ + benchmarks/reproduce.sh \ + --only longmemeval,locomo,beam \ + --no-ablation \ + $QUICK_FLAG \ + >"${CELL_DIR}/reproduce.log" 2>&1 +rc=$? + +END_SNAPSHOT="$(snapshot_json)" + +if [[ $rc -eq 0 ]]; then + echo "cell W=${W}: OK" | tee -a "$LOG" + STATUS="complete" +else + echo "cell W=${W}: FAILED rc=${rc} (see ${CELL_DIR}/reproduce.log)" | tee -a "$LOG" + STATUS="failed" +fi + +# reproduce.sh writes into benchmarks/results/repro//; +# record which one belongs to this cell so the summary can be rebuilt +# without guessing from timestamps. +latest_repro="$(ls -td benchmarks/results/repro/*/ 2>/dev/null | head -1)" +echo "${latest_repro}" > "${CELL_DIR}/repro_dir.txt" +echo "cell W=${W} results: ${latest_repro}" | tee -a "$LOG" + +uv run python -c " +import json, sys +sys.path.insert(0, 'benchmarks/lib') +from sweep_progress import record_cell_result + +start = json.loads('''${START_SNAPSHOT}''') +end = json.loads('''${END_SNAPSHOT}''') +record_cell_result( + '${OUT_ROOT}', + ${W}, + status='${STATUS}', + repro_dir='${latest_repro}' or None, + snapshots={ + 'machine_load_at_start': start['machine_load'], + 'disk_space_at_start': start['disk_space'], + 'machine_load_at_end': end['machine_load'], + 'disk_space_at_end': end['disk_space'], + }, +) +" echo "" | tee -a "$LOG" -echo "sweep finished $(date -u +%Y-%m-%dT%H:%M:%SZ) — results under ${OUT_ROOT}" | tee -a "$LOG" -echo "NEXT: apply the decision rule from the pre-registration and record the" | tee -a "$LOG" -echo "chosen W in docs/provenance/trust-factor-calibration.md §Results." | tee -a "$LOG" +echo "cell W=${W} recorded to ${OUT_ROOT}/PROGRESS.json — control returned." | tee -a "$LOG" +remaining="$(next_pending_w)" +if [[ -n "$remaining" ]]; then + echo "NEXT PENDING CELL: W=${remaining}. Re-run this script to continue." | tee -a "$LOG" +else + echo "GRID COMPLETE — every cell has reported." | tee -a "$LOG" +fi diff --git a/docs/provenance/trust-factor-calibration.md b/docs/provenance/trust-factor-calibration.md index 4cfff8bc..66bc163d 100644 --- a/docs/provenance/trust-factor-calibration.md +++ b/docs/provenance/trust-factor-calibration.md @@ -176,3 +176,100 @@ Two provenance notes, stated rather than smoothed over: correction. What the gate requires still holds: one shared condition across all five cells, and a control arm that reproduces the published numbers exactly. + +## Re-measurement after the capture_origin fix (PR #410, issue #368) + +The paragraph above diagnosed a defect, not a residual unknown: the gated arm +could not discriminate W because `core.memory_ingest.ingest_memory` accepted +a caller-supplied `capture_origin` but never forwarded it to +`store.insert_memory` — every LME/LoCoMo/BEAM memory landed on the column +default `'unknown'`, `core.capture_origin.trust_factor` demoted all of them +by the same uniform multiplier, and a uniform rescale cannot change WRRF +order. **PR #410** (`b280fd25`, merged to `main` 2026-08-10) fixed the +forwarding and added `benchmarks/lib/capture_origin_mix.py` — a deterministic +sampler that assigns benchmark memories a realistic `capture_origin` mixture +(`local_action` .966 / `network` .025 / `deliberate` .009, measured over this +machine's own 886 Claude Code session transcripts) instead of leaving every +memory at `'unknown'`. Wired into `benchmarks/lib/bench_db.py` via +`_apply_capture_origin_mix`, this is the first harness run in which W can +discriminate anything on these three benchmarks. + +### What was re-run + +Five cells, `W ∈ {1.0, 0.8, 0.7, 0.6, 0.5}` — the same grid as the original +sweep, on the branch that carries PR #410. Each cell is one +`benchmarks/trust_factor_sweep.sh` invocation in `--quick` mode (per-cell +`--limit`: 10 LongMemEval questions, 1 LoCoMo conversation, 2 BEAM windows — +`reproduce.sh:421`), sequential, own ephemeral pgvector container, at +`git_sha f87bf6e3`. Machine load and free disk space were snapshotted at the +start and end of every cell (`benchmarks/lib/machine_load_snapshot.py`, +`benchmarks/lib/disk_space_snapshot.py`) and are recorded alongside each +result — load ran 6.2–13.1 (1m avg) across the run, consistent with a shared, +busy machine rather than an isolated benchmark host; nothing in the deltas +below correlates with the load swing. + +| W | LME MRR | LME R@10 | LoCoMo MRR | LoCoMo R@10 | BEAM MRR | Artifacts | +|---|---|---|---|---|---|---| +| 1.0 (control) | 0.8500 | 1.0000 | 0.8209 | 0.9594 | 0.6579 | `benchmarks/results/repro/20260810T140209Z/` | +| 0.8 | 0.8500 | 1.0000 | 0.8043 | 0.9391 | 0.6575 | `benchmarks/results/repro/20260810T141001Z/` | +| **0.7** | **0.8500** | **1.0000** | **0.8044** | **0.9340** | **0.6575** | `benchmarks/results/repro/20260810T141800Z/` | +| 0.6 | 0.8500 | 1.0000 | 0.8044 | 0.9340 | 0.6575 | `benchmarks/results/repro/20260810T142503Z/` | +| 0.5 | 0.8500 | 1.0000 | 0.7892 | 0.9188 | 0.6575 | `benchmarks/results/repro/20260810T144349Z/` | + +Per-cell load/disk snapshots and the exact `--limit`-derived sample sizes +(LME `n_questions=10`, LoCoMo `n_conversations=1`/`n_questions=197`) are in +`benchmarks/results/trust-factor-sweep/active/PROGRESS.json` and each cell's +`reproduce.log`; the raw per-benchmark JSON (with full `MANIFEST.json` +provenance — git sha, package versions, reranker sha256) is under the +`Artifacts` paths above. + +### What this establishes + +Relative to each other — same harness, same `--quick` mode, same corpus, +five consecutive cells, one shared provenance — **the trust factor costs +relevance on LoCoMo, monotonically in W**. W = 1.0 (disabled) gives the best +scores; at the shipped production value, W = 0.7, LoCoMo loses 0.0165 MRR +(0.8209 → 0.8044) and 2.54 points of Recall@10 (0.9594 → 0.9340) relative to +the disabled control. LongMemEval is flat across the whole grid (0.8500 / +1.0000 identically at every W) and BEAM MRR moves 0.0004 (0.6579 → 0.6575) — +neither is large enough, at this sample size, to call a W effect. + +### What this does NOT establish + +Two gaps, stated as such rather than folded into the table above: + +1. **Conformance to the published floors.** `reproduce.sh:94-95` states + explicitly that `--quick`/`--limit` runs skip the floor gate — "partial + runs are not comparable to n=500 / n=1986 figures" — and every cell above + ran `--quick` (LME n=10, LoCoMo n=1 conversation / 197 questions, vs the + published n=500/n=1986). Comparing these numbers to + `FLOOR_LME_R10`/`FLOOR_LME_MRR`/`FLOOR_LOCOMO_R10`/`FLOOR_LOCOMO_MRR` + would be exactly the protocol-mismatch error PR #414 spent four review + rounds correcting — different sample size, different protocol, not a + comparable number. A full-mode re-run against the floors is the open + item; it would tell us whether the LoCoMo cost measured here still holds + (or is smaller/larger) at the published sample size. +2. **Whether the adversarial defense justifies the cost.** The adversarial + arm (§ above) shows 4/4 scenarios defended at W ≤ 0.70, but that corpus + (`benchmarks/lib/adversarial_corpus.py`) is constructed specifically to + exercise the attack families — it says nothing about whether that + defense buys anything on LME/LoCoMo/BEAM's real question distributions. + No measurement in this document links the adversarial-arm defense rate to + a benefit on the real benchmarks; the relevance cost above and the + security benefit in § Adversarial arm are two separate, unlinked + measurements. Closing this gap needs a single corpus or evaluation that + scores both relevance and adversarial robustness together. + +### Decision (owner call, not derived from this measurement alone) + +**W stays at 0.7.** The relevance cost measured above is real and is +published, with both reserves above stated rather than buried: this +re-measurement does not confirm floor conformance (quick mode only) and does +not establish that the adversarial defense is worth the cost (no linking +measurement exists). Publishing the honest number now — rather than +withholding it until a full-mode run and a linking measurement both exist — +is the position consistent with § Scientific Implementation Standard: report +what was measured and the uncertainty that remains, rather than delay or +omit. Superseded by a full-mode re-run against the floors, or a measurement +linking adversarial defense to real-benchmark relevance, whichever lands +first. From fb28eb541ab4f90b994690cd4cbac603841abcf2 Mon Sep 17 00:00:00 2001 From: cdeust Date: Mon, 10 Aug 2026 17:47:25 +0200 Subject: [PATCH 3/3] style(benchmarks): ruff format sweep_progress.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI Lint failure on PR #415 — sweep_progress.py was committed from the worktree without ever going through the formatter. No logic change: one line collapsed under the ruff line-length rule. Co-Authored-By: Claude --- benchmarks/lib/sweep_progress.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/lib/sweep_progress.py b/benchmarks/lib/sweep_progress.py index fede62d9..7ec327a5 100644 --- a/benchmarks/lib/sweep_progress.py +++ b/benchmarks/lib/sweep_progress.py @@ -59,9 +59,7 @@ def read_progress(sweep_dir: str) -> dict: def completed_w_values(sweep_dir: str) -> set[float]: progress = read_progress(sweep_dir) - return { - c["w"] for c in progress.get("cells", []) if c.get("status") == "complete" - } + return {c["w"] for c in progress.get("cells", []) if c.get("status") == "complete"} def next_pending_w(grid: list[float], sweep_dir: str) -> float | None: