chore(deps)!: 7-day dependency cooldown (uv exclude-newer + CI assert)

AGENTS.md

@@ -17,6 +17,7 @@ uv run ruff check .
 - **Regression tests must name the PR/commit they guard** in the docstring (e.g. `Guards the fix from PR #198 against the regression introduced by PR #193`).
 - **Human review before `main`.** PRs only. No force-pushes to `main`. Self-approval doesn't count.
 - **Trunk-based:** branch off `main`, PR back to `main`. No long-lived release branches.
+- **Dependency cooldown:** `uv lock` never resolves to a release younger than 7 days. Don't hand-edit `[tool.uv] exclude-newer` — run `python tools/lock.py` to set it to `now - 7d` and re-lock; `tests/test_dep_cooldown.py` fails if the committed cutoff is younger than the window. A genuinely urgent fix gets a `[tool.uv.exclude-newer-package]` override plus a matching `[tool.benchflow.dependency-cooldown.override-rationale]` entry.
 - **Releases:** current mechanics live in [`docs/release.md`](./docs/release.md). Merges to `main` publish internal preview `.devN` builds after CI passes; public releases require a reviewed stable-version PR, a matching `v<version>` tag on `main`, then a bump back to the next `.dev0`.
 
 ## Experiment guidance (when using benchflow to run batch tasks experiments)

pyproject.toml

@@ -108,6 +108,10 @@ build-backend = "hatchling.build"
 only-include = [
     "src",
     "tests",
+    # `tests/test_dep_cooldown.py` does `from tools.lock import ...`, so the
+    # sdist must ship `tools` too or the test suite fails to import from a
+    # source distribution.
+    "tools",
     "README.md",
     "CHANGELOG.md",
     "LICENSE",
@@ -185,3 +189,29 @@ exclude = [
     "src/benchflow/rewards/rubric_config.py",
     "src/benchflow/experimental/mcp/reviewer_server.py",
 ]
+
+[tool.uv]
+# The per-package cooldown override (`exclude-newer-package` below) is only
+# honored by uv >= 0.8.4. Pin a floor so an older uv can't silently ignore the
+# override table and resolve a different (vulnerable / un-vetted) lock.
+required-version = ">=0.8.4"
+# Dependency cooldown (hard rule): never resolve to a release younger than ~7
+# days. `exclude-newer` caps every `uv lock` at this timestamp. Don't hand-edit
+# it — run `python tools/lock.py` to set it to `now - 7d` and re-lock in one
+# step (the value stays static so CI's `uv sync --locked` is deterministic; the
+# helper just rolls it forward). The CI check `tests/test_dep_cooldown.py` fails
+# if this date is ever within the last 7 days.
+#
+# Security exception: a freshly-published CVE fix (younger than the cap) is
+# allowed via a per-package override in `exclude-newer-package` below, with a
+# matching machine-checkable rationale in
+# `[tool.benchflow.dependency-cooldown.override-rationale]`. The CI gate fails
+# missing/stale rationales and stale overrides once their package cutoff ages
+# past the cooldown window, so they cannot become a permanent package allowlist.
+exclude-newer = "2026-07-06T00:00:00Z"
+
+[tool.uv.exclude-newer-package]
+trl = "2026-07-10T00:00:00Z"
+
+[tool.benchflow.dependency-cooldown.override-rationale]
+trl = "Temporary mainline compatibility override: the TRL-SFT export extra requires trl>=1.8.0, and 1.8.0 was published after the current global cooldown cutoff. Remove once the release ages past the cooldown window."

src/benchflow/task/prompts.py

@@ -650,7 +650,7 @@ def _str_or_none(value: object) -> str | None:
 def _document_user_model_kind(
     model: str | None,
 ) -> Literal["scripted-linear", "model-linear"] | None:
-    if model in {None, "scripted", "deterministic"}:
+    if model is None or model in {"scripted", "deterministic"}:
         return "scripted-linear"
     if model.startswith(
         (

tests/test_dep_cooldown.py

@@ -0,0 +1,106 @@
+"""Hard rule: the dependency cooldown — never resolve to a release younger than
+a 7-day window.
+
+Two layers, both evaluated against *today*:
+
+1. `test_uv_exclude_newer_caps_resolution` — `[tool.uv] exclude-newer` is the
+   static cutoff `uv lock` applies at resolve time; it must be >= 7 days old so
+   the default resolution can't reach a brand-new release. Set it with
+   `python tools/lock.py` (computes `now - 7d` and re-locks).
+2. `test_locked_packages_respect_cooldown` — the dynamic half: it reads the
+   `upload-time` uv bakes into `uv.lock` and fails if any *resolved* package is
+   younger than `now - 7d`. This tracks the current date with no stored date to
+   trust and no network calls.
+
+A genuinely urgent (e.g. security) fix younger than the window is allowed only
+via `[tool.uv.exclude-newer-package]` plus a matching structured rationale in
+`[tool.benchflow.dependency-cooldown.override-rationale]`, which exempts that
+package from layer 2.
+"""
+
+from __future__ import annotations
+
+import datetime
+import tomllib
+from pathlib import Path
+
+from tools.lock import (
+    COOLDOWN_DAYS,
+    find_cooldown_override_rationale_issues,
+    find_cooldown_violations,
+    find_expired_cooldown_overrides,
+)
+
+_PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml"
+_LOCK = Path(__file__).resolve().parents[1] / "uv.lock"
+
+
+def test_uv_exclude_newer_caps_resolution() -> None:
+    cfg = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8"))
+    raw = cfg.get("tool", {}).get("uv", {}).get("exclude-newer")
+    assert raw, (
+        "[tool.uv] exclude-newer is missing from pyproject.toml — it enforces the "
+        f"{COOLDOWN_DAYS}-day dependency cooldown and must be set."
+    )
+    cutoff = datetime.datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
+    if cutoff.tzinfo is None:
+        cutoff = cutoff.replace(tzinfo=datetime.UTC)
+    age = datetime.datetime.now(datetime.UTC) - cutoff
+    age_days = age.total_seconds() / 86400
+    assert age >= datetime.timedelta(days=COOLDOWN_DAYS), (
+        f"[tool.uv] exclude-newer ({raw}) is only {age_days:.1f}d old; the dependency "
+        f"cooldown requires it to be >= {COOLDOWN_DAYS} days in the past. Re-lock "
+        "with `python tools/lock.py` when taking dependency updates."
+    )
+
+
+def test_locked_packages_respect_cooldown() -> None:
+    cfg = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8"))
+    exempt = set(cfg.get("tool", {}).get("uv", {}).get("exclude-newer-package", {}))
+    violations = find_cooldown_violations(
+        _LOCK.read_text(encoding="utf-8"),
+        exempt=exempt,
+        now=datetime.datetime.now(datetime.UTC),
+    )
+    assert not violations, (
+        f"uv.lock resolves packages younger than the {COOLDOWN_DAYS}-day cooldown "
+        "without a documented [tool.uv.exclude-newer-package] override:\n"
+        + "\n".join(f"  {n} {v} (uploaded {ts:%Y-%m-%d})" for n, v, ts in violations)
+        + "\nRe-lock with `python tools/lock.py`, or add an override with rationale."
+    )
+
+
+def test_exclude_newer_package_overrides_have_rationales() -> None:
+    cfg = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8"))
+    tool = cfg.get("tool", {})
+    uv = tool.get("uv", {})
+    benchflow = tool.get("benchflow", {})
+    cooldown = benchflow.get("dependency-cooldown", {})
+    overrides = uv.get("exclude-newer-package", {})
+    rationales = cooldown.get("override-rationale", {})
+    issues = find_cooldown_override_rationale_issues(overrides, rationales)
+    assert not issues, (
+        "[tool.uv.exclude-newer-package] overrides must carry a non-empty "
+        "machine-checkable rationale in "
+        "[tool.benchflow.dependency-cooldown.override-rationale]:\n"
+        + "\n".join(f"  {issue}" for issue in issues)
+    )
+
+
+def test_exclude_newer_package_overrides_expire_with_cooldown() -> None:
+    cfg = tomllib.loads(_PYPROJECT.read_text(encoding="utf-8"))
+    overrides = cfg.get("tool", {}).get("uv", {}).get("exclude-newer-package", {})
+    now = datetime.datetime.now(datetime.UTC)
+    expired = find_expired_cooldown_overrides(overrides, now=now)
+    floor = now - datetime.timedelta(days=COOLDOWN_DAYS)
+    assert not expired, (
+        "[tool.uv.exclude-newer-package] overrides are temporary security/urgent "
+        f"escape hatches and must be removed once their cutoff is older than the "
+        f"{COOLDOWN_DAYS}-day cooldown floor ({floor:%Y-%m-%dT%H:%M:%SZ}):\n"
+        + "\n".join(
+            f"  {name} = {raw!r} (override cutoff {cutoff:%Y-%m-%dT%H:%M:%SZ})"
+            for name, raw, cutoff in expired
+        )
+        + "\nRemove the stale override and re-lock with `python tools/lock.py`, "
+        "or keep an override only for a still-fresh security/urgent pin."
+    )

tests/test_lock_tool.py

@@ -0,0 +1,222 @@
+"""Unit tests for the dependency-cooldown re-lock helper (`tools/lock.py`).
+
+These cover the pure date/text transforms and the lock-freshness audit against
+synthetic lock text; `uv lock` itself is not invoked.
+"""
+
+from __future__ import annotations
+
+import datetime
+import tomllib
+
+import pytest
+
+from tools import lock as lock_tool
+from tools.lock import (
+    COOLDOWN_DAYS,
+    LockError,
+    compute_cooldown_cutoff,
+    find_cooldown_override_rationale_issues,
+    find_cooldown_violations,
+    find_expired_cooldown_overrides,
+    main,
+    newest_upload_times,
+    rewrite_exclude_newer,
+)
+
+_NOW = datetime.datetime(2026, 6, 15, 12, 0, tzinfo=datetime.UTC)
+
+# floor (now - 7d) = 2026-06-08T12:00Z: old-pkg is older, fresh-pkg is younger.
+_SYNTH_LOCK = """
+[options]
+exclude-newer = "2026-06-08T00:00:00Z"
+
+[[package]]
+name = "old-pkg"
+version = "1.0.0"
+sdist = { url = "https://example/old.tar.gz", upload-time = "2026-05-01T00:00:00Z" }
+
+[[package]]
+name = "fresh-pkg"
+version = "2.0.0"
+sdist = { url = "https://example/fresh.tar.gz", upload-time = "2026-06-13T00:00:00Z" }
+wheels = [
+    { url = "https://example/fresh.whl", upload-time = "2026-06-14T08:00:00Z" },
+]
+
+[[package]]
+name = "git-pkg"
+version = "0.1.0"
+source = { git = "https://example/repo.git" }
+"""
+
+
+def test_cutoff_trails_today_by_cooldown_window_at_midnight() -> None:
+    now = datetime.datetime(2026, 6, 15, 9, 30, tzinfo=datetime.UTC)
+    assert compute_cooldown_cutoff(now) == "2026-06-08T00:00:00Z"
+
+
+def test_cutoff_is_idempotent_within_a_day() -> None:
+    morning = datetime.datetime(2026, 6, 15, 0, 1, tzinfo=datetime.UTC)
+    night = datetime.datetime(2026, 6, 15, 23, 59, tzinfo=datetime.UTC)
+    assert compute_cooldown_cutoff(morning) == compute_cooldown_cutoff(night)
+
+
+def test_cutoff_normalizes_non_utc_input() -> None:
+    # 2026-06-15 02:00 in UTC+9 is still 2026-06-14 17:00 UTC -> minus 7d.
+    tz = datetime.timezone(datetime.timedelta(hours=9))
+    now = datetime.datetime(2026, 6, 15, 2, 0, tzinfo=tz)
+    assert compute_cooldown_cutoff(now) == "2026-06-07T00:00:00Z"
+
+
+def test_cutoff_default_window_is_the_cooldown_constant() -> None:
+    now = datetime.datetime(2026, 6, 15, 12, 0, tzinfo=datetime.UTC)
+    expected = (now - datetime.timedelta(days=COOLDOWN_DAYS)).date()
+    assert compute_cooldown_cutoff(now).startswith(expected.isoformat())
+
+
+def test_cutoff_rejects_negative_window() -> None:
+    now = datetime.datetime(2026, 6, 15, tzinfo=datetime.UTC)
+    with pytest.raises(LockError):
+        compute_cooldown_cutoff(now, cooldown_days=-1)
+
+
+def test_rewrite_replaces_only_the_exclude_newer_value() -> None:
+    text = (
+        "[tool.uv]\n"
+        'exclude-newer = "2026-06-01T00:00:00Z"\n'
+        "\n"
+        "[tool.uv.exclude-newer-package]\n"
+        'litellm = "2026-06-14T00:00:00Z"\n'
+    )
+    out = rewrite_exclude_newer(text, "2026-06-08T00:00:00Z")
+    assert 'exclude-newer = "2026-06-08T00:00:00Z"' in out
+    # The per-package override table is untouched.
+    assert 'litellm = "2026-06-14T00:00:00Z"' in out
+
+
+def test_rewrite_requires_exactly_one_assignment() -> None:
+    with pytest.raises(LockError):
+        rewrite_exclude_newer("[tool.uv]\n", "2026-06-08T00:00:00Z")
+
+
+def test_newest_upload_time_picks_latest_artifact_and_skips_sourceless() -> None:
+    newest = newest_upload_times(tomllib.loads(_SYNTH_LOCK))
+    # The wheel (08:00) beats the sdist (00:00) for the same package.
+    assert newest["fresh-pkg"][1] == datetime.datetime(
+        2026, 6, 14, 8, 0, tzinfo=datetime.UTC
+    )
+    # A git-sourced package has no upload-time and is omitted entirely.
+    assert "git-pkg" not in newest
+
+
+def test_find_cooldown_violations_flags_only_young_unexempted_packages() -> None:
+    violations = find_cooldown_violations(_SYNTH_LOCK, exempt=set(), now=_NOW)
+    assert [name for name, _, _ in violations] == ["fresh-pkg"]
+
+
+def test_find_cooldown_violations_honors_exemptions() -> None:
+    violations = find_cooldown_violations(_SYNTH_LOCK, exempt={"fresh-pkg"}, now=_NOW)
+    assert violations == []
+
+
+def test_find_cooldown_violations_normalizes_exemption_names() -> None:
+    """Guards the fix from PR #788 so overrides are not spelling-sensitive."""
+    violations = find_cooldown_violations(_SYNTH_LOCK, exempt={"Fresh_Pkg"}, now=_NOW)
+    assert violations == []
+
+
+def test_find_cooldown_violations_only_grows_more_lenient_over_time() -> None:
+    # The same lock, evaluated a month later: fresh-pkg has aged past the window.
+    later = _NOW + datetime.timedelta(days=30)
+    assert find_cooldown_violations(_SYNTH_LOCK, exempt=set(), now=later) == []
+
+
+def test_expired_cooldown_overrides_flags_aged_out_entries() -> None:
+    """Guards the PR #788 override escape hatch from becoming a package allowlist."""
+    overrides = {
+        "old-pkg": "2026-06-08T00:00:00Z",
+        "fresh-pkg": "2026-06-14T00:00:00Z",
+    }
+    expired = find_expired_cooldown_overrides(overrides, now=_NOW)
+    assert [(name, raw) for name, raw, _ in expired] == [
+        ("old-pkg", "2026-06-08T00:00:00Z")
+    ]
+
+
+def test_expired_cooldown_overrides_rejects_invalid_timestamp() -> None:
+    """Guards the PR #788 override gate against unclear TOML diagnostics."""
+    with pytest.raises(
+        LockError,
+        match=r"\[tool\.uv\.exclude-newer-package\] 'bad-pkg' must be an ISO timestamp",
+    ):
+        find_expired_cooldown_overrides({"bad-pkg": "not-a-date"}, now=_NOW)
+
+
+def test_override_rationale_gate_accepts_normalized_matching_entry() -> None:
+    """Guards PR #788: cooldown overrides require machine-checkable rationale."""
+    issues = find_cooldown_override_rationale_issues(
+        {"Fresh_Pkg": "2026-06-14T00:00:00Z"},
+        {"fresh-pkg": "GHSA fix published inside the cooldown window."},
+    )
+    assert issues == []
+
+
+def test_override_rationale_gate_rejects_missing_or_empty_rationale() -> None:
+    """Guards PR #788 against bare package allowlist overrides."""
+    issues = find_cooldown_override_rationale_issues(
+        {
+            "missing-rationale": "2026-06-14T00:00:00Z",
+            "empty-rationale": "2026-06-14T00:00:00Z",
+        },
+        {"empty-rationale": "  "},
+    )
+    assert issues == [
+        "'empty-rationale' override rationale must be a non-empty string.",
+        "'missing-rationale' has [tool.uv.exclude-newer-package] but no "
+        "[tool.benchflow.dependency-cooldown.override-rationale] entry.",
+    ]
+
+
+def test_override_rationale_gate_rejects_stale_rationale_entries() -> None:
+    """Guards PR #788 so rationale entries expire with their overrides."""
+    issues = find_cooldown_override_rationale_issues(
+        {},
+        {"old-pkg": "no longer has an override"},
+    )
+    assert issues == [
+        "'old-pkg' has an override rationale but no matching "
+        "[tool.uv.exclude-newer-package] entry."
+    ]
+
+
+def test_override_rationale_gate_rejects_non_table() -> None:
+    """Guards PR #788 against unclear pyproject policy shapes."""
+    issues = find_cooldown_override_rationale_issues(
+        {"fresh-pkg": "2026-06-14T00:00:00Z"},
+        "fresh-pkg: security fix",
+    )
+    assert issues == [
+        "[tool.benchflow.dependency-cooldown.override-rationale] must be a TOML table."
+    ]
+
+
+def test_main_wraps_pyproject_read_oserror(tmp_path, capsys) -> None:
+    """Guards the PR #788 lock helper from leaking raw OSError tracebacks."""
+    with pytest.raises(SystemExit) as exc_info:
+        main(["--pyproject", str(tmp_path), "--no-lock"])
+
+    assert exc_info.value.code == 1
+    assert "Could not read" in capsys.readouterr().err
+
+
+def test_run_uv_lock_wraps_oserror(monkeypatch, tmp_path) -> None:
+    """Guards the PR #788 lock helper from leaking subprocess OSError."""
+
+    def raise_oserror(*args, **kwargs):
+        raise OSError("spawn failed")
+
+    monkeypatch.setattr(lock_tool.subprocess, "run", raise_oserror)
+
+    with pytest.raises(LockError, match="Could not run `uv lock`"):
+        lock_tool._run_uv_lock(tmp_path / "pyproject.toml")