Add the cheap first-real-run on-ramp between toy_calc and the ~$148 tau2 run (closes #124)

core/tests/test_cheap_real_preset.py

@@ -0,0 +1,136 @@
+"""The cheap_real preset's honesty invariants — the things a "make it cheaper" edit breaks.
+
+The preset (``examples/cheap_real/capevolve.yaml``) exists to be the CHEAP rung, so the
+obvious future edit is to shrink it further. Several of its values are load-bearing and
+would fail silently or confusingly if changed:
+
+* the dataset size and split ratios must land ``val`` at or above the floors in
+  ``splits`` — below ``MIN_VAL_TASKS`` a run dies mid-flight inside ``gate.decide``
+  (#113) *after* the budget is spent, and below ``LOW_CONFIDENCE_VAL_TASKS`` every
+  decision is branded low-confidence. So val is pinned by a test, not by a comment.
+* ``protected_paths`` must stay ABSENT. Since #142/#197 an empty list is a hard error
+  and a present list REPLACES the layout defaults. Precisely: an omitted key resolves
+  to ``[*_DEFAULT_GLOBS, *spec['dataset_source']]`` (``protect.py:168-170``), and
+  ``tasks.jsonl`` matches **none** of the default globs (``adapters``,
+  ``capevolve.yaml``, ``*gold*`` suffixes) — the answer key is guarded *by the
+  dataset_source fold-in*, which only happens when the key is absent. Omission is
+  load-bearing for the dataset specifically, not merely tidier.
+* ``optimizer_max_turns`` must stay at or above the MEASURED floor: at 12 the agent
+  exits ``Reached max turns``, which cap-evolve correctly reports as a failed
+  iteration, so every candidate is discarded and the run looks like "the optimizer
+  proposed nothing".
+* ``run.sh``'s YAML rewrite is a line-PREFIX string rewriter over this preset, so it
+  silently no-ops if a key it targets is ever indented or renamed. Pinned here because
+  a no-op means the paid rung runs with ``optimizer_skill: mock``.
+
+All are cheap file reads: no model, no run.
+"""
+
+from pathlib import Path
+import sys
+
+REPO = Path(__file__).resolve().parents[2]
+PRESET = REPO / "examples" / "cheap_real" / "capevolve.yaml"
+TASKS = REPO / "examples" / "cheap_real" / "tasks.jsonl"
+
+sys.path.insert(0, str(REPO / "core"))
+
+
+def _spec() -> dict:
+    from cap_evolve.specfile import read_yaml
+    return read_yaml(PRESET.read_text(encoding="utf-8"))
+
+
+def _task_ids() -> list[str]:
+    import json
+    return [json.loads(ln)["id"]
+            for ln in TASKS.read_text(encoding="utf-8").splitlines() if ln.strip()]
+
+
+def test_split_clears_the_honest_gate_floors():
+    from cap_evolve.splits import make_splits
+    import cap_evolve.splits as splits
+
+    spec = _spec()
+    ids = _task_ids()
+    assert len(ids) == len(set(ids)), "duplicate task ids would corrupt the split"
+    sp = make_splits(ids, seed=int(spec["split_seed"]),
+                     ratios=(float(spec["split_train"]), float(spec["split_val"]),
+                             float(spec["split_test"])))
+    # MIN_VAL_TASKS is the hard floor (gate.decide refuses below it); it only exists
+    # once #113 lands, so tolerate its absence rather than skip the whole assertion.
+    hard = getattr(splits, "MIN_VAL_TASKS", 2)
+    soft = getattr(splits, "LOW_CONFIDENCE_VAL_TASKS", 5)
+    assert len(sp.val) >= hard, (
+        f"cheap_real val split is {len(sp.val)}, below the hard floor {hard}: the run "
+        "would die inside gate.decide after spending its budget")
+    assert len(sp.val) >= soft, (
+        f"cheap_real val split is {len(sp.val)}, below {soft}: every gate decision "
+        "would be branded LOW CONFIDENCE. Add tasks rather than lowering this.")
+    assert sp.test, "the sealed test split must be non-empty"
+    assert not (set(sp.train) & set(sp.val)), "train/val overlap makes val a fit metric"
+    assert not (set(sp.test) & (set(sp.train) | set(sp.val))), "the test split leaked"
+
+
+def test_protected_paths_is_omitted_not_empty():
+    # An empty list is a hard error since #142/#197, and a present list replaces the
+    # layout defaults — omission is what protects adapters/, the dataset and the spec.
+    for line in PRESET.read_text(encoding="utf-8").splitlines():
+        assert not line.strip().startswith("protected_paths:"), (
+            "cheap_real must OMIT protected_paths: an empty list is a hard error and a "
+            "declared list would silently replace the defaults that cover the grader.")
+
+
+def test_dataset_source_names_the_real_file():
+    # #142/#197's guard hashes the path `dataset_source` names, so `adapter` here would
+    # leave the answer key unprotected.
+    assert _spec()["dataset_source"] == "tasks.jsonl"
+
+
+def test_budget_is_bounded():
+    # The whole point of the rung: it cannot silently become the multi-hour run.
+    spec = _spec()
+    assert 0 < float(spec["max_usd"]) <= 5.0
+    assert 0 < float(spec["max_optimizer_usd"]) <= float(spec["max_usd"])
+    assert 0 < int(spec["max_iterations"]) <= 5
+
+
+def test_optimizer_max_turns_clears_the_measured_floor():
+    # MEASURED, not guessed: at 12 the agent exits `Reached max turns (12)`, cap-evolve
+    # reports a failed iteration, and every candidate is discarded — the run looks like
+    # "the optimizer proposed nothing". A "tighten the caps" edit must not go back there.
+    assert int(_spec()["optimizer_max_turns"]) >= 40, (
+        "optimizer_max_turns below 40 was MEASURED to make every iteration fail with "
+        "`Reached max turns`, so no candidate survives. Raise it back.")
+
+
+def test_run_sh_yaml_rewrite_still_matches_the_preset():
+    # run.sh flips the optimizer keys with a line-PREFIX rewriter, so it silently
+    # no-ops if a targeted key is ever indented or renamed — and a no-op means the
+    # paid rung quietly runs `optimizer_skill: mock`. Run the real heredoc, not a copy.
+    import re
+    import tempfile
+    run_sh = (REPO / "examples" / "cheap_real" / "run.sh").read_text(encoding="utf-8")
+    m = re.search(r"<<'SED'\n(.*?)\nSED\n", run_sh, re.S)
+    assert m, "run.sh no longer has the SED heredoc this test pins"
+
+    with tempfile.TemporaryDirectory() as td:
+        target = Path(td) / "capevolve.yaml"
+        target.write_text(PRESET.read_text(encoding="utf-8"), encoding="utf-8")
+        old_argv = sys.argv
+        sys.argv = ["-", str(target), "claude-code", "claude-haiku-4-5"]
+        try:
+            exec(compile(m.group(1), "run.sh:SED", "exec"), {"__name__": "__main__"})
+        finally:
+            sys.argv = old_argv
+        out = target.read_text(encoding="utf-8")
+
+    assert "optimizer_skill: claude-code\n" in out, "the optimizer_skill flip no-opped"
+    assert "optimizer_model: claude-haiku-4-5\n" in out
+    assert "proposer_model: claude-haiku-4-5\n" in out
+    instr = [ln for ln in out.splitlines()
+             if ln.startswith("optimizer_instructions_file:")]
+    assert len(instr) == 1, f"expected exactly one instructions line, got {instr}"
+    assert Path(instr[0].split(":", 1)[1].strip()).is_absolute(), (
+        "the instructions path must be ABSOLUTE: cli.py resolves a relative one against "
+        "its own cwd and silently falls back to the generic template (see #252)")

docs/GETTING_STARTED.md

@@ -1,6 +1,62 @@
 # Getting started
 
-Your first successful cap-evolve run, in two minutes, with **no API key**.
+Your first successful cap-evolve run, in two minutes, with **no API key** — then a
+real one, cheaply, before committing to anything expensive.
+
+## The ladder
+
+Three rungs, so "see it work" and "run it for real" are not the same decision. Every
+figure is either **measured** on the run it describes or labelled an **estimate** with
+its arithmetic — no rung is advertised on a guess.
+
+| Rung | What is real | Runtime | Cost | How |
+|---|---|---|---|---|
+| **1. free** | pipeline, gate, sealed test — **no model** | seconds | **$0** | [`examples/toy_calc`](../examples/toy_calc/) · §3 |
+| **2. cheap-real** | **a real LLM**, a real accept/reject decision, a real sealed test | **~30 s**–**5.4 min** measured | **$0** local, **$0.0115** with a hosted runner, or **$0.54** with a hosted proposer — all measured | [`examples/cheap_real`](../examples/cheap_real/) · §4 |
+| **3. full** | a published benchmark (τ²-bench airline) | hours | ~$148 | [`REPRODUCE_tau2.md`](REPRODUCE_tau2.md) |
+
+### Where rung 2's numbers come from
+
+The preset is 20 tasks → train 10 / val 5 / test 5, `num_trials: 1`,
+`max_iterations: 3`. So the call counts are fixed and checkable:
+
+```
+runner calls   = val 5 x trials 1 x (1 baseline + 3 candidates)  = 20
+               + test 5 x (best + baseline seed)                 = 10   → 30 calls
+proposer calls = max_iterations                                  =  3
+```
+
+- **Free variant** — local `ollama/llama3.2:3b` runner + `mock` proposer.
+  **MEASURED: ~30 s wall clock, $0.00** (30.4 s, 33.9 s and 34.2 s across three runs on
+  two machines — single-run timing is noise at this scale, so treat it as "about half a
+  minute", not as a figure). A local model is not metered and the `mock` proposer makes
+  no network call at all, so the $0 is by construction, not by luck.
+- **Cheap variant** — same local runner, `claude-code`/Haiku proposer. The 30 runner
+  calls stay $0; only the 3 proposals cost anything. **MEASURED** from the run's own
+  accounting (`state.json` → `spent`): **$0.5427 total, 321 s (5.4 min) wall clock,
+  exactly the 30 `metric_calls` the formula above predicts.** Per-proposal spend was
+  $0.159 / $0.199 / $0.185, so budget **~$0.15–0.35 per iteration**. Proposal latency
+  dominates: 293 s of the 321 s was the optimizer, and only 27 s the runner.
+- **Hosted-runner variant** — no local model available, so the runner is
+  `claude-haiku-4-5` and `mock` proposes for free. **MEASURED: $0.011478 total, 98 s
+  wall clock, the same 30 `metric_calls`, `baseline_val 0.0` → sealed `test_reward
+  1.0`.** The six eval costs sum to the total exactly
+  (`0.003466 + 3 x 0.000681 + 0.000687 + 0.005282 = 0.011478`); the two seed evals
+  dominate because the unoptimized prompt answers in prose and burns output tokens.
+  This is **20x cheaper than the prior estimate** of $0.21, which assumed
+  `pricing.py`'s generic 3 000-in/800-out rollout — a one-line date is far smaller than
+  that. The estimate is retired: this row is now measured.
+
+The preset's `max_usd: 3.0` / `max_optimizer_usd: 2.5` are hard stops, so rung 2
+cannot quietly become rung 3. Rung 3's ~$148 is the committed τ² run's own reported
+optimizer spend ([`RESULTS.md`](RESULTS.md)).
+
+**What rung 2 does *not* show.** Val saturates at 1.0 on iteration 1 in every variant,
+so the two candidates that follow are rejected against a ceiling, not on merit — and at
+`num_trials: 1` the decision is `Δ > 0`, not a significance test (the run says so:
+`SE=0 → STRICT fallback`). The gate runs and decides for real; it is never asked a hard
+question. For the significance machinery under load, see `examples/skillsbench`
+(`num_trials: 3`).
 
 ## Prerequisites
 - Python **3.10+** and **git**.
@@ -43,11 +99,59 @@ This is exactly what `core/tests/test_e2e_slice.py` asserts. The script prints a
 directory; open the `dashboard.html` it writes in any browser to see the run (KPIs,
 per-iteration diffs, the tasks × iterations heatmap).
 
-## 4. Where to next
+## 4. Run the cheap FIRST REAL example (rung 2)
+
+`toy_calc` proves the machinery but calls no model. This one calls a real one — and
+still finishes in minutes for $0, so there is a rung between it and the multi-hour
+τ² run.
+
+The task: normalize a human-written date (`"the 22nd of November, 1963"`) to ISO
+(`1963-11-22`), scored by exact match. The seed prompt is a generic "you are a helpful
+assistant", so a small model replies in prose and scores 0; the fix is an output
+contract — exactly what an optimizer is good at proposing.
+
+```bash
+pip install litellm                      # the runner's provider shim
+ollama pull llama3.2:3b                  # ~2 GB, free, local
+
+bash examples/cheap_real/run.sh          # rung 2, free variant
+```
+
+For the paid variant, let a real agent propose the edit instead of `mock` (the runner
+stays local and free):
+
+```bash
+CHEAP_REAL_OPTIMIZER=claude-code CHEAP_REAL_OPT_MODEL=claude-haiku-4-5 \
+  bash examples/cheap_real/run.sh
+```
+
+No local model? Point the runner at a cheap hosted one — one env var, no code change.
+Needs that provider's credential in the environment (e.g. `ANTHROPIC_API_KEY`).
+**MEASURED: 98 s, $0.0115.**
+
+```bash
+CHEAP_REAL_MODEL=claude-haiku-4-5 bash examples/cheap_real/run.sh
+```
+
+`run.sh` is also the **programmatic entry point**: every knob is an env var with a
+default and **all of stdout is the run's summary JSON** — progress goes to stderr
+(#116) and the script keeps only the last object, so `json.loads(stdout)` works even
+under `CAPEVOLVE_DASHBOARD=auto`, where the CLI prints a second object (#217).
+
+It preflights **every** variant before spending anything, because the adapter
+(correctly) turns a failed model call into reward `0.0`, which otherwise makes a broken
+run look like a clean one. Local: the endpoint answers *and* the model is actually
+pulled. Hosted: one real 1-token completion through the same wiring the adapter uses,
+so a rejected parameter, a missing credential or a typo'd model name fails loudly
+instead of scoring 0.0 thirty times. Details, all variants and the honest limits:
+[`examples/cheap_real/README.md`](../examples/cheap_real/README.md).
+
+## 5. Where to next
 
 | You want to… | Go to |
 |---|---|
 | Understand what cap-evolve optimizes and how | [`../README.md`](../README.md) · [`ARCHITECTURE.md`](ARCHITECTURE.md) |
+| See a real (cheap) LLM run end to end | [`../examples/cheap_real/`](../examples/cheap_real/) |
 | Set up a real optimizer/runner (credentials, dashboard) | [`INSTALL.md`](INSTALL.md) |
 | Optimize your own agent + benchmark | [`OPTIMIZE_YOUR_OWN.md`](OPTIMIZE_YOUR_OWN.md) |
 | See real benchmark results | [`RESULTS.md`](RESULTS.md) |

docs/OPTIMIZATION_EXAMPLES.md

@@ -8,6 +8,11 @@ under the champion `cand_0007`. Rollouts live at
 `<run_dir>/rollouts/val/<task_id>__<tag>__t<k>.json` (full message transcript +
 `reward_info`); every example below was read directly from those files.
 
+> These come from the **full** ~$148 run. To watch an optimizer propose an edit on a
+> real model for cents and minutes first, run rung 2 of the ladder —
+> [`examples/cheap_real/`](../examples/cheap_real/) — and read the `candidates/*/`
+> diffs it writes. Same mechanism, three orders of magnitude cheaper.
+
 The throughline: **argument-level feedback from failing rollouts is converted into
 executable guards inside the existing tools** — code where the agent already "knew"
 the rule but skipped it, prose only for genuine knowledge gaps. Across the run