diff --git a/core/tests/test_cheap_real_preset.py b/core/tests/test_cheap_real_preset.py new file mode 100644 index 00000000..89613812 --- /dev/null +++ b/core/tests/test_cheap_real_preset.py @@ -0,0 +1,136 @@ +"""The cheap_real preset's honesty invariants — the things a "make it cheaper" edit breaks. + +The preset (``examples/cheap_real/capevolve.yaml``) exists to be the CHEAP rung, so the +obvious future edit is to shrink it further. Several of its values are load-bearing and +would fail silently or confusingly if changed: + +* the dataset size and split ratios must land ``val`` at or above the floors in + ``splits`` — below ``MIN_VAL_TASKS`` a run dies mid-flight inside ``gate.decide`` + (#113) *after* the budget is spent, and below ``LOW_CONFIDENCE_VAL_TASKS`` every + decision is branded low-confidence. So val is pinned by a test, not by a comment. +* ``protected_paths`` must stay ABSENT. Since #142/#197 an empty list is a hard error + and a present list REPLACES the layout defaults. Precisely: an omitted key resolves + to ``[*_DEFAULT_GLOBS, *spec['dataset_source']]`` (``protect.py:168-170``), and + ``tasks.jsonl`` matches **none** of the default globs (``adapters``, + ``capevolve.yaml``, ``*gold*`` suffixes) — the answer key is guarded *by the + dataset_source fold-in*, which only happens when the key is absent. Omission is + load-bearing for the dataset specifically, not merely tidier. +* ``optimizer_max_turns`` must stay at or above the MEASURED floor: at 12 the agent + exits ``Reached max turns``, which cap-evolve correctly reports as a failed + iteration, so every candidate is discarded and the run looks like "the optimizer + proposed nothing". +* ``run.sh``'s YAML rewrite is a line-PREFIX string rewriter over this preset, so it + silently no-ops if a key it targets is ever indented or renamed. Pinned here because + a no-op means the paid rung runs with ``optimizer_skill: mock``. + +All are cheap file reads: no model, no run. +""" + +from pathlib import Path +import sys + +REPO = Path(__file__).resolve().parents[2] +PRESET = REPO / "examples" / "cheap_real" / "capevolve.yaml" +TASKS = REPO / "examples" / "cheap_real" / "tasks.jsonl" + +sys.path.insert(0, str(REPO / "core")) + + +def _spec() -> dict: + from cap_evolve.specfile import read_yaml + return read_yaml(PRESET.read_text(encoding="utf-8")) + + +def _task_ids() -> list[str]: + import json + return [json.loads(ln)["id"] + for ln in TASKS.read_text(encoding="utf-8").splitlines() if ln.strip()] + + +def test_split_clears_the_honest_gate_floors(): + from cap_evolve.splits import make_splits + import cap_evolve.splits as splits + + spec = _spec() + ids = _task_ids() + assert len(ids) == len(set(ids)), "duplicate task ids would corrupt the split" + sp = make_splits(ids, seed=int(spec["split_seed"]), + ratios=(float(spec["split_train"]), float(spec["split_val"]), + float(spec["split_test"]))) + # MIN_VAL_TASKS is the hard floor (gate.decide refuses below it); it only exists + # once #113 lands, so tolerate its absence rather than skip the whole assertion. + hard = getattr(splits, "MIN_VAL_TASKS", 2) + soft = getattr(splits, "LOW_CONFIDENCE_VAL_TASKS", 5) + assert len(sp.val) >= hard, ( + f"cheap_real val split is {len(sp.val)}, below the hard floor {hard}: the run " + "would die inside gate.decide after spending its budget") + assert len(sp.val) >= soft, ( + f"cheap_real val split is {len(sp.val)}, below {soft}: every gate decision " + "would be branded LOW CONFIDENCE. Add tasks rather than lowering this.") + assert sp.test, "the sealed test split must be non-empty" + assert not (set(sp.train) & set(sp.val)), "train/val overlap makes val a fit metric" + assert not (set(sp.test) & (set(sp.train) | set(sp.val))), "the test split leaked" + + +def test_protected_paths_is_omitted_not_empty(): + # An empty list is a hard error since #142/#197, and a present list replaces the + # layout defaults — omission is what protects adapters/, the dataset and the spec. + for line in PRESET.read_text(encoding="utf-8").splitlines(): + assert not line.strip().startswith("protected_paths:"), ( + "cheap_real must OMIT protected_paths: an empty list is a hard error and a " + "declared list would silently replace the defaults that cover the grader.") + + +def test_dataset_source_names_the_real_file(): + # #142/#197's guard hashes the path `dataset_source` names, so `adapter` here would + # leave the answer key unprotected. + assert _spec()["dataset_source"] == "tasks.jsonl" + + +def test_budget_is_bounded(): + # The whole point of the rung: it cannot silently become the multi-hour run. + spec = _spec() + assert 0 < float(spec["max_usd"]) <= 5.0 + assert 0 < float(spec["max_optimizer_usd"]) <= float(spec["max_usd"]) + assert 0 < int(spec["max_iterations"]) <= 5 + + +def test_optimizer_max_turns_clears_the_measured_floor(): + # MEASURED, not guessed: at 12 the agent exits `Reached max turns (12)`, cap-evolve + # reports a failed iteration, and every candidate is discarded — the run looks like + # "the optimizer proposed nothing". A "tighten the caps" edit must not go back there. + assert int(_spec()["optimizer_max_turns"]) >= 40, ( + "optimizer_max_turns below 40 was MEASURED to make every iteration fail with " + "`Reached max turns`, so no candidate survives. Raise it back.") + + +def test_run_sh_yaml_rewrite_still_matches_the_preset(): + # run.sh flips the optimizer keys with a line-PREFIX rewriter, so it silently + # no-ops if a targeted key is ever indented or renamed — and a no-op means the + # paid rung quietly runs `optimizer_skill: mock`. Run the real heredoc, not a copy. + import re + import tempfile + run_sh = (REPO / "examples" / "cheap_real" / "run.sh").read_text(encoding="utf-8") + m = re.search(r"<<'SED'\n(.*?)\nSED\n", run_sh, re.S) + assert m, "run.sh no longer has the SED heredoc this test pins" + + with tempfile.TemporaryDirectory() as td: + target = Path(td) / "capevolve.yaml" + target.write_text(PRESET.read_text(encoding="utf-8"), encoding="utf-8") + old_argv = sys.argv + sys.argv = ["-", str(target), "claude-code", "claude-haiku-4-5"] + try: + exec(compile(m.group(1), "run.sh:SED", "exec"), {"__name__": "__main__"}) + finally: + sys.argv = old_argv + out = target.read_text(encoding="utf-8") + + assert "optimizer_skill: claude-code\n" in out, "the optimizer_skill flip no-opped" + assert "optimizer_model: claude-haiku-4-5\n" in out + assert "proposer_model: claude-haiku-4-5\n" in out + instr = [ln for ln in out.splitlines() + if ln.startswith("optimizer_instructions_file:")] + assert len(instr) == 1, f"expected exactly one instructions line, got {instr}" + assert Path(instr[0].split(":", 1)[1].strip()).is_absolute(), ( + "the instructions path must be ABSOLUTE: cli.py resolves a relative one against " + "its own cwd and silently falls back to the generic template (see #252)") diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 5d16e597..7e8ca6e4 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -1,6 +1,62 @@ # Getting started -Your first successful cap-evolve run, in two minutes, with **no API key**. +Your first successful cap-evolve run, in two minutes, with **no API key** — then a +real one, cheaply, before committing to anything expensive. + +## The ladder + +Three rungs, so "see it work" and "run it for real" are not the same decision. Every +figure is either **measured** on the run it describes or labelled an **estimate** with +its arithmetic — no rung is advertised on a guess. + +| Rung | What is real | Runtime | Cost | How | +|---|---|---|---|---| +| **1. free** | pipeline, gate, sealed test — **no model** | seconds | **$0** | [`examples/toy_calc`](../examples/toy_calc/) · §3 | +| **2. cheap-real** | **a real LLM**, a real accept/reject decision, a real sealed test | **~30 s**–**5.4 min** measured | **$0** local, **$0.0115** with a hosted runner, or **$0.54** with a hosted proposer — all measured | [`examples/cheap_real`](../examples/cheap_real/) · §4 | +| **3. full** | a published benchmark (τ²-bench airline) | hours | ~$148 | [`REPRODUCE_tau2.md`](REPRODUCE_tau2.md) | + +### Where rung 2's numbers come from + +The preset is 20 tasks → train 10 / val 5 / test 5, `num_trials: 1`, +`max_iterations: 3`. So the call counts are fixed and checkable: + +``` +runner calls = val 5 x trials 1 x (1 baseline + 3 candidates) = 20 + + test 5 x (best + baseline seed) = 10 → 30 calls +proposer calls = max_iterations = 3 +``` + +- **Free variant** — local `ollama/llama3.2:3b` runner + `mock` proposer. + **MEASURED: ~30 s wall clock, $0.00** (30.4 s, 33.9 s and 34.2 s across three runs on + two machines — single-run timing is noise at this scale, so treat it as "about half a + minute", not as a figure). A local model is not metered and the `mock` proposer makes + no network call at all, so the $0 is by construction, not by luck. +- **Cheap variant** — same local runner, `claude-code`/Haiku proposer. The 30 runner + calls stay $0; only the 3 proposals cost anything. **MEASURED** from the run's own + accounting (`state.json` → `spent`): **$0.5427 total, 321 s (5.4 min) wall clock, + exactly the 30 `metric_calls` the formula above predicts.** Per-proposal spend was + $0.159 / $0.199 / $0.185, so budget **~$0.15–0.35 per iteration**. Proposal latency + dominates: 293 s of the 321 s was the optimizer, and only 27 s the runner. +- **Hosted-runner variant** — no local model available, so the runner is + `claude-haiku-4-5` and `mock` proposes for free. **MEASURED: $0.011478 total, 98 s + wall clock, the same 30 `metric_calls`, `baseline_val 0.0` → sealed `test_reward + 1.0`.** The six eval costs sum to the total exactly + (`0.003466 + 3 x 0.000681 + 0.000687 + 0.005282 = 0.011478`); the two seed evals + dominate because the unoptimized prompt answers in prose and burns output tokens. + This is **20x cheaper than the prior estimate** of $0.21, which assumed + `pricing.py`'s generic 3 000-in/800-out rollout — a one-line date is far smaller than + that. The estimate is retired: this row is now measured. + +The preset's `max_usd: 3.0` / `max_optimizer_usd: 2.5` are hard stops, so rung 2 +cannot quietly become rung 3. Rung 3's ~$148 is the committed τ² run's own reported +optimizer spend ([`RESULTS.md`](RESULTS.md)). + +**What rung 2 does *not* show.** Val saturates at 1.0 on iteration 1 in every variant, +so the two candidates that follow are rejected against a ceiling, not on merit — and at +`num_trials: 1` the decision is `Δ > 0`, not a significance test (the run says so: +`SE=0 → STRICT fallback`). The gate runs and decides for real; it is never asked a hard +question. For the significance machinery under load, see `examples/skillsbench` +(`num_trials: 3`). ## Prerequisites - Python **3.10+** and **git**. @@ -43,11 +99,59 @@ This is exactly what `core/tests/test_e2e_slice.py` asserts. The script prints a directory; open the `dashboard.html` it writes in any browser to see the run (KPIs, per-iteration diffs, the tasks × iterations heatmap). -## 4. Where to next +## 4. Run the cheap FIRST REAL example (rung 2) + +`toy_calc` proves the machinery but calls no model. This one calls a real one — and +still finishes in minutes for $0, so there is a rung between it and the multi-hour +τ² run. + +The task: normalize a human-written date (`"the 22nd of November, 1963"`) to ISO +(`1963-11-22`), scored by exact match. The seed prompt is a generic "you are a helpful +assistant", so a small model replies in prose and scores 0; the fix is an output +contract — exactly what an optimizer is good at proposing. + +```bash +pip install litellm # the runner's provider shim +ollama pull llama3.2:3b # ~2 GB, free, local + +bash examples/cheap_real/run.sh # rung 2, free variant +``` + +For the paid variant, let a real agent propose the edit instead of `mock` (the runner +stays local and free): + +```bash +CHEAP_REAL_OPTIMIZER=claude-code CHEAP_REAL_OPT_MODEL=claude-haiku-4-5 \ + bash examples/cheap_real/run.sh +``` + +No local model? Point the runner at a cheap hosted one — one env var, no code change. +Needs that provider's credential in the environment (e.g. `ANTHROPIC_API_KEY`). +**MEASURED: 98 s, $0.0115.** + +```bash +CHEAP_REAL_MODEL=claude-haiku-4-5 bash examples/cheap_real/run.sh +``` + +`run.sh` is also the **programmatic entry point**: every knob is an env var with a +default and **all of stdout is the run's summary JSON** — progress goes to stderr +(#116) and the script keeps only the last object, so `json.loads(stdout)` works even +under `CAPEVOLVE_DASHBOARD=auto`, where the CLI prints a second object (#217). + +It preflights **every** variant before spending anything, because the adapter +(correctly) turns a failed model call into reward `0.0`, which otherwise makes a broken +run look like a clean one. Local: the endpoint answers *and* the model is actually +pulled. Hosted: one real 1-token completion through the same wiring the adapter uses, +so a rejected parameter, a missing credential or a typo'd model name fails loudly +instead of scoring 0.0 thirty times. Details, all variants and the honest limits: +[`examples/cheap_real/README.md`](../examples/cheap_real/README.md). + +## 5. Where to next | You want to… | Go to | |---|---| | Understand what cap-evolve optimizes and how | [`../README.md`](../README.md) · [`ARCHITECTURE.md`](ARCHITECTURE.md) | +| See a real (cheap) LLM run end to end | [`../examples/cheap_real/`](../examples/cheap_real/) | | Set up a real optimizer/runner (credentials, dashboard) | [`INSTALL.md`](INSTALL.md) | | Optimize your own agent + benchmark | [`OPTIMIZE_YOUR_OWN.md`](OPTIMIZE_YOUR_OWN.md) | | See real benchmark results | [`RESULTS.md`](RESULTS.md) | diff --git a/docs/OPTIMIZATION_EXAMPLES.md b/docs/OPTIMIZATION_EXAMPLES.md index 32b31ee3..4fcb1165 100644 --- a/docs/OPTIMIZATION_EXAMPLES.md +++ b/docs/OPTIMIZATION_EXAMPLES.md @@ -8,6 +8,11 @@ under the champion `cand_0007`. Rollouts live at `/rollouts/val/____t.json` (full message transcript + `reward_info`); every example below was read directly from those files. +> These come from the **full** ~$148 run. To watch an optimizer propose an edit on a +> real model for cents and minutes first, run rung 2 of the ladder — +> [`examples/cheap_real/`](../examples/cheap_real/) — and read the `candidates/*/` +> diffs it writes. Same mechanism, three orders of magnitude cheaper. + The throughline: **argument-level feedback from failing rollouts is converted into executable guards inside the existing tools** — code where the agent already "knew" the rule but skipped it, prose only for genuine knowledge gaps. Across the run diff --git a/examples/cheap_real/README.md b/examples/cheap_real/README.md new file mode 100644 index 00000000..45d47f94 --- /dev/null +++ b/examples/cheap_real/README.md @@ -0,0 +1,126 @@ +# Example: cheap_real — a REAL LLM run in minutes, for $0 or cents + +The missing middle rung. `toy_calc` is free but calls no model; the τ²-bench airline +example calls a real model but [takes hours and ~$148](../../docs/RESULTS.md). This +one is a **real** run — real LLM, a real accept/reject decision, a real sealed test +split — that finishes in **minutes** and costs **$0** on a local model. + +The task: normalize a human-written date (`"the 22nd of November, 1963"`) to ISO +(`1963-11-22`), scored by exact match. The seed prompt is a generic "you are a helpful +assistant", so a small model answers in prose and scores **0**; the fix is an output +contract, which is exactly the kind of edit an optimizer proposes. + +## The three rungs + +| Rung | Runner | Optimizer | Runtime | Cost | +|---|---|---|---|---| +| free | `ollama/llama3.2:3b` | `mock` | **~30 s** measured (30.4 / 33.9 / 34.2 s on three runs) | **$0** measured | +| hosted-runner | `claude-haiku-4-5` | `mock` | **98 s** measured | **$0.0115** measured | +| cheap-real | `ollama/llama3.2:3b` | `claude-code` (Haiku) | **5.4 min** measured | **$0.54** measured | +| full | tau2-bench airline | `claude-code` | hours | ~$148 | + +All three measured rungs reached the same honest outcome — `baseline_val 0.0` → +`test_reward 1.0` on the sealed test split. The cheap rung's $0.54 is entirely the 3 +proposals ($0.159 / $0.199 / $0.185 from the run's own `opt_cost_usd`); its 30 runner +calls were **metered at $0** because the runner is local — the runner's cost there is +*time* (29.5 s, 3 863 tokens), not zero across the board. The hosted-runner rung is the +mirror image: `mock` proposes for $0 and the 30 runner calls cost $0.0115 in total. + +Derivation and provenance of every figure: [`../../docs/GETTING_STARTED.md`](../../docs/GETTING_STARTED.md). + +## Run it + +```bash +# Rung 1 — free. Needs Ollama (`ollama pull llama3.2:3b`, ~2 GB) and `pip install litellm`. +bash examples/cheap_real/run.sh + +# Rung 2 — real agent proposes the edit; the runner stays free and local. +CHEAP_REAL_OPTIMIZER=claude-code CHEAP_REAL_OPT_MODEL=claude-haiku-4-5 \ + bash examples/cheap_real/run.sh + +# No local model? Use a cheap hosted one as the runner instead. Needs that provider's +# credential in the environment (e.g. ANTHROPIC_API_KEY). MEASURED: 98 s, $0.0115. +CHEAP_REAL_MODEL=claude-haiku-4-5 bash examples/cheap_real/run.sh +``` + +`run.sh` is the **programmatic entry point** — every knob is an env var with a default +and **all of stdout is the run's summary JSON**: progress goes to stderr (#116) and the +script filters `cap-evolve run`'s output down to the last object, so a plain +`json.loads(stdout)` works even with `CAPEVOLVE_DASHBOARD=auto`, which makes the CLI +print a second object (#217). See its header for the full knob list +(`CHEAP_REAL_MODEL`, `CHEAP_REAL_API_BASE`, `CHEAP_REAL_OPTIMIZER`, +`CHEAP_REAL_OPT_MODEL`, `CHEAP_REAL_WORKDIR`, `CHEAP_REAL_MAX_USD`, +`CHEAP_REAL_PYTHON`). + +It **preflights every variant** before spending anything, because the adapter turns a +failed model call into reward `0.0` — a broken run otherwise looks like an honest one. +Local: the endpoint answers *and* the model is actually pulled. Hosted: one real +1-token completion through the same wiring the adapter uses. `run.sh` also exports +`LITELLM_DROP_PARAMS=1`, because the bundled adapter forwards `seed=` and Anthropic +rejects it — without that every hosted rollout scored `0.0` behind a clean-looking run. +`API_BASE` is exported **only for an `ollama/` model**: `model_config.py` reads the +generic `API_BASE` before any provider special-casing, so setting it unconditionally +pointed a hosted model at localhost. + +## Files + +- `capevolve.yaml` — the preset. Only the keys that differ from + [`templates/project/capevolve.yaml`](../../templates/project/capevolve.yaml) are set. + Note `protected_paths` is **omitted, not `[]`** — and what omission buys is not just + the default globs: `protect.py:168-170` resolves an omitted key to + `[*_DEFAULT_GLOBS, *spec['dataset_source']]`. `tasks.jsonl` matches **none** of the + default globs (those are `adapters`, `capevolve.yaml`, and `*gold*` suffixes) — the + answer key is covered *because* `dataset_source` is folded in when the key is absent. + Declare the key and that fold-in never happens. +- `tasks.jsonl` — 20 dates. **20 is a floor, not a preference:** at the default + 0.5/0.25/0.25 ratios that is train 10 / val 5 / test 5, and val 5 is the smallest + split that clears *both* of #113's bars — `MIN_VAL_TASKS = 2` (below which + `gate.decide` refuses outright) and `LOW_CONFIDENCE_VAL_TASKS = 5` (below which + every decision is stamped LOW CONFIDENCE). Shrinking the task count to save money + would buy a cheaper run by giving up the honest gate. +- `capability/prompt.txt` — the seed system prompt that gets optimized. +- `mock_script.json` — the deterministic edit the `mock` optimizer applies, so rung 1 + is reproducible and zero-API. +- `optimizer_INSTRUCTIONS.md` — the per-iteration prompt for the real-agent rung. +- **No adapter.** `run.sh` copies + [`templates/adapters/jsonl_litellm/adapter.py`](../../templates/adapters/jsonl_litellm/adapter.py) + and `model_config.py` verbatim; a JSONL dataset scored by exact match is precisely + what that bundled template is for. + +## Why this is not a benchmark-zoo entry + +The benchmark zoo (`docs/BENCHMARK_ZOO.md`, landing in #233) would be the natural home +— declarative manifest, one code file, a `verify` command. It cannot host this one: +`verify`'s step 5 runs every val task **twice** and fails the benchmark if any rollout +fingerprint differs (`NON-DETERMINISTIC: … cannot produce a reproducible number`). +That guard is correct and worth keeping, but a real LLM cannot satisfy it even at +`temperature=0` and a fixed seed. This example exists to call a real model, so it is +a standalone example and the zoo's exactness guard stays as strict as it is today. + +That is the answer for *this* PR, not the durable one: `cheap_real` is the first of a +class, and every future live-model benchmark hits the same wall. The open shape is an +explicit `determinism: exact | statistical | none` grade in the manifest, where +`statistical` substitutes a variance check for the byte-exact fingerprint and the zoo +*reports* the grade rather than implying everything in it is exact. Deferred to #233 or +a follow-up — not settled. + +## Honest limits + +- The gate reports `SE=0 → STRICT fallback` at `num_trials: 1`, because a single + deterministic-ish trial per task gives the paired gate no variance to work with. + That is the documented behavior, not a defect of the preset — but it means the + accept decision here is "Δ > 0", not a significance test. Raise `num_trials` to get + a real bar, at a proportional increase in runner calls. +- **This run never exercises a rejection on merit.** Val saturates at 1.0 on iteration + 1, so the two candidates that follow are rejected against a ceiling — there was + nothing left to improve, not a plausible candidate found wanting. The gate *ran* and + made real decisions; it was never asked a hard question. A task the model still gets + partly wrong *with* the output contract would land baseline ~0.0 → best ~0.6 at the + same cost and give the gate something to decide. Do not read this example as a + demonstration that the significance machinery works — read `examples/skillsbench` + (`num_trials: 3`) for that. +- `test_pass_k` reports `{"2": 0.0}` even though `num_trials: 1`, so pass^2 is not + measurable here. That is pre-existing #112 (pass^k above `num_trials` should read + N/A), not a property of this preset — ignore the `"2"` entry. +- A 3B local model is a weak reader. The point is to see the machinery work end to + end on a real model, not to produce a publishable number. diff --git a/examples/cheap_real/capability/prompt.txt b/examples/cheap_real/capability/prompt.txt new file mode 100644 index 00000000..0f4d6a5c --- /dev/null +++ b/examples/cheap_real/capability/prompt.txt @@ -0,0 +1 @@ +You are a helpful assistant. Answer the user as best you can. diff --git a/examples/cheap_real/capevolve.yaml b/examples/cheap_real/capevolve.yaml new file mode 100644 index 00000000..5db367e0 --- /dev/null +++ b/examples/cheap_real/capevolve.yaml @@ -0,0 +1,80 @@ +# cheap_real — the "step 2" preset: a REAL LLM run for cents and minutes. +# +# Only the values that differ from templates/project/capevolve.yaml are set; the +# omitted keys keep their documented defaults. In particular `protected_paths` is +# OMITTED, not set to []: since #142/#197 an empty list is a hard error, and omitting +# the key resolves to [*_DEFAULT_GLOBS, *dataset_source] (protect.py:168-170). The +# distinction matters: the default globs are `adapters`, `capevolve.yaml` and *gold* +# suffixes — `tasks.jsonl` matches NONE of them. The answer key is guarded because +# `dataset_source` is folded in when the key is absent, so declaring the key would +# silently drop it. +capabilities: [system-prompt] +capability_path: seed_capability +actions: [edit] + +# --- who proposes edits ---------------------------------------------------- +# `mock` is the zero-cost default so `run.sh` is free out of the box. Set +# CHEAP_REAL_OPTIMIZER=claude-code (see run.sh) for the cents-scale agent rung. +optimizer_skill: mock +# The proposer is set DELIBERATELY to a cheap tier (#132): adding an output-format rule +# to a prompt does not need the frontier model the full tau2 run uses, and Haiku is +# ~5x cheaper per proposal. BOTH keys are set on purpose — `optimizer_model` is what +# ships today, `proposer_model` is #132's tier name, and #132 falls back to +# `optimizer_model` when the tier is blank, so setting both is correct before and +# after that merge. `aux_model` is deliberately NOT set: every auxiliary step in +# cap-evolve is still pure Python, so an aux tier would route nothing and cost $0. +optimizer_model: claude-haiku-4-5 +proposer_model: claude-haiku-4-5 +# Per-iteration WORK cap (claude-code → --max-turns). MEASURED, not guessed: at 12 the +# agent reliably exits `Reached max turns (12)` — a non-zero exit cap-evolve correctly +# reports as a failed iteration, so every candidate was discarded and the run looked +# like "the optimizer proposed nothing" rather than "the cap was too tight". 40 leaves +# room to read the trajectories, edit, and write its journal. +optimizer_max_turns: 40 +# Per-iteration $ cap, enforced by the agent CLI itself (claude-code → +# --max-budget-usd). Set ABOVE the observed ~$0.10-0.40 per proposal on purpose: the +# CLI exits non-zero when it trips this, which cap-evolve correctly reports as a failed +# iteration — a cap set at the expected spend turns normal variance into a dead run. +# `max_optimizer_usd` below is the real ceiling. +optimizer_usd_per_iter: 1.0 + +algorithm_skill: hill-climb +algorithm_focus: all + +# --- data + splits --------------------------------------------------------- +# 20 tasks at the default 0.5/0.25/0.25 → train 10 / val 5 / test 5. Deliberately +# NOT smaller: #113's floor refuses a gate on fewer than MIN_VAL_TASKS=2 matched val +# pairs, and warns below LOW_CONFIDENCE_VAL_TASKS=5. val=5 is the smallest split that +# clears both, so this is the cheapest honest configuration, not merely a cheap one. +# Named as the FILE, not `adapter`: #142/#197's guard hashes the path the spec's +# `dataset_source` names, so naming tasks.jsonl is what gets the answer key covered. +# run.sh copies it inside the project dir for exactly that reason. (Nothing in core +# reads this key at runtime — the adapter loads TASKS_FILE either way — so this is a +# truthful declaration, not a behavior change.) +dataset_source: tasks.jsonl +split_seed: 0 +split_train: 0.5 +split_val: 0.25 +split_test: 0.25 + +num_trials: 1 + +gate_mode: paired +gate_k_se: 1.0 + +# --- budget ---------------------------------------------------------------- +# 30 runner calls, plus 3 proposer calls. The full derivation — the baseline val eval +# and both test evals are easy to forget: +# val 5 x trials 1 x (1 baseline + 3 candidates) = 20 +# test 5 x (best FINAL + baseline FINAL_seed) = 10 +# total = 30 (= state.json metric_calls) +# These are hard stops, so the run cannot become the thing this example exists to +# avoid even if a model is priced differently than assumed. Sized at roughly 2x the +# measured spend (~$0.15-0.40 per Haiku proposal x 3), not at it: a cap set at the +# expected number makes ordinary variance look like a budget failure. +max_iterations: 3 +stall: 2 +max_usd: 3.0 +max_optimizer_usd: 2.5 + +store: copy diff --git a/examples/cheap_real/mock_script.json b/examples/cheap_real/mock_script.json new file mode 100644 index 00000000..61287537 --- /dev/null +++ b/examples/cheap_real/mock_script.json @@ -0,0 +1,5 @@ +{ + "edits": [ + {"file": "prompt.txt", "op": "ensure_contains", "text": "\n[ISO] The user's message contains one date. Reply with ONLY that date in YYYY-MM-DD form: four-digit year, two-digit month, two-digit day, hyphen-separated. No prose, no explanation, no trailing punctuation."} + ] +} diff --git a/examples/cheap_real/optimizer_INSTRUCTIONS.md b/examples/cheap_real/optimizer_INSTRUCTIONS.md new file mode 100644 index 00000000..ee97b33a --- /dev/null +++ b/examples/cheap_real/optimizer_INSTRUCTIONS.md @@ -0,0 +1,34 @@ +# Optimize the date-normalization system prompt + +{{FOCUS_SUMMARY}} + +You are editing **one file**: `prompt.txt` in your working directory. It is the system +prompt of an agent that is shown a date written in some human format and must reply +with that same date in ISO form, `YYYY-MM-DD`. That is the ONLY job. The agent is not +a trivia assistant: nothing about the date's historical significance is wanted, and +any such text makes the answer wrong. + +The scorer is **exact match**, case-insensitively, after stripping whitespace. So a +correct date wrapped in prose scores **0** — a reply must be the bare date and nothing +else. Read `./trajectories/` to see exactly what the agent said instead. + +{{FAILURES}} + +{{TARGET_READER}} + +## Rules +- Edit **only** `prompt.txt`. Nothing else here is yours to change; there is no tool + code in this project, so ignore any generic advice about editing tools. +- **Never** hard-code an answer, a task input, or a specific date into the prompt. The + prompt is scored on held-out dates it has never seen, so a memorized date earns + nothing and is reward hacking — the tamper guard and the sealed test split catch it. +- Prefer one precise, general rule over a list. The reader is a small local model + (~3B): short, unambiguous, imperative instructions land; nuance does not. +- Keep the prompt under ~10 lines, then STOP. The harness re-scores you; do not run + the evaluation yourself. + +## What works here +State the output contract explicitly: the exact format, that it is the *only* thing to +emit, and that no explanation, greeting, or trailing punctuation may be added. Say +"four-digit year, two-digit month, two-digit day" out loud — small models otherwise +emit `2021-3-3`, which exact-match scores 0. diff --git a/examples/cheap_real/run.sh b/examples/cheap_real/run.sh new file mode 100755 index 00000000..4ef492cc --- /dev/null +++ b/examples/cheap_real/run.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# cheap_real — the "step 2" rung: a REAL LLM run in minutes, for $0 (local) or cents. +# +# Usage: cd && bash examples/cheap_real/run.sh +# +# Programmatic entry point (this is the contract #133's quickstart calls). Every knob +# is an env var with a default; the last object on stdout is the run's summary JSON: +# +# CHEAP_REAL_MODEL litellm model string (default ollama/llama3.2:3b — free, local) +# CHEAP_REAL_API_BASE endpoint for that model (default http://localhost:11434 for an +# ollama/ MODEL; UNSET for any other, so a +# hosted model keeps its provider default) +# CHEAP_REAL_OPTIMIZER who proposes the edit (default mock — zero-API/$0) +# CHEAP_REAL_OPT_MODEL optimizer/proposer model (default "" — mock needs none) +# CHEAP_REAL_WORKDIR where to run (default a fresh mktemp dir) +# CHEAP_REAL_MAX_USD hard $ stop (default from capevolve.yaml: 3.0) +# CHEAP_REAL_PYTHON interpreter (default python3 — must have litellm) +# +# The three rungs, and how to get each (costs + derivation in docs/GETTING_STARTED.md): +# free ollama + mock optimizer — the default below +# cheap ollama + a real agent — CHEAP_REAL_OPTIMIZER=claude-code +# hosted a hosted runner model — CHEAP_REAL_MODEL=claude-haiku-4-5 (+ its credential) +set -euo pipefail + +REPO="$(cd "$(dirname "$0")/../.." && pwd)" +EX="$REPO/examples/cheap_real" +PY="${CHEAP_REAL_PYTHON:-python3}" + +export CAPEVOLVE_CORE="$REPO/core" +export PYTHONPATH="$REPO/core" +export CAPEVOLVE_SKILLS_DIR="$REPO/skills" +export CAPEVOLVE_MOCK_SCRIPT="$EX/mock_script.json" + +# The runner's model wiring. `model_config.py` (copied in below) reads these; the +# provider-agnostic MODEL= switch is the whole point — no adapter edit to change model. +export MODEL="${CHEAP_REAL_MODEL:-ollama/llama3.2:3b}" +# API_BASE only for a LOCAL model. `model_config.py:95` reads the GENERIC `API_BASE` +# before any provider special-casing, so exporting it unconditionally pointed a hosted +# MODEL at localhost:11434 — a second silent all-zeros trap on the hosted path. +case "$MODEL" in + ollama/*) export API_BASE="${CHEAP_REAL_API_BASE:-http://localhost:11434}" ;; + *) if [ -n "${CHEAP_REAL_API_BASE:-}" ]; then export API_BASE="$CHEAP_REAL_API_BASE"; else unset API_BASE; fi ;; +esac +export SCORING=exact +# The bundled adapter forwards `seed=` so distinct trials are independent draws, but +# Anthropic rejects it (`UnsupportedParamsError`) — and the adapter turns that into +# reward 0.0, i.e. the whole hosted rung silently scored 0. litellm honours this env var +# by dropping kwargs the provider does not support (`litellm/__init__.py:227`). +# A warning inside the adapter would not have saved us: per #251 the optimizer's stderr +# is discarded on success by three separate layers, so the only signal was the reward. +export LITELLM_DROP_PARAMS=1 + +# Preflight, because the adapter turns a failed model call into `error:` + reward 0.0 — +# correct behavior (infra noise must not be optimized against), but it means a missing +# dep, a stopped Ollama or an unroutable hosted model yields a clean-looking run whose +# every number is 0.0. A real run must be distinguishable from a broken one BEFORE +# anything is spent, on EVERY documented variant — the hosted path was uncovered and +# that is exactly where the bug shipped. The endpoint is never echoed: since #134 a +# non-default base URL is confidential, and neither is any credential. +# +# The whole block is redirected to stderr: it is progress/diagnostic output (#116), and +# litellm writes some of its own error banners straight to stdout, which would otherwise +# land in the caller's JSON stream. +{ "$PY" - "$REPO/templates/adapters" <<'PRE' +import os, sys, urllib.error, urllib.request +try: + import litellm # noqa: F401 +except ImportError: + sys.exit("cheap_real preflight: litellm is not importable by this interpreter. " + "`pip install litellm`, or point CHEAP_REAL_PYTHON at one that has it.") +MODEL = os.environ.get("MODEL", "") +if MODEL.startswith("ollama/"): + base, model = os.environ["API_BASE"].rstrip("/"), MODEL.split("/", 1)[1] + try: + with urllib.request.urlopen(base + "/api/tags", timeout=5) as r: + names = {m.get("name", "") for m in __import__("json").load(r).get("models", [])} + except (urllib.error.URLError, OSError, ValueError) as e: + sys.exit(f"cheap_real preflight: the local Ollama endpoint did not answer " + f"({type(e).__name__}). Start it (`ollama serve`) or set CHEAP_REAL_MODEL " + f"to a hosted model. Endpoint withheld (see docs/TROUBLESHOOTING.md).") + if model not in names and f"{model}:latest" not in names: + sys.exit(f"cheap_real preflight: Ollama is up but does not have {model!r}. " + f"Run `ollama pull {model}` (~2 GB), or set CHEAP_REAL_MODEL.") +else: + # Hosted: one real 1-token completion through the SAME wiring the adapter uses + # (model_config.llm_kwargs() + the seed kwarg), so a rejected param, a missing + # credential or a bad model name fails here instead of scoring 0.0 thirty times. + # Costs a fraction of a cent — far less than a full silently-dead run. + sys.path.insert(0, sys.argv[1]) + import model_config + try: + litellm.completion(model=MODEL, messages=[{"role": "user", "content": "ping"}], + max_tokens=1, seed=0, **model_config.llm_kwargs()) + except Exception as e: # noqa: BLE001 — any failure here means the run would be 0.0 + msg = str(e) + for secret in (os.environ.get("API_BASE"), os.environ.get("ANTHROPIC_BASE_URL"), + os.environ.get("API_KEY"), os.environ.get("ANTHROPIC_API_KEY"), + os.environ.get("ANTHROPIC_AUTH_TOKEN"), os.environ.get("OPENAI_API_KEY")): + if secret: + msg = msg.replace(secret, "") + sys.exit(f"cheap_real preflight: the hosted model {MODEL!r} did not answer a " + f"1-token probe ({type(e).__name__}). Every rollout would score 0.0. " + f"Check the model name and its credential; endpoint/keys withheld " + f"(see docs/TROUBLESHOOTING.md). Detail: {msg[:300]}") +print("cheap_real preflight: OK", file=sys.stderr) +PRE +} >&2 + +D="${CHEAP_REAL_WORKDIR:-$(mktemp -d -t cheap_real.XXXXXX)}" +mkdir -p "$D/.capevolve/project/adapters" +# Reuse the bundled generic template verbatim — this example adds NO adapter code. +# Both files land in adapters/ and the dataset inside the project dir, deliberately: +# #142/#197's guard can only hash paths under the project dir, and `adapters/` + +# the spec's `dataset_source` are two of its layout DEFAULTS. So every file the +# grader depends on is protected without the preset declaring `protected_paths` — +# which is what lets it omit that key (an empty list is now a hard error). +cp "$REPO/templates/adapters/jsonl_litellm/adapter.py" "$D/.capevolve/project/adapters/" +cp "$REPO/templates/adapters/model_config.py" "$D/.capevolve/project/adapters/" +cp "$EX/tasks.jsonl" "$D/.capevolve/project/tasks.jsonl" +cp -R "$EX/capability" "$D/.capevolve/project/seed_capability" +cp "$EX/capevolve.yaml" "$D/.capevolve/project/capevolve.yaml" +export TASKS_FILE="$D/.capevolve/project/tasks.jsonl" + +OPT="${CHEAP_REAL_OPTIMIZER:-mock}" +if [ "$OPT" != "mock" ]; then + # Flip the two optimizer keys in place; everything else in the preset is unchanged. + cp "$EX/optimizer_INSTRUCTIONS.md" "$D/.capevolve/project/INSTRUCTIONS.md" + "$PY" - "$D/.capevolve/project/capevolve.yaml" "$OPT" "${CHEAP_REAL_OPT_MODEL:-}" <<'SED' +import pathlib, sys +p = pathlib.Path(sys.argv[1]); opt, model = sys.argv[2], sys.argv[3] +out = [] +for line in p.read_text(encoding="utf-8").splitlines(True): + if line.startswith("optimizer_skill:"): + line = f"optimizer_skill: {opt}\n" + elif model and line.startswith(("optimizer_model:", "proposer_model:")): + line = f"{line.split(':', 1)[0]}: {model}\n" + elif line.startswith("optimizer_instructions_file:"): + continue + out.append(line) +# ABSOLUTE, deliberately — this is the workaround for #252. cli.py resolves a relative +# `optimizer_instructions_file` against ITS OWN cwd and then against a cwd-relative +# `.capevolve/project`, neither of which is this run's project dir when the workdir is +# elsewhere. A relative path there silently falls back to the GENERIC template — the +# optimizer then gets tau2-flavored "edit the tool code" advice for a project that has +# no tools, and proposes a prompt for the wrong task entirely. Observed; not +# hypothetical. Worse (per #252): pipeline_selftest.py:73 resolves the SAME key +# project-relative and *reports a problem*, so `cap-evolve check` passes what +# `cap-evolve run` silently ignores. An absolute path satisfies both resolvers. +out.append(f"optimizer_instructions_file: {p.parent / 'INSTRUCTIONS.md'}\n") +p.write_text("".join(out), encoding="utf-8") +SED +fi + +# Progress goes to STDERR, per #116's convention, so ALL of stdout is parseable JSON. +echo "Working directory: $D" >&2 +echo "Runner model: $MODEL Optimizer: $OPT" >&2 + +# The last-object filter is what makes the contract hold unconditionally. `cap-evolve run` +# prints a SECOND object on stdout when the dashboard mode is `auto` (#217: cli.py:205-207 +# prints the launch status there instead of stderr), so a caller who sets +# CAPEVOLVE_DASHBOARD=auto would otherwise get two objects and a broken parse. The `off` +# default alone is not the guard — this filter is. Delete it when #217 lands. +"$PY" -m cap_evolve.cli run \ + --spec "$D/.capevolve/project/capevolve.yaml" \ + --project "$D/.capevolve/project" \ + --run-ts cheap \ + ${CHEAP_REAL_MAX_USD:+--max-usd "$CHEAP_REAL_MAX_USD"} \ + --dashboard "${CAPEVOLVE_DASHBOARD:-off}" \ + | "$PY" -c ' +import json, sys +text, dec, i, last = sys.stdin.read(), json.JSONDecoder(), 0, None +while i < len(text): + try: + last, i = dec.raw_decode(text, i) + except ValueError: + i += 1 +if last is not None: + print(json.dumps(last, indent=2)) +' diff --git a/examples/cheap_real/tasks.jsonl b/examples/cheap_real/tasks.jsonl new file mode 100644 index 00000000..9f44159d --- /dev/null +++ b/examples/cheap_real/tasks.jsonl @@ -0,0 +1,20 @@ +{"id": "d01", "input": "March 3rd, 2021", "target": "2021-03-03"} +{"id": "d02", "input": "the 5th of July 1999", "target": "1999-07-05"} +{"id": "d03", "input": "12/25/2020", "target": "2020-12-25"} +{"id": "d04", "input": "Jan 1 2000", "target": "2000-01-01"} +{"id": "d05", "input": "29 Feb 2024", "target": "2024-02-29"} +{"id": "d06", "input": "7 August 1985", "target": "1985-08-07"} +{"id": "d07", "input": "October 31, 1993", "target": "1993-10-31"} +{"id": "d08", "input": "2/9/2011", "target": "2011-02-09"} +{"id": "d09", "input": "the 22nd of November, 1963", "target": "1963-11-22"} +{"id": "d10", "input": "Sept 30 2018", "target": "2018-09-30"} +{"id": "d11", "input": "1 Apr 2005", "target": "2005-04-01"} +{"id": "d12", "input": "June 6th 1944", "target": "1944-06-06"} +{"id": "d13", "input": "11/11/1918", "target": "1918-11-11"} +{"id": "d14", "input": "20 January 2009", "target": "2009-01-20"} +{"id": "d15", "input": "Dec 8, 1980", "target": "1980-12-08"} +{"id": "d16", "input": "the 4th of March 1997", "target": "1997-03-04"} +{"id": "d17", "input": "5/17/2016", "target": "2016-05-17"} +{"id": "d18", "input": "14 July 1789", "target": "1789-07-14"} +{"id": "d19", "input": "Feb 2 2002", "target": "2002-02-02"} +{"id": "d20", "input": "August 15th, 1947", "target": "1947-08-15"} diff --git a/site/getting-started.html b/site/getting-started.html index c25810eb..a78feba5 100644 --- a/site/getting-started.html +++ b/site/getting-started.html @@ -107,8 +107,50 @@

What you saw

+
+

Run the cheap FIRST REAL example

+

+ toy_calc proves the machinery but calls no model. This is the rung between it + and the multi-hour benchmark run: a real LLM, a real accept/reject decision + and a real sealed test split, in minutes, for $0 on a local model. The task is + normalizing a human-written date ("the 22nd of November, 1963") to ISO + (1963-11-22), scored by exact match. +

+
pip install litellm            # the runner's provider shim
+ollama pull llama3.2:3b        # ~2 GB, free, local
+
+bash examples/cheap_real/run.sh
+

+ To have a real agent propose the edit instead of the deterministic mock + optimizer (the runner stays local and free — measured ~$0.15–0.35 per iteration): +

+
CHEAP_REAL_OPTIMIZER=claude-code CHEAP_REAL_OPT_MODEL=claude-haiku-4-5 \
+  bash examples/cheap_real/run.sh
+

+ No local model? CHEAP_REAL_MODEL=claude-haiku-4-5 points the runner at a cheap + hosted one instead — one env var, no code change, plus that provider's credential in the + environment. Measured: 98 s, $0.0115 for the whole run. +

+
+ +

The three rungs, with costs

+

+ So "see it work" and "run it for real" are not the same decision. Every figure is either + measured on the run it describes or labelled an estimate with its arithmetic — the full + derivation is in + docs/GETTING_STARTED.md. +

+ + + + + + + +
RungWhat is realRuntimeCost
1. free — toy_calcpipeline, gate, sealed test — no modelseconds$0
2. cheap-real — cheap_reala real LLM, a real accept/reject decision, a real sealed test~30 s – 5.4 min measured$0 local, $0.0115 with a hosted runner, or $0.54 with a hosted proposer — all measured
3. full — τ²-bench airlinea published benchmarkhours~$148
+

Where to next