diff --git a/deploy/harbor/gen_tb_docs.py b/deploy/harbor/gen_tb_docs.py new file mode 100644 index 0000000..378f5e3 --- /dev/null +++ b/deploy/harbor/gen_tb_docs.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Generate the Terminal-Bench 2.0 BASELINE results doc from a rows.json + task +metadata, mirroring the SWE-bench result pages (totals + cache-aware cost model + +per-task table) but adding TB-specific breakdowns by difficulty and category and a +first-class treatment of timeouts — a baseline characterization of *where* the +claude-code agent is strong/weak, so the later compaction arms have a like-for-like +reference. + +All 89 attempted tasks are included. A task that hit AgentTimeout/VerifierTimeout is a +genuine failure (reward 0, agent did not finish within its 1.5× wall-clock budget on +this ~26 s/request gateway) and is flagged as a `timeout`; its partial spend still +counts toward cost. The primary solve rate is solved / attempted (the standard +Terminal-Bench metric); a secondary rate over completed-only tasks is also reported. + +Usage: + gen_tb_docs.py [--meta /tmp/tb-runs/task_meta.json] [--summary summary.json] +""" +import argparse, json +from collections import defaultdict + +IN, OUT, CREAD, CWRITE = 2e-6, 10e-6, 0.2e-6, 2.5e-6 # same cache-aware price model as the SWE study + + +def billed(r): + return (r.get("fresh_input", 0) * IN + r.get("cache_read", 0) * CREAD + + r.get("cache_write", 0) * CWRITE + r.get("completion_tokens", 0) * OUT) + + +def agg(rows): + n = len(rows) + solved = sum(1 for r in rows if (r.get("reward") or 0) >= 1) + cost = sum(billed(r) for r in rows) + steps = sum(r.get("steps", 0) or 0 for r in rows) + wall = sum(r.get("agent_wall_s", 0) or 0 for r in rows) + return n, solved, cost, steps, wall + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("rows"); ap.add_argument("out") + ap.add_argument("--meta", default="/tmp/tb-runs/task_meta.json") + ap.add_argument("--summary", default="") + ap.add_argument("--label", default="baseline", help="arm label for the page header") + ap.add_argument("--kind", default="baseline", choices=["baseline", "arm"], + help="baseline emits the full narrative; arm emits a compact per-arm page") + a = ap.parse_args() + rows = json.load(open(a.rows)) + rows.sort(key=lambda r: r["task"]) + meta = {} + try: + meta = json.load(open(a.meta)) + except Exception: + pass + for r in rows: + m = meta.get(r["task"], {}) + r["_diff"] = m.get("difficulty", "unknown") + r["_cat"] = m.get("category", "unknown") + r["_timeout"] = bool(r.get("exception")) + + scored = [r for r in rows if not r["_timeout"]] # completed within budget + timeouts = [r for r in rows if r["_timeout"]] + + is_base = a.kind == "baseline" + L = [] + L.append(f"# Full results — {a.label} (Terminal-Bench 2.0, 89 tasks)\n") + if is_base: + L.append("Baseline arm: **no compaction** — the `claude-code` agent on `aws/claude-sonnet-5`, " + "run LIVE through the harness against Terminal-Bench 2.0's 89 tasks. Routing goes " + "through the context-guru `off` transparent passthrough proxy (identical plumbing to " + "the compaction arms; zero content change), so this is the like-for-like reference the " + "framework arms are measured against. Cache-aware billed input cost (fresh $2/M · " + "cache-read $0.20/M · cache-write $2.50/M) + output $10/M, recomputed from each trial's " + "own token tiers — the same model as the SWE-bench study. See [REPRODUCE.md](REPRODUCE.md).\n") + else: + L.append(f"Full per-task results for the **{a.label}** arm on Terminal-Bench 2.0 (`claude-code` on " + "`aws/claude-sonnet-5`, live). Same cache-aware cost model and 4× budget as the other arms. " + "For the four-way analysis (cost decomposition, per-component, verdict) see the " + "**[Terminal-Bench comparison](terminal-bench-comparison.md)**; the reference arm is the " + "**[baseline](terminal-bench-baseline.md)**. See [REPRODUCE.md](REPRODUCE.md).\n") + + n = len(rows) + solved = sum(1 for r in rows if (r.get("reward") or 0) >= 1) + cost = sum(billed(r) for r in rows) + steps_c = sum(r.get("steps", 0) or 0 for r in scored) + tcr = sum(r.get("cache_read", 0) for r in rows) + tcw = sum(r.get("cache_write", 0) for r in rows) + tfresh = sum(r.get("fresh_input", 0) for r in rows) + tout = sum(r.get("completion_tokens", 0) for r in rows) + hit = 100 * tcr / max(tcr + tcw + tfresh, 1) + wall_c = sum(r.get("agent_wall_s", 0) or 0 for r in scored) + + L.append("## Totals\n") + L.append("| attempted | solved | solve rate | completed | timed out | total billed cost | mean steps* | cache-hit |") + L.append("|--:|--:|--:|--:|--:|--:|--:|--:|") + L.append(f"| {n} | {solved} | **{solved/max(n,1):.1%}** | {len(scored)} | {len(timeouts)} | " + f"${cost:.2f} | {steps_c/max(len(scored),1):.1f} | {hit:.1f}% |") + L.append(f"\n\\* mean steps over the {len(scored)} completed tasks (timed-out runs are truncated). " + f"Solve rate over **completed-only** tasks: **{sum(1 for r in scored if (r.get('reward') or 0)>=1)}/{len(scored)} " + f"= {sum(1 for r in scored if (r.get('reward') or 0)>=1)/max(len(scored),1):.1%}**.\n") + if is_base: + L.append("**Time-budget policy.** Wall-clock budget = the task-authored timeout × a multiplier. Most tasks " + "ran at **1.5×**; the long-horizon tasks that first timed out were retried at low concurrency and, " + "if still short, given an extended **4×** budget (up to ~4 h) to measure capability rather than a " + "latency-truncated result. 3 tasks solved only under the 4× budget (counted as solved here); the " + f"{len(timeouts)} below exhausted even 4×.\n") + + L.append("### Token & cost accounting (cache-aware, all 89 tasks)\n") + L.append("| tier | tokens | $/M | billed |") + L.append("|---|--:|--:|--:|") + L.append(f"| cache-read (input) | {tcr:,} | 0.20 | ${tcr*CREAD:.2f} |") + L.append(f"| cache-write (input) | {tcw:,} | 2.50 | ${tcw*CWRITE:.2f} |") + L.append(f"| fresh (input) | {tfresh:,} | 2.00 | ${tfresh*IN:.2f} |") + L.append(f"| completion (output) | {tout:,} | 10.00 | ${tout*OUT:.2f} |") + L.append(f"| **total** | | | **${cost:.2f}** |") + L.append(f"\nCache-read is **{100*tcr*CREAD/max(cost,1e-9):.0f}%** of the bill at a **{hit:.1f}%** cache-hit " + "rate — as on SWE-bench, a heavily-cached agent, so the lever a compaction layer must pull is " + "cache-read tokens.\n") + + # timeouts — first-class bucket + L.append(f"## Timeouts ({len(timeouts)} long-horizon tasks)\n") + L.append("These tasks still hit the wall-clock budget under the **extended 4×** timeout (up to ~4 h each) " + "and scored **reward 0** — counted as failures in the solve rate above. A large part of the cause " + "is **gateway latency, not only agent capability**: Terminal-Bench's timeouts assume a fast " + "endpoint (~2–5 s/request), but this IBM LiteLLM gateway runs **~26 s/request** (5–10× slower), so " + "long-horizon tasks that need many round-trips run out of clock (concurrency is *not* the cause — " + "latency was flat ~23–30 s/req from n=1 to n=24). They are all `hard`/long software-engineering " + "and compute tasks (path-tracing, a MIPS Doom port, a metacircular evaluator, COBOL modernization, " + "GPT-2 code-golf, CIFAR training). A compaction arm that cuts round-trips could bring some under " + "budget, so the timeout count is itself a comparison metric.\n") + L.append("| task | difficulty | category | steps before timeout | partial billed | budget (4×) |") + L.append("|---|---|---|--:|--:|--:|") + for r in sorted(timeouts, key=lambda x: x["task"]): + bud = (meta.get(r["task"], {}) or {}).get("agent_timeout_sec") + bud = f"{bud*4/60:.0f} min" if bud else "—" + L.append(f"| {r['task']} | {r['_diff']} | {r['_cat']} | {r.get('steps')} | ${billed(r):.2f} | {bud} |") + + # by difficulty (all 89; timeouts count as failures) + L.append("\n## By difficulty (all 89 tasks; timeouts = failures)\n") + L.append("| difficulty | tasks | solved | rate | timed out | mean $/task |") + L.append("|---|--:|--:|--:|--:|--:|") + order = {"easy": 0, "medium": 1, "hard": 2, "unknown": 3} + byd = defaultdict(list) + for r in rows: + byd[r["_diff"]].append(r) + for d in sorted(byd, key=lambda x: order.get(x, 9)): + rs = byd[d] + nn, sv, cc, st, wl = agg(rs) + to = sum(1 for r in rs if r["_timeout"]) + L.append(f"| {d} | {nn} | {sv} | {sv/max(nn,1):.0%} | {to} | ${cc/max(nn,1):.3f} |") + + # by category + L.append("\n## By category (all 89 tasks)\n") + L.append("| category | tasks | solved | rate | mean $/task | mean steps* |") + L.append("|---|--:|--:|--:|--:|--:|") + byc = defaultdict(list) + for r in rows: + byc[r["_cat"]].append(r) + for c in sorted(byc, key=lambda x: (-agg(byc[x])[1] / max(len(byc[x]), 1), x)): + rs = byc[c] + nn, sv, cc, st, wl = agg(rs) + rs_c = [r for r in rs if not r["_timeout"]] + st_c = sum(r.get("steps", 0) or 0 for r in rs_c) + L.append(f"| {c} | {nn} | {sv} | {sv/max(nn,1):.0%} | ${cc/max(nn,1):.3f} | " + f"{st_c/max(len(rs_c),1):.1f} |") + + # per-task (all 89) + L.append("\n## Per-task (all 89)\n") + L.append("| task | difficulty | category | outcome | steps | cache_read | cache_write | billed | wall |") + L.append("|---|---|---|:--:|--:|--:|--:|--:|--:|") + for r in rows: + w = r.get("agent_wall_s") + if r["_timeout"]: + outcome = "⏱ timeout" + elif (r.get("reward") or 0) >= 1: + outcome = "✅ solved" + else: + outcome = "❌ failed" + L.append(f"| {r['task']} | {r['_diff']} | {r['_cat']} | {outcome} | " + f"{r.get('steps')} | {r.get('cache_read',0):,} | {r.get('cache_write',0):,} | " + f"${billed(r):.3f} | {(str(round(w/60,1))+' min') if w else '—'} |") + + open(a.out, "w").write("\n".join(L) + "\n") + print(f"wrote {a.out}: {n} attempted, {solved} solved ({solved/max(n,1):.1%}), " + f"{len(scored)} completed, {len(timeouts)} timeout, ${cost:.2f}") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/swebench.py b/deploy/harbor/swebench.py index 42c0251..d0e5711 100644 --- a/deploy/harbor/swebench.py +++ b/deploy/harbor/swebench.py @@ -108,6 +108,12 @@ def stop_proxy(): " llm_every_n_requests: 1\n" " llm_max_per_request: 4\n" ), + # cacheinject ALONE. Isolates the prompt-cache lever from token reduction: no + # component here removes a single content token, so any cost delta vs `off` is + # purely breakpoint placement. This is the arm that tests whether cacheinject + # earns its place — the offline model says placement headroom against + # claude-code's own breakpoints is ~0%, and this is the live check of that. + "cacheonly": "pipeline: [cacheinject]\n", # conservative deterministic-only (no LLM, no mask): safe control "codesafe": ( "pipeline: [format, dedup, failed_run, cmdfilter, extract, collapse, cacheinject]\n" @@ -272,8 +278,13 @@ def main(): for cfg in a.configs: jobs = f"{a.jobs_root}/{cfg}" subprocess.run(f"rm -rf {jobs}", shell=True) - cap = f"/tmp/cg-runs/capture-swebench.jsonl" if cfg == a.capture_config else None - dump = f"/tmp/cg-runs/dump-swebench-{cfg}.jsonl" if cfg in a.dump_configs else None + # Captures go under the RUN's jobs_root, not a shared /tmp path. A fixed path + # is silently destructive: start_proxy unlinks it, so launching any new run + # truncates the capture an earlier analysis was computed from — which is + # exactly how the 472-request capture behind docs/cache-optimization.md was + # lost mid-analysis. Run-scoped means a new run can never clobber an old one. + cap = f"{a.jobs_root}/capture-{cfg}.jsonl" if cfg == a.capture_config else None + dump = f"{a.jobs_root}/dump-{cfg}.jsonl" if cfg in a.dump_configs else None print(f"### config={cfg} (capture={'yes' if cap else 'no'} dump={'yes' if dump else 'no'}) ...", flush=True) start_proxy(cfg, base, token, capture=cap, dump=dump) t0 = time.time() diff --git a/deploy/harbor/terminalbench.py b/deploy/harbor/terminalbench.py new file mode 100644 index 0000000..278c474 --- /dev/null +++ b/deploy/harbor/terminalbench.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Terminal-Bench 2.0 benchmark harness: run the baseline (`off` transparent +passthrough) — and, later, context-guru/headroom/rtk arms — LIVE through the proxy +with the claude-code agent on aws/claude-sonnet-5, collecting the SAME full metric +set as the SWE-bench study (reward, steps, wall-time, cache-aware token/cost +accounting, cache-hit rate). + +This is a thin adaptation of `swebench.py`: the ONLY benchmark-specific change is the +Harbor dataset (`terminal-bench@2.0` instead of `swebench-verified@1.0`) and the +default jobs-root. The claude-code trajectory parser, the cache-aware cost model, and +the summarizer are agent-specific (not benchmark-specific), so every number is +computed identically to the SWE arms and is directly comparable in methodology. + +Baseline routing note: like the SWE baseline arm, "baseline" runs through the `off` +passthrough proxy on :4000 (transparent — no compaction) so routing/model-forcing is +byte-identical to how the compaction arms will later run. The only difference between +baseline and a framework arm is the compaction, never the plumbing. + +Usage: + # 1-task smoke: + python3 deploy/harbor/terminalbench.py --tasks /tmp/tb-runs/tb1.txt --configs off \ + --jobs-root /tmp/tb-runs/smoke --n 1 + # full 89-task baseline: + python3 deploy/harbor/terminalbench.py --tasks /tmp/tb-runs/tb89.txt --configs off \ + --jobs-root /tmp/tb-runs/tb89 --n 2 +""" +import argparse, glob, json, os, subprocess, sys, time, urllib.request +from pathlib import Path + +CG = Path("/home/vpcuser/projects/context-engineering/context-guru") +HB = Path("/home/vpcuser/projects/context-engineering/harbor") +BIN = "/tmp/cg-runs/cg-proxy-d1" +PORT = 4000 +LAN = "9.47.170.83" +DATASET = "terminal-bench@2.0" # <-- the only benchmark-specific difference vs swebench.py +PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +MODEL = "aws/claude-sonnet-5" +CHEAP_MODEL = "aws/claude-haiku-4-5" # for CG's own compaction LLM (extract_llm) in later arms + + +def creds(): + e = json.load(open(Path("~/.claude/settings.json").expanduser()))["env"] + return e["ANTHROPIC_BASE_URL"], e["ANTHROPIC_AUTH_TOKEN"] + + +def price(model): + fb = (2e-6, 1e-5, 2e-7, 2.5e-6) # in, out, cache_read, cache_write(≈1.25×in) + try: + d = json.load(urllib.request.urlopen(PRICES_URL, timeout=15)) + c = d.get(model) or d.get(model.split("/")[-1]) + if c: + return (c.get("input_cost_per_token") or fb[0], c.get("output_cost_per_token") or fb[1], + c.get("cache_read_input_token_cost") or fb[2], + c.get("cache_creation_input_token_cost") or (c.get("input_cost_per_token") or fb[0]) * 1.25) + except Exception as e: + print(f"[price] {e}; fallback", file=sys.stderr) + return fb + + +def stop_proxy(): + for _ in range(3): + subprocess.run("pkill -x cg-proxy-d1", shell=True) + time.sleep(1) + r = subprocess.run("pgrep -x cg-proxy-d1", shell=True, capture_output=True) + if not r.stdout.strip(): + return + time.sleep(2) # let the port fully release (avoids bind race on restart) + + +# Custom (non-preset) configs for the LATER framework arms — kept identical to the SWE +# harness so a terminal-bench framework run is a one-flag change. Baseline uses `off`. +CUSTOM_CONFIGS = { + # cacheinject ALONE. Removes no content tokens, so any delta vs `off` is purely + # cache mechanics (breakpoint placement + the cross-session prefix repairs that + # apply/prefixorder.go gates on this component). Confirmed by + # proxy_tokens_before == proxy_tokens_after in the summary. + "cacheonly": "pipeline: [cacheinject]\n", + "codesmart": ( + "pipeline: [format, dedup, failed_run, cmdfilter, extract_llm, extract, cacheinject]\n" + "components:\n" + " extract:\n" + " min_tokens: 400\n" + " extract_llm:\n" + " strategy: code\n" + " model:\n" + " source: config\n" + " min_tokens: 3000\n" + " trigger:\n" + " min_request_tokens: 3000\n" + " llm_every_n_requests: 1\n" + " llm_max_per_request: 4\n" + ), +} + + +def start_proxy(preset, base, token, capture=None, dump=None): + stop_proxy() + env = dict(os.environ, ANTHROPIC_UPSTREAM=base, ANTHROPIC_API_KEY=token, + OPENAI_UPSTREAM=base, OPENAI_API_KEY=token, FORCE_MODEL=MODEL, + LISTEN_ADDR=f":{PORT}", INJECT_EXPAND="auto", CONTEXT_GURU_DEBUG="1", + CHEAP_MODEL=CHEAP_MODEL, CHEAP_MODEL_PROVIDER="anthropic", + CHEAP_MODEL_BASE=base, CHEAP_MODEL_KEY=token, CHEAP_MODEL_AUTH="bearer") + if capture: + env["CONTEXT_GURU_CAPTURE"] = capture + Path(capture).unlink(missing_ok=True) + if dump: + env["CONTEXT_GURU_DUMP"] = dump + Path(dump).unlink(missing_ok=True) + log = open(f"/tmp/tb-runs/proxy-terminalbench-{preset}.log", "w") + PRESETS = {"off", "safe", "balanced", "aggressive", "coding", "mcp", "agent", "general"} + if preset in CUSTOM_CONFIGS: + cfgp = f"/tmp/tb-runs/cfg-{preset}.yaml" + Path(cfgp).write_text(CUSTOM_CONFIGS[preset]) + args = [BIN, "--config", cfgp] + elif preset in PRESETS: + args = [BIN, "--preset", preset] + else: # single component name -> one-component pipeline + cfgp = f"/tmp/tb-runs/cfg-{preset}.yaml" + Path(cfgp).write_text(f"pipeline: [{preset}]\n") + args = [BIN, "--config", cfgp] + p = subprocess.Popen(args, cwd=str(CG), stdout=log, stderr=log, + env=env, preexec_fn=os.setsid) + for _ in range(30): + try: + urllib.request.urlopen(f"http://localhost:{PORT}/healthz", timeout=3).read() + return p + except Exception: + time.sleep(0.5) + raise RuntimeError(f"proxy for {preset} did not come up") + + +def run_harbor(tasks, jobs_dir, n, setup_mult, build_mult, agent_mult, max_retries=2): + proxy_url = f"http://{LAN}:{PORT}/anthropic" + inc = " ".join(f"-i {t}" for t in tasks) + home = os.path.expanduser("~") + abs_path = f"{home}/.local/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + cmd = (f"cd {HB} && ANTHROPIC_BASE_URL='{proxy_url}' ANTHROPIC_API_KEY='sk-proxy' " + f"ANTHROPIC_AUTH_TOKEN='sk-proxy' PATH='{abs_path}' HOME='{home}' " + f"{home}/.local/bin/uv run harbor run -y -d {DATASET} -a claude-code -m '{MODEL}' " + f"--env docker {inc} -n {n} --jobs-dir '{jobs_dir}' " + # --no-delete keeps each task's image after the trial so its base layers are + # NOT re-pulled on later runs — avoids exhausting the Docker Hub anonymous quota. + f"--no-delete " + f"--agent-setup-timeout-multiplier {setup_mult} --environment-build-timeout-multiplier {build_mult} " + f"--agent-timeout-multiplier {agent_mult} --max-retries {max_retries} " + f"--ae ANTHROPIC_BASE_URL='{proxy_url}' --ae ANTHROPIC_API_KEY='sk-proxy' --ae ANTHROPIC_AUTH_TOKEN='sk-proxy'") + log = f"/tmp/tb-runs/run-terminalbench-{Path(jobs_dir).name}.log" + with open(log, "w") as f: + subprocess.run(["sg", "docker", "-c", cmd], stdout=f, stderr=f) + return log + + +def iso(s): + from datetime import datetime + return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +def parse_trials(jobs_dir, pr): + rows = [] + for rf in glob.glob(f"{jobs_dir}/*/*/result.json"): + try: + d = json.load(open(rf)) + except Exception: + continue + if "verifier_result" not in d: + continue # job-summary file, skip + tdir = Path(rf).parent + reward = ((d.get("verifier_result") or {}).get("rewards") or {}).get("reward") + fm = {} + traj = tdir / "agent" / "trajectory.json" + if traj.exists(): + try: + fm = (json.load(open(traj)) or {}).get("final_metrics") or {} + except Exception: + fm = {} + ex = fm.get("extra") or {} + pt = fm.get("total_prompt_tokens") or 0 + ct = fm.get("total_completion_tokens") or 0 + cached = fm.get("total_cached_tokens") or 0 + cwrite = ex.get("total_cache_creation_input_tokens") or 0 + cread = ex.get("total_cache_read_input_tokens") or cached + fresh = max(pt - cread - cwrite, 0) # uncached fresh input + norm_cost = fresh * pr[0] + ct * pr[1] + cread * pr[2] + cwrite * pr[3] + wall = None + try: + wall = iso(d["finished_at"]) - iso(d["started_at"]) + except Exception: + pass + agent_wall = None + try: + ae = d.get("agent_execution") or {} + agent_wall = iso(ae["finished_at"]) - iso(ae["started_at"]) + except Exception: + pass + rows.append(dict(task=d.get("task_name", tdir.parent.name), reward=reward, + steps=fm.get("total_steps"), prompt_tokens=pt, completion_tokens=ct, + cached_tokens=cached, cache_read=cread, cache_write=cwrite, fresh_input=fresh, + agent_cost=fm.get("total_cost_usd"), norm_cost=round(norm_cost, 5), + wall_s=round(wall, 1) if wall else None, + agent_wall_s=round(agent_wall, 1) if agent_wall else None, + exception=bool(d.get("exception_info")))) + return rows + + +def summarize(cfg, rows): + n = len(rows) + got = [r for r in rows if r["reward"] is not None] + solved = sum(1 for r in got if r["reward"] and r["reward"] >= 1) + def avg(k): + vs = [r[k] for r in rows if isinstance(r.get(k), (int, float))] + return round(sum(vs) / len(vs), 3) if vs else None + tot = lambda k: sum(r[k] for r in rows if isinstance(r.get(k), (int, float))) + cacheable = tot("cache_read") + tot("fresh_input") + tot("cache_write") + return dict(config=cfg, trials=n, scored=len(got), solved=solved, + solve_rate=round(solved / len(got), 3) if got else None, + exceptions=sum(1 for r in rows if r["exception"]), + mean_steps=avg("steps"), mean_prompt_tokens=avg("prompt_tokens"), + mean_completion_tokens=avg("completion_tokens"), + cache_hit_rate=round(tot("cache_read") / cacheable, 4) if cacheable else None, + total_fresh_input=tot("fresh_input"), total_cache_read=tot("cache_read"), + total_cache_write=tot("cache_write"), total_completion=tot("completion_tokens"), + total_norm_cost=round(tot("norm_cost"), 4), mean_norm_cost=avg("norm_cost"), + mean_agent_cost=avg("agent_cost"), mean_wall_s=avg("wall_s"), + mean_agent_wall_s=avg("agent_wall_s")) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--tasks", required=True) + ap.add_argument("--configs", nargs="+", default=["off"]) + ap.add_argument("--jobs-root", default="/tmp/tb-runs/tb89") + ap.add_argument("--n", type=int, default=2, help="harbor concurrency") + ap.add_argument("--setup-mult", type=float, default=4.0, help="agent-setup timeout multiplier") + ap.add_argument("--build-mult", type=float, default=4.0, help="environment-build timeout multiplier") + # keep methodology identical to the SWE study (setup 4 / build 4 / agent 1.5) so the two + # benchmarks are directly comparable; the task.yaml's own max_agent_timeout_sec still applies. + ap.add_argument("--agent-mult", type=float, default=1.5, help="agent-execution timeout multiplier") + ap.add_argument("--max-retries", type=int, default=2, help="harbor per-trial retries (bump under high concurrency to absorb transient 429s)") + ap.add_argument("--capture-config", default=None, help="which config also captures the stream for replay") + ap.add_argument("--dump-configs", nargs="*", default=[], help="configs that DUMP before→after change logs") + a = ap.parse_args() + base, token = creds() + pr = price(MODEL) + cpr = price(CHEAP_MODEL) + tasks = [t.strip() for t in open(a.tasks) if t.strip()] + print(f"Terminal-Bench 2.0: {len(tasks)} tasks × {len(a.configs)} configs (n={a.n}) | " + f"price in=${pr[0]*1e6:.2f} out=${pr[1]*1e6:.2f} cread=${pr[2]*1e6:.2f} cwrite=${pr[3]*1e6:.2f} /M\n") + all_summ = [] + for cfg in a.configs: + jobs = f"{a.jobs_root}/{cfg}" + subprocess.run(f"rm -rf {jobs}", shell=True) + cap = f"/tmp/tb-runs/capture-terminalbench.jsonl" if cfg == a.capture_config else None + dump = f"/tmp/tb-runs/dump-terminalbench-{cfg}.jsonl" if cfg in a.dump_configs else None + print(f"### config={cfg} (capture={'yes' if cap else 'no'} dump={'yes' if dump else 'no'}) ...", flush=True) + start_proxy(cfg, base, token, capture=cap, dump=dump) + t0 = time.time() + run_harbor(tasks, jobs, a.n, a.setup_mult, a.build_mult, a.agent_mult, a.max_retries) + rows = parse_trials(jobs, pr) + Path(f"{a.jobs_root}/rows-{cfg}.json").write_text(json.dumps(rows, indent=1)) + st = {} + try: + st = json.load(urllib.request.urlopen(f"http://localhost:{PORT}/stats", timeout=5)) + except Exception: + pass + stop_proxy() + s = summarize(cfg, rows) + s["proxy_savings_pct"] = round(st.get("savings_pct", 0), 2) + s["proxy_bounces"] = st.get("bounces") + s["wall_total_min"] = round((time.time() - t0) / 60, 1) + comps = st.get("components", {}) or {} + s["per_component"] = {k: {"runs": v.get("runs"), "acted": v.get("acted"), + "saved_tokens": v.get("saved_tokens"), + "saved_tokens_unique": v.get("saved_tokens_unique"), + "overcount_ratio": v.get("overcount_ratio"), + "duration_ms": round(v.get("duration_ms", 0), 1)} + for k, v in comps.items()} + s["cg_added_ms_avg"] = st.get("cg_added_ms_avg") + s["upstream_ms_avg"] = st.get("upstream_ms_avg") + lc, li, lo = st.get("llm_calls", 0), st.get("llm_input_tokens", 0), st.get("llm_output_tokens", 0) + s["cg_llm_calls"] = lc + s["cg_llm_cost"] = round(li * cpr[0] + lo * cpr[1], 4) + s["cg_total_latency_s"] = round(sum(v.get("duration_ms", 0) for v in comps.values()) / 1000, 1) + if s.get("total_norm_cost") is not None: + s["total_cost_incl_cg"] = round(s["total_norm_cost"] + s["cg_llm_cost"], 4) + all_summ.append(s) + print(json.dumps(s, indent=1), flush=True) + Path(f"{a.jobs_root}/summary.json").write_text(json.dumps(dict(model=MODEL, price=pr, dataset=DATASET, tasks=len(tasks), configs=all_summ), indent=1)) + print("\n==== SUMMARY ====") + hdr = (f"{'config':<10}{'solved':>8}{'rate':>7}{'steps':>7}{'cache_hit':>10}" + f"{'agent$/t':>9}{'wall_s/t':>9}{'excs':>6}") + print(hdr) + for s in all_summ: + print(f"{s['config']:<10}{str(s['solved'])+'/'+str(s['scored']):>8}{str(s['solve_rate']):>7}" + f"{str(s['mean_steps']):>7}{str(s['cache_hit_rate']):>10}{str(s['mean_norm_cost']):>9}" + f"{str(s['mean_agent_wall_s']):>9}{str(s['exceptions']):>6}") + print(f"\nwrote {a.jobs_root}/summary.json + rows-*.json") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/terminalbench_headroom.py b/deploy/harbor/terminalbench_headroom.py new file mode 100644 index 0000000..3bee2b3 --- /dev/null +++ b/deploy/harbor/terminalbench_headroom.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""SWE-bench Verified harness for HEADROOM — mirror of context-guru's +deploy/harbor/swebench.py so results are directly comparable. + +Per config it: (1) starts the headroom proxy on :4010 pointed at the IBM +LiteLLM gateway (upstream via ANTHROPIC_TARGET_API_URL + a bearer token +injected through --anthropic-extra-headers so the containerized claude-code +agent can send a dummy key); (2) runs harbor over the task list with the +claude-code agent on aws/claude-sonnet-5; (3) parses each trial's result.json +(reward, timings) and agent/trajectory.json final_metrics (cache-aware tokens, +cost, steps) — IDENTICAL accounting to the context-guru harness; (4) reads the +headroom /stats + /metrics endpoints for savings %, token before/after, +per-transform breakdown, proxy-added latency, and CCR retrieval (bounce) +counts. Writes rows-.json + summary.json under the jobs-root. + +The `off` baseline is NOT re-run here — reuse context-guru's validated +/tmp/cg-runs/final50/rows-off.json. Use config `hdoff` (headroom passthrough, +--no-optimize) only to validate proxy wiring end-to-end. + +Usage: + swebench_headroom.py --tasks /tmp/cg-runs/swe3-verify.txt \ + --configs hd-cache --jobs-root /tmp/tb-runs/swe3 --n 2 +""" +import argparse, glob, json, os, re, subprocess, sys, time, urllib.request +from pathlib import Path + +HD = Path("/home/vpcuser/projects/context-engineering/headroom") +HB = Path("/home/vpcuser/projects/context-engineering/harbor") +HEADROOM_BIN = os.path.expanduser("~/.local/bin/headroom") +PORT = 4010 +LAN = "9.47.170.83" +PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +MODEL = "aws/claude-sonnet-5" + +# Headroom proxy configs. Each is (extra CLI args, extra env). All route to the +# gateway; the difference is the compaction policy. +# hd-cache : cache mode (freeze prior turns; only newest turn mutable) + +# code-aware AST compression on — the fair analog of context-guru's +# cache-aware `codesmart`. This is what a coding-agent user gets. +# hd-token : token mode (max compression, prior history may be rewritten). +# hdoff : passthrough (--no-optimize) — wiring sanity / matched control. +# NOTE: claude-code always streams (SSE). Headroom's CCR response-interception +# buffers+re-emits the stream to catch `headroom_retrieve` calls, and that +# re-emission corrupts the content-block sequence ("API Error: Content block +# not found" -> agent aborts turn 1). Headroom's --help says --no-ccr is +# "right for streaming", so the streaming-safe config disables CCR (compression +# stays fully active; only the reversible retrieve tool is dropped -> no +# restoration/bounce metric, by design). +CONFIGS = { + "hd-cache": (["--mode", "cache", "--code-aware", "--no-ccr"], {}), + "hd-token": (["--mode", "token", "--code-aware", "--no-ccr"], {}), + "hd-ccr": (["--mode", "cache", "--code-aware"], {}), # CCR on (breaks streaming); reference only + "hdoff": (["--no-optimize"], {}), +} + + +def creds(): + e = json.load(open(Path("~/.claude/settings.json").expanduser()))["env"] + return e["ANTHROPIC_BASE_URL"], e["ANTHROPIC_AUTH_TOKEN"] + + +def price(model): + fb = (2e-6, 1e-5, 2e-7, 2.5e-6) # in, out, cache_read, cache_write + try: + d = json.load(urllib.request.urlopen(PRICES_URL, timeout=15)) + c = d.get(model) or d.get(model.split("/")[-1]) + if c: + return (c.get("input_cost_per_token") or fb[0], c.get("output_cost_per_token") or fb[1], + c.get("cache_read_input_token_cost") or fb[2], + c.get("cache_creation_input_token_cost") or (c.get("input_cost_per_token") or fb[0]) * 1.25) + except Exception as e: + print(f"[price] {e}; fallback", file=sys.stderr) + return fb + + +_PROXY = {"p": None} + + +def stop_proxy(): + # Kill the tracked proxy by its process GROUP (setsid), then free the port with + # fuser. Never use `pkill -f ` where appears in this command + # line — it self-matches the killing shell and can leave the proxy orphaned. + import signal + p = _PROXY.get("p") + if p is not None and p.poll() is None: + try: + os.killpg(os.getpgid(p.pid), signal.SIGTERM) + except Exception: + pass + for _ in range(4): + # fuser operates on the socket, not on cmdline text -> no self-match. + subprocess.run(f"fuser -k {PORT}/tcp >/dev/null 2>&1 || true", shell=True) + time.sleep(1) + r = subprocess.run(f"fuser {PORT}/tcp 2>/dev/null", shell=True, capture_output=True) + if not r.stdout.strip(): + break + _PROXY["p"] = None + time.sleep(2) + + +def start_proxy(cfg, base, token): + stop_proxy() + extra_args, extra_env = CONFIGS[cfg] + # Upstream = IBM gateway. base already looks like https://host[/path]; headroom + # appends /v1/messages to ANTHROPIC_TARGET_API_URL. The gateway expects a bearer + # token; inject it via extra-headers so the container can send a dummy key. + # inject BOTH forms — LiteLLM accepts Authorization: Bearer and x-api-key. + hdrs = json.dumps({"Authorization": f"Bearer {token}", "x-api-key": token}) + state = f"/tmp/tb-runs/hdstate-{cfg}" + subprocess.run(f"rm -rf {state} && mkdir -p {state}", shell=True) + env = dict(os.environ, + ANTHROPIC_TARGET_API_URL=base, + HEADROOM_WORKSPACE_DIR=state, + HEADROOM_CONFIG_DIR=f"{state}/config", + HEADROOM_TELEMETRY="off", + HEADROOM_UPDATE_CHECK="off", + # output shaper OFF (default) so we compare INPUT compaction fairly vs CG + HEADROOM_OUTPUT_SHAPER="0", + # CRITICAL: disable server-side Tool Search deferral. Headroom auto-enables + # it for the detected claude-code client and treats our gateway as first-party + # Anthropic, injecting first-party-only tool_search_tool_* / defer_loading. + # The Bedrock-backed gateway can't honor them -> deferred tools become + # unreachable -> claude-code aborts turn 1 with "API Error: Content block + # not found". Force it off; also disable tool-schema/desc mutation and + # memory-tool injection so headroom does not alter the tool surface. + HEADROOM_TOOL_SEARCH="0", + HEADROOM_TOOL_DESC_MAX_CHARS="0", + HEADROOM_TOOL_DESC_STRIP_SEMANTIC="0", + HEADROOM_NO_MEMORY_TOOLS="1") + env.update(extra_env) + args = [HEADROOM_BIN, "proxy", "--host", "0.0.0.0", "--port", str(PORT), + "--anthropic-api-url", base, + "--anthropic-extra-headers", hdrs] + extra_args + log = open(f"/tmp/tb-runs/proxy-{cfg}.log", "w") + p = subprocess.Popen(args, cwd=str(HD), stdout=log, stderr=log, env=env, preexec_fn=os.setsid) + _PROXY["p"] = p + for _ in range(60): + try: + urllib.request.urlopen(f"http://localhost:{PORT}/livez", timeout=3).read() + return p + except Exception: + if p.poll() is not None: + raise RuntimeError(f"headroom proxy for {cfg} exited early; see /tmp/tb-runs/proxy-{cfg}.log") + time.sleep(1) + raise RuntimeError(f"headroom proxy for {cfg} did not come up; see /tmp/tb-runs/proxy-{cfg}.log") + + +def run_harbor(tasks, jobs_dir, n, setup_mult, build_mult, agent_mult, max_retries=4): + # headroom serves the Anthropic messages endpoint at the ROOT (/v1/messages), + # so the client base URL is just http://host:port (no /anthropic suffix). + proxy_url = f"http://{LAN}:{PORT}" + inc = " ".join(f"-i {t}" for t in tasks) + home = os.path.expanduser("~") + abs_path = f"{home}/.local/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + cmd = (f"cd {HB} && ANTHROPIC_BASE_URL='{proxy_url}' ANTHROPIC_API_KEY='sk-proxy' " + f"ANTHROPIC_AUTH_TOKEN='sk-proxy' PATH='{abs_path}' HOME='{home}' " + f"{home}/.local/bin/uv run harbor run -y -d terminal-bench@2.0 -a claude-code -m '{MODEL}' " + f"--env docker {inc} -n {n} --jobs-dir '{jobs_dir}' --no-delete " + f"--agent-setup-timeout-multiplier {setup_mult} --environment-build-timeout-multiplier {build_mult} " + f"--agent-timeout-multiplier {agent_mult} --max-retries {max_retries} " + f"--ae ANTHROPIC_BASE_URL='{proxy_url}' --ae ANTHROPIC_API_KEY='sk-proxy' --ae ANTHROPIC_AUTH_TOKEN='sk-proxy'") + log = f"/tmp/tb-runs/run-{Path(jobs_dir).name}.log" + with open(log, "w") as f: + subprocess.run(["sg", "docker", "-c", cmd], stdout=f, stderr=f) + return log + + +def iso(s): + from datetime import datetime + return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +def parse_trials(jobs_dir, pr): + rows = [] + for rf in glob.glob(f"{jobs_dir}/*/*/result.json"): + try: + d = json.load(open(rf)) + except Exception: + continue + if "verifier_result" not in d: + continue + tdir = Path(rf).parent + reward = ((d.get("verifier_result") or {}).get("rewards") or {}).get("reward") + fm = {} + traj = tdir / "agent" / "trajectory.json" + if traj.exists(): + try: + fm = (json.load(open(traj)) or {}).get("final_metrics") or {} + except Exception: + fm = {} + ex = fm.get("extra") or {} + pt = fm.get("total_prompt_tokens") or 0 + ct = fm.get("total_completion_tokens") or 0 + cached = fm.get("total_cached_tokens") or 0 + cwrite = ex.get("total_cache_creation_input_tokens") or 0 + cread = ex.get("total_cache_read_input_tokens") or cached + fresh = max(pt - cread - cwrite, 0) + norm_cost = fresh * pr[0] + ct * pr[1] + cread * pr[2] + cwrite * pr[3] + wall = None + try: + wall = iso(d["finished_at"]) - iso(d["started_at"]) + except Exception: + pass + agent_wall = None + try: + ae = d.get("agent_execution") or {} + agent_wall = iso(ae["finished_at"]) - iso(ae["started_at"]) + except Exception: + pass + rows.append(dict(task=d.get("task_name", tdir.parent.name), reward=reward, + steps=fm.get("total_steps"), prompt_tokens=pt, completion_tokens=ct, + cached_tokens=cached, cache_read=cread, cache_write=cwrite, fresh_input=fresh, + agent_cost=fm.get("total_cost_usd"), norm_cost=round(norm_cost, 5), + wall_s=round(wall, 1) if wall else None, + agent_wall_s=round(agent_wall, 1) if agent_wall else None, + exception=bool(d.get("exception_info")))) + return rows + + +def summarize(cfg, rows): + got = [r for r in rows if r["reward"] is not None] + solved = sum(1 for r in got if r["reward"] and r["reward"] >= 1) + def avg(k): + vs = [r[k] for r in rows if isinstance(r.get(k), (int, float))] + return round(sum(vs) / len(vs), 3) if vs else None + tot = lambda k: sum(r[k] for r in rows if isinstance(r.get(k), (int, float))) + cacheable = tot("cache_read") + tot("fresh_input") + tot("cache_write") + return dict(config=cfg, trials=len(rows), scored=len(got), solved=solved, + solve_rate=round(solved / len(got), 3) if got else None, + exceptions=sum(1 for r in rows if r["exception"]), + mean_steps=avg("steps"), mean_prompt_tokens=avg("prompt_tokens"), + mean_completion_tokens=avg("completion_tokens"), + cache_hit_rate=round(tot("cache_read") / cacheable, 4) if cacheable else None, + total_fresh_input=tot("fresh_input"), total_cache_read=tot("cache_read"), + total_cache_write=tot("cache_write"), total_completion=tot("completion_tokens"), + total_norm_cost=round(tot("norm_cost"), 4), mean_norm_cost=avg("norm_cost"), + mean_agent_cost=avg("agent_cost"), mean_wall_s=avg("wall_s"), + mean_agent_wall_s=avg("agent_wall_s")) + + +def fetch(path): + try: + return urllib.request.urlopen(f"http://localhost:{PORT}{path}", timeout=8).read().decode() + except Exception as e: + return None + + +def deep_find(obj, keys): + """Return the first value under any of `keys` anywhere in a nested dict/list.""" + out = {} + def walk(o): + if isinstance(o, dict): + for k, v in o.items(): + if k in keys and k not in out and isinstance(v, (int, float)): + out[k] = v + walk(v) + elif isinstance(o, list): + for v in o: + walk(v) + walk(obj) + return out + + +def parse_headroom_stats(cfg): + """Best-effort extraction of headroom proxy metrics from /stats + /metrics. + Saves the raw dumps for later inspection.""" + stats_txt = fetch("/stats") + metrics_txt = fetch("/metrics") + Path(f"/tmp/tb-runs/stats-{cfg}.json").write_text(stats_txt or "") + Path(f"/tmp/tb-runs/metrics-{cfg}.txt").write_text(metrics_txt or "") + res = {} + st = None + if stats_txt: + try: + st = json.loads(stats_txt) + except Exception: + st = None + if st: + tk = st.get("tokens", {}) if isinstance(st.get("tokens"), dict) else {} + # headline savings figures (exact keys per proxy/server.py _build_stats_payload) + res["tokens"] = tk + res["tokens_saved"] = tk.get("saved") + res["savings_percent"] = tk.get("savings_percent") # all layers, whole request + res["active_savings_percent"] = tk.get("active_savings_percent") # saved / compressible (headline) + res["proxy_savings_percent"] = tk.get("proxy_savings_percent") + res["proxy_compression_saved"] = tk.get("proxy_compression_saved") + res["total_before_compression"] = tk.get("total_before_compression") + res["tokens_input"] = tk.get("input") + res["tokens_output"] = tk.get("output") + res["output_reduction_percent"] = tk.get("output_reduction_percent") + # proxy-added latency (Headroom optimization time only) + total + ttfb + res["overhead"] = st.get("overhead") + res["latency"] = st.get("latency") + res["ttfb"] = st.get("ttfb") + # per-transform timings + per-strategy savings + res["pipeline_timing"] = st.get("pipeline_timing") + res["compressions_by_strategy"] = st.get("compressions_by_strategy") + res["tokens_saved_by_strategy"] = st.get("tokens_saved_by_strategy") + res["extension_savings"] = st.get("extension_savings") + # CCR retrieval (bounce / restoration) counters + comp = st.get("compression", {}) if isinstance(st.get("compression"), dict) else {} + res["ccr_retrievals"] = comp.get("ccr_retrievals") + res["ccr_entries"] = comp.get("ccr_entries") + res["original_tokens_cached"] = comp.get("original_tokens_cached") + res["compressed_tokens_cached"] = comp.get("compressed_tokens_cached") + res["compression"] = comp + res["prefix_cache"] = st.get("prefix_cache") + res["compression_cache"] = st.get("compression_cache") + rq = st.get("requests", {}) if isinstance(st.get("requests"), dict) else {} + res["requests_total"] = rq.get("total") + res["requests_cached"] = rq.get("cached") + res["requests_failed"] = rq.get("failed") + res["requests_by_provider"] = rq.get("by_provider") + # Prometheus scrape for durable counters + if metrics_txt: + prom = {} + for line in metrics_txt.splitlines(): + if line.startswith("#") or not line.strip(): + continue + m = re.match(r"^(\S+?)(\{[^}]*\})?\s+([0-9eE.+-]+)$", line.strip()) + if not m: + continue + name, labels, val = m.group(1), m.group(2) or "", m.group(3) + try: + v = float(val) + except Exception: + continue + key = name + labels + prom[key] = v + # pull the interesting ones + want = [k for k in prom if any(s in k for s in ( + "tokens_saved", "transform_timing", "latency_ms", "overhead", + "ccr", "retrieve", "expansion", "requests_total", "cache_bust", + "provider_cache", "ttfb"))] + res["prom"] = {k: prom[k] for k in want} + return res + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--tasks", required=True) + ap.add_argument("--configs", nargs="+", default=["hd-cache"]) + ap.add_argument("--jobs-root", default="/tmp/tb-runs/hd") + ap.add_argument("--n", type=int, default=2) + ap.add_argument("--setup-mult", type=float, default=4.0) + ap.add_argument("--build-mult", type=float, default=4.0) + ap.add_argument("--agent-mult", type=float, default=4.0) # TB: 4x flat budget (matches baseline's long-horizon budget) + ap.add_argument("--max-retries", type=int, default=4) + a = ap.parse_args() + base, token = creds() + pr = price(MODEL) + tasks = [t.strip() for t in open(a.tasks) if t.strip()] + Path(a.jobs_root).mkdir(parents=True, exist_ok=True) + print(f"Terminal-Bench(headroom): {len(tasks)} tasks x {len(a.configs)} configs (n={a.n}) | " + f"price in=${pr[0]*1e6:.2f} out=${pr[1]*1e6:.2f} cread=${pr[2]*1e6:.2f} cwrite=${pr[3]*1e6:.2f}/M\n", flush=True) + all_summ = [] + for cfg in a.configs: + jobs = f"{a.jobs_root}/{cfg}" + subprocess.run(f"rm -rf {jobs}", shell=True) + print(f"### config={cfg} ...", flush=True) + start_proxy(cfg, base, token) + t0 = time.time() + run_harbor(tasks, jobs, a.n, a.setup_mult, a.build_mult, a.agent_mult, a.max_retries) + rows = parse_trials(jobs, pr) + Path(f"{a.jobs_root}/rows-{cfg}.json").write_text(json.dumps(rows, indent=1)) + hd = parse_headroom_stats(cfg) + stop_proxy() + s = summarize(cfg, rows) + s["wall_total_min"] = round((time.time() - t0) / 60, 1) + s["headroom"] = hd + all_summ.append(s) + print(json.dumps(s, indent=1), flush=True) + Path(f"{a.jobs_root}/summary.json").write_text(json.dumps( + dict(model=MODEL, price=pr, tasks=len(tasks), configs=all_summ), indent=1)) + print("\n==== SUMMARY ====") + for s in all_summ: + hd = s.get("headroom", {}) + print(f"{s['config']:<10} solved={s['solved']}/{s['scored']} rate={s['solve_rate']} " + f"steps={s['mean_steps']} cache_hit={s['cache_hit_rate']} " + f"norm$/t={s['mean_norm_cost']} save%={hd.get('savings_percent')} " + f"saved_tok={hd.get('tokens_saved')}") + print(f"\nwrote {a.jobs_root}/summary.json + rows-*.json ; raw stats-*.json/metrics-*.txt in /tmp/tb-runs") + + +if __name__ == "__main__": + main() diff --git a/deploy/harbor/terminalbench_rtk.py b/deploy/harbor/terminalbench_rtk.py new file mode 100644 index 0000000..e4f9c93 --- /dev/null +++ b/deploy/harbor/terminalbench_rtk.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""SWE-bench Verified harness for RTK (Rust Token Killer) — a 4th arm for the +three-way study, using IDENTICAL trajectory accounting to swebench.py so results +are directly comparable to baseline / context-guru / headroom. + +rtk is NOT a request-stream proxy. It is a Claude Code ``PreToolUse`` hook that +rewrites Bash commands (``pytest`` -> ``rtk pytest``, ``cat`` -> ``rtk read``, +``git status`` -> ``rtk git status``) INSIDE the task container, compressing bash +output at the shell before it enters the model context. So there is nothing to +proxy for compaction — model routing is made IDENTICAL to the baseline by +running the same context-guru ``off`` passthrough proxy on :4000. The ONLY +difference from baseline is the in-container bash compression, delivered by the +custom ``claude-code-rtk`` Harbor agent (see +harbor/src/harbor/agents/installed/claude_code_rtk.py). + +Per config it: (1) starts cg-proxy-d1 with preset ``off`` (pure passthrough, +FORCE_MODEL=sonnet-5) — same routing as the baseline ``rows-off.json``; +(2) runs harbor with ``-a claude-code-rtk`` (uploads the rtk binary + installs +the PreToolUse hook in-container), passing RTK_BIN_HOST so the agent finds the +host-built static-musl binary; (3) parses each trial's result.json + trajectory +final_metrics for reward/steps/cache-aware tokens/cost (identical to swebench.py); +(4) reads each trial's /logs/agent/rtk-gain.json for rtk's OWN bash-output +savings ledger. Writes rows-.json + summary.json under the jobs-root. + +The ``off`` baseline is NOT re-run — reuse /tmp/cg-runs/final50/rows-off.json. + +Usage: + swebench_rtk.py --tasks /tmp/cg-runs/swe3-verify.txt --jobs-root /tmp/rtk-runs/swe3 --n 2 +""" +import argparse, glob, json, os, subprocess, sys, time, urllib.request +from pathlib import Path + +CG = Path("/home/vpcuser/projects/context-engineering/context-guru") +HB = Path("/home/vpcuser/projects/context-engineering/harbor") +BIN = "/tmp/cg-runs/cg-proxy-d1" # context-guru proxy (off = passthrough) +RTK_BIN = os.environ.get("RTK_BIN_HOST", "/tmp/rtk-runs/rtk") # host static-musl rtk +PORT = 4000 +LAN = "9.47.170.83" +PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +MODEL = "aws/claude-sonnet-5" + + +def creds(): + e = json.load(open(Path("~/.claude/settings.json").expanduser()))["env"] + return e["ANTHROPIC_BASE_URL"], e["ANTHROPIC_AUTH_TOKEN"] + + +def price(model): + fb = (2e-6, 1e-5, 2e-7, 2.5e-6) # in, out, cache_read, cache_write + try: + d = json.load(urllib.request.urlopen(PRICES_URL, timeout=15)) + c = d.get(model) or d.get(model.split("/")[-1]) + if c: + return (c.get("input_cost_per_token") or fb[0], c.get("output_cost_per_token") or fb[1], + c.get("cache_read_input_token_cost") or fb[2], + c.get("cache_creation_input_token_cost") or (c.get("input_cost_per_token") or fb[0]) * 1.25) + except Exception as e: + print(f"[price] {e}; fallback", file=sys.stderr) + return fb + + +def stop_proxy(): + for _ in range(3): + subprocess.run("pkill -x cg-proxy-d1", shell=True) + time.sleep(1) + r = subprocess.run("pgrep -x cg-proxy-d1", shell=True, capture_output=True) + if not r.stdout.strip(): + return + time.sleep(2) + + +def start_proxy(base, token): + """Start cg-proxy-d1 as a pure passthrough (preset off) — identical routing + to the baseline, so the only variable is rtk's in-container compression.""" + stop_proxy() + env = dict(os.environ, ANTHROPIC_UPSTREAM=base, ANTHROPIC_API_KEY=token, + OPENAI_UPSTREAM=base, OPENAI_API_KEY=token, FORCE_MODEL=MODEL, + LISTEN_ADDR=f":{PORT}") + log = open("/tmp/tb-runs/proxy-rtk-off.log", "w") + p = subprocess.Popen([BIN, "--preset", "off"], cwd=str(CG), stdout=log, stderr=log, + env=env, preexec_fn=os.setsid) + for _ in range(30): + try: + urllib.request.urlopen(f"http://localhost:{PORT}/healthz", timeout=3).read() + return p + except Exception: + time.sleep(0.5) + raise RuntimeError("off proxy did not come up") + + +def run_harbor(tasks, jobs_dir, n, setup_mult, build_mult, agent_mult, max_retries=4): + proxy_url = f"http://{LAN}:{PORT}/anthropic" + inc = " ".join(f"-i {t}" for t in tasks) + home = os.path.expanduser("~") + abs_path = f"{home}/.local/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + # RTK_BIN_HOST tells the claude-code-rtk agent where the host rtk binary is. + cmd = (f"cd {HB} && ANTHROPIC_BASE_URL='{proxy_url}' ANTHROPIC_API_KEY='sk-proxy' " + f"ANTHROPIC_AUTH_TOKEN='sk-proxy' RTK_BIN_HOST='{RTK_BIN}' PATH='{abs_path}' HOME='{home}' " + f"{home}/.local/bin/uv run harbor run -y -d terminal-bench@2.0 -a claude-code-rtk -m '{MODEL}' " + f"--env docker {inc} -n {n} --jobs-dir '{jobs_dir}' --no-delete " + f"--agent-setup-timeout-multiplier {setup_mult} --environment-build-timeout-multiplier {build_mult} " + f"--agent-timeout-multiplier {agent_mult} --max-retries {max_retries} " + f"--ae ANTHROPIC_BASE_URL='{proxy_url}' --ae ANTHROPIC_API_KEY='sk-proxy' --ae ANTHROPIC_AUTH_TOKEN='sk-proxy'") + log = f"/tmp/tb-runs/run-{Path(jobs_dir).name}.log" + with open(log, "w") as f: + subprocess.run(["sg", "docker", "-c", cmd], stdout=f, stderr=f) + return log + + +def iso(s): + from datetime import datetime + return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() + + +def read_rtk_gain(tdir): + """rtk's OWN bash-output savings ledger for this trial (bytes/4 estimate).""" + gp = tdir / "agent" / "rtk-gain.json" + if not gp.exists(): + return None + try: + g = json.load(open(gp)) + except Exception: + return None + s = g.get("summary") or {} + by_cmd = g.get("byCommand") or g.get("by_command") or [] + return dict(commands=s.get("total_commands"), input_tokens=s.get("total_input"), + output_tokens=s.get("total_output"), saved_tokens=s.get("total_saved"), + savings_pct=s.get("avg_savings_pct"), total_time_ms=s.get("total_time_ms"), + by_command=by_cmd) + + +def parse_trials(jobs_dir, pr): + rows = [] + for rf in glob.glob(f"{jobs_dir}/*/*/result.json"): + try: + d = json.load(open(rf)) + except Exception: + continue + if "verifier_result" not in d: + continue + tdir = Path(rf).parent + reward = ((d.get("verifier_result") or {}).get("rewards") or {}).get("reward") + fm = {} + traj = tdir / "agent" / "trajectory.json" + if traj.exists(): + try: + fm = (json.load(open(traj)) or {}).get("final_metrics") or {} + except Exception: + fm = {} + ex = fm.get("extra") or {} + pt = fm.get("total_prompt_tokens") or 0 + ct = fm.get("total_completion_tokens") or 0 + cached = fm.get("total_cached_tokens") or 0 + cwrite = ex.get("total_cache_creation_input_tokens") or 0 + cread = ex.get("total_cache_read_input_tokens") or cached + fresh = max(pt - cread - cwrite, 0) + norm_cost = fresh * pr[0] + ct * pr[1] + cread * pr[2] + cwrite * pr[3] + wall = None + try: + wall = iso(d["finished_at"]) - iso(d["started_at"]) + except Exception: + pass + agent_wall = None + try: + ae = d.get("agent_execution") or {} + agent_wall = iso(ae["finished_at"]) - iso(ae["started_at"]) + except Exception: + pass + rows.append(dict(task=d.get("task_name", tdir.parent.name), reward=reward, + steps=fm.get("total_steps"), prompt_tokens=pt, completion_tokens=ct, + cached_tokens=cached, cache_read=cread, cache_write=cwrite, fresh_input=fresh, + agent_cost=fm.get("total_cost_usd"), norm_cost=round(norm_cost, 5), + wall_s=round(wall, 1) if wall else None, + agent_wall_s=round(agent_wall, 1) if agent_wall else None, + exception=bool(d.get("exception_info")), + rtk=read_rtk_gain(tdir))) + return rows + + +def summarize(cfg, rows): + got = [r for r in rows if r["reward"] is not None] + solved = sum(1 for r in got if r["reward"] and r["reward"] >= 1) + def avg(k): + vs = [r[k] for r in rows if isinstance(r.get(k), (int, float))] + return round(sum(vs) / len(vs), 3) if vs else None + tot = lambda k: sum(r[k] for r in rows if isinstance(r.get(k), (int, float))) + cacheable = tot("cache_read") + tot("fresh_input") + tot("cache_write") + # aggregate rtk's own ledger across trials + rk = [r["rtk"] for r in rows if r.get("rtk")] + rtk_before = sum((x.get("input_tokens") or 0) for x in rk) + rtk_after = sum((x.get("output_tokens") or 0) for x in rk) + rtk_saved = sum((x.get("saved_tokens") or 0) for x in rk) + rtk_cmds = sum((x.get("commands") or 0) for x in rk) + return dict(config=cfg, trials=len(rows), scored=len(got), solved=solved, + solve_rate=round(solved / len(got), 3) if got else None, + exceptions=sum(1 for r in rows if r["exception"]), + mean_steps=avg("steps"), mean_prompt_tokens=avg("prompt_tokens"), + mean_completion_tokens=avg("completion_tokens"), + cache_hit_rate=round(tot("cache_read") / cacheable, 4) if cacheable else None, + total_fresh_input=tot("fresh_input"), total_cache_read=tot("cache_read"), + total_cache_write=tot("cache_write"), total_completion=tot("completion_tokens"), + total_norm_cost=round(tot("norm_cost"), 4), mean_norm_cost=avg("norm_cost"), + mean_agent_cost=avg("agent_cost"), mean_wall_s=avg("wall_s"), + mean_agent_wall_s=avg("agent_wall_s"), + rtk_trials_with_ledger=len(rk), rtk_commands=rtk_cmds, + rtk_bash_tokens_before=rtk_before, rtk_bash_tokens_after=rtk_after, + rtk_bash_tokens_saved=rtk_saved, + rtk_bash_savings_pct=round(100 * rtk_saved / rtk_before, 2) if rtk_before else None) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--tasks", required=True) + ap.add_argument("--config", default="rtk", help="label for this arm") + ap.add_argument("--jobs-root", default="/tmp/tb-runs/rtk") + ap.add_argument("--n", type=int, default=2) + ap.add_argument("--setup-mult", type=float, default=4.0) + ap.add_argument("--build-mult", type=float, default=4.0) + ap.add_argument("--agent-mult", type=float, default=4.0) # TB: 4x flat budget (matches baseline's long-horizon budget) + ap.add_argument("--max-retries", type=int, default=4) + a = ap.parse_args() + base, token = creds() + pr = price(MODEL) + tasks = [t.strip() for t in open(a.tasks) if t.strip()] + Path(a.jobs_root).mkdir(parents=True, exist_ok=True) + print(f"Terminal-Bench(rtk): {len(tasks)} tasks (n={a.n}) | rtk_bin={RTK_BIN} | " + f"price in=${pr[0]*1e6:.2f} out=${pr[1]*1e6:.2f} cread=${pr[2]*1e6:.2f} cwrite=${pr[3]*1e6:.2f}/M\n", flush=True) + if not Path(RTK_BIN).exists(): + raise SystemExit(f"rtk binary not found at {RTK_BIN}; set RTK_BIN_HOST") + cfg = a.config + jobs = f"{a.jobs_root}/{cfg}" + subprocess.run(f"rm -rf {jobs}", shell=True) + print(f"### config={cfg} (rtk in-container hook; off-proxy routing) ...", flush=True) + start_proxy(base, token) + t0 = time.time() + run_harbor(tasks, jobs, a.n, a.setup_mult, a.build_mult, a.agent_mult, a.max_retries) + rows = parse_trials(jobs, pr) + Path(f"{a.jobs_root}/rows-{cfg}.json").write_text(json.dumps(rows, indent=1)) + stop_proxy() + s = summarize(cfg, rows) + s["wall_total_min"] = round((time.time() - t0) / 60, 1) + Path(f"{a.jobs_root}/summary.json").write_text(json.dumps( + dict(model=MODEL, price=pr, tasks=len(tasks), configs=[s]), indent=1)) + print(json.dumps(s, indent=1), flush=True) + print("\n==== SUMMARY ====") + print(f"{cfg:<12} solved={s['solved']}/{s['scored']} rate={s['solve_rate']} " + f"steps={s['mean_steps']} cache_hit={s['cache_hit_rate']} " + f"norm$/t={s['mean_norm_cost']} exceptions={s['exceptions']}") + print(f" rtk ledger: {s['rtk_trials_with_ledger']}/{len(rows)} trials, {s['rtk_commands']} cmds, " + f"bash tokens {s['rtk_bash_tokens_before']}->{s['rtk_bash_tokens_after']} " + f"(saved {s['rtk_bash_tokens_saved']} = {s['rtk_bash_savings_pct']}%)") + print(f"\nwrote {a.jobs_root}/summary.json + rows-{cfg}.json") + + +if __name__ == "__main__": + main() diff --git a/docs/results/REPRODUCE.md b/docs/results/REPRODUCE.md index f877fa3..af11ee5 100644 --- a/docs/results/REPRODUCE.md +++ b/docs/results/REPRODUCE.md @@ -164,10 +164,178 @@ output $10/M), cache-hit rate, proxy savings %, per-component savings + own late context-guru's own cheap-model cost (priced at the haiku rate), and expand/restoration bounces. -## 6. Result docs +## 7. Terminal-Bench 2.0 (second benchmark) -- [`baseline.md`](baseline.md) — baseline (`off`) full results. +The same harness pattern extends to **Terminal-Bench 2.0** (89 open-ended terminal tasks, +harder/longer-horizon than SWE-bench). Only the Harbor dataset and jobs-root change; the +`claude-code` trajectory parser, cache-aware cost model, and summarizer are agent-specific, +so every metric is computed identically and is methodologically comparable to the SWE study. +Harness: [`deploy/harbor/terminalbench.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/terminalbench.py) +(a thin adaptation of `swebench.py` with `-d terminal-bench@2.0`). + +``` +cd /home/vpcuser/projects/context-engineering/context-guru +# task list: all 89 task names from the terminal-bench registry entry (see below) +# baseline (off passthrough), parallel: +python3 -u deploy/harbor/terminalbench.py \ + --tasks /tmp/tb-runs/tb89.txt --configs off --jobs-root /tmp/tb-runs/tb89 \ + --n 24 --agent-mult 1.5 --max-retries 4 +``` + +Build the 89-task list from the registry: +``` +python3 - <<'PY' +import json +d=json.load(open('/home/vpcuser/projects/context-engineering/harbor/registry.json')) +def find(o): + r=[] + if isinstance(o,list): + for e in o: + if isinstance(e,dict) and e.get('name')=='terminal-bench': r.append(e) + r+=find(e) + elif isinstance(o,dict): + for v in o.values(): r+=find(v) + return r +open('/tmp/tb-runs/tb89.txt','w').write('\n'.join(t['name'] for t in find(d)[0]['tasks'])+'\n') +PY +``` + +**Concurrency (feasibility on this 16-core / 62 GB box).** `--n` is Harbor's `--n-concurrent`. +Agents are network-bound (~300 MB, ~0.5% CPU while waiting on the ~26 s/request gateway), so RAM +is not the limit — **build-phase CPU** is (compile tasks run `make -j` and saturate cores; Harbor +self-limits build ramp), and **disk** (`--no-delete` image accumulation). `--n 24` is the sweet +spot (~10× faster than `n=2`, ~2.7 h for all 89); `n=45` only widens the disk/rate-limit blast +radius without building faster. Prune unused images (`docker image prune -af`, safe — protects +in-flight containers) if disk gets tight. + +**Timeouts / time-budget policy.** TB's task timeouts assume a fast endpoint; this gateway is +~26 s/request, so long-horizon tasks can exhaust the wall-clock budget. Two effects, separated: +(1) at `n=24`, CPU oversubscription (load ~45) inflated wall time and timed out ~9 edge tasks — +**rerun those at `n=6`** (no oversubscription) to clear the artifact; (2) genuinely long tasks were +given an extended **`--agent-mult 4.0`** budget to measure capability. Merge the best clean result +per task (`/tmp/tb-runs/merge_tb.py`). Any task that still times out at 4× is a genuine failure +(reward 0). All 89 tasks carry a scored outcome (solved / failed / timeout). + +**Baseline row provenance (needed to reproduce the published $100.81).** The published baseline is +a *two-stage* merge, and the intermediate file alone does not reproduce the totals: + +1. `merge_tb.py` overlays the `n=6` rerun onto the clean `n=24` rows → `rows-off-final.json` + (sums to $71.44 — **not** the published figure); +2. the 11 long-horizon tasks rerun at `--agent-mult 4.0` (`/tmp/tb-runs/tb-rerun2/rows-off.json`: + `caffe-cifar-10`, `circuit-fibsqrt`, `cobol-modernization`, `feal-linear-cryptanalysis`, + `gpt2-codegolf`, `make-doom-for-mips`, `make-mips-interpreter`, `path-tracing`, + `path-tracing-reverse`, `schemelike-metacircular-eval`, `write-compressor`) are then overlaid + on top → the published 89-task baseline (215,971,427 cache-read / 4,011,068 cache-write / + 58,893 fresh / 4,746,887 output = **$100.81**). + +Do stage 2 explicitly; `merge_tb.py` does not do it. Keep the merged file so the totals stay +reconstructible: + +``` +python3 - <<'PY' +import json +fin={r['task']:r for r in json.load(open('/tmp/tb-runs/tb89/rows-off-final.json'))} +for r in json.load(open('/tmp/tb-runs/tb-rerun2/rows-off.json')): fin[r['task']]=r +json.dump(list(fin.values()), open('/tmp/tb-runs/tb89/rows-off-published.json','w'), indent=1) +PY +``` + +**Excluding degenerate trials.** A trial where the *baseline* aborted in a handful of steps is not +a measurement: the per-task delta then reflects the baseline not doing the work. Six tasks are in +this class (`extract-moves-from-video`, `polyglot-rust-c`, `mteb-leaderboard`, `regex-chess`, +`write-compressor`, `code-from-image`). Compare **per task, paired**, and report the clean subset +alongside the raw total — per-arm sums hide this entirely. See the +[comparison](terminal-bench-comparison.md) correction box. + +Analyze → doc (totals, cache-aware cost, by-difficulty/category, timeouts, per-task): +``` +# task metadata (difficulty/category) from Harbor's task cache — needs py3.11+ (tomllib): +/home/vpcuser/projects/context-engineering/harbor/.venv/bin/python - <<'PY' +import glob, json, os, tomllib +names=set(l.strip() for l in open('/tmp/tb-runs/tb89.txt') if l.strip()); meta={} +for f in glob.glob('/home/vpcuser/.cache/harbor/tasks/*/*/task.toml'): + t=os.path.basename(os.path.dirname(f)) + if t not in names: continue + d=tomllib.load(open(f,'rb')); m=d.get('metadata',{}) + if t in meta and (meta[t]['difficulty']!='unknown' or m.get('difficulty') is None): continue + meta[t]=dict(difficulty=m.get('difficulty','unknown'),category=m.get('category','unknown'), + agent_timeout_sec=(d.get('agent',{}) or {}).get('timeout_sec')) +json.dump(meta, open('/tmp/tb-runs/task_meta.json','w'), indent=1) +PY +python3 deploy/harbor/gen_tb_docs.py /tmp/tb-runs/tb89/rows-off.json \ + docs/results/terminal-bench-baseline.md --meta /tmp/tb-runs/task_meta.json +``` + +### 7b. Terminal-Bench framework arms (context-guru / headroom / rtk) + +The three compaction arms reuse the SWE harnesses, re-pointed at `terminal-bench@2.0` +([`terminalbench.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/terminalbench.py) +for context-guru's `codesmart`, +[`terminalbench_headroom.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/terminalbench_headroom.py), +[`terminalbench_rtk.py`](https://github.com/rossoctl/context-guru/blob/main/deploy/harbor/terminalbench_rtk.py)). +All ran at a **flat `--agent-mult 4.0`** budget (see the [comparison](terminal-bench-comparison.md) +for why this is fair vs the baseline's mixed budget), `n=12` (headroom + rtk in parallel on +their separate ports 4010/4000; context-guru on 4000): + +``` +cd /home/vpcuser/projects/context-engineering/context-guru +# context-guru (codesmart) — dumps the change log for the per-component analysis +python3 -u deploy/harbor/terminalbench.py --tasks /tmp/tb-runs/tb89.txt --configs codesmart \ + --jobs-root /tmp/tb-runs/cg --n 12 --agent-mult 4.0 --max-retries 4 --dump-configs codesmart +# headroom (hd-cache; --no-ccr + HEADROOM_TOOL_SEARCH=0 baked in, as on SWE) +python3 -u deploy/harbor/terminalbench_headroom.py --tasks /tmp/tb-runs/tb89.txt --configs hd-cache \ + --jobs-root /tmp/tb-runs/hd --n 12 --agent-mult 4.0 --max-retries 4 +# rtk (claude-code-rtk agent + off-proxy routing; needs the rtk binary + registered agent) +RTK_BIN_HOST=/tmp/rtk-runs/rtk python3 -u deploy/harbor/terminalbench_rtk.py --tasks /tmp/tb-runs/tb89.txt \ + --jobs-root /tmp/tb-runs/rtk --n 12 --agent-mult 4.0 --max-retries 4 +``` + +Each writes `rows-.json` + `summary.json` under its jobs-root (headroom also dumps +`/tmp/tb-runs/stats-hd-cache.json`; rtk's ledger is in each trial's `agent/rtk-gain.json`). +Per-arm pages via `gen_tb_docs.py --kind arm --label ""`; the four-way +[comparison](terminal-bench-comparison.md) is assembled from the four `rows-*.json`. + +### 7c. The merged-system arm (2026-08-10) + +After the cache/filter/observe work landed on `main` (15 PRs), context-guru was re-measured as a +fifth arm. Two things differ from §7b and both matter for reproduction. + +**The config is not `codesmart`.** It is `cgfinal = [format, dedup, cmdfilter, extract, cachesplit]`, +chosen on per-component evidence rather than maximal token reduction. `extract_llm` (82× underwater +at cache-read prices), `failed_run` (`acted=0`, 28.8 s of scanning) and `cacheinject` (removed from +every preset by #36) are excluded. See the +[comparison](terminal-bench-comparison.md#the-merged-system-a-fifth-arm-2026-08-10). + +**Verify the mechanism fired before crediting it.** Each claim must name a counter: + +``` +# after the run, from /stats: +llm_calls == 0 # extract_llm genuinely never ran +cmdfilter.acted / requests # #42 firing rate (0.9% before, 28.7% after) +sse_buffered_pct # #33 fast path (100% before, 44.2% after) +frozen_* # #40 -- ZERO here, because its only callers are excluded. + # That is the expected result, NOT evidence about #40. +cachesplit.acted # 0 on TB: the Agent SDK sends no volatile tail to split +``` + +Two analysis requirements, both learned from earlier errors in this study: + +- **Report the median per-task ratio and a leave-one-out beside any aggregate.** The −16.4% + aggregate becomes −9.1% without `path-tracing` alone. A sum over heterogeneous tasks can be + one task. +- **Use unique, never cumulative, token figures.** Measured overcount was **44.5×** for + `cmdfilter` and **18.4×** for `extract` — a cumulative number is wrong by an order of magnitude. + +The per-arm run commands are otherwise identical to §7b, with a distinct binary name, port and +jobs-root. Note that the harnesses `pkill` by binary **name**, so two concurrent runs sharing a +name will kill each other's proxy — this happened twice during the study and invalidated two arms. + +## 8. Result docs + +- [`baseline.md`](baseline.md) — SWE-bench baseline (`off`) full results. - [`context-guru.md`](context-guru.md) — context-guru `codesmart` full results. - [`headroom.md`](headroom.md) — headroom full results. - [`rtk.md`](rtk.md) — rtk (Rust Token Killer) full results. -- [`comparison.md`](comparison.md) — the four-way comparison across all metrics. +- [`comparison.md`](comparison.md) — the four-way SWE-bench comparison across all metrics. +- [`terminal-bench-comparison.md`](terminal-bench-comparison.md) — the four-way **Terminal-Bench 2.0** comparison. +- [`terminal-bench-baseline.md`](terminal-bench-baseline.md) · [`-context-guru`](terminal-bench-context-guru.md) · [`-headroom`](terminal-bench-headroom.md) · [`-rtk`](terminal-bench-rtk.md) — TB per-arm full results. diff --git a/docs/results/improvement-plan.md b/docs/results/improvement-plan.md new file mode 100644 index 0000000..490e038 --- /dev/null +++ b/docs/results/improvement-plan.md @@ -0,0 +1,463 @@ +# context-guru improvement plan — deep analysis across SWE-bench + Terminal-Bench + +Synthesis of a 6-way deep analysis (context-guru components, headroom, rtk, per-task empirics, +the change-log dumps, and cache_control mechanics measured against real captured Claude Code +request bodies). Goal: understand what helped / hurt on both benchmarks and lay out, with +examples, how to make context-guru **much** cheaper and higher-reward across *all* workloads — +not just these two — and where our durable edge is. + +--- + +## 0. The one thing to internalise: what the bill is actually made of + +On a modern Claude Code agent the request is **~99.95% cached** (measured: fresh/uncached input +is 0.05% of the TB bill). So "remove tokens from the request" is almost the wrong objective — +**context-guru's unique token removal is a rounding error: 0.024% of billed input on SWE, 0.127% +on TB.** What actually moves the dollar figure, in order: + +1. **Agent steps.** `corr(Δsteps, Δcost) = +0.95` on *every* arm and *both* benchmarks. Each extra + turn re-reads the whole accumulated prefix at $0.20/M and emits more output at $10/M. On SWE, + context-guru's win was almost entirely a **−13.7% step reduction** that multiplied 165k removed + tokens into **−18.3M cache-read tokens (110× leverage)**. Tokens are the lever *only* insofar as + they change steps. +2. **Cache-write.** One cache-write costs **11.5 cache-reads** (`($2.50−$0.20)/$0.20`). Any mutation + of already-cached content re-writes the whole suffix. This is the entire "TB regression" story + for the proxies (see §1). +3. **Output tokens.** On TB, output is **47% of the bill** ($47 of $100) — *larger than cache-read*. + The only way to cut it is shorter trajectories (fewer steps). On SWE, cache-read is 64% and + output is minor. **The two benchmarks are different cost regimes and must be tuned separately.** +4. **Breakpoint placement.** `cache_control` is **metadata, not hashed content** — adding/moving a + breakpoint is **free and lossless**, yet it decides whether a turn is a cheap read or a full + re-write. This is the single largest *unused* lever (§3A). + +Content compaction (our current focus) matters, but mostly as a *means* to (1) and (2), and it is +currently our **most expensive and least reliable** lever. + +--- + +## 1. What happened, corrected — the results are not what the headline said + +### 1a. The Terminal-Bench "regression" was largely a measurement artifact +Six TB tasks have **degenerate baseline trials**: the baseline agent aborted in 2–6 steps / +16–800 s (almost certainly n=24 CPU-contention early-aborts that were not flagged as timeouts), +while every compaction arm ran the task properly for 50–160 steps. Comparing a 16-second no-op to +a 2-hour genuine attempt is not a cost comparison. `extract-moves-from-video` alone is **$24.10 of +headroom's $24.65 nominal regression**, and it is a task where the baseline did 2 steps. + +**Recomputed on the 83 clean tasks the story matches SWE:** + +| arm | all-in $ | Δ vs baseline | steps | +|---|--:|--:|--:| +| baseline | 100.17 | — | 38.1 | +| context-guru | 90.34 | **−9.8%** | 34.9 (−3.1) | +| headroom | 84.19 | **−16.0%** | 34.1 | +| rtk | 106.59 | +6.4% | 39.7 | + +Reproduced independently from the row files: baseline **$100.17**, context-guru **$87.47** model cost +(**−12.7%**) or **$90.34** including its own haiku cost (**−9.8%**), solving **56 vs 54** with +**2,899 vs 3,160** total steps (**−8.3%**). Note the published baseline is a *two-stage* merge +(`rows-off-final.json` + the 11-task 4x rerun); the intermediate file alone sums to $71.44 and does +not reproduce the published $100.81 — see REPRODUCE.md §7. + +So: **both proxies save ~10–16% on TB too; only rtk genuinely regresses; and context-guru +*reduces* steps on clean TB, it does not raise them.** (Action item F0: re-run those 6 baselines and +regenerate the published TB docs — the current numbers are wrong.) + +### 1b. Where compaction earns, by task size (the cleanest signal in the data) +Every arm **loses money on small tasks and makes it on large ones**: + +| bench | tercile | base prompt-tok | context-guru | headroom | rtk | +|---|---|--:|--:|--:|--:| +| SWE | small | 0.72M | **+8.1%** | +19.8% | +19.1% | +| SWE | large | 4.21M | **−25.7%** | −18.9% | −27.2% | +| TB | small | 0.41M | **+52.1%** | +49.7% | +24.0% | +| TB | large | 6.30M | **−19.7%** | −20.9% | +0.5% | + +The marker/overhead and (for us) the haiku call are fixed costs; below ~1M prompt tokens they +exceed the savings. **Gating compaction on conversation size is the cheapest large win and carries +zero reward risk** (§3B1). + +### 1c. What helped reward, and the honest read on headroom +Reward is **neutral on every arm except headroom-on-TB (+8, p≈0.096, n=1)** — suggestive, not +established. Examined per task, headroom's 5 unique hard-task solves decompose mostly into +**persistence** (it kept trying longer) and **baseline artifacts**; only `path-tracing-reverse` is a +clean "compaction freed context so the agent finished in time" case. The transferable lesson is a +**posture, not a technique**: headroom compressed only 2.64% of content, touched only the newest +turn, never dropped a whole message, excluded Read/Grep/Glob/Write/Edit, and freed ~825 tok/req of +tool-schema overhead. That is *"baseline plus free headroom,"* not *"better compression."* + +### 1d. Where context-guru specifically loses today +- **Small tasks and short/exact-output categories.** cg's TB losses concentrate in `security` + (+121%, 0 reward gain) and `debugging` (+51%) — short, exact-output-sensitive families. +- **`extract_llm` is 8× underwater.** It saved 197,548 unique tokens (~$0.04 at cache-read) for + **$3.26 + 26 minutes of blocking wall time**. Its per-call economics are *identical* on both + benchmarks (~$0.0166 / 1k unique tokens); TB just fires it 3.7× more often because 7% of TB + requests clear the 3000-token floor vs 0% on SWE. **93% of its realised savings come from the + replay result-cache, not from the model** — only 0.24% of tool outputs ever reached haiku. +- **The cache-write tax.** cg's write/read ratio is 2.82% on TB vs baseline 1.86% (+52%), from + mutating content inside the cached prefix (§2, §3A). +- ~~**`context_guru_expand` is referenced 1,496× and callable 0×.**~~ **REFUTED — see §B2.** The + tool *is* registered, the SSE loop *is* armed, and a live agent restored 3,372 tokens through the + streaming path. The 4.8M was a cumulative re-count; unique is 234,119 tokens (21× smaller). The + genuine defect in this area was a latency tautology that buffered every SSE response. + +--- + +## 2. How headroom and rtk handle the cache (you asked explicitly) + +**headroom** — *rewrites the newest turn, and pays for it.* On the path that actually ran (Python, +not the in-progress Rust rewrite) it **ignores incoming `cache_control` as a boundary** and derives +its frozen floor from observed provider usage, then in `cache` mode freezes "everything but the last +user turn." Because it compresses the live zone, it forwards bytes the client never sent and must +**replay** them next turn; every replay-state failure (600 s tracker TTL, 32-lineage cap under +subagent fan-out, any thinking/tool-id divergence) is a **full prefix re-write**. It also +**collapses Claude Code's multi-breakpoint ladder to a single tail marker**, removing the durable +older anchor — so a miss is total, not partial. Net: **its tripled cache-write is a tripled miss +rate.** The lesson is a warning: *do not rewrite the live zone; do not collapse the breakpoint +ladder.* + +**rtk** — *cache-safe by construction, but a weak one-sided lever.* It compresses Bash output at the +shell **before** it enters the transcript, so there is exactly one immutable version of every tool +result from turn 1 — no replay machinery, no idempotence requirement, $0, 0 ms. But it only ever +shrinks the *new tool-result write*; everything already in the transcript is re-read forever. Its +guard is **per-command (byte-level)** and structurally blind to the global bill, so on long horizons +it happily takes lossy compressions that trigger an extra step, and one extra step (a full-prefix +re-read) wipes out a dozen compressed commands. Its regression is **step-driven, not cache-driven.** +The lesson: *operate "at the source" and freeze on first sight (cache-safe), but never let a +byte-local guard authorize a step-costly loss.* + +**context-guru today** — *the right idea, three bugs.* It reads the breakpoint as a **boolean** and +throws away the position; it infers the cache boundary from **message count** (agreed with the true +last-breakpoint index on **0/73** TB turns); it **fails open** on a store miss (mutates the whole +prefix); and its freeze-store **expires frozen decisions mid-task**. Fixing these (§3A) is worth more +than any new compressor. + +--- + +## 3. The plan — prioritized, with mechanism, economic condition, and example + +### A. Cache-control exploitation — biggest, free, risk-free (your core question) + +**A1. Sticky-anchor breakpoint. ★ highest single lever, measured −23.2% on TB, ±0% on SWE.** +*Mechanism:* Claude Code leaves its 4th breakpoint slot empty and pins BP3 to the **last block every +turn**. When a turn adds ≥20 blocks (parallel tool calls — **28% of TB turns**, one message had 36 +`tool_use` blocks), BP3's 20-block lookback can't reach the previous entry and the whole history is +re-written. Fix: record the previous turn's breakpoint block (by **content hash**, not index) and +stamp the free 4th breakpoint there, giving the lookback a reachable anchor. Because `cache_control` +is metadata, this touches no cached bytes. +*Condition:* free; pays whenever `Pr(turn adds ≥20 blocks) > 0`. +*Example (real TB traffic, simulator):* write tokens **1.195M → 0.669M**, cost **$5.22 → $4.01 +(−23.2%)**; on the 35-turn chain alone −42%. SWE unaffected (median growth 2 blocks). + +**A2. Fix the freeze-store lifetime — this *is* the TB cache-write regression.** +*Mechanism:* `store/store.go` `Get` refreshes LRU recency but **not** `e.expires`, and the default +TTL is 1800 s. So a frozen compaction dies ~69 turns after it was written regardless of how often +it's replayed; on expiry it reverts full→compacted and re-writes the suffix. Three fixes: (a) sliding +TTL — refresh `expires` on `Get`; (b) raise the default well past a task's wall time (TB averaged +1975 s > 1800 s); (c) **never revert on a store miss** — once compacted, reverting is the expensive +move; keep replaying or fail closed. +*Condition:* a revert costs `W·$2.30/M` and buys nothing — always fix. +*Example:* eliminates ~15 mid-depth reverts ≈ **2.5M cache-write tokens ≈ $6.3** on TB — essentially +the entire 4.0M→6.5M gap — with no loss of compaction. + +**A3. Key the tail gate on the *real* breakpoint index, and fail *closed*.** +*Mechanism:* extend `hasCacheBreakpoint` (`apply/apply.go:254`) to return the last breakpoint's +message index; set `MaxCachedIdx = min(markerFloor, growthFloor)`. Invert `TailOnly` +(`components/component.go:129`): on an unknown boundary with `CacheAware`, return **false** (mutate +nothing) instead of true. +*Condition:* the current fail-open branch risks up to **$0.63/turn** (250k-token suffix rewrite) to +save `S·$0.20/M`. Sign is unambiguous. + +**A4. Promote the two static breakpoints to `ttl:"1h"`.** +*Mechanism:* rewrite `system[1]/system[2]` `cache_control` to `{"type":"ephemeral","ttl":"1h"}` +(longer-TTL must precede shorter — automatically satisfied). Protects the ~32k-token tools+system +prefix from re-creation whenever the agent stalls >5 min (test suites, training runs). +*Condition:* worth it when `Pr(idle gap > 5 min) > ~1%` — near-certain on agentic workloads. + +**A5. Retire `cacheinject` in its current form.** It stamps the 4th breakpoint one message behind +BP3, *inside the same 20-block window* → **provably inert** (simulated $5.22 → $5.22) while consuming +the slot A1 needs. Replace it with the sticky-anchor component (they are mutually exclusive at the +4-breakpoint ceiling). + +**A6. Do we ever want to harm the cache on purpose? Essentially no.** Break-even for deliberately +mutating cached content: `N > 11.5·(W/S)` (turns-remaining vs suffix/saving ratio). Compacting a +typical mid-depth output needs **N > 434 turns**; only removing a *large fraction of a near-tail +region* (`S ≈ 0.55·W`) clears it at `N > 21`. And tail-only + freeze-replay already propagates a +compaction into the prefix on later turns, so it captures the same savings **without** a bust: +simulated tail-only −44.2% equals compact-everything −44.2%. **Rule: never mutate at depth; there is +no realistic case for deliberate cache-busting on this traffic.** Stacking A1+A2+tail-only reached +**−57%** in simulation on the exact traffic where cg currently measures +1.7%. + +### B. Step & output reduction — the real cost/reward driver (corr 0.95) + +**B1. Gate compaction on context size / turn count.** Below ~1M prompt tokens compaction is pure +loss (+8% SWE-small, +52% TB-small). A `min_conversation_tokens` / `min_turns` gate before any +offloader recovers most of the small-task regression at zero reward risk. Above the gate, escalate +aggressiveness with size. + +**B2. ~~Register `context_guru_expand` on the streaming path~~ — REFUTED; the real bug was a +latency tautology.** ([#26](https://github.com/rossoctl/context-guru/issues/26) / +[PR #33](https://github.com/rossoctl/context-guru/pull/33)) + +Both halves of the original claim were wrong: + +- **The tool IS registered and the SSE loop IS armed.** `expand.Inject` fires on real streaming + requests (24 → 25 tools, idempotent on the next turn), and `proxy.serve` buffers + aggregates SSE + whenever markers are present. There is no streaming short-circuit. +- **A real agent DID invoke restoration through the streaming path.** Live SWE run: `bounces=1`, + **3,372 tokens restored**. `RecordExpand` has exactly one reachable call site (`proxy.go:488`), + inside the continuation loop, only after `expand.ResponseCalls` finds a model-issued expand call + *and* `expand.Resolve` succeeds. All traffic was SSE. +- **The 4.8M figure was cumulative**, re-counting each compaction on every turn history is re-sent. + Unique: **234,119 tokens behind 103 distinct markers** (TB) and **15,457 behind 29** (SWE) — + **21× and 8× smaller**. Restoration demand is genuinely low (~1 recoverable compaction per 3 + sessions), not blocked. + +**The actual bug, and it is a real one.** `hasMarkers` tested the raw body for `\u003ccg:`, and the +*injected tool description itself* contains that sequence (`toolDesc` mentions `<>`; Go's +`encoding/json` HTML-escapes `<`). So from the moment `Inject` ran, the check was a **tautology** and +**every SSE response was fully buffered** — defeating the documented zero-added-latency fast path. + +Fixed by scoping the check to `messages` + `system` (`expand.HasMarkersInMessages`). Measured: +marker-free TTFB **1007 ms → 43 ms (23×)**, marker-bearing correctly unchanged. On live traffic +buffering fell from an implied **100% → 27.3%**, and the transitions confirm the intended semantics — +17 requests streamed before the first offload, buffering began exactly as `saved` went non-zero, and +a fresh session resumed the fast path. + +**Lesson for this document:** a "component never fires" finding must be checked against the raw wire +bytes before it becomes a plan item. Two of this plan's entries (B2 here, C1 in §C) were premise +errors of exactly this kind, both caused by trusting a derived artifact — the change-log dumps — over +the request captures. + +**B3. Make `extract_llm` cost-aware and get it off the hot path.** +(a) **Prompt-cache its 852-token fixed preamble** (`cheapmodel/anthropic.go:45` — put the invariant +rules in a cached `system` block): ~90% of its input tokens become cache-reads. +(b) **Global content-hash-keyed result cache** (drop the session prefix in `resultKey`): 82/103 +unique contents recurred across sessions and are re-derived for nothing. +(c) **Economic gate:** only call when `expected_saving_$ > haiku_call_$ (~$0.012)` — on a cached +backend this correctly suppresses ~all TB calls; on a non-caching backend it lets them through. +(d) **Async:** compact in the background and let the next turn's replay cache pick it up — 93% of +value is already replay, so this removes 450 ms/req and $3.26 with negligible savings loss. + +**B4. Cut expand round-trips with a head-peek in the marker.** `mask` already exposes +`keep_head_chars`; add the same ~15-token peek to `extract`/`dedup`/`extract_llm` markers so the +model can decide whether an expand is worth a turn instead of bouncing blindly. + +### C. Cross-turn dedup — ~~the biggest untapped token lever~~ **REFUTED by measurement** + +!!! failure "C1 was wrong. The re-send factor is real; the interpretation was not." + **`xdedup` is unbuildable, and would be harmful if built.** Measured on the raw request + captures (1,325 requests / 51 sessions across `capture-tb`, `capture-swe`, `capture-swebench` + — *not* the change-log dumps, which only record messages a component already acted on and so + cannot answer this question): + + - **232 of 232** re-sent large outputs live at exactly **one stable message index** for their + entire session. Independently re-checked: of **77** distinct >4 KB tool outputs tracked per + session, **0 ever appeared at a second index**. + - **1124/1124 = 100%** of consecutive turn pairs have the entire previous turn as a + byte-identical prefix (comparing *content*, ignoring `cache_control` annotations). + - **Zero** compaction/rewind events on swebench — the message count never shrank. + + So a "re-read" is not a re-read. **The agent appends.** Turn N's copy of a file sits at the + index it has always occupied, inside the byte-identical cached prefix, billing at the + **cache-read** rate. The load-bearing claim — "content genuinely re-sent as new bytes" — is + false. + + `xdedup` could only act where the first copy is absent *and* the repeat is in the mutable tail. + Across 1,325 requests that intersection is **empty**: 5,314 re-send events (5.46M tokens) sit in + the cached prefix, where `TailOnly` correctly refuses — and rewriting them is precisely the + full → referenced flip that converts cache-reads into cache-writes at **11.5×**. The genuinely + duplicated cases are already caught by `dedup` (23 acts on the TB run, **0 tokens missed** above + the size gate). Implemented with its guards intact, the component is a permanent no-op. + + **Where the token mass actually is:** those 5.46M tokens are real but **already cheap**. The + lever is not removing them, it is keeping the prefix stable so they *stay* cache-reads — which + is `cacheinject`'s territory, and consistent with cross-session prefix repair measuring ≈−14% + while placement tuning measured ≈0%. See the finding that `cacheinject`'s breakpoints never + reach the wire at all (46 applied, 0 forwarded). + + **One caveat left open:** `capture-tb` showed 19 turns where the message count shrank. + Compaction is the one regime that could make cross-turn dedup viable, since it removes the first + copy while later re-reads land in the tail. Even there 0 actionable cases were measured, but the + premise is worth re-testing if aggressive compaction is enabled. + + Tracked as [#27](https://github.com/rossoctl/context-guru/issues/27), closed as + measurement-refuted. C2 and C3 below are unaffected and still stand. + +**C2. Recurrence-aware floor.** Track `seenCount[hash]`; effective floor `= floor / max(1,seenCount)`. +A 298-token output re-sent 40× is worth 12k tokens but is invisible to the 3000-token floor today — +this unlocks the **64% of tool-token mass currently below the floor** via the free deterministic path. + +**C3. Wire `freeze`/`reapplyFrozen` into *every* offloader.** Today only `mask` and `failed_run` use +it (both 0 acts on TB); `extract`, `dedup`, `cmdfilter`, `extract_llm` do not — which is why the dump +shows **101 compacted→full→compacted flip turns** and 15 non-byte-stable replays. This is the +documented cache-safety invariant, simply not implemented for 5 of 7 components. + +### D. Steal from headroom — the lossless layers we lack + +**D1. Tool-schema compaction. ★ highest-ROI steal (~825 tok/req, ~30 lines, lossless, cache-safe).** +Recursively drop annotation keys (`$schema,$id,title,examples,readOnly,…`), whitespace-collapse +descriptions, memoize on `sha256(tools)`, never-worse guard. That is ~3× headroom's entire *content* +savings on SWE. Splice point already exists (`expand/inject.go:57`); compact before injecting the +expand tool and apply the same transform to it. Instrument it as a first-class metric. +**Do not** adopt headroom's system-prompt compaction (lossy, unmemoized, byte-unstable). + +**D2. Structural log / diff / search compressors** (TB tool output is dominated by these). Lift two +invariants verbatim: **verbatim under 50 lines** (a hard floor that kills the "compressed something +tiny and lost the one line that mattered" class) and **stack-frame collapse** (keep 3 head + 5 app +frames, collapse runtime runs to a marker pinned above the drop threshold). For diffs: **never drop a +`+`/`-` line** — only whole hunks. + +**D3. `TextCrusher`-style extractive prose selection** (not the ONNX scorer first). Sentence +segmentation + `recency + 2·BM25 + 1.5·salience`, near-dup rejection, **emit kept segments verbatim in +original order** — pure selection, no model, no 209 ms, no newline destruction. Add the ModernBERT +scorer later only behind a size gate with their defence-in-depth (canary, wall-clock deadline, +verbatim-tail fallback). + +**D4. Kneedle adaptive-K + lossless-first in `smartcrush`** (replaces the fixed keep-first-3/last-2 +that our own code flags as unimplemented): re-render CSV/markdown-KV losslessly, adopt only if ≥15% +shrink, *then* consider lossy. + +**D5. Symbol-importance + query-boost in `skeleton`.** Rank symbols by `ref_count + is_public + +0.5·fan_out (+3.0 if the goal names the symbol)`, proportional per-symbol line budget, re-parse to +verify — far better reward preservation than uniform body elision. + +**D6. In-process cache-miss telemetry.** We have none — accounting is offline. Classify every turn as +`hit / ttl_expiry / prefix_change / cold_start` by comparing idle gap to the 300 s TTL and checking +prefix stability. We are flying blind on our single largest cost line; this is how headroom *knew* +"prefix_change = 100% of its misses." + +### E. Steal from rtk — cache-safe structural per-command compressors + +**E1. Pair each tool output with its originating command.** Walk back from a tool message via +`ToolCallID` to the assistant `ToolCall` and read `arguments.command` — the literal shell string, +rtk's dispatch key. This beats `cmdfilter`'s "match the output's first line" (which misses because +pytest starts with a platform banner) and, via `match_tool: Read|Grep|Glob`, lets us **cover +Claude Code's built-in tools — exactly rtk's structural blind spot.** + +**E2. Port the near-lossless per-command compressors, refuse the lossy ones.** rtk's SWE win came +from near-lossless structure (grep grouping, `git status` porcelain, all-green pytest→one line, the +63 pre-tested `*.toml` filters — directly translatable to our DSL with their inline tests); its TB +loss came from high-loss compressors (aggressive body elision, pytest failure-gutting to 3×100 +chars, silent `ls`/`git log` semantic rewrites). Port the first class; **never** the second. Keep +the real openable path (not rtk's `compact_path`), and for pytest keep the **full first traceback**, +compacting only the 2nd..Nth failure. + +**E3. Two invariants from rtk worth adopting verbatim.** (i) **"Never emit an unrecoverable +elision"** — if a filter is lossy and no resolvable marker can be written, **skip the filter** (today +`cmdfilter` degrades to dropping content with no way back). (ii) **Re-runnable hints** where the +source still exists (`re-run with: git diff --no-compact`) — no store entry, no TTL, never dangles; +composes with E1 since we know the command. Plus **partial/tail restore** (`expand(id, from_line=N)`) +so recovering is cheaper than re-running. + +**E4. One global "cap class" knob + a step-aware/cache-aware savings guard.** Centralise all filter +limits into `CAP_ERRORS/WARNINGS/LIST/INVENTORY` so a single per-preset dial moves every deterministic +compressor from aggressive→conservative. And make our own metric **cache-aware**: a component that +shrinks bytes but adds a turn (or a cache-write) must score **negative** — that single accounting +change would have caught rtk's TB regression before it shipped, and stops us optimising the raw byte +ratio (which overcounts 22–42×). + +### F. Hygiene / methodology + +!!! danger "F-1. The rule this whole document was written the hard way to learn" + **An aggregate moving in the predicted direction is not evidence that the predicted mechanism + operated.** Verify the mechanism fired *before* believing the outcome. + + **Eight** premises in this effort were wrong, and nearly all failed the same way — a derived + artifact was trusted over the raw request stream, or an outcome was credited to a mechanism + nobody checked had run: + + | premise | what was believed | what was true | + |---|---|---| + | §C1 `xdedup` | a 39.8× re-send factor meant tokens re-sent as new bytes | 232/232 large outputs sit at **one stable index**; the agent *appends*. Those tokens are cached-prefix reads, already cheap. Component would have had **zero** legal opportunity to act | + | §B2 expand tool | "never registered on the streaming path", 4.8M tokens stranded | tool IS registered, loop IS armed, a live agent restored 3,372 tokens. 4.8M was a **cumulative re-count**; unique is 234k (21× smaller). Real bug was a latency tautology | + | `prefixpin` | early messages mutate in place, worth ~31% of input cost | **0 mutations in ~6,500 comparisons** on claude-code. The 52% churn first measured was concurrent sessions sharing a first message, diffed against each other | + | §31 async cache-write | −45%/−39% cache-write proved the tail-protection worked | the protection **only stripped context-guru's own breakpoints**, never the agent's — so it did nothing on the primary workload. Lower cache-write came from writing *fewer breakpoints*, not from protecting the tail | + | §A5 `cacheinject` "provably inert" | placement has no headroom; the component does nothing | it applied **46 breakpoints and forwarded 0** — the writeback layer discarded every one. Inert for a reason nobody had checked. Two benchmark studies concluded things about placement while measuring a suppressed component | + | §A5 follow-on: placement is *harmful* | +61.9% cache-write/step once the marks reached the wire | **0 of 106 marks land above** the agent's own breakpoint, so the proposed mechanism (shortening the readable prefix) is ruled out. `acted=0` in that arm is a *tautology* of the arm's design, not proof the delta was placement | + | `cachesplit` on Terminal-Bench | the volatile-tail split carries a −34.1% win | TB runs the Agent **SDK**, which never appends the git/env snapshot the CLI does. All 73 captured TB requests: 3 system blocks, **zero** volatile-tail markers. **Zero legal opportunity** — the same shape as `xdedup` | + | the split on Bedrock Converse | `Changed: true`, `cachesplit` active in `/stats` | `cachePoint` is its own array entry *after* the block, so the split inserts the volatile half **before** it and the breakpoint still covers the churn. It reports success and achieves nothing | + + Most of these produced a number that pointed the *right* way for the *wrong* reason, which is + why they survived review. Two were mine as orchestrator, both from accepting a number without + checking the mechanism — and one of those was an *unfavourable* number, which is the tell that + the bias is not optimism but incuriosity. **Skepticism applied only to good news is not + skepticism.** The countermeasures, in order of value: + + 1. **Group request lineages by append-only prefix match**, never by a hash of `messages[0]` — a + benchmark harness runs concurrent sessions whose first message is byte-identical. + 2. **Distinguish cumulative from unique on every token figure.** The overcount is 8–42×, so a + cumulative number is off by an order of magnitude, not a rounding error. + 3. **Instrument "did this component act?" separately from "did the metric improve?"** and require + both before claiming a mechanism. `acted=0` beside a favourable delta is the tell. + 4. **A trial where the baseline aborts is not a measurement** — exclude it, don't average it. Six + such trials inverted the sign of the entire TB cost conclusion (§1a). + 5. **Read the raw wire bytes.** The change-log dumps only record messages a component *already + acted on*, so they structurally cannot answer "was this ever sent?" + 6. **A component reporting that it acted is not evidence it acted usefully.** `Changed: true` and + a non-zero `acted` counter both survive a mechanism that achieves nothing — see the Converse + split. Assert the *effect* (did the breakpoint stop covering the churn?), not the activity. + 7. **Check that a favourable metric had the opportunity to be caused by your change.** Three + arms credited components that could not fire on that workload at all. Before attributing, ask + what counter would be non-zero if the mechanism ran, and confirm it is. + 8. **Verify the verifier.** Two "defects" in this effort were bugs in the checking script (a + regex scraping the wrong table column; a stale binary one commit behind). A check that + over-matches manufactures exactly the findings that waste a review cycle. + 9. **A sum over heterogeneous tasks can be one task.** An interim TB delta read −40.2%; a single + trial carried half of it, and dropping that one task gave −19.2%. Report the **median + per-task ratio** and a **leave-one-out** on the top contributors beside any aggregate. + +- **F0. Re-run the 6 degenerate TB baselines** and regenerate the comparison/baseline docs (§1a). +- **F1. Report `saved_tokens_unique` and cache-aware $, never raw cumulative byte ratio** (overcounts + 22–42×; unusable for tuning). +- **F2. Kill dead components** — *revised twice*: `failed_run` (0 acts, burns 28.8 s scanning) — hoist + the `CacheAware` check so the regexes never run. **`cacheinject` is no longer in this list**, and the + story behind it is the clearest example of F-1 in the whole document: + 1. It read as "0 acts, inert", which is *expected* for a Reformat that removes no tokens — so the + reading was accepted. + 2. It was in fact discarding **every** breakpoint before the wire (46 applied, 0 forwarded): the + writeback layer dropped them because its only targets are `tool_use` messages that bifrost + cannot round-trip. Two full benchmark studies drew conclusions about placement while measuring + a component whose output never left the process. Fixed in + [#36](https://github.com/rossoctl/context-guru/pull/36). + 3. The first live measurement then looked *harmful* — +61.9% cache-write per step — and that was + accepted too, by me, until review showed the proposed mechanism is **ruled out**: 0 of 106 + marks land above the agent's own breakpoint, and the arm's `acted=0` is a tautology of its + design rather than proof the delta was placement. + **Net position: `cacheinject` is removed from all nine presets** (#36) — not because it is proven + harmful, but because there is a live cost signal with no explanation and no demonstrated benefit, + which is not a defensible default. A `cachesplit` marker component now carries the volatile-tail + split, which *does* have measured savings on CLI traffic. Whether placement helps at all remains + **genuinely unanswered** and needs a properly-powered study, not another n=1. +- **F3. Tune per regime:** SWE = cache-read regime (optimise steps; note cross-turn dedup is refuted, §C); TB = output + regime (optimise trajectory length). Same knobs, different settings, selected by measured context + size. + +--- + +## 4. Our edge — how context-guru wins on *every* workload + +The competitors each own one idea and pay for it elsewhere: **rtk** is cache-safe-by-construction but +a weak one-sided lever with a byte-local guard that costs steps; **headroom** has the best lossless +layers and prose scorer but rewrites the live zone and collapses the breakpoint ladder, tripling +cache-write. context-guru is the only one positioned to hold **all** of the good ideas at once: + +1. **Own breakpoint placement** (A1/A4) — a free −23% nobody else exploits; we already sit on the + request and can read the real breakpoints. +2. **Be strictly cache-safe** (A2/A3/C3) — freeze-and-replay, tail-only, fail-closed: never rewrite + the live zone (headroom's mistake), never expire a frozen decision. +3. **Compact *more* than headroom without the penalty** — add its lossless layers (D1–D5) *and* keep + `mask`'s aggressive-but-reward-neutral offload, because our freeze-replay makes aggression safe. +4. **Unlock the sub-floor token mass** (C2) — a recurrence-aware floor reaches the 64% of tool-token + mass currently invisible to the 3000-token threshold, LLM-free. (C1/`xdedup` is refuted — the + re-sends it targeted are cached-prefix reads, already cheap; see §C.) +5. **Cover the built-in tools** (E1) — rtk's structural ceiling, our free extension. +6. **Reduce steps, and account in the currency of the bill** (B, E4) — the only thing correlated 0.95 + with cost, and the path to beating headroom on reward: register the expand tool (stop the re-work), + gate small tasks, free context so long tasks finish within the timeout. + +**Plausible stacked outcome:** sticky-anchor (−23%) + freeze-fix (recovers the ~$6 cache-write gap) + +tail-only (−44%) compose to **−57%** in simulation, before adding tool-schema compaction, the +recurrence-aware floor, and the step-reduction levers — i.e. a path to **substantially beyond headroom's +−16%**, while being strictly cache-safe and reward-neutral-to-positive. The prerequisites are the +three cache bugs (A2/A3) and the SSE buffering fix (B2, shipped); everything else compounds on top. diff --git a/docs/results/terminal-bench-baseline.md b/docs/results/terminal-bench-baseline.md new file mode 100644 index 0000000..d630ce2 --- /dev/null +++ b/docs/results/terminal-bench-baseline.md @@ -0,0 +1,185 @@ +# Full results — baseline (Terminal-Bench 2.0, 89 tasks) + +Baseline arm: **no compaction** — the `claude-code` agent on `aws/claude-sonnet-5`, run LIVE through the harness against Terminal-Bench 2.0's 89 tasks. Routing goes through the context-guru `off` transparent passthrough proxy (identical plumbing to the compaction arms; zero content change), so this is the like-for-like reference the framework arms are measured against. Cache-aware billed input cost (fresh $2/M · cache-read $0.20/M · cache-write $2.50/M) + output $10/M, recomputed from each trial's own token tiers — the same model as the SWE-bench study. See [REPRODUCE.md](REPRODUCE.md). + +## Totals + +| attempted | solved | solve rate | completed | timed out | total billed cost | mean steps* | cache-hit | +|--:|--:|--:|--:|--:|--:|--:|--:| +| 89 | 56 | **62.9%** | 82 | 7 | $100.81 | 31.5 | 98.2% | + +\* mean steps over the 82 completed tasks (timed-out runs are truncated). Solve rate over **completed-only** tasks: **56/82 = 68.3%**. + +**Time-budget policy.** Wall-clock budget = the task-authored timeout × a multiplier. Most tasks ran at **1.5×**; the long-horizon tasks that first timed out were retried at low concurrency and, if still short, given an extended **4×** budget (up to ~4 h) to measure capability rather than a latency-truncated result. 3 tasks solved only under the 4× budget (counted as solved here); the 7 below exhausted even 4×. + +## Analysis — where the agent is strong / weak + +- **Terminal-Bench 2.0 is a much harder, longer-horizon benchmark than SWE-bench Verified.** The + baseline solves **62.9%** of tasks vs 86% on SWE-bench, at **~$1.13/task** (vs $0.64) and **~1.7M + prompt tokens/task** — TB tasks are open-ended terminal goals (build/compile/train/exploit), not a + localized patch, so the agent runs longer and reads far more. +- **Difficulty is the dominant axis.** Easy/medium solve at **71–75%**; `hard` drops to **47%** and + costs **3.4× more per task** ($2.04 vs $0.61). Every one of the 7 unrecoverable timeouts is a + hard/long task. +- **Category tells the same story from the other side.** The agent is reliable on bounded, + verifiable goals — `debugging` (100%), `system-administration` (78%), `security` (75%), + `data-processing`/`model-training` (75%) — and weak on sprawling build/implement tasks: + `software-engineering` (38%, the largest bucket at 26 tasks) and `video-processing` (0%). +- **It is still a ~98%-cached agent** (98.2% cache-hit), so — exactly as on SWE-bench — + **cache-read is the single biggest cost term (43% of the bill)**. That is the lever the compaction + arms must pull; a layer that shrinks the cached context should move TB cost the same way it moved + SWE cost. +- **Latency, not just capability, caps the ceiling.** The gateway's ~26 s/request (5–10× a normal + endpoint) means long-horizon tasks can run out of wall-clock before finishing. 3 tasks that first + timed out **solved once given a 4× budget** — so **62.9% is a floor**, and a compaction arm that + reduces round-trips could recover more of the remaining 7. The timeout count is therefore itself a + comparison metric, not just noise. + +### Token & cost accounting (cache-aware, all 89 tasks) + +| tier | tokens | $/M | billed | +|---|--:|--:|--:| +| cache-read (input) | 215,971,427 | 0.20 | $43.19 | +| cache-write (input) | 4,011,068 | 2.50 | $10.03 | +| fresh (input) | 58,893 | 2.00 | $0.12 | +| completion (output) | 4,746,887 | 10.00 | $47.47 | +| **total** | | | **$100.81** | + +Cache-read is **43%** of the bill at a **98.2%** cache-hit rate — as on SWE-bench, a heavily-cached agent, so the lever a compaction layer must pull is cache-read tokens. + +## Timeouts (7 long-horizon tasks) + +These tasks still hit the wall-clock budget under the **extended 4×** timeout (up to ~4 h each) and scored **reward 0** — counted as failures in the solve rate above. A large part of the cause is **gateway latency, not only agent capability**: Terminal-Bench's timeouts assume a fast endpoint (~2–5 s/request), but this IBM LiteLLM gateway runs **~26 s/request** (5–10× slower), so long-horizon tasks that need many round-trips run out of clock (concurrency is *not* the cause — latency was flat ~23–30 s/req from n=1 to n=24). They are all `hard`/long software-engineering and compute tasks (path-tracing, a MIPS Doom port, a metacircular evaluator, COBOL modernization, GPT-2 code-golf, CIFAR training). A compaction arm that cuts round-trips could bring some under budget, so the timeout count is itself a comparison metric. + +| task | difficulty | category | steps before timeout | partial billed | budget (4×) | +|---|---|---|--:|--:|--:| +| caffe-cifar-10 | medium | machine-learning | 12 | $0.24 | 80 min | +| cobol-modernization | easy | software-engineering | 133 | $5.11 | 60 min | +| gpt2-codegolf | hard | software-engineering | 43 | $1.48 | 60 min | +| make-doom-for-mips | hard | software-engineering | 160 | $6.39 | 60 min | +| path-tracing-reverse | hard | software-engineering | 170 | $9.62 | 120 min | +| schemelike-metacircular-eval | medium | software-engineering | 74 | $4.37 | 160 min | +| write-compressor | hard | software-engineering | 6 | $0.36 | 60 min | + +## By difficulty (all 89 tasks; timeouts = failures) + +| difficulty | tasks | solved | rate | timed out | mean $/task | +|---|--:|--:|--:|--:|--:| +| easy | 4 | 3 | 75% | 1 | $1.561 | +| medium | 55 | 39 | 71% | 2 | $0.606 | +| hard | 30 | 14 | 47% | 4 | $2.040 | + +## By category (all 89 tasks) + +| category | tasks | solved | rate | mean $/task | mean steps* | +|---|--:|--:|--:|--:|--:| +| data-querying | 1 | 1 | 100% | $1.151 | 17.0 | +| debugging | 5 | 5 | 100% | $0.878 | 40.0 | +| games | 1 | 1 | 100% | $0.570 | 36.0 | +| personal-assistant | 1 | 1 | 100% | $0.329 | 8.0 | +| system-administration | 9 | 7 | 78% | $0.903 | 46.3 | +| data-processing | 4 | 3 | 75% | $0.309 | 15.0 | +| mathematics | 4 | 3 | 75% | $1.396 | 36.0 | +| model-training | 4 | 3 | 75% | $0.553 | 24.8 | +| security | 8 | 6 | 75% | $0.334 | 15.4 | +| machine-learning | 3 | 2 | 67% | $1.248 | 34.5 | +| data-science | 8 | 5 | 62% | $0.892 | 30.5 | +| file-operations | 5 | 3 | 60% | $0.577 | 26.0 | +| scientific-computing | 8 | 4 | 50% | $0.678 | 24.1 | +| software-engineering | 26 | 12 | 46% | $2.043 | 37.9 | +| optimization | 1 | 0 | 0% | $0.145 | 8.0 | +| video-processing | 1 | 0 | 0% | $2.094 | 80.0 | + +## Per-task (all 89) + +| task | difficulty | category | outcome | steps | cache_read | cache_write | billed | wall | +|---|---|---|:--:|--:|--:|--:|--:|--:| +| adaptive-rejection-sampler | medium | scientific-computing | ❌ failed | 11 | 462,474 | 17,913 | $0.351 | 6.7 min | +| bn-fit-modify | hard | scientific-computing | ✅ solved | 19 | 806,095 | 12,961 | $0.304 | 6.0 min | +| break-filter-js-from-html | medium | security | ✅ solved | 13 | 524,418 | 6,145 | $0.236 | 11.0 min | +| build-cython-ext | medium | debugging | ✅ solved | 75 | 5,004,079 | 52,076 | $1.342 | 17.8 min | +| build-pmars | medium | software-engineering | ❌ failed | 32 | 1,614,846 | 25,937 | $0.478 | 6.2 min | +| build-pov-ray | medium | software-engineering | ❌ failed | 46 | 2,311,209 | 22,883 | $0.597 | 10.9 min | +| caffe-cifar-10 | medium | machine-learning | ⏱ timeout | 12 | 530,037 | 22,399 | $0.245 | 5.8 min | +| cancel-async-tasks | hard | software-engineering | ✅ solved | 9 | 334,567 | 4,618 | $0.224 | 3.3 min | +| chess-best-move | medium | games | ✅ solved | 36 | 1,676,787 | 16,013 | $0.570 | 8.8 min | +| circuit-fibsqrt | hard | software-engineering | ✅ solved | 50 | 3,331,130 | 69,283 | $3.849 | 109.3 min | +| cobol-modernization | easy | software-engineering | ⏱ timeout | 133 | 9,627,750 | 62,902 | $5.107 | 60.0 min | +| code-from-image | medium | software-engineering | ✅ solved | 5 | 165,696 | 4,230 | $0.049 | 0.4 min | +| compile-compcert | medium | system-administration | ✅ solved | 63 | 3,765,545 | 152,729 | $1.294 | 51.6 min | +| configure-git-webserver | hard | system-administration | ✅ solved | 26 | 1,145,910 | 12,625 | $0.356 | 4.7 min | +| constraints-scheduling | medium | personal-assistant | ✅ solved | 8 | 309,526 | 9,879 | $0.329 | 4.7 min | +| count-dataset-tokens | medium | model-training | ✅ solved | 10 | 410,163 | 10,860 | $0.140 | 7.4 min | +| crack-7z-hash | medium | security | ✅ solved | 21 | 904,687 | 10,512 | $0.232 | 9.4 min | +| custom-memory-heap-crash | medium | debugging | ✅ solved | 37 | 2,605,935 | 55,794 | $1.394 | 20.1 min | +| db-wal-recovery | medium | file-operations | ❌ failed | 34 | 1,645,855 | 26,864 | $1.083 | 20.2 min | +| distribution-search | medium | machine-learning | ✅ solved | 11 | 439,452 | 9,718 | $0.291 | 4.2 min | +| dna-assembly | hard | scientific-computing | ❌ failed | 28 | 1,443,693 | 25,382 | $0.689 | 13.9 min | +| dna-insert | medium | scientific-computing | ❌ failed | 31 | 1,617,092 | 26,810 | $0.854 | 13.5 min | +| extract-elf | medium | file-operations | ✅ solved | 12 | 488,419 | 9,549 | $0.272 | 4.2 min | +| extract-moves-from-video | hard | file-operations | ❌ failed | 2 | 38,801 | 2,561 | $0.017 | 0.3 min | +| feal-differential-cryptanalysis | hard | mathematics | ✅ solved | 25 | 1,158,755 | 16,179 | $1.430 | 28.9 min | +| feal-linear-cryptanalysis | hard | mathematics | ✅ solved | 54 | 3,170,572 | 140,161 | $2.436 | 69.2 min | +| filter-js-from-html | medium | security | ❌ failed | 12 | 496,608 | 11,505 | $0.530 | 7.6 min | +| financial-document-processor | medium | data-processing | ❌ failed | 35 | 1,217,231 | 63,060 | $0.547 | 5.7 min | +| fix-code-vulnerability | hard | security | ✅ solved | 10 | 392,091 | 19,737 | $0.144 | 1.3 min | +| fix-git | easy | software-engineering | ✅ solved | 10 | 387,268 | 17,054 | $0.146 | 3.3 min | +| fix-ocaml-gc | hard | software-engineering | ✅ solved | 52 | 3,262,223 | 91,713 | $1.251 | 25.1 min | +| gcode-to-text | medium | file-operations | ✅ solved | 64 | 3,465,831 | 84,253 | $1.151 | 15.4 min | +| git-leak-recovery | medium | software-engineering | ✅ solved | 11 | 422,092 | 5,796 | $0.120 | 1.5 min | +| git-multibranch | medium | system-administration | ✅ solved | 38 | 1,947,898 | 23,971 | $0.545 | 9.0 min | +| gpt2-codegolf | hard | software-engineering | ⏱ timeout | 43 | 2,163,376 | 27,198 | $1.476 | 60.0 min | +| headless-terminal | medium | software-engineering | ✅ solved | 19 | 860,979 | 14,958 | $0.329 | 6.3 min | +| hf-model-inference | medium | data-science | ✅ solved | 10 | 380,807 | 5,907 | $0.113 | 3.0 min | +| install-windows-3.11 | hard | system-administration | ❌ failed | 102 | 6,968,317 | 56,473 | $2.002 | 30.4 min | +| kv-store-grpc | medium | software-engineering | ✅ solved | 9 | 341,540 | 6,033 | $0.103 | 2.5 min | +| large-scale-text-editing | medium | file-operations | ✅ solved | 18 | 752,118 | 8,475 | $0.360 | 7.3 min | +| largest-eigenval | medium | mathematics | ✅ solved | 25 | 1,138,904 | 15,330 | $0.429 | 6.1 min | +| llm-inference-batching-scheduler | hard | machine-learning | ✅ solved | 58 | 5,548,609 | 156,412 | $3.209 | 42.0 min | +| log-summary-date-ranges | medium | data-processing | ✅ solved | 7 | 253,952 | 5,872 | $0.082 | 1.3 min | +| mailman | medium | system-administration | ✅ solved | 91 | 7,495,690 | 72,125 | $2.326 | 26.6 min | +| make-doom-for-mips | hard | software-engineering | ⏱ timeout | 160 | 18,623,369 | 227,155 | $6.391 | 60.0 min | +| make-mips-interpreter | hard | software-engineering | ❌ failed | 99 | 10,602,706 | 145,552 | $4.276 | 47.2 min | +| mcmc-sampling-stan | hard | data-science | ✅ solved | 47 | 3,348,755 | 70,024 | $0.944 | 27.0 min | +| merge-diff-arc-agi-task | medium | debugging | ✅ solved | 33 | 1,489,291 | 54,882 | $0.550 | 16.4 min | +| model-extraction-relu-logits | hard | mathematics | ❌ failed | 40 | 2,186,378 | 33,983 | $1.288 | 17.4 min | +| modernize-scientific-stack | medium | scientific-computing | ✅ solved | 8 | 309,419 | 8,771 | $0.108 | 1.3 min | +| mteb-leaderboard | medium | data-science | ✅ solved | 4 | 124,910 | 325 | $0.075 | 2.2 min | +| mteb-retrieve | medium | data-science | ✅ solved | 9 | 338,332 | 7,062 | $0.112 | 3.0 min | +| multi-source-data-merger | medium | data-processing | ✅ solved | 8 | 299,556 | 7,022 | $0.142 | 2.8 min | +| nginx-request-logging | medium | system-administration | ✅ solved | 11 | 439,741 | 8,750 | $0.137 | 1.8 min | +| openssl-selfsigned-cert | medium | security | ❌ failed | 11 | 426,962 | 6,501 | $0.125 | 2.2 min | +| overfull-hbox | easy | debugging | ✅ solved | 45 | 2,464,056 | 30,433 | $0.901 | 16.1 min | +| password-recovery | hard | security | ✅ solved | 33 | 1,579,302 | 19,069 | $0.874 | 15.0 min | +| path-tracing | hard | software-engineering | ✅ solved | 262 | 26,460,778 | 346,898 | $10.795 | 109.7 min | +| path-tracing-reverse | hard | software-engineering | ⏱ timeout | 170 | 17,971,303 | 373,040 | $9.621 | 120.0 min | +| polyglot-c-py | medium | software-engineering | ❌ failed | 11 | 422,257 | 5,552 | $0.370 | 5.8 min | +| polyglot-rust-c | hard | software-engineering | ❌ failed | 3 | 80,536 | 2,653 | $0.080 | 13.3 min | +| portfolio-optimization | medium | optimization | ❌ failed | 8 | 313,695 | 11,094 | $0.145 | 4.3 min | +| protein-assembly | hard | scientific-computing | ✅ solved | 41 | 2,573,426 | 50,569 | $1.849 | 30.7 min | +| prove-plus-comm | easy | software-engineering | ✅ solved | 7 | 239,431 | 12,162 | $0.090 | 0.9 min | +| pypi-server | medium | software-engineering | ✅ solved | 15 | 603,607 | 6,981 | $0.165 | 3.9 min | +| pytorch-model-cli | medium | model-training | ✅ solved | 32 | 1,505,970 | 19,926 | $0.438 | 9.3 min | +| pytorch-model-recovery | medium | model-training | ✅ solved | 18 | 814,226 | 16,239 | $0.344 | 14.1 min | +| qemu-alpine-ssh | medium | system-administration | ✅ solved | 39 | 2,026,593 | 30,041 | $0.724 | 21.2 min | +| qemu-startup | medium | system-administration | ✅ solved | 25 | 1,086,800 | 12,472 | $0.470 | 12.1 min | +| query-optimize | medium | data-science | ❌ failed | 14 | 564,257 | 8,717 | $0.262 | 14.3 min | +| raman-fitting | medium | scientific-computing | ❌ failed | 34 | 1,731,134 | 29,874 | $0.894 | 13.9 min | +| regex-chess | hard | software-engineering | ❌ failed | 4 | 129,443 | 1,453 | $0.057 | 2.1 min | +| regex-log | medium | data-processing | ✅ solved | 10 | 380,065 | 5,785 | $0.463 | 7.4 min | +| reshard-c4-data | medium | data-science | ❌ failed | 59 | 3,513,094 | 70,607 | $3.219 | 64.2 min | +| rstan-to-pystan | medium | data-science | ✅ solved | 48 | 2,690,995 | 95,456 | $0.950 | 35.5 min | +| sam-cell-seg | hard | data-science | ❌ failed | 53 | 3,259,944 | 66,464 | $1.458 | 39.5 min | +| sanitize-git-repo | medium | security | ✅ solved | 12 | 712,805 | 57,064 | $0.395 | 2.9 min | +| schemelike-metacircular-eval | medium | software-engineering | ⏱ timeout | 74 | 5,585,032 | 175,235 | $4.370 | 160.0 min | +| sparql-university | hard | data-querying | ✅ solved | 17 | 773,789 | 15,944 | $1.151 | 15.3 min | +| sqlite-db-truncate | medium | debugging | ✅ solved | 10 | 380,755 | 7,161 | $0.203 | 2.8 min | +| sqlite-with-gcov | medium | system-administration | ❌ failed | 22 | 980,315 | 15,320 | $0.275 | 7.2 min | +| torch-pipeline-parallelism | hard | software-engineering | ❌ failed | 33 | 1,964,911 | 41,056 | $0.863 | 16.9 min | +| torch-tensor-parallelism | hard | software-engineering | ❌ failed | 24 | 1,086,495 | 18,363 | $0.440 | 18.5 min | +| train-fasttext | hard | model-training | ❌ failed | 39 | 2,647,796 | 260,108 | $1.289 | 53.2 min | +| tune-mjcf | medium | scientific-computing | ✅ solved | 21 | 927,646 | 13,076 | $0.374 | 8.5 min | +| video-processing | hard | video-processing | ❌ failed | 80 | 5,345,972 | 55,105 | $2.094 | 30.7 min | +| vulnerable-secret | medium | security | ✅ solved | 11 | 434,737 | 12,474 | $0.140 | 1.9 min | +| winning-avg-corewars | medium | software-engineering | ✅ solved | 57 | 3,293,598 | 38,085 | $1.507 | 29.5 min | +| write-compressor | hard | software-engineering | ⏱ timeout | 6 | 208,218 | 3,710 | $0.357 | 60.0 min | diff --git a/docs/results/terminal-bench-comparison.md b/docs/results/terminal-bench-comparison.md new file mode 100644 index 0000000..1fd3fa8 --- /dev/null +++ b/docs/results/terminal-bench-comparison.md @@ -0,0 +1,292 @@ +# Benchmark: Terminal-Bench 2.0 — baseline vs context-guru vs headroom vs rtk + +**Terminal-Bench 2.0 · 89 tasks · `claude-code` agent on `aws/claude-sonnet-5`**, run live +through the harness. This is the second benchmark of the study; the [SWE-bench Verified +four-way](comparison.md) is the first. + +!!! tip "A fifth arm was added on 2026-08-10" + The [merged-system section](#the-merged-system-a-fifth-arm-2026-08-10) below re-measures + context-guru after 15 PRs of cache/filter/observe work. Headline: **61 solved vs baseline 53**, + **cache-write back to baseline parity** (1.86% vs the previous arm's 2.86%), **$0** own-LLM + cost, and **−7.8% median per task** — read the median rather than the −16.4% aggregate, which + one task dominates. The original four-arm study is unchanged below it. + +The four original arms: + +- **baseline** — no compaction (context-guru `off` passthrough; identical routing). +- **context-guru** (`codesmart`) — cache-aware request-stream proxy, hybrid deterministic + a + cheap **haiku** LLM (`extract_llm`). +- **headroom** (`hd-cache`) — deterministic request-stream proxy (`--no-ccr`, streaming-safe). +- **rtk** — shell-level `PreToolUse` hook, compresses Bash output in-container. + +**Methodology.** Cache-aware billed **input** cost (fresh $2/M · cache-read $0.20/M · +cache-write $2.50/M) + output $10/M, recomputed from each trial's own token tiers; **total** +adds the tool's own compaction-LLM cost (context-guru's haiku calls). All 89 tasks carry a +scored outcome; timeouts (agent exceeded its wall-clock budget) count as reward-0 failures. +See [REPRODUCE.md](REPRODUCE.md) and the [baseline page](terminal-bench-baseline.md). + +!!! danger "Read the 89-task cost figures with this correction" + **Six baseline trials are degenerate.** On these tasks the baseline aborted almost + immediately while the compaction arms did the real work, so the per-task cost delta + measures *the baseline not doing the job*, not the cost of compaction: + + | task | baseline | context-guru | + |---|--:|--:| + | `mteb-leaderboard` | $0.08 (4 steps) | $6.05 (147 steps) | + | `polyglot-rust-c` | $0.08 (3 steps) | $2.81 (50 steps) | + | `extract-moves-from-video` | $0.02 (2 steps) | $2.69 (112 steps) | + | `regex-chess` · `write-compressor` · `code-from-image` | 4–6 steps each | comparable | + + Those three rows alone are **$11.5 of apparent regression**. Recomputed over the + **83 clean tasks** (same cache-aware model, context-guru's own haiku cost included): + + | | baseline | context-guru | delta | + |---|--:|--:|--:| + | billed model cost | $100.17 | $87.47 | **−12.7%** | + | + context-guru's LLM cost | — | $90.34 | **−9.8%** | + | solved | 54 | **56** | +2 | + | total steps | 3,160 | **2,899** | **−8.3%** | + + So on the clean set context-guru is **cheaper, solves more, and takes fewer steps** — + the same direction as its SWE-bench result, not the reversal the 89-task total implies. + headroom recomputes to ≈**−16%** and rtk remains a genuine regression (≈**+6%**). + + The tables below are the raw 89-task figures, kept as measured. They will be + regenerated once the six baselines are re-run at low concurrency; that re-run is + tracked as follow-up work. See [improvement-plan.md](improvement-plan.md) §1. + +!!! warning "Two caveats that shape how to read this" + **1. Single trial per task (`n-attempts=1`).** Unlike the SWE study (2 trials), each task + ran once per arm. **Solve-rate deltas carry real run-to-run noise** — the per-task flip + churn below (e.g. context-guru +10/−8) shows the net reward differences are mostly within + that noise. The **cost, cache, and step aggregates are robust** (sums over 89 tasks), and + are where the real signal is. + **2. Budget policy.** Framework arms ran at a flat **4× wall-clock budget**; the baseline + used 1.5× for most tasks and 4× for the long-horizon set. This is fair: every task a + framework "gained" over baseline was one the baseline *completed and got wrong* at 1.5× + (more time would not have changed it), not a baseline timeout. + +## The merged system — a fifth arm (2026-08-10) + +Everything below this section is the **original four-way study**. This section is the +re-measurement of context-guru after the cache/filter/observe work landed on `main` +(15 PRs: SSE fast path, `cacheinject` reaching the wire, freeze TTL, 24 cmdfilter filters, +observe mode, and the `extract_llm` economic gate). + +**Configuration `cgfinal` = `[format, dedup, cmdfilter, extract, cachesplit]`.** Chosen on +per-component evidence, not on maximal token reduction. Three components excluded: + +| excluded | measurement | +|---|---| +| `extract_llm` | saved 197,548 unique tokens — worth **$0.0395** at the cache-read rate they actually bill at — for **$3.26** and 1,592,467 ms of blocking time. **82× underwater.** (The plan's earlier "8×" priced those tokens as *fresh*; they sit in the cached prefix.) | +| `failed_run` | `acted=0`, 28,757 ms spent scanning. Pure latency. | +| `cacheinject` | removed from all nine presets — see [cacheinject](../components/cacheinject.md). Now that its breakpoints reach the wire, enabling it pushes **cache-write**, the deciding term here, the wrong way. | + +`mask` was deliberately **not** added despite being the largest known token lever (~29.5%): +that figure is a single-task replay, it drops whole messages, and it is the one offloader that +reaches inside the cached prefix. + +### Results (81 clean tasks, paired per task) + +| | baseline | context-guru (old) | **context-guru (merged)** | +|---|--:|--:|--:| +| solved | 53 | 55 | **61** | +| steps | 3,067 | 2,811 | 2,815 | +| cache-write | 3.90M | **5.04M** | **3.36M** | +| **cache-write / cache-read** | **1.86%** | **2.86%** | **1.86%** | +| cache-hit | 98.16% | 97.18% | 98.12% | +| own LLM cost | $0 | $2.97 | **$0** | +| context-guru added latency | — | 449.8 ms | **38.5 ms** | +| **total billed** | **$94.85** | $85.72 (−9.6%) | **$79.32 (−16.4%)** | + +!!! warning "Read the median, not the aggregate" + **−16.4% is single-task sensitive.** `path-tracing` alone contributes most of it: dropping + that one task gives **−9.1%**, and an independent re-derivation with a stricter degenerate + rule (76 clean tasks) gave **−13.7% → −2.8%** on the same exclusion. + + The **median per-task ratio is 0.922, i.e. −7.8%**, with **49/81 tasks cheaper**. That is the + figure to quote for "what this does to a normal task"; the aggregate answers the different + question "what would the whole benchmark have cost". Both are reported because they differ + by 9 points. + + Reward is **+11 / −3 = net +8** at a single trial per task, so the churn matters more than + the net. All three losses were read from their verifier output and are **capability + failures, not information loss**: an HTTP 404, a wrong-cased flag + (`gcod3_iz_ch4llenging` vs `gc0d3_iz_ch4LLenGiNg`), and a rejected non-fast-forward push. + Two of the three used *fewer* steps than baseline, which argues against the + "compaction hid something, agent redid work" mechanism. + +### The one result that needs no caveat + +**cache-write / cache-read returns to 1.86% — identical to baseline** — where the previous arm +ran at **2.86% (+54%)**. That is the "cache-write tax" this study identified as the deciding +term on Terminal-Bench, eliminated. It holds under every exclusion rule tried, because it is a +ratio rather than a sum. + +### Mechanisms verified fired — and one that could not be + +Per the [F-1 rule](improvement-plan.md), an aggregate moving the predicted way is not evidence +the predicted mechanism operated. Each claim below names the counter that proves it: + +| PR | status | evidence | +|---|---|---| +| #33 SSE fast path | **verified** | 44.2% of streams buffered, down from an unconditional **100%**; a marker-free probe streamed through, which was impossible before | +| #42 cmdfilter | **verified, large** | `acted` **950/3,311 (28.7%)** vs the old arm's **34/3,827 (0.9%)** — a 32× firing rate, 35.7× unique tokens | +| #36 cacheinject wire | code verified, **value inert here** | `cachesplit acted=0` for a *structural* reason: TB runs the Claude Agent **SDK**, which never appends the git/env snapshot the CLI does, so all 73 captured requests carry 3 system blocks and **zero** volatile-tail markers. The −34.1% split figure came from SWE-bench CLI traffic and does not transfer | +| #40 freeze TTL | **NOT EXERCISED — unverified** | all five `frozen_*` counters are 0, because its only callers are `mask`, `failed_run` and `extract_llm` — all three excluded by this config. **This arm is not evidence for or against #40 in either direction**, and none of the cost improvement above may be credited to it | +| #43 observe | off by design | `mode: sync` | + +### Regressions, stated plainly + +1. **`system-administration`: +17.2% cost *and* −2 solved (7→5).** The only category losing on + both axes, and the clearest genuine regression in the run. +2. **`security`: +25.6%.** Better than the previous arm's +121%, still the wrong direction. +3. **`fresh_input` is 3.8× baseline** (101,152 vs 26,901) — worth ~$0.20 against $0.05, under + 0.3% of the bill, but directionally wrong. +4. **Small tasks still inflate**, exactly as §1b predicts: `optimization` +311% (n=1), + `games` +50.7%, `model-training` +28.1%. Marker overhead is not recovered below ~1M prompt + tokens, so **size-gating remains an unclaimed win**. +5. **The honest framing of the headline.** `cgfinal`'s raw *model* cost is nearly tied with the + old arm ($79.32 vs $82.75) and its cache-read is **higher** (180.4M vs 176.4M) — the LLM pass + genuinely removed more content. `cgfinal` wins mainly by **not spending $2.97** on haiku. + +### Limitations + +- **headroom and rtk cannot be re-derived.** Their trial artifacts have been pruned from disk, + so those two columns are **cited from the original study, not recomputed**. Baseline and the + old context-guru arm both reproduce their published totals exactly. +- **Single trial per task**, as in the original study. +- Every token figure here is **unique**, never cumulative — the measured overcount ratios were + **44.5× (cmdfilter)** and **18.4× (extract)**. +- One task (`circuit-fibsqrt`) was still running at report time; 88 of 89 are scored. + +--- + +## Headline + +**On long-horizon terminal tasks the raw 89-task totals show no arm beating baseline on cost — +but that is dominated by six degenerate baseline trials** (see the correction above). On the +83 clean tasks **both proxies save**: context-guru −9.8% (solving +2, with 8.3% fewer steps) +and headroom ≈−16%. **rtk is the one genuine cost regression**, the mirror image of its +SWE-bench result. Read the tables below as measured, and the clean-set figures as the +conclusion. + +| dimension | baseline | **context-guru** | headroom | **rtk** | best | +|---|--:|--:|--:|--:|:--| +| solved / 89 | 56 (62.9%) | 58 (65.2%) | **64 (71.9%)** | 55 (61.8%) | headroom | +| **total billed cost** | **$100.81** | $102.55 (+1.7%) | $114.75 (+13.8%) | $118.83 (+17.9%) | **baseline** | +| cache-read tokens | 216.0M | 204.6M (−5.3%) | **198.0M (−8.3%)** | 254.0M (+17.6%) | headroom | +| cache-write tokens | **4.01M** | 6.53M | 12.37M | 7.05M | **baseline** | +| cache-hit rate | **98.2%** | 96.9% | 94.1% | 97.3% | **baseline** | +| mean steps (completed) | **31.5** | 34.7 | 32.0 | 40.0 | **baseline** | +| timeouts (of 89) | **7** | 11 | 11 | **7** | baseline / rtk | +| tool's own LLM cost | $0 | $3.26 | **$0** | **$0** | headroom / rtk | +| content removed / req | 0 | 10.7% (whole req) | 1.27% (whole req) | 94.4% *of bash only* | — (diff. denominators) | + +### Verdict + +- **baseline has the best cache behaviour** — a 98.2% cache-hit and the lowest cache-write — + and on the *raw* 89-task total it is the cheapest arm. On the 83 clean tasks it is not. +- **context-guru saves on the clean set: −9.8% including its own LLM cost** (−12.7% on model + cost alone), while solving **+2** and taking **8.3% fewer steps** — the same shape as its + −13% SWE-bench win. Its `extract_llm` haiku pass still costs **$3.26** and adds **450 ms/req**, + which is a real drag on the margin (and is why that component is being reworked), but it does + not erase the saving. The **+1.7%** in the table below is the six degenerate baselines. +- **headroom solves the most (+8, and +5 on `hard` tasks)** — the clearest real reward signal — + and recomputes to ≈**−16%** on the clean set. Its **3× cache-write blow-up** (12.4M vs 4.0M + tokens) from rewriting the live zone is nonetheless real, and is the mechanism to avoid. +- **rtk backfires (+17.9%, −1 solved).** On SWE-bench it was the −9% efficiency surprise; here + its 94% Bash-output compression **discards information the agent needs on open-ended tasks**, + so the agent takes **+27% more steps (40 vs 31.5)** → more round-trips → the **highest + cache-read of any arm (+17.6%)** and the highest cost. + +## Why cost goes up here but down on SWE-bench + +The decomposition points at one mechanism: **cache-write**. + +| arm | fresh $ | cache-read $ | **cache-write $** | output $ | + tool-LLM | total | +|---|--:|--:|--:|--:|--:|--:| +| baseline | 0.12 | 43.19 | **10.03** | 47.47 | — | $100.81 | +| context-guru | 0.21 | 40.93 | **16.31** | 41.83 | 3.26 | $102.55 | +| headroom | 0.20 | 39.60 | **30.92** | 44.03 | — | $114.75 | +| rtk | 0.11 | 50.80 | **17.62** | 50.30 | — | $118.83 | + +- On **SWE-bench**, contexts are smaller and the proxies' *freeze-and-replay* keeps the cached + prefix byte-stable, so cache-write stayed within 1% of baseline and the cache-read savings + won → context-guru −13%. +- On **Terminal-Bench**, contexts are ~1.7M tokens and outputs are large and varied. Any layer + that mutates cached content pays for it in **cache-write**: headroom's live-zone rewriting + triples it; context-guru's freeze-replay is less clean on huge outputs (+63%); rtk's extra + steps inflate every tier. The cache-write premium (plus context-guru's LLM cost) swamps the + cache-read saving. Cache-write, a rounding error on SWE-bench, is the deciding term here. + +## Per-component / per-compressor + +**context-guru** — unique tokens saved (whole-request savings **10.7%**): + +| component | acts | unique tokens saved | note | +|---|--:|--:|---| +| `extract_llm` | 684 | 197,548 | haiku skeletonization of big reads/logs — 271 calls, $3.26, ~1,592 s cumulative latency (the cost + latency source) | +| `extract` | 1,446 | 59,728 | deterministic ANSI/CR + noise, ~0 latency | +| `format` | 51 | 6,929 | JSON repack | +| `dedup` | 99 | 3,828 | duplicate tool outputs | +| `cmdfilter` | 34 | 886 | DSL log/test trims | +| `failed_run` / `cacheinject` | 0 | 0 | cache-aware auto-off / systemic | + +**headroom** — tokens saved by strategy (live-zone content; the headline `saved`=971k is mostly +tool-schema compaction, content savings **1.27%**): + +| strategy | events | tokens saved | +|---|--:|--:| +| `text` (Kompress) | 435 | 34,994 | +| `code_aware` (AST) | 96 | 17,551 | +| `html` | 26 | 6,015 | +| `search` | 15 | 4,917 | +| `smart_crusher` (JSON) | 60 | 1,922 | +| `tabular` / `log` | 30 | 877 | + +**rtk** — in-container Bash-output compression (its own `bytes/4` estimate, bash-output +denominator): **1,185 commands, 49.4M → 2.78M tokens (94.4% of bash output)**. The compression +is real and huge — but it is exactly what drives the agent to re-issue commands and take more +steps on open-ended tasks, so the net effect on billed cost is **negative**. + +## Reward: where the arms win and lose + +Solve rate by difficulty (solved / n): + +| arm | easy (4) | medium (55) | hard (30) | +|---|--:|--:|--:| +| baseline | 3 | 39 | 14 | +| context-guru | 4 | 41 | 13 | +| headroom | 3 | **42** | **19** | +| rtk | 3 | 38 | 14 | + +Net solve vs baseline (gains / losses — note the churn, i.e. single-trial noise): + +- **context-guru** +10 / −8 → **net +2** (within noise) +- **headroom** +13 / −5 → **net +8** (gains cluster on `hard`; the clearest real signal) +- **rtk** +8 / −9 → **net −1** (reward-neutral; the cost regression is the real story) + +Both compaction proxies solved `path-tracing-reverse` — a task the **baseline timed out on even +at 4×** — showing that a smaller context can let a long task finish in time. That is the +upside; on Terminal-Bench it is not (yet) enough to offset the cache-write and LLM costs. + +## Bottom line + +Terminal-Bench 2.0 does **not** invert the SWE-bench story once the six degenerate baselines are +removed. On the 83 clean tasks **context-guru saves −9.8%** while solving +2 with 8.3% fewer +steps, and **headroom saves ≈−16%** while solving +8 (its gains clustered on `hard`). **rtk is +the one real regression.** + +What *is* genuinely different here is the **mechanism**, and it survives the correction: +**cache-write, a rounding error on SWE-bench, is the deciding term on Terminal-Bench's ~1.7M-token +contexts.** Any layer that mutates already-cached content pays 11.5× for it — headroom's live-zone +rewriting triples cache-write, and rtk's information loss costs +27% more steps. The transferable +lesson is therefore not "compaction fails on long horizons" but **"on long horizons, cache-write +avoidance and step count dominate token removal"** — which is exactly what the +[improvement plan](improvement-plan.md) is built around. + +Two methodological lessons worth carrying forward, both learned the hard way here: +**a trial where the baseline aborts is not a measurement**, and it must be excluded rather than +averaged; and **per-arm totals hide this**, so a per-task paired comparison is the only honest +default. diff --git a/docs/results/terminal-bench-context-guru.md b/docs/results/terminal-bench-context-guru.md new file mode 100644 index 0000000..c60cce7 --- /dev/null +++ b/docs/results/terminal-bench-context-guru.md @@ -0,0 +1,164 @@ +# Full results — context-guru (codesmart) (Terminal-Bench 2.0, 89 tasks) + +Full per-task results for the **context-guru (codesmart)** arm on Terminal-Bench 2.0 (`claude-code` on `aws/claude-sonnet-5`, live). Same cache-aware cost model and 4× budget as the other arms. For the four-way analysis (cost decomposition, per-component, verdict) see the **[Terminal-Bench comparison](terminal-bench-comparison.md)**; the reference arm is the **[baseline](terminal-bench-baseline.md)**. See [REPRODUCE.md](REPRODUCE.md). + +## Totals + +| attempted | solved | solve rate | completed | timed out | total billed cost | mean steps* | cache-hit | +|--:|--:|--:|--:|--:|--:|--:|--:| +| 89 | 58 | **65.2%** | 78 | 11 | $99.29 | 34.7 | 96.9% | + +\* mean steps over the 78 completed tasks (timed-out runs are truncated). Solve rate over **completed-only** tasks: **57/78 = 73.1%**. + +### Token & cost accounting (cache-aware, all 89 tasks) + +| tier | tokens | $/M | billed | +|---|--:|--:|--:| +| cache-read (input) | 204,631,836 | 0.20 | $40.93 | +| cache-write (input) | 6,525,522 | 2.50 | $16.31 | +| fresh (input) | 107,482 | 2.00 | $0.21 | +| completion (output) | 4,183,424 | 10.00 | $41.83 | +| **total** | | | **$99.29** | + +Cache-read is **41%** of the bill at a **96.9%** cache-hit rate — as on SWE-bench, a heavily-cached agent, so the lever a compaction layer must pull is cache-read tokens. + +## Timeouts (11 long-horizon tasks) + +These tasks still hit the wall-clock budget under the **extended 4×** timeout (up to ~4 h each) and scored **reward 0** — counted as failures in the solve rate above. A large part of the cause is **gateway latency, not only agent capability**: Terminal-Bench's timeouts assume a fast endpoint (~2–5 s/request), but this IBM LiteLLM gateway runs **~26 s/request** (5–10× slower), so long-horizon tasks that need many round-trips run out of clock (concurrency is *not* the cause — latency was flat ~23–30 s/req from n=1 to n=24). They are all `hard`/long software-engineering and compute tasks (path-tracing, a MIPS Doom port, a metacircular evaluator, COBOL modernization, GPT-2 code-golf, CIFAR training). A compaction arm that cuts round-trips could bring some under budget, so the timeout count is itself a comparison metric. + +| task | difficulty | category | steps before timeout | partial billed | budget (4×) | +|---|---|---|--:|--:|--:| +| caffe-cifar-10 | medium | machine-learning | None | $0.00 | 80 min | +| cobol-modernization | easy | software-engineering | 110 | $4.40 | 60 min | +| gpt2-codegolf | hard | software-engineering | 28 | $1.13 | 60 min | +| make-doom-for-mips | hard | software-engineering | 141 | $5.76 | 60 min | +| mteb-retrieve | medium | data-science | None | $0.00 | 120 min | +| path-tracing-reverse | hard | software-engineering | 58 | $2.51 | 120 min | +| polyglot-rust-c | hard | software-engineering | 50 | $2.81 | 60 min | +| protein-assembly | hard | scientific-computing | 8 | $0.12 | 120 min | +| pytorch-model-recovery | medium | model-training | 21 | $0.44 | 60 min | +| schemelike-metacircular-eval | medium | software-engineering | 96 | $4.77 | 160 min | +| write-compressor | hard | software-engineering | 4 | $0.19 | 60 min | + +## By difficulty (all 89 tasks; timeouts = failures) + +| difficulty | tasks | solved | rate | timed out | mean $/task | +|---|--:|--:|--:|--:|--:| +| easy | 4 | 4 | 100% | 1 | $1.451 | +| medium | 55 | 41 | 75% | 4 | $0.781 | +| hard | 30 | 13 | 43% | 6 | $1.685 | + +## By category (all 89 tasks) + +| category | tasks | solved | rate | mean $/task | mean steps* | +|---|--:|--:|--:|--:|--:| +| data-processing | 4 | 4 | 100% | $0.226 | 11.5 | +| data-querying | 1 | 1 | 100% | $0.818 | 16.0 | +| debugging | 5 | 5 | 100% | $1.324 | 56.6 | +| optimization | 1 | 1 | 100% | $0.609 | 23.0 | +| personal-assistant | 1 | 1 | 100% | $0.229 | 6.0 | +| file-operations | 5 | 4 | 80% | $0.801 | 36.2 | +| mathematics | 4 | 3 | 75% | $0.813 | 15.5 | +| security | 8 | 6 | 75% | $0.740 | 25.8 | +| machine-learning | 3 | 2 | 67% | $0.739 | 29.5 | +| system-administration | 9 | 6 | 67% | $0.725 | 38.8 | +| data-science | 8 | 5 | 62% | $1.682 | 56.9 | +| software-engineering | 26 | 15 | 58% | $1.662 | 33.7 | +| model-training | 4 | 2 | 50% | $0.599 | 33.0 | +| scientific-computing | 8 | 3 | 38% | $0.596 | 26.9 | +| games | 1 | 0 | 0% | $0.386 | 17.0 | +| video-processing | 1 | 0 | 0% | $3.961 | 129.0 | + +## Per-task (all 89) + +| task | difficulty | category | outcome | steps | cache_read | cache_write | billed | wall | +|---|---|---|:--:|--:|--:|--:|--:|--:| +| adaptive-rejection-sampler | medium | scientific-computing | ❌ failed | 32 | 1,674,712 | 28,817 | $1.053 | 26.5 min | +| bn-fit-modify | hard | scientific-computing | ✅ solved | 23 | 1,010,421 | 13,731 | $0.382 | 7.2 min | +| break-filter-js-from-html | medium | security | ✅ solved | 56 | 2,925,793 | 26,244 | $1.603 | 34.7 min | +| build-cython-ext | medium | debugging | ✅ solved | 116 | 8,772,811 | 73,085 | $2.208 | 26.5 min | +| build-pmars | medium | software-engineering | ✅ solved | 38 | 1,922,393 | 24,994 | $0.530 | 10.5 min | +| build-pov-ray | medium | software-engineering | ❌ failed | 30 | 1,500,560 | 24,878 | $0.428 | 5.2 min | +| caffe-cifar-10 | medium | machine-learning | ⏱ timeout | None | 0 | 0 | $0.000 | — | +| cancel-async-tasks | hard | software-engineering | ❌ failed | 9 | 336,627 | 5,252 | $0.172 | 2.1 min | +| chess-best-move | medium | games | ❌ failed | 17 | 704,277 | 10,456 | $0.386 | 6.0 min | +| circuit-fibsqrt | hard | software-engineering | ✅ solved | 60 | 4,235,103 | 81,094 | $3.590 | 91.2 min | +| cobol-modernization | easy | software-engineering | ⏱ timeout | 110 | 7,425,465 | 59,332 | $4.395 | 60.0 min | +| code-from-image | medium | software-engineering | ✅ solved | 5 | 165,696 | 4,236 | $0.049 | 0.7 min | +| compile-compcert | medium | system-administration | ✅ solved | 69 | 3,870,275 | 180,987 | $1.425 | 74.0 min | +| configure-git-webserver | hard | system-administration | ✅ solved | 21 | 911,693 | 11,638 | $0.281 | 6.7 min | +| constraints-scheduling | medium | personal-assistant | ✅ solved | 6 | 215,662 | 6,897 | $0.229 | 3.5 min | +| count-dataset-tokens | medium | model-training | ✅ solved | 10 | 389,414 | 7,591 | $0.129 | 6.0 min | +| crack-7z-hash | medium | security | ✅ solved | 19 | 788,652 | 24,805 | $0.245 | 17.9 min | +| custom-memory-heap-crash | medium | debugging | ✅ solved | 75 | 5,344,207 | 56,082 | $2.507 | 36.1 min | +| db-wal-recovery | medium | file-operations | ✅ solved | 10 | 387,762 | 6,433 | $0.129 | 1.6 min | +| distribution-search | medium | machine-learning | ✅ solved | 21 | 958,627 | 16,830 | $0.654 | 11.5 min | +| dna-assembly | hard | scientific-computing | ❌ failed | 26 | 1,250,596 | 26,031 | $0.749 | 15.2 min | +| dna-insert | medium | scientific-computing | ❌ failed | 29 | 1,499,781 | 28,732 | $0.793 | 14.9 min | +| extract-elf | medium | file-operations | ✅ solved | 18 | 801,751 | 27,891 | $0.610 | 9.3 min | +| extract-moves-from-video | hard | file-operations | ❌ failed | 112 | 7,449,833 | 227,396 | $2.691 | 108.7 min | +| feal-differential-cryptanalysis | hard | mathematics | ✅ solved | 13 | 531,990 | 9,751 | $0.851 | 13.5 min | +| feal-linear-cryptanalysis | hard | mathematics | ✅ solved | 20 | 984,819 | 32,867 | $1.602 | 55.6 min | +| filter-js-from-html | medium | security | ❌ failed | 8 | 302,660 | 7,716 | $0.239 | 3.2 min | +| financial-document-processor | medium | data-processing | ✅ solved | 25 | 974,052 | 35,433 | $0.426 | 6.1 min | +| fix-code-vulnerability | hard | security | ✅ solved | 11 | 461,737 | 23,400 | $0.179 | 1.3 min | +| fix-git | easy | software-engineering | ✅ solved | 12 | 475,724 | 16,806 | $0.160 | 1.3 min | +| fix-ocaml-gc | hard | software-engineering | ✅ solved | 21 | 1,209,789 | 86,820 | $0.707 | 19.5 min | +| gcode-to-text | medium | file-operations | ✅ solved | 33 | 1,226,297 | 37,655 | $0.441 | 7.2 min | +| git-leak-recovery | medium | software-engineering | ✅ solved | 10 | 382,074 | 6,004 | $0.113 | 1.0 min | +| git-multibranch | medium | system-administration | ❌ failed | 37 | 1,860,897 | 21,763 | $0.581 | 5.6 min | +| gpt2-codegolf | hard | software-engineering | ⏱ timeout | 28 | 1,320,855 | 16,647 | $1.128 | 60.0 min | +| headless-terminal | medium | software-engineering | ✅ solved | 55 | 3,000,428 | 31,593 | $1.066 | 15.5 min | +| hf-model-inference | medium | data-science | ✅ solved | 10 | 383,229 | 6,288 | $0.119 | 2.4 min | +| install-windows-3.11 | hard | system-administration | ❌ failed | 49 | 2,592,143 | 29,114 | $0.729 | 8.1 min | +| kv-store-grpc | medium | software-engineering | ✅ solved | 11 | 429,312 | 6,723 | $0.126 | 1.3 min | +| large-scale-text-editing | medium | file-operations | ✅ solved | 8 | 307,912 | 7,638 | $0.135 | 3.2 min | +| largest-eigenval | medium | mathematics | ✅ solved | 20 | 881,834 | 13,592 | $0.333 | 4.0 min | +| llm-inference-batching-scheduler | hard | machine-learning | ✅ solved | 38 | 2,496,481 | 54,992 | $1.562 | 27.8 min | +| log-summary-date-ranges | medium | data-processing | ✅ solved | 6 | 210,077 | 5,648 | $0.071 | 0.6 min | +| mailman | medium | system-administration | ❌ failed | 81 | 6,947,541 | 75,092 | $2.097 | 23.8 min | +| make-doom-for-mips | hard | software-engineering | ⏱ timeout | 141 | 13,556,629 | 354,083 | $5.760 | 60.0 min | +| make-mips-interpreter | hard | software-engineering | ✅ solved | 171 | 16,819,007 | 906,773 | $7.554 | 71.4 min | +| mcmc-sampling-stan | hard | data-science | ✅ solved | 44 | 2,662,323 | 51,716 | $0.790 | 22.6 min | +| merge-diff-arc-agi-task | medium | debugging | ✅ solved | 25 | 1,114,428 | 13,596 | $0.406 | 4.6 min | +| model-extraction-relu-logits | hard | mathematics | ❌ failed | 9 | 357,509 | 13,370 | $0.467 | 7.3 min | +| modernize-scientific-stack | medium | scientific-computing | ✅ solved | 6 | 215,145 | 6,818 | $0.083 | 0.7 min | +| mteb-leaderboard | medium | data-science | ✅ solved | 147 | 12,277,618 | 1,122,303 | $6.045 | 73.5 min | +| mteb-retrieve | medium | data-science | ⏱ timeout | None | 0 | 0 | $0.000 | — | +| multi-source-data-merger | medium | data-processing | ✅ solved | 7 | 256,234 | 6,751 | $0.103 | 3.1 min | +| nginx-request-logging | medium | system-administration | ✅ solved | 12 | 477,493 | 7,266 | $0.143 | 2.1 min | +| openssl-selfsigned-cert | medium | security | ✅ solved | 10 | 383,978 | 6,629 | $0.122 | 1.9 min | +| overfull-hbox | easy | debugging | ✅ solved | 53 | 3,388,975 | 42,344 | $1.168 | 13.1 min | +| password-recovery | hard | security | ❌ failed | 69 | 3,995,130 | 40,768 | $2.480 | 39.3 min | +| path-tracing | hard | software-engineering | ✅ solved | 114 | 9,509,036 | 442,884 | $4.820 | 77.1 min | +| path-tracing-reverse | hard | software-engineering | ⏱ timeout | 58 | 4,628,243 | 258,646 | $2.507 | 34.2 min | +| polyglot-c-py | medium | software-engineering | ❌ failed | 12 | 464,910 | 5,731 | $0.360 | 6.3 min | +| polyglot-rust-c | hard | software-engineering | ⏱ timeout | 50 | 2,642,242 | 30,164 | $2.810 | 60.0 min | +| portfolio-optimization | medium | optimization | ✅ solved | 23 | 1,127,622 | 22,185 | $0.609 | 11.5 min | +| protein-assembly | hard | scientific-computing | ⏱ timeout | 8 | 269,038 | 4,867 | $0.116 | 2.4 min | +| prove-plus-comm | easy | software-engineering | ✅ solved | 6 | 197,256 | 12,101 | $0.079 | 0.7 min | +| pypi-server | medium | software-engineering | ✅ solved | 16 | 653,138 | 7,711 | $0.174 | 8.0 min | +| pytorch-model-cli | medium | model-training | ✅ solved | 25 | 1,124,334 | 57,946 | $0.461 | 5.5 min | +| pytorch-model-recovery | medium | model-training | ⏱ timeout | 21 | 943,457 | 14,990 | $0.437 | 8.6 min | +| qemu-alpine-ssh | medium | system-administration | ✅ solved | 37 | 1,717,574 | 21,608 | $0.579 | 8.0 min | +| qemu-startup | medium | system-administration | ✅ solved | 24 | 1,024,044 | 9,915 | $0.459 | 7.7 min | +| query-optimize | medium | data-science | ❌ failed | 44 | 2,213,453 | 25,584 | $0.877 | 30.2 min | +| raman-fitting | medium | scientific-computing | ❌ failed | 48 | 2,626,488 | 36,884 | $1.164 | 19.3 min | +| regex-chess | hard | software-engineering | ❌ failed | 2 | 41,954 | 0 | $0.031 | 1.3 min | +| regex-log | medium | data-processing | ✅ solved | 8 | 301,024 | 9,175 | $0.303 | 4.3 min | +| reshard-c4-data | medium | data-science | ✅ solved | 25 | 1,264,046 | 22,514 | $1.226 | 19.9 min | +| rstan-to-pystan | medium | data-science | ✅ solved | 73 | 4,191,501 | 588,048 | $2.717 | 118.7 min | +| sam-cell-seg | hard | data-science | ❌ failed | 55 | 3,065,248 | 37,729 | $1.683 | 29.0 min | +| sanitize-git-repo | medium | security | ✅ solved | 22 | 1,591,443 | 186,042 | $0.893 | 7.4 min | +| schemelike-metacircular-eval | medium | software-engineering | ⏱ timeout | 96 | 6,646,069 | 120,462 | $4.768 | 160.0 min | +| sparql-university | hard | data-querying | ✅ solved | 16 | 734,172 | 13,592 | $0.818 | 12.5 min | +| sqlite-db-truncate | medium | debugging | ✅ solved | 14 | 576,165 | 11,206 | $0.332 | 5.1 min | +| sqlite-with-gcov | medium | system-administration | ✅ solved | 19 | 827,372 | 13,175 | $0.236 | 4.8 min | +| torch-pipeline-parallelism | hard | software-engineering | ✅ solved | 8 | 293,023 | 16,998 | $0.278 | 4.4 min | +| torch-tensor-parallelism | hard | software-engineering | ❌ failed | 10 | 405,241 | 12,627 | $0.290 | 4.9 min | +| train-fasttext | hard | model-training | ❌ failed | 64 | 3,191,706 | 210,917 | $1.371 | 113.6 min | +| tune-mjcf | medium | scientific-computing | ✅ solved | 24 | 1,086,103 | 13,996 | $0.426 | 10.7 min | +| video-processing | hard | video-processing | ❌ failed | 129 | 10,928,251 | 200,113 | $3.961 | 69.7 min | +| vulnerable-secret | medium | security | ✅ solved | 11 | 439,504 | 12,575 | $0.162 | 2.2 min | +| winning-avg-corewars | medium | software-engineering | ✅ solved | 51 | 2,757,923 | 30,691 | $1.125 | 15.8 min | +| write-compressor | hard | software-engineering | ⏱ timeout | 4 | 123,825 | 3,055 | $0.194 | 60.0 min | diff --git a/docs/results/terminal-bench-headroom.md b/docs/results/terminal-bench-headroom.md new file mode 100644 index 0000000..b209bcd --- /dev/null +++ b/docs/results/terminal-bench-headroom.md @@ -0,0 +1,164 @@ +# Full results — headroom (hd-cache) (Terminal-Bench 2.0, 89 tasks) + +Full per-task results for the **headroom (hd-cache)** arm on Terminal-Bench 2.0 (`claude-code` on `aws/claude-sonnet-5`, live). Same cache-aware cost model and 4× budget as the other arms. For the four-way analysis (cost decomposition, per-component, verdict) see the **[Terminal-Bench comparison](terminal-bench-comparison.md)**; the reference arm is the **[baseline](terminal-bench-baseline.md)**. See [REPRODUCE.md](REPRODUCE.md). + +## Totals + +| attempted | solved | solve rate | completed | timed out | total billed cost | mean steps* | cache-hit | +|--:|--:|--:|--:|--:|--:|--:|--:| +| 89 | 64 | **71.9%** | 78 | 11 | $114.75 | 32.0 | 94.1% | + +\* mean steps over the 78 completed tasks (timed-out runs are truncated). Solve rate over **completed-only** tasks: **64/78 = 82.1%**. + +### Token & cost accounting (cache-aware, all 89 tasks) + +| tier | tokens | $/M | billed | +|---|--:|--:|--:| +| cache-read (input) | 198,020,429 | 0.20 | $39.60 | +| cache-write (input) | 12,369,660 | 2.50 | $30.92 | +| fresh (input) | 97,763 | 2.00 | $0.20 | +| completion (output) | 4,402,777 | 10.00 | $44.03 | +| **total** | | | **$114.75** | + +Cache-read is **35%** of the bill at a **94.1%** cache-hit rate — as on SWE-bench, a heavily-cached agent, so the lever a compaction layer must pull is cache-read tokens. + +## Timeouts (11 long-horizon tasks) + +These tasks still hit the wall-clock budget under the **extended 4×** timeout (up to ~4 h each) and scored **reward 0** — counted as failures in the solve rate above. A large part of the cause is **gateway latency, not only agent capability**: Terminal-Bench's timeouts assume a fast endpoint (~2–5 s/request), but this IBM LiteLLM gateway runs **~26 s/request** (5–10× slower), so long-horizon tasks that need many round-trips run out of clock (concurrency is *not* the cause — latency was flat ~23–30 s/req from n=1 to n=24). They are all `hard`/long software-engineering and compute tasks (path-tracing, a MIPS Doom port, a metacircular evaluator, COBOL modernization, GPT-2 code-golf, CIFAR training). A compaction arm that cuts round-trips could bring some under budget, so the timeout count is itself a comparison metric. + +| task | difficulty | category | steps before timeout | partial billed | budget (4×) | +|---|---|---|--:|--:|--:| +| caffe-cifar-10 | medium | machine-learning | 38 | $0.77 | 80 min | +| cobol-modernization | easy | software-engineering | 69 | $2.35 | 60 min | +| extract-moves-from-video | hard | file-operations | 160 | $24.12 | 120 min | +| make-doom-for-mips | hard | software-engineering | 101 | $4.24 | 60 min | +| path-tracing | hard | software-engineering | 142 | $6.16 | 120 min | +| protein-assembly | hard | scientific-computing | 2 | $0.00 | 120 min | +| pytorch-model-cli | medium | model-training | 35 | $0.47 | 60 min | +| query-optimize | medium | data-science | 19 | $0.31 | 60 min | +| schemelike-metacircular-eval | medium | software-engineering | 77 | $4.66 | 160 min | +| tune-mjcf | medium | scientific-computing | 25 | $0.42 | 60 min | +| write-compressor | hard | software-engineering | 5 | $0.47 | 60 min | + +## By difficulty (all 89 tasks; timeouts = failures) + +| difficulty | tasks | solved | rate | timed out | mean $/task | +|---|--:|--:|--:|--:|--:| +| easy | 4 | 3 | 75% | 1 | $0.878 | +| medium | 55 | 42 | 76% | 5 | $0.631 | +| hard | 30 | 19 | 63% | 5 | $2.551 | + +## By category (all 89 tasks) + +| category | tasks | solved | rate | mean $/task | mean steps* | +|---|--:|--:|--:|--:|--:| +| data-querying | 1 | 1 | 100% | $0.351 | 8.0 | +| debugging | 5 | 5 | 100% | $0.909 | 42.8 | +| games | 1 | 1 | 100% | $0.383 | 20.0 | +| mathematics | 4 | 4 | 100% | $0.703 | 10.8 | +| optimization | 1 | 1 | 100% | $0.464 | 19.0 | +| personal-assistant | 1 | 1 | 100% | $0.244 | 6.0 | +| video-processing | 1 | 1 | 100% | $3.080 | 90.0 | +| system-administration | 9 | 8 | 89% | $0.933 | 47.2 | +| data-processing | 4 | 3 | 75% | $0.212 | 13.8 | +| data-science | 8 | 6 | 75% | $1.105 | 49.7 | +| security | 8 | 6 | 75% | $0.371 | 19.6 | +| machine-learning | 3 | 2 | 67% | $1.060 | 28.0 | +| software-engineering | 26 | 17 | 65% | $1.696 | 34.0 | +| file-operations | 5 | 3 | 60% | $5.065 | 15.8 | +| model-training | 4 | 2 | 50% | $0.801 | 33.3 | +| scientific-computing | 8 | 3 | 38% | $0.752 | 30.3 | + +## Per-task (all 89) + +| task | difficulty | category | outcome | steps | cache_read | cache_write | billed | wall | +|---|---|---|:--:|--:|--:|--:|--:|--:| +| adaptive-rejection-sampler | medium | scientific-computing | ❌ failed | 30 | 1,438,504 | 52,222 | $1.253 | 26.4 min | +| bn-fit-modify | hard | scientific-computing | ✅ solved | 22 | 928,502 | 13,410 | $0.375 | 7.2 min | +| break-filter-js-from-html | medium | security | ✅ solved | 35 | 1,600,416 | 17,679 | $0.879 | 21.7 min | +| build-cython-ext | medium | debugging | ✅ solved | 62 | 4,496,202 | 63,447 | $1.296 | 14.0 min | +| build-pmars | medium | software-engineering | ✅ solved | 39 | 1,926,499 | 29,817 | $0.560 | 7.3 min | +| build-pov-ray | medium | software-engineering | ✅ solved | 36 | 1,752,704 | 28,039 | $0.517 | 11.2 min | +| caffe-cifar-10 | medium | machine-learning | ⏱ timeout | 38 | 1,748,587 | 131,625 | $0.769 | 57.0 min | +| cancel-async-tasks | hard | software-engineering | ✅ solved | 8 | 279,969 | 5,770 | $0.200 | 2.5 min | +| chess-best-move | medium | games | ✅ solved | 20 | 806,656 | 9,186 | $0.383 | 4.9 min | +| circuit-fibsqrt | hard | software-engineering | ✅ solved | 53 | 3,341,016 | 46,604 | $2.881 | 178.9 min | +| cobol-modernization | easy | software-engineering | ⏱ timeout | 69 | 3,877,615 | 36,673 | $2.353 | 60.0 min | +| code-from-image | medium | software-engineering | ✅ solved | 5 | 156,544 | 4,231 | $0.047 | 0.5 min | +| compile-compcert | medium | system-administration | ✅ solved | 85 | 5,515,699 | 227,890 | $1.922 | 78.9 min | +| configure-git-webserver | hard | system-administration | ✅ solved | 13 | 494,249 | 7,317 | $0.166 | 7.2 min | +| constraints-scheduling | medium | personal-assistant | ✅ solved | 6 | 204,738 | 8,481 | $0.244 | 3.2 min | +| count-dataset-tokens | medium | model-training | ✅ solved | 13 | 526,046 | 17,052 | $0.196 | 5.3 min | +| crack-7z-hash | medium | security | ✅ solved | 20 | 814,384 | 11,301 | $0.221 | 6.3 min | +| custom-memory-heap-crash | medium | debugging | ✅ solved | 52 | 3,359,686 | 49,983 | $1.714 | 30.4 min | +| db-wal-recovery | medium | file-operations | ❌ failed | 17 | 689,904 | 13,305 | $0.408 | 5.6 min | +| distribution-search | medium | machine-learning | ✅ solved | 16 | 663,356 | 14,575 | $0.538 | 7.5 min | +| dna-assembly | hard | scientific-computing | ✅ solved | 56 | 3,250,476 | 53,252 | $2.386 | 43.6 min | +| dna-insert | medium | scientific-computing | ❌ failed | 18 | 773,481 | 13,546 | $0.320 | 5.5 min | +| extract-elf | medium | file-operations | ✅ solved | 18 | 763,108 | 13,337 | $0.390 | 5.2 min | +| extract-moves-from-video | hard | file-operations | ⏱ timeout | 160 | 7,363,596 | 7,806,411 | $24.118 | 120.0 min | +| feal-differential-cryptanalysis | hard | mathematics | ✅ solved | 13 | 502,491 | 9,262 | $1.003 | 16.9 min | +| feal-linear-cryptanalysis | hard | mathematics | ✅ solved | 13 | 532,046 | 12,513 | $1.316 | 45.3 min | +| filter-js-from-html | medium | security | ❌ failed | 22 | 989,347 | 16,881 | $0.496 | 5.7 min | +| financial-document-processor | medium | data-processing | ❌ failed | 26 | 533,975 | 61,704 | $0.360 | 2.5 min | +| fix-code-vulnerability | hard | security | ✅ solved | 16 | 633,232 | 20,508 | $0.206 | 1.5 min | +| fix-git | easy | software-engineering | ✅ solved | 11 | 412,077 | 17,470 | $0.151 | 1.0 min | +| fix-ocaml-gc | hard | software-engineering | ✅ solved | 27 | 1,340,872 | 145,817 | $0.778 | 27.9 min | +| gcode-to-text | medium | file-operations | ✅ solved | 17 | 692,582 | 11,878 | $0.227 | 4.1 min | +| git-leak-recovery | medium | software-engineering | ✅ solved | 16 | 611,482 | 7,048 | $0.172 | 1.5 min | +| git-multibranch | medium | system-administration | ✅ solved | 33 | 1,485,333 | 16,544 | $0.470 | 4.5 min | +| gpt2-codegolf | hard | software-engineering | ❌ failed | 2 | 0 | 39,156 | $0.138 | 0.9 min | +| headless-terminal | medium | software-engineering | ✅ solved | 16 | 652,521 | 15,294 | $0.289 | 13.3 min | +| hf-model-inference | medium | data-science | ✅ solved | 12 | 448,133 | 12,428 | $0.149 | 11.6 min | +| install-windows-3.11 | hard | system-administration | ❌ failed | 89 | 6,294,323 | 136,137 | $2.271 | 40.8 min | +| kv-store-grpc | medium | software-engineering | ✅ solved | 11 | 407,203 | 6,197 | $0.121 | 1.8 min | +| large-scale-text-editing | medium | file-operations | ✅ solved | 11 | 406,201 | 6,239 | $0.180 | 3.3 min | +| largest-eigenval | medium | mathematics | ✅ solved | 10 | 370,777 | 8,085 | $0.168 | 2.8 min | +| llm-inference-batching-scheduler | hard | machine-learning | ✅ solved | 40 | 2,384,925 | 52,778 | $1.873 | 28.7 min | +| log-summary-date-ranges | medium | data-processing | ✅ solved | 8 | 280,917 | 5,643 | $0.086 | 0.6 min | +| mailman | medium | system-administration | ✅ solved | 71 | 5,039,376 | 54,115 | $1.495 | 15.4 min | +| make-doom-for-mips | hard | software-engineering | ⏱ timeout | 101 | 11,022,228 | 111,639 | $4.237 | 60.0 min | +| make-mips-interpreter | hard | software-engineering | ❌ failed | 181 | 18,796,281 | 222,145 | $7.696 | 83.6 min | +| mcmc-sampling-stan | hard | data-science | ✅ solved | 39 | 2,322,900 | 54,635 | $0.702 | 20.2 min | +| merge-diff-arc-agi-task | medium | debugging | ✅ solved | 37 | 1,639,043 | 15,823 | $0.472 | 6.6 min | +| model-extraction-relu-logits | hard | mathematics | ✅ solved | 7 | 240,960 | 7,470 | $0.325 | 5.7 min | +| modernize-scientific-stack | medium | scientific-computing | ✅ solved | 6 | 202,237 | 6,055 | $0.067 | 0.6 min | +| mteb-leaderboard | medium | data-science | ✅ solved | 91 | 6,713,015 | 81,960 | $2.041 | 55.8 min | +| mteb-retrieve | medium | data-science | ✅ solved | 15 | 591,826 | 9,506 | $0.184 | 8.3 min | +| multi-source-data-merger | medium | data-processing | ✅ solved | 9 | 328,735 | 7,246 | $0.130 | 1.5 min | +| nginx-request-logging | medium | system-administration | ✅ solved | 12 | 458,387 | 9,220 | $0.144 | 2.4 min | +| openssl-selfsigned-cert | medium | security | ✅ solved | 16 | 620,007 | 8,407 | $0.184 | 1.6 min | +| overfull-hbox | easy | debugging | ✅ solved | 55 | 2,900,215 | 28,336 | $0.918 | 19.9 min | +| password-recovery | hard | security | ✅ solved | 24 | 1,018,613 | 12,598 | $0.475 | 6.0 min | +| path-tracing | hard | software-engineering | ⏱ timeout | 142 | 13,062,719 | 184,070 | $6.155 | 120.0 min | +| path-tracing-reverse | hard | software-engineering | ✅ solved | 112 | 11,500,154 | 209,909 | $6.088 | 102.2 min | +| polyglot-c-py | medium | software-engineering | ❌ failed | 11 | 400,585 | 5,841 | $0.396 | 8.5 min | +| polyglot-rust-c | hard | software-engineering | ✅ solved | 7 | 390,788 | 39,597 | $0.573 | 11.7 min | +| portfolio-optimization | medium | optimization | ✅ solved | 19 | 859,707 | 20,442 | $0.464 | 8.9 min | +| protein-assembly | hard | scientific-computing | ⏱ timeout | 2 | 0 | 0 | $0.000 | 0.1 min | +| prove-plus-comm | easy | software-engineering | ✅ solved | 7 | 225,644 | 12,207 | $0.091 | 5.8 min | +| pypi-server | medium | software-engineering | ✅ solved | 15 | 571,478 | 7,504 | $0.160 | 1.9 min | +| pytorch-model-cli | medium | model-training | ⏱ timeout | 35 | 1,595,679 | 20,587 | $0.471 | 11.1 min | +| pytorch-model-recovery | medium | model-training | ✅ solved | 19 | 844,676 | 18,634 | $0.377 | 12.7 min | +| qemu-alpine-ssh | medium | system-administration | ✅ solved | 79 | 4,459,313 | 41,132 | $1.345 | 39.3 min | +| qemu-startup | medium | system-administration | ✅ solved | 21 | 832,903 | 9,683 | $0.324 | 10.1 min | +| query-optimize | medium | data-science | ⏱ timeout | 19 | 757,494 | 15,322 | $0.307 | 10.2 min | +| raman-fitting | medium | scientific-computing | ❌ failed | 50 | 2,706,005 | 43,519 | $1.196 | 15.4 min | +| regex-chess | hard | software-engineering | ✅ solved | 75 | 6,457,203 | 482,484 | $3.309 | 79.9 min | +| regex-log | medium | data-processing | ✅ solved | 12 | 450,419 | 8,037 | $0.270 | 3.9 min | +| reshard-c4-data | medium | data-science | ✅ solved | 33 | 1,497,470 | 15,842 | $0.924 | 16.5 min | +| rstan-to-pystan | medium | data-science | ✅ solved | 88 | 5,680,153 | 442,469 | $2.661 | 106.6 min | +| sam-cell-seg | hard | data-science | ❌ failed | 70 | 4,414,374 | 60,442 | $1.873 | 35.5 min | +| sanitize-git-repo | medium | security | ❌ failed | 8 | 403,241 | 45,268 | $0.235 | 1.1 min | +| schemelike-metacircular-eval | medium | software-engineering | ⏱ timeout | 77 | 5,741,094 | 99,381 | $4.664 | 160.0 min | +| sparql-university | hard | data-querying | ✅ solved | 8 | 301,163 | 10,650 | $0.351 | 4.2 min | +| sqlite-db-truncate | medium | debugging | ✅ solved | 8 | 277,629 | 6,865 | $0.146 | 1.6 min | +| sqlite-with-gcov | medium | system-administration | ✅ solved | 22 | 924,943 | 16,688 | $0.264 | 8.7 min | +| torch-pipeline-parallelism | hard | software-engineering | ✅ solved | 41 | 2,156,546 | 25,746 | $0.994 | 35.2 min | +| torch-tensor-parallelism | hard | software-engineering | ❌ failed | 12 | 469,160 | 11,589 | $0.337 | 8.7 min | +| train-fasttext | hard | model-training | ❌ failed | 68 | 3,683,278 | 478,817 | $2.159 | 136.9 min | +| tune-mjcf | medium | scientific-computing | ⏱ timeout | 25 | 1,121,523 | 17,848 | $0.420 | 13.2 min | +| video-processing | hard | video-processing | ✅ solved | 90 | 6,490,786 | 73,613 | $3.080 | 70.4 min | +| vulnerable-secret | medium | security | ✅ solved | 16 | 654,879 | 15,453 | $0.270 | 2.8 min | +| winning-avg-corewars | medium | software-engineering | ✅ solved | 28 | 1,288,502 | 20,289 | $0.717 | 11.7 min | +| write-compressor | hard | software-engineering | ⏱ timeout | 5 | 156,724 | 3,842 | $0.471 | 60.0 min | diff --git a/docs/results/terminal-bench-rtk.md b/docs/results/terminal-bench-rtk.md new file mode 100644 index 0000000..5c84aaf --- /dev/null +++ b/docs/results/terminal-bench-rtk.md @@ -0,0 +1,160 @@ +# Full results — rtk (Terminal-Bench 2.0, 89 tasks) + +Full per-task results for the **rtk** arm on Terminal-Bench 2.0 (`claude-code` on `aws/claude-sonnet-5`, live). Same cache-aware cost model and 4× budget as the other arms. For the four-way analysis (cost decomposition, per-component, verdict) see the **[Terminal-Bench comparison](terminal-bench-comparison.md)**; the reference arm is the **[baseline](terminal-bench-baseline.md)**. See [REPRODUCE.md](REPRODUCE.md). + +## Totals + +| attempted | solved | solve rate | completed | timed out | total billed cost | mean steps* | cache-hit | +|--:|--:|--:|--:|--:|--:|--:|--:| +| 89 | 55 | **61.8%** | 82 | 7 | $118.83 | 40.0 | 97.3% | + +\* mean steps over the 82 completed tasks (timed-out runs are truncated). Solve rate over **completed-only** tasks: **55/82 = 67.1%**. + +### Token & cost accounting (cache-aware, all 89 tasks) + +| tier | tokens | $/M | billed | +|---|--:|--:|--:| +| cache-read (input) | 253,994,766 | 0.20 | $50.80 | +| cache-write (input) | 7,047,955 | 2.50 | $17.62 | +| fresh (input) | 55,363 | 2.00 | $0.11 | +| completion (output) | 5,029,700 | 10.00 | $50.30 | +| **total** | | | **$118.83** | + +Cache-read is **43%** of the bill at a **97.3%** cache-hit rate — as on SWE-bench, a heavily-cached agent, so the lever a compaction layer must pull is cache-read tokens. + +## Timeouts (7 long-horizon tasks) + +These tasks still hit the wall-clock budget under the **extended 4×** timeout (up to ~4 h each) and scored **reward 0** — counted as failures in the solve rate above. A large part of the cause is **gateway latency, not only agent capability**: Terminal-Bench's timeouts assume a fast endpoint (~2–5 s/request), but this IBM LiteLLM gateway runs **~26 s/request** (5–10× slower), so long-horizon tasks that need many round-trips run out of clock (concurrency is *not* the cause — latency was flat ~23–30 s/req from n=1 to n=24). They are all `hard`/long software-engineering and compute tasks (path-tracing, a MIPS Doom port, a metacircular evaluator, COBOL modernization, GPT-2 code-golf, CIFAR training). A compaction arm that cuts round-trips could bring some under budget, so the timeout count is itself a comparison metric. + +| task | difficulty | category | steps before timeout | partial billed | budget (4×) | +|---|---|---|--:|--:|--:| +| cobol-modernization | easy | software-engineering | 88 | $4.11 | 60 min | +| make-doom-for-mips | hard | software-engineering | 149 | $5.51 | 60 min | +| query-optimize | medium | data-science | 21 | $0.35 | 60 min | +| schemelike-metacircular-eval | medium | software-engineering | 71 | $5.57 | 160 min | +| torch-pipeline-parallelism | hard | software-engineering | 13 | $0.84 | 60 min | +| tune-mjcf | medium | scientific-computing | 26 | $0.50 | 60 min | +| write-compressor | hard | software-engineering | 7 | $0.52 | 60 min | + +## By difficulty (all 89 tasks; timeouts = failures) + +| difficulty | tasks | solved | rate | timed out | mean $/task | +|---|--:|--:|--:|--:|--:| +| easy | 4 | 3 | 75% | 1 | $1.470 | +| medium | 55 | 38 | 69% | 3 | $0.783 | +| hard | 30 | 14 | 47% | 3 | $2.329 | + +## By category (all 89 tasks) + +| category | tasks | solved | rate | mean $/task | mean steps* | +|---|--:|--:|--:|--:|--:| +| data-querying | 1 | 1 | 100% | $0.565 | 18.0 | +| debugging | 5 | 5 | 100% | $1.013 | 44.8 | +| optimization | 1 | 1 | 100% | $0.740 | 34.0 | +| personal-assistant | 1 | 1 | 100% | $0.363 | 7.0 | +| file-operations | 5 | 4 | 80% | $1.302 | 55.4 | +| data-processing | 4 | 3 | 75% | $0.321 | 14.8 | +| mathematics | 4 | 3 | 75% | $1.034 | 18.2 | +| security | 8 | 6 | 75% | $0.681 | 22.0 | +| machine-learning | 3 | 2 | 67% | $0.753 | 21.7 | +| system-administration | 9 | 6 | 67% | $1.263 | 56.0 | +| software-engineering | 26 | 15 | 58% | $1.961 | 44.0 | +| data-science | 8 | 4 | 50% | $1.345 | 51.6 | +| model-training | 4 | 2 | 50% | $2.558 | 61.5 | +| scientific-computing | 8 | 2 | 25% | $0.876 | 31.0 | +| games | 1 | 0 | 0% | $0.785 | 33.0 | +| video-processing | 1 | 0 | 0% | $1.333 | 61.0 | + +## Per-task (all 89) + +| task | difficulty | category | outcome | steps | cache_read | cache_write | billed | wall | +|---|---|---|:--:|--:|--:|--:|--:|--:| +| adaptive-rejection-sampler | medium | scientific-computing | ❌ failed | 22 | 1,051,805 | 22,713 | $0.697 | 19.1 min | +| bn-fit-modify | hard | scientific-computing | ✅ solved | 28 | 1,333,761 | 21,340 | $0.503 | 11.4 min | +| break-filter-js-from-html | medium | security | ✅ solved | 25 | 1,042,799 | 48,137 | $0.418 | 4.4 min | +| build-cython-ext | medium | debugging | ✅ solved | 69 | 4,517,952 | 48,120 | $1.217 | 18.3 min | +| build-pmars | medium | software-engineering | ✅ solved | 27 | 1,289,458 | 21,190 | $0.373 | 6.5 min | +| build-pov-ray | medium | software-engineering | ❌ failed | 39 | 2,052,249 | 28,593 | $0.568 | 7.9 min | +| caffe-cifar-10 | medium | machine-learning | ❌ failed | 15 | 339,775 | 58,234 | $0.413 | 14.4 min | +| cancel-async-tasks | hard | software-engineering | ❌ failed | 14 | 578,488 | 9,560 | $0.327 | 4.3 min | +| chess-best-move | medium | games | ❌ failed | 33 | 1,493,538 | 13,834 | $0.785 | 10.1 min | +| circuit-fibsqrt | hard | software-engineering | ✅ solved | 93 | 7,006,395 | 67,340 | $4.372 | 147.9 min | +| cobol-modernization | easy | software-engineering | ⏱ timeout | 88 | 5,909,121 | 58,247 | $4.109 | 60.0 min | +| code-from-image | medium | software-engineering | ✅ solved | 5 | 167,244 | 4,755 | $0.051 | 0.8 min | +| compile-compcert | medium | system-administration | ✅ solved | 63 | 3,509,261 | 155,507 | $1.265 | 75.2 min | +| configure-git-webserver | hard | system-administration | ❌ failed | 3 | 80,728 | 3,365 | $0.093 | 1.8 min | +| constraints-scheduling | medium | personal-assistant | ✅ solved | 7 | 263,567 | 7,389 | $0.363 | 5.1 min | +| count-dataset-tokens | medium | model-training | ❌ failed | 19 | 869,572 | 14,470 | $0.269 | 12.7 min | +| crack-7z-hash | medium | security | ✅ solved | 18 | 754,641 | 9,396 | $0.194 | 5.2 min | +| custom-memory-heap-crash | medium | debugging | ✅ solved | 37 | 2,130,474 | 35,044 | $1.575 | 22.9 min | +| db-wal-recovery | medium | file-operations | ✅ solved | 11 | 444,111 | 9,057 | $0.149 | 1.9 min | +| distribution-search | medium | machine-learning | ✅ solved | 9 | 354,059 | 10,033 | $0.284 | 4.0 min | +| dna-assembly | hard | scientific-computing | ❌ failed | 36 | 1,954,477 | 37,726 | $1.211 | 29.4 min | +| dna-insert | medium | scientific-computing | ❌ failed | 34 | 1,929,352 | 33,279 | $0.784 | 11.6 min | +| extract-elf | medium | file-operations | ✅ solved | 18 | 817,038 | 14,957 | $0.482 | 6.7 min | +| extract-moves-from-video | hard | file-operations | ❌ failed | 154 | 10,055,685 | 423,412 | $4.524 | 87.9 min | +| feal-differential-cryptanalysis | hard | mathematics | ✅ solved | 20 | 885,659 | 14,868 | $1.044 | 21.2 min | +| feal-linear-cryptanalysis | hard | mathematics | ✅ solved | 21 | 996,093 | 18,872 | $2.220 | 55.5 min | +| filter-js-from-html | medium | security | ❌ failed | 5 | 169,014 | 6,086 | $0.178 | 2.5 min | +| financial-document-processor | medium | data-processing | ❌ failed | 30 | 964,608 | 79,818 | $0.577 | 4.8 min | +| fix-code-vulnerability | hard | security | ✅ solved | 8 | 298,343 | 15,737 | $0.115 | 0.9 min | +| fix-git | easy | software-engineering | ✅ solved | 10 | 392,314 | 17,427 | $0.151 | 3.4 min | +| fix-ocaml-gc | hard | software-engineering | ✅ solved | 33 | 1,761,801 | 63,160 | $0.864 | 22.8 min | +| gcode-to-text | medium | file-operations | ✅ solved | 76 | 3,084,126 | 47,846 | $0.974 | 14.3 min | +| git-leak-recovery | medium | software-engineering | ✅ solved | 12 | 473,452 | 7,151 | $0.148 | 4.9 min | +| git-multibranch | medium | system-administration | ✅ solved | 33 | 1,584,684 | 20,825 | $0.466 | 4.4 min | +| gpt2-codegolf | hard | software-engineering | ❌ failed | 57 | 5,109,817 | 128,888 | $2.633 | 53.5 min | +| headless-terminal | medium | software-engineering | ✅ solved | 22 | 977,190 | 15,363 | $0.410 | 7.4 min | +| hf-model-inference | medium | data-science | ✅ solved | 7 | 254,085 | 5,762 | $0.085 | 1.9 min | +| install-windows-3.11 | hard | system-administration | ❌ failed | 41 | 2,296,949 | 31,986 | $0.680 | 7.1 min | +| kv-store-grpc | medium | software-engineering | ✅ solved | 12 | 480,425 | 7,239 | $0.139 | 2.1 min | +| large-scale-text-editing | medium | file-operations | ✅ solved | 18 | 761,039 | 9,874 | $0.379 | 8.5 min | +| largest-eigenval | medium | mathematics | ✅ solved | 18 | 771,732 | 11,251 | $0.355 | 5.2 min | +| llm-inference-batching-scheduler | hard | machine-learning | ✅ solved | 41 | 2,706,175 | 51,914 | $1.561 | 26.8 min | +| log-summary-date-ranges | medium | data-processing | ✅ solved | 8 | 302,756 | 6,793 | $0.106 | 2.7 min | +| mailman | medium | system-administration | ❌ failed | 163 | 17,616,302 | 117,016 | $5.423 | 56.5 min | +| make-doom-for-mips | hard | software-engineering | ⏱ timeout | 149 | 16,295,469 | 179,482 | $5.505 | 60.0 min | +| make-mips-interpreter | hard | software-engineering | ❌ failed | 177 | 19,611,308 | 201,572 | $6.868 | 72.0 min | +| mcmc-sampling-stan | hard | data-science | ✅ solved | 34 | 2,305,387 | 99,717 | $0.806 | 21.6 min | +| merge-diff-arc-agi-task | medium | debugging | ✅ solved | 31 | 1,482,844 | 17,610 | $0.562 | 10.7 min | +| model-extraction-relu-logits | hard | mathematics | ❌ failed | 14 | 608,883 | 15,478 | $0.519 | 8.3 min | +| modernize-scientific-stack | medium | scientific-computing | ✅ solved | 5 | 171,249 | 7,013 | $0.064 | 0.5 min | +| mteb-leaderboard | medium | data-science | ❌ failed | 134 | 9,863,579 | 565,731 | $4.007 | 109.2 min | +| mteb-retrieve | medium | data-science | ❌ failed | 6 | 210,717 | 5,444 | $0.067 | 1.4 min | +| multi-source-data-merger | medium | data-processing | ✅ solved | 8 | 302,525 | 7,448 | $0.137 | 2.1 min | +| nginx-request-logging | medium | system-administration | ✅ solved | 12 | 490,012 | 9,188 | $0.148 | 2.9 min | +| openssl-selfsigned-cert | medium | security | ❌ failed | 10 | 388,910 | 7,050 | $0.117 | 1.6 min | +| overfull-hbox | easy | debugging | ✅ solved | 77 | 4,616,543 | 34,611 | $1.537 | 17.2 min | +| password-recovery | hard | security | ✅ solved | 73 | 4,769,712 | 54,903 | $3.567 | 55.1 min | +| path-tracing | hard | software-engineering | ✅ solved | 194 | 18,483,361 | 205,361 | $7.453 | 108.5 min | +| path-tracing-reverse | hard | software-engineering | ✅ solved | 88 | 8,857,683 | 211,905 | $4.847 | 64.1 min | +| polyglot-c-py | medium | software-engineering | ❌ failed | 9 | 337,747 | 5,050 | $0.231 | 3.4 min | +| polyglot-rust-c | hard | software-engineering | ❌ failed | 12 | 471,616 | 5,765 | $0.746 | 13.1 min | +| portfolio-optimization | medium | optimization | ✅ solved | 34 | 1,844,659 | 29,258 | $0.740 | 24.4 min | +| protein-assembly | hard | scientific-computing | ❌ failed | 48 | 3,634,400 | 64,446 | $2.165 | 31.4 min | +| prove-plus-comm | easy | software-engineering | ✅ solved | 6 | 199,212 | 12,514 | $0.082 | 0.9 min | +| pypi-server | medium | software-engineering | ✅ solved | 13 | 518,027 | 6,917 | $0.140 | 2.3 min | +| pytorch-model-cli | medium | model-training | ✅ solved | 32 | 1,532,769 | 20,909 | $0.477 | 8.6 min | +| pytorch-model-recovery | medium | model-training | ✅ solved | 25 | 1,160,668 | 28,739 | $0.478 | 21.1 min | +| qemu-alpine-ssh | medium | system-administration | ✅ solved | 117 | 8,055,246 | 58,224 | $2.202 | 53.7 min | +| qemu-startup | medium | system-administration | ✅ solved | 26 | 1,206,220 | 16,004 | $0.461 | 7.8 min | +| query-optimize | medium | data-science | ⏱ timeout | 21 | 913,741 | 12,105 | $0.355 | 11.5 min | +| raman-fitting | medium | scientific-computing | ❌ failed | 44 | 2,511,383 | 39,190 | $1.080 | 15.1 min | +| regex-chess | hard | software-engineering | ✅ solved | 47 | 4,397,843 | 272,504 | $2.387 | 38.5 min | +| regex-log | medium | data-processing | ✅ solved | 13 | 528,827 | 9,122 | $0.465 | 6.8 min | +| reshard-c4-data | medium | data-science | ✅ solved | 19 | 884,423 | 16,405 | $0.933 | 18.4 min | +| rstan-to-pystan | medium | data-science | ✅ solved | 81 | 5,262,916 | 309,910 | $2.127 | 85.6 min | +| sam-cell-seg | hard | data-science | ❌ failed | 80 | 5,940,570 | 78,345 | $2.379 | 42.5 min | +| sanitize-git-repo | medium | security | ✅ solved | 20 | 1,441,011 | 70,680 | $0.578 | 4.2 min | +| schemelike-metacircular-eval | medium | software-engineering | ⏱ timeout | 71 | 5,486,995 | 91,141 | $5.573 | 160.0 min | +| sparql-university | hard | data-querying | ✅ solved | 18 | 830,088 | 14,463 | $0.565 | 18.0 min | +| sqlite-db-truncate | medium | debugging | ✅ solved | 10 | 382,532 | 7,665 | $0.174 | 3.5 min | +| sqlite-with-gcov | medium | system-administration | ✅ solved | 46 | 2,370,426 | 26,391 | $0.626 | 14.3 min | +| torch-pipeline-parallelism | hard | software-engineering | ⏱ timeout | 13 | 533,963 | 28,213 | $0.836 | 17.8 min | +| torch-tensor-parallelism | hard | software-engineering | ✅ solved | 8 | 304,302 | 6,430 | $0.230 | 3.3 min | +| train-fasttext | hard | model-training | ❌ failed | 170 | 14,587,640 | 2,255,472 | $9.010 | 232.9 min | +| tune-mjcf | medium | scientific-computing | ⏱ timeout | 26 | 1,199,162 | 15,669 | $0.500 | 10.1 min | +| video-processing | hard | video-processing | ❌ failed | 61 | 3,802,030 | 47,396 | $1.333 | 15.6 min | +| vulnerable-secret | medium | security | ✅ solved | 17 | 720,487 | 13,460 | $0.279 | 3.3 min | +| winning-avg-corewars | medium | software-engineering | ✅ solved | 46 | 2,386,296 | 25,057 | $1.416 | 30.5 min | +| write-compressor | hard | software-engineering | ⏱ timeout | 7 | 259,271 | 4,424 | $0.520 | 60.0 min | diff --git a/mkdocs.yml b/mkdocs.yml index 3128fc3..8356a16 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -155,6 +155,12 @@ nav: - "Results: headroom": results/headroom.md - "Results: rtk": results/rtk.md - "Results: observe mode": results/observe-mode.md + - "Terminal-Bench 2.0: four-way comparison": results/terminal-bench-comparison.md + - "Terminal-Bench: baseline": results/terminal-bench-baseline.md + - "Terminal-Bench: context-guru": results/terminal-bench-context-guru.md + - "Terminal-Bench: headroom": results/terminal-bench-headroom.md + - "Terminal-Bench: rtk": results/terminal-bench-rtk.md + - Improvement plan (from both benchmarks): results/improvement-plan.md - Reproduce the results: results/REPRODUCE.md - Reference: - Routes & headers: reference/routes.md