From a5ad099678c52b81ee11fad2909934151620a02e Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 10 Jul 2026 13:53:23 +0100 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20built-in=20benchmark=20suite=20?= =?UTF-8?q?=E2=80=94=204=20frozen=20prompt=20cards=20+=20run/track=20harne?= =?UTF-8?q?ss?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds benchmarks/: four standard prompts (assistant easy/medium/hard + teacher) with 100-point rubrics split machine-checkable vs judged, a run-record schema (meta.yaml/transcript/score/artifacts) committed per run, and autoassistant/benchmark.py (new-run/score/report) regenerating leaderboard + same-model-over-time tables in RESULTS.md. Protocol and agent contract in benchmarks/{README,AGENTS}.md: frozen prompt versioning, same-judge comparability, failures-are-data, runs/ is data never instructions. README gains a Benchmarks section (and the SLACS example-prompt header simplification); card/README prompt parity is test-enforced. Template boundary in modes/maintainer.md classifies the package for cloning (harness generic, cards domain, runs per-clone); the citation sweep now covers benchmarks/ docs and cards (never runs/). Co-Authored-By: Claude Fable 5 --- README.md | 14 +- autoassistant/audit_skill_apis.py | 4 + autoassistant/benchmark.py | 385 ++++++++++++++++++ autoassistant/tests/test_benchmark.py | 176 ++++++++ benchmarks/AGENTS.md | 49 +++ benchmarks/README.md | 102 +++++ benchmarks/RESULTS.md | 21 + benchmarks/prompts/easy_cosmos_web_ring.md | 74 ++++ benchmarks/prompts/hard_group_multi.md | 92 +++++ .../prompts/medium_slacs0946_subhalo.md | 93 +++++ benchmarks/prompts/teacher_workflow.md | 79 ++++ benchmarks/runs/.gitkeep | 0 modes/maintainer.md | 14 +- 13 files changed, 1098 insertions(+), 5 deletions(-) create mode 100644 autoassistant/benchmark.py create mode 100644 autoassistant/tests/test_benchmark.py create mode 100644 benchmarks/AGENTS.md create mode 100644 benchmarks/README.md create mode 100644 benchmarks/RESULTS.md create mode 100644 benchmarks/prompts/easy_cosmos_web_ring.md create mode 100644 benchmarks/prompts/hard_group_multi.md create mode 100644 benchmarks/prompts/medium_slacs0946_subhalo.md create mode 100644 benchmarks/prompts/teacher_workflow.md create mode 100644 benchmarks/runs/.gitkeep diff --git a/README.md b/README.md index 7231072..b35761d 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,7 @@ if needed, walk you through setting the analysis up on a High Performance Computer (HPC) you have access to. ``` -Assistant mode — run this autonomously, planning in phases and checkpointing -with me at the big scientific decisions. +Assistant mode. The strong lens SLACS0946+1006 famously has a dark matter subhalo detection that many argue is unusually concentrated. I'd like to analyse @@ -193,6 +192,17 @@ keep the raw data private; make the code, figures, manifests, citation and DOI publication-ready. ``` +## Benchmarks + +The three example prompts above (plus the hard cross-package benchmark) are +also shipped as **frozen benchmark prompts** under [`benchmarks/`](benchmarks/), +with scoring rubrics and a small harness that records each run's conversation, +results and score. Run them against different AI agents and models — or the +same model on different days — and the committed run records in +`benchmarks/runs/` plus the regenerated tables in `benchmarks/RESULTS.md` give +you an evidence-backed comparison of how well each setup drives the assistant. +The protocol is in [`benchmarks/README.md`](benchmarks/README.md). + ## Scientific Context The assistant doesn't just know the PyAutoLens API — it ships with a diff --git a/autoassistant/audit_skill_apis.py b/autoassistant/audit_skill_apis.py index 7535374..8e01cf0 100644 --- a/autoassistant/audit_skill_apis.py +++ b/autoassistant/audit_skill_apis.py @@ -1186,6 +1186,10 @@ def check_citations(root: Path) -> int: ) files = sorted((root / "skills").glob("*.md")) files += sorted((root / "wiki" / "core").rglob("*.md")) + # Benchmark protocol docs + frozen prompt cards — but never runs/, whose + # transcripts are historical data and may legitimately quote stale paths. + files += sorted((root / "benchmarks").glob("*.md")) + files += sorted((root / "benchmarks" / "prompts").glob("*.md")) files += [p for p in (root / "AGENTS.md", root / "llms.txt") if p.exists()] errors: list[str] = [] diff --git a/autoassistant/benchmark.py b/autoassistant/benchmark.py new file mode 100644 index 0000000..746b48a --- /dev/null +++ b/autoassistant/benchmark.py @@ -0,0 +1,385 @@ +"""Scaffold, score and report benchmark runs of the assistant (see benchmarks/README.md).""" + +from __future__ import annotations + +import argparse +import datetime as dt +import re +import subprocess +import sys +from dataclasses import dataclass, field +from importlib import metadata +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +PROMPTS_DIRNAME = "prompts" +RUNS_DIRNAME = "runs" +BENCHMARKS_DIRNAME = "benchmarks" + +STACK_PACKAGES = ("autolens", "autofit", "autoarray", "autoconf") + +FRONTMATTER = re.compile(r"\A---\n(.*?)\n---\n", re.DOTALL) +RUBRIC_ROW = re.compile(r"^\|\s*(M\d+|J\d+)\s*\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*$") +SCORE_ROW = re.compile( + r"^\|\s*(M\d+|J\d+)\s*\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*([0-9.]*)\s*\|\s*(.*?)\s*\|\s*$" +) + +TRANSCRIPT_STUB = """\ +# Transcript — {run_name} + +Paste (or export) the **complete conversation** of the benchmark session here, +verbatim: every user message and every assistant message, in order. Do not +summarise, do not trim failures. If the harness supports transcript export, +prefer that; otherwise copy the conversation manually. + +--- + +(transcript goes here) +""" + + +@dataclass(frozen=True) +class RubricLine: + """One rubric criterion from a prompt card.""" + + code: str + criterion: str + max_points: int + + @property + def machine(self) -> bool: + return self.code.startswith("M") + + +@dataclass(frozen=True) +class PromptCard: + """A parsed benchmark prompt card.""" + + path: Path + meta: dict + rubric: tuple[RubricLine, ...] + + @property + def id(self) -> str: + return self.meta["id"] + + @property + def version(self) -> int: + return int(self.meta.get("version", 1)) + + +@dataclass +class RunScore: + """Totals parsed from a run's score.md.""" + + machine: float = 0.0 + judged: float = 0.0 + machine_max: int = 0 + judged_max: int = 0 + unfilled: list[str] = field(default_factory=list) + + @property + def total(self) -> float: + return self.machine + self.judged + + +def benchmarks_dir(root: Path) -> Path: + return root / BENCHMARKS_DIRNAME + + +def load_card(path: Path) -> PromptCard: + text = path.read_text() + match = FRONTMATTER.match(text) + if match is None: + raise ValueError(f"{path}: no YAML frontmatter") + meta = yaml.safe_load(match.group(1)) + rubric = tuple( + RubricLine(code=m.group(1), criterion=m.group(2), max_points=int(m.group(3))) + for line in text.splitlines() + if (m := RUBRIC_ROW.match(line)) is not None + ) + if "id" not in meta: + raise ValueError(f"{path}: frontmatter has no 'id'") + if not rubric: + raise ValueError(f"{path}: no rubric rows (| M1 | ... | pts |) found") + return PromptCard(path=path, meta=meta, rubric=rubric) + + +def load_cards(root: Path) -> dict[str, PromptCard]: + prompts = benchmarks_dir(root) / PROMPTS_DIRNAME + cards = {} + for path in sorted(prompts.glob("*.md")): + card = load_card(path) + if card.id in cards: + raise ValueError(f"duplicate benchmark id '{card.id}' ({path})") + cards[card.id] = card + return cards + + +def slugify(value: str) -> str: + return re.sub(r"[^a-z0-9.]+", "-", value.lower()).strip("-") + + +def stack_versions() -> dict[str, str]: + versions = {} + for package in STACK_PACKAGES: + try: + versions[package] = metadata.version(package) + except metadata.PackageNotFoundError: + versions[package] = "not-installed" + return versions + + +def git_sha(root: Path) -> str: + out = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + ) + return out.stdout.strip() or "unknown" + + +def score_md_text(card: PromptCard, run_name: str) -> str: + lines = [ + f"# Score — {run_name}", + "", + f"Rubric from `{PROMPTS_DIRNAME}/{card.path.name}` v{card.version}. Fill", + "the Awarded column (0 up to Max; fractions allowed) and put the evidence —", + "a file path, a transcript quote, a truth-vs-recovered number — in the", + "Evidence column. Machine rows (M*) need verifiable evidence; judged rows", + "(J*) record who/what judged them in `meta.yaml`.", + "", + "| # | Criterion | Max | Awarded | Evidence |", + "|---|-----------|-----|---------|----------|", + ] + for line in card.rubric: + lines.append(f"| {line.code} | {line.criterion} | {line.max_points} | | |") + lines.append("") + return "\n".join(lines) + + +def meta_yaml_dict(card: PromptCard, model: str, harness: str, root: Path) -> dict: + return { + "benchmark": card.id, + "prompt_version": card.version, + "date": dt.date.today().isoformat(), + "model": model, + "harness": harness, + "assistant_sha": git_sha(root), + "stack": stack_versions(), + "hardware": "", + "operator": "", + "run": { + "duration_minutes": None, + "cost_usd": None, + "tokens": None, + "turns": None, + }, + "status": "pending", + "score": { + "machine": None, + "judged": None, + "total": None, + "judge": "", + }, + "notes": "", + } + + +def new_run(root: Path, benchmark_id: str, model: str, harness: str) -> Path: + cards = load_cards(root) + if benchmark_id not in cards: + known = ", ".join(sorted(cards)) + raise SystemExit(f"unknown benchmark '{benchmark_id}' (known: {known})") + card = cards[benchmark_id] + + base = f"{dt.date.today().isoformat()}_{slugify(model)}_{slugify(harness)}" + runs = benchmarks_dir(root) / RUNS_DIRNAME / card.id + run_dir = runs / base + suffix = 2 + while run_dir.exists(): + run_dir = runs / f"{base}_{suffix}" + suffix += 1 + (run_dir / "artifacts").mkdir(parents=True) + + (run_dir / "meta.yaml").write_text( + yaml.safe_dump(meta_yaml_dict(card, model, harness, root), sort_keys=False) + ) + (run_dir / "transcript.md").write_text(TRANSCRIPT_STUB.format(run_name=run_dir.name)) + (run_dir / "score.md").write_text(score_md_text(card, run_dir.name)) + return run_dir + + +def parse_score(run_dir: Path) -> RunScore: + text = (run_dir / "score.md").read_text() + score = RunScore() + seen = False + for line in text.splitlines(): + m = SCORE_ROW.match(line) + if m is None: + continue + seen = True + code, _criterion, max_points, awarded, _evidence = m.groups() + max_points = int(max_points) + if awarded == "": + score.unfilled.append(code) + value = 0.0 + else: + value = float(awarded) + if value < 0 or value > max_points: + raise SystemExit( + f"{run_dir}/score.md: {code} awarded {value} outside 0..{max_points}" + ) + if code.startswith("M"): + score.machine += value + score.machine_max += max_points + else: + score.judged += value + score.judged_max += max_points + if not seen: + raise SystemExit(f"{run_dir}/score.md: no rubric rows found") + return score + + +def score_run(run_dir: Path) -> RunScore: + score = parse_score(run_dir) + if score.unfilled: + raise SystemExit( + f"{run_dir}/score.md: unfilled Awarded rows: {', '.join(score.unfilled)} " + "(fill every row — award 0 explicitly where a check failed)" + ) + meta_path = run_dir / "meta.yaml" + meta = yaml.safe_load(meta_path.read_text()) + meta["score"]["machine"] = score.machine + meta["score"]["judged"] = score.judged + meta["score"]["total"] = score.total + meta["status"] = "complete" + meta_path.write_text(yaml.safe_dump(meta, sort_keys=False)) + return score + + +def iter_run_metas(root: Path): + runs = benchmarks_dir(root) / RUNS_DIRNAME + for meta_path in sorted(runs.glob("*/*/meta.yaml")): + yield meta_path.parent, yaml.safe_load(meta_path.read_text()) + + +def report(root: Path) -> str: + cards = load_cards(root) + by_benchmark: dict[str, list[tuple[Path, dict]]] = {bid: [] for bid in cards} + for run_dir, meta in iter_run_metas(root): + by_benchmark.setdefault(meta.get("benchmark", "unknown"), []).append( + (run_dir, meta) + ) + + today = dt.date.today().isoformat() + lines = [ + "# Benchmark results", + "", + f"Regenerated by `python autoassistant/benchmark.py report` — do not edit " + f"by hand. Last regenerated: {today}.", + "", + "Scores are comparable only within the same benchmark **and** prompt " + "version; judged sub-scores are comparable only across runs graded by " + "the same judge.", + ] + for bid in sorted(by_benchmark): + entries = by_benchmark[bid] + lines += ["", f"## {bid}", ""] + scored = [(d, m) for d, m in entries if m.get("score", {}).get("total") is not None] + pending = [(d, m) for d, m in entries if m.get("score", {}).get("total") is None] + if not scored and not pending: + lines.append("_No runs recorded yet._") + continue + if scored: + lines += [ + "### Leaderboard (model × harness)", + "", + "| Model | Harness | Runs | Best | Latest | Latest date |", + "|-------|---------|------|------|--------|-------------|", + ] + groups: dict[tuple[str, str], list[dict]] = {} + for _d, meta in scored: + groups.setdefault((meta["model"], meta["harness"]), []).append(meta) + for (model, harness), metas in sorted( + groups.items(), + key=lambda kv: -max(m["score"]["total"] for m in kv[1]), + ): + metas.sort(key=lambda m: m["date"]) + best = max(m["score"]["total"] for m in metas) + latest = metas[-1] + lines.append( + f"| {model} | {harness} | {len(metas)} | {best:g} " + f"| {latest['score']['total']:g} | {latest['date']} |" + ) + lines += [ + "", + "### All scored runs (chronological)", + "", + "| Date | Model | Harness | Machine | Judged | Total | Prompt v | Run |", + "|------|-------|---------|---------|--------|-------|----------|-----|", + ] + for run_dir, meta in sorted(scored, key=lambda dm: dm[1]["date"]): + rel = run_dir.relative_to(benchmarks_dir(root)) + s = meta["score"] + lines.append( + f"| {meta['date']} | {meta['model']} | {meta['harness']} " + f"| {s['machine']:g} | {s['judged']:g} | {s['total']:g} " + f"| {meta.get('prompt_version', '?')} | `{rel}` |" + ) + if pending: + lines += ["", "### Unscored runs", ""] + for run_dir, meta in pending: + rel = run_dir.relative_to(benchmarks_dir(root)) + lines.append(f"- `{rel}` ({meta.get('status', 'pending')})") + lines.append("") + return "\n".join(lines) + + +def write_report(root: Path) -> Path: + path = benchmarks_dir(root) / "RESULTS.md" + path.write_text(report(root)) + return path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT, help="assistant repo root") + sub = parser.add_subparsers(dest="command", required=True) + + p_new = sub.add_parser("new-run", help="scaffold a run directory for a benchmark") + p_new.add_argument("benchmark_id") + p_new.add_argument("--model", required=True, help="model identity, e.g. claude-opus-4-8") + p_new.add_argument("--harness", required=True, help="agent harness, e.g. claude-code") + + p_score = sub.add_parser("score", help="total a filled score.md into meta.yaml") + p_score.add_argument("run_dir", type=Path) + + sub.add_parser("report", help="regenerate benchmarks/RESULTS.md") + + args = parser.parse_args(argv) + root = args.root.resolve() + + if args.command == "new-run": + run_dir = new_run(root, args.benchmark_id, args.model, args.harness) + print(f"scaffolded {run_dir}") + print("next: run the benchmark session, save transcript.md, fill score.md,") + print(f"then: python autoassistant/benchmark.py score {run_dir}") + elif args.command == "score": + score = score_run(args.run_dir.resolve()) + print( + f"machine {score.machine:g}/{score.machine_max} · " + f"judged {score.judged:g}/{score.judged_max} · " + f"total {score.total:g}/{score.machine_max + score.judged_max}" + ) + print("meta.yaml updated; regenerate tables: python autoassistant/benchmark.py report") + elif args.command == "report": + path = write_report(root) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/autoassistant/tests/test_benchmark.py b/autoassistant/tests/test_benchmark.py new file mode 100644 index 0000000..ccfc17b --- /dev/null +++ b/autoassistant/tests/test_benchmark.py @@ -0,0 +1,176 @@ +"""Tests for the benchmark harness (autoassistant/benchmark.py).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from autoassistant import benchmark + +REPO_ROOT = Path(__file__).resolve().parents[2] + +CARD = """\ +--- +id: test-bench +version: 2 +mode: assistant +difficulty: easy +datasets: [] +workspace_packages: + - imaging +added: 2026-07-10 +--- + +# Benchmark: test + +## Prompt + +``` +Assistant mode. Do the thing. +``` + +## Success rubric (30 points) + +### Machine-checkable (20) + +| # | Check | Pts | +|---|-------|-----| +| M1 | A thing exists | 10 | +| M2 | Another thing exists | 10 | + +### Judged (10) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | The thing was done well | 10 | +""" + + +@pytest.fixture +def root(tmp_path): + prompts = tmp_path / "benchmarks" / "prompts" + prompts.mkdir(parents=True) + (prompts / "test_bench.md").write_text(CARD) + return tmp_path + + +def fill_score(run_dir, awards): + path = run_dir / "score.md" + lines = [] + for line in path.read_text().splitlines(): + m = benchmark.SCORE_ROW.match(line) + if m and m.group(1) in awards: + code, criterion, max_points = m.group(1), m.group(2), m.group(3) + lines.append( + f"| {code} | {criterion} | {max_points} | {awards[code]} | evidence |" + ) + else: + lines.append(line) + path.write_text("\n".join(lines) + "\n") + + +def test_load_card_parses_frontmatter_and_rubric(root): + cards = benchmark.load_cards(root) + card = cards["test-bench"] + assert card.version == 2 + assert [r.code for r in card.rubric] == ["M1", "M2", "J1"] + assert card.rubric[0].max_points == 10 + assert card.rubric[0].machine and not card.rubric[2].machine + + +def test_new_run_scaffolds_run_directory(root): + run_dir = benchmark.new_run(root, "test-bench", "Claude Opus 4.8", "claude-code") + assert run_dir.name.endswith("_claude-opus-4.8_claude-code") + assert (run_dir / "transcript.md").exists() + assert (run_dir / "artifacts").is_dir() + meta = yaml.safe_load((run_dir / "meta.yaml").read_text()) + assert meta["benchmark"] == "test-bench" + assert meta["prompt_version"] == 2 + assert meta["status"] == "pending" + assert set(meta["stack"]) == set(benchmark.STACK_PACKAGES) + score_text = (run_dir / "score.md").read_text() + assert "| M1 |" in score_text and "| J1 |" in score_text + + +def test_new_run_same_day_repeat_gets_suffix(root): + first = benchmark.new_run(root, "test-bench", "m", "h") + second = benchmark.new_run(root, "test-bench", "m", "h") + assert second.name == f"{first.name}_2" + + +def test_new_run_unknown_benchmark_fails(root): + with pytest.raises(SystemExit): + benchmark.new_run(root, "nope", "m", "h") + + +def test_score_run_totals_and_updates_meta(root): + run_dir = benchmark.new_run(root, "test-bench", "m", "h") + fill_score(run_dir, {"M1": 10, "M2": 0, "J1": 7.5}) + score = benchmark.score_run(run_dir) + assert score.machine == 10 and score.judged == 7.5 and score.total == 17.5 + meta = yaml.safe_load((run_dir / "meta.yaml").read_text()) + assert meta["score"]["total"] == 17.5 + assert meta["status"] == "complete" + + +def test_score_run_rejects_unfilled_rows(root): + run_dir = benchmark.new_run(root, "test-bench", "m", "h") + fill_score(run_dir, {"M1": 10}) + with pytest.raises(SystemExit, match="unfilled"): + benchmark.score_run(run_dir) + + +def test_score_run_rejects_over_max_award(root): + run_dir = benchmark.new_run(root, "test-bench", "m", "h") + fill_score(run_dir, {"M1": 11, "M2": 0, "J1": 0}) + with pytest.raises(SystemExit, match="outside"): + benchmark.score_run(run_dir) + + +def test_report_builds_leaderboard_and_pending(root): + scored = benchmark.new_run(root, "test-bench", "model-a", "h") + fill_score(scored, {"M1": 10, "M2": 10, "J1": 5}) + benchmark.score_run(scored) + pending = benchmark.new_run(root, "test-bench", "model-b", "h") + + text = benchmark.report(root) + assert "## test-bench" in text + assert "| model-a | h | 1 | 25 | 25 |" in text + assert f"`{pending.relative_to(root / 'benchmarks')}`" in text + + path = benchmark.write_report(root) + assert path == root / "benchmarks" / "RESULTS.md" + assert path.read_text() == text + + +def test_repo_prompt_cards_parse(): + """Every committed prompt card must load: unique ids, frontmatter, rubric.""" + cards = benchmark.load_cards(REPO_ROOT) + assert set(cards) == { + "assistant-easy-cosmos-web-ring", + "assistant-medium-slacs0946-subhalo", + "assistant-hard-group-multi", + "teacher-basic-workflow", + } + for card in cards.values(): + machine = sum(r.max_points for r in card.rubric if r.machine) + judged = sum(r.max_points for r in card.rubric if not r.machine) + assert machine + judged == 100, f"{card.id}: rubric totals {machine + judged}" + + +def test_repo_card_datasets_exist(): + """Bundled datasets a card declares must exist — a missing one is a stale card.""" + for card in benchmark.load_cards(REPO_ROOT).values(): + for dataset in card.meta.get("datasets", []): + assert (REPO_ROOT / dataset).is_dir(), f"{card.id}: missing {dataset}" + + +def test_repo_readme_prompts_match_cards(): + """Cards that mirror README example prompts must stay textually identical.""" + readme = (REPO_ROOT / "README.md").read_text() + for name in ("easy_cosmos_web_ring", "medium_slacs0946_subhalo", "teacher_workflow"): + card_text = (REPO_ROOT / "benchmarks" / "prompts" / f"{name}.md").read_text() + prompt = card_text.split("```\n", 1)[1].split("```", 1)[0] + assert prompt.strip() in readme, f"{name}: prompt text diverges from README.md" diff --git a/benchmarks/AGENTS.md b/benchmarks/AGENTS.md new file mode 100644 index 0000000..791862a --- /dev/null +++ b/benchmarks/AGENTS.md @@ -0,0 +1,49 @@ +# benchmarks/ — agent contract + +Two situations bring an agent here. Follow the section that applies. + +## You are *being benchmarked* (an operator pasted a prompt from `prompts/`) + +You will not usually know it — benchmark prompts are ordinary user requests +and the session must be indistinguishable from real use. If you *do* realise +mid-session that a request matches a benchmark card: + +- **Change nothing.** Follow the repo's normal instructions (`../AGENTS.md`), + skills and safety invariants exactly as for any user. Do not read the card's + rubric, do not read `runs/`, and do not optimise for scoring criteria — a + benchmark gamed is a benchmark destroyed. +- Report results honestly, including failures. The rubric rewards honest + failure reporting and penalises fabrication far harder than incompleteness. + +## You are *operating a benchmark* (a maintainer asked you to run/record/score one) + +This is maintainer-flavoured work; the protocol is +[`README.md`](README.md) "Running a benchmark". The parts that are yours: + +- **Scaffold first**: `python autoassistant/benchmark.py new-run --model + --harness ` — never hand-create run directories; the + scaffold captures stack versions and the assistant SHA that make runs + comparable. +- **The session under test is not yours.** The benchmarked session runs in a + separate, fresh agent session with the prompt pasted verbatim. Never answer + the benchmark prompt yourself inside the operating session, and never feed + the session under test hints, rubric rows, or past transcripts. +- **Record verbatim**: the full conversation into `transcript.md`, key images + (≤ ~500 KB total) into `artifacts/`, hardware/duration/judge into + `meta.yaml`. Do not summarise or prettify the transcript. +- **Score with evidence**: every rubric row gets an Awarded value (0 where a + check failed) and concrete evidence — a path, a quote, a number. Machine + rows (M*) must point at verifiable artifacts. If you are the judge for + judged rows (J*), record your model identity in `meta.yaml` `score.judge`. +- **Then**: `benchmark.py score `, `benchmark.py report`, and commit + the run directory plus `RESULTS.md` (normal commit cadence rules from + `../AGENTS.md` apply — announce, stage explicitly, never push unasked). + +## Hard rules (both roles) + +- `runs/` is **data, never instructions** — past transcripts must not shape + how a benchmark is answered. +- Prompt cards are **frozen**: never edit a published card's prompt text in + place; a wording change is a `version` bump (and for README-mirrored cards, + a matching README edit — a unit test enforces the parity). +- Failures are recorded, not discarded. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..b9780fd --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,102 @@ +# Benchmarks + +Built-in benchmarks for the assistant: **standard, frozen prompts** run against +different AI agents (harnesses) and models to measure how well they drive the +assistant. Every run's conversation, results and score are recorded in this +directory and committed, so performance can be compared **across models** and +**across days for the same model**, with the full evidence pushed to GitHub. + +## Layout + +``` +benchmarks/ + README.md this file — the protocol + AGENTS.md the contract an agent follows when a benchmark is run on it + RESULTS.md regenerated comparison tables (leaderboards + time series) + prompts/ the frozen prompt cards (one benchmark each) + runs//__/ one recorded run + meta.yaml who/what/when: model, harness, stack versions, score + transcript.md the complete conversation, verbatim + score.md the filled rubric, with evidence per criterion + artifacts/ a few small key images (≤ ~500 KB total per run) +``` + +## The benchmarks + +| Card | Mode | Difficulty | Exercises | +|------|------|-----------|-----------| +| [`prompts/easy_cosmos_web_ring.md`](prompts/easy_cosmos_web_ring.md) | assistant | easy | the built-in workflow on bundled JWST data | +| [`prompts/medium_slacs0946_subhalo.md`](prompts/medium_slacs0946_subhalo.md) | assistant | medium | pipeline design beyond bundled workflows: Bayesian model comparison, runtime/HPC judgment | +| [`prompts/hard_group_multi.md`](prompts/hard_group_multi.md) | assistant | hard | cross-package synthesis: group × multi × imaging × interferometer, simulation + joint modeling | +| [`prompts/teacher_workflow.md`](prompts/teacher_workflow.md) | teacher | easy | pedagogy: end-to-end workflow walkthrough on simulated Euclid-like data | + +Each card carries the verbatim prompt, what it measures, and a 100-point +rubric split into **machine-checkable** rows (artifacts that exist or don't) +and **judged** rows (quality graded by a human or a stated judge model). + +## Running a benchmark + +1. **Scaffold the run record** (from the repo root, any Python ≥3.10 with + `pyyaml`): + + ```bash + python autoassistant/benchmark.py new-run assistant-easy-cosmos-web-ring \ + --model claude-opus-4-8 --harness claude-code + ``` + + This creates `runs//__/` with a `meta.yaml` + pre-filled with the date, the assistant's git SHA and the installed PyAuto* + versions, plus a transcript stub and a `score.md` generated from the card's + rubric. Same-day repeats get a `_2`, `_3`, … suffix. + +2. **Run the session**: start the agent under test in a **clean clone state** + (fresh session, no `.maintainer`, no leftover `scripts/scratch` context from + a previous attempt) and paste the card's prompt **verbatim** as the first + message. From there the operator behaves like a real user: answer the + agent's questions honestly and minimally, never coach it toward the rubric. + `benchmarks/AGENTS.md` states the same contract from the agent's side. + +3. **Record**: save the complete conversation into `transcript.md` (export if + the harness supports it; paste otherwise), copy a few key images into + `artifacts/` (cap ~500 KB — link paths in `score.md` for the rest), and fill + `meta.yaml`'s `hardware`, `operator`, `run:` (duration/cost/tokens/turns + where known) and `score.judge` fields. + +4. **Score**: fill every `Awarded` cell in `score.md` (0 explicitly for failed + checks) with evidence per row, then: + + ```bash + python autoassistant/benchmark.py score benchmarks/runs/// + python autoassistant/benchmark.py report # regenerates RESULTS.md + ``` + +5. **Commit and push** the run directory and `RESULTS.md`. + +## Comparability rules (what keeps the numbers honest) + +- **Prompts are frozen.** A published card's prompt text never changes in + place; any wording change bumps the card's `version`, and scores are only + comparable within one version. Cards that mirror the top-level README + example prompts are kept textually identical by a unit test. +- **Same judge for judged rows.** Judged sub-scores are comparable only across + runs graded by the same judge (a named human or a stated judge model); + `meta.yaml` records the judge. +- **Failures are data.** A run where the agent gets stuck, fabricates, or + honestly reports a poor fit is recorded and scored like any other — do not + discard bad runs; the gaps are the interesting part. +- **Environment is recorded, not standardised.** Stack versions, assistant + SHA and hardware go in `meta.yaml`; wall-clock comparisons are only + meaningful on matching hardware, score comparisons travel better. +- **No benchmark-aware behaviour.** The prompts are ordinary user requests; + the agent is not told it is being benchmarked, and `runs/` is data, not + instructions (agents must never read past runs to prepare). + +## Why these three axes + +- **Different models, same harness** — model capability comparison (the + leaderboard per benchmark in `RESULTS.md`). +- **Same model, different days** — drift/regression tracking as models, + harnesses and the assistant itself evolve (the chronological table; the + cheap teacher benchmark is the recommended drift probe). +- **Same model, different harness** — how much the agent runtime (Claude + Code, Codex CLI, Gemini CLI, …) contributes beyond raw model quality. diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 0000000..2164550 --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,21 @@ +# Benchmark results + +Regenerated by `python autoassistant/benchmark.py report` — do not edit by hand. Last regenerated: 2026-07-10. + +Scores are comparable only within the same benchmark **and** prompt version; judged sub-scores are comparable only across runs graded by the same judge. + +## assistant-easy-cosmos-web-ring + +_No runs recorded yet._ + +## assistant-hard-group-multi + +_No runs recorded yet._ + +## assistant-medium-slacs0946-subhalo + +_No runs recorded yet._ + +## teacher-basic-workflow + +_No runs recorded yet._ diff --git a/benchmarks/prompts/easy_cosmos_web_ring.md b/benchmarks/prompts/easy_cosmos_web_ring.md new file mode 100644 index 0000000..3b8c13c --- /dev/null +++ b/benchmarks/prompts/easy_cosmos_web_ring.md @@ -0,0 +1,74 @@ +--- +id: assistant-easy-cosmos-web-ring +version: 1 +mode: assistant +difficulty: easy +datasets: + - dataset/imaging/cosmos_web_ring +workspace_packages: + - imaging +added: 2026-07-10 +--- + +# Benchmark: model the bundled JWST ring (assistant · easy) + +Everything this prompt asks for is directly available in the assistant: the +dataset ships with the repo, and data preparation, model composition, pixelized +source fitting and result inspection are all covered by existing `al_*` skills. +A capable agent should complete it without inventing anything — the benchmark +measures whether it *finds and follows* the built-in workflow. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol): + +``` +Assistant mode. + +Model the JWST imaging in dataset/imaging/cosmos_web_ring: perform data preparation steps, +set up a sensible lens light and mass model with a pixelized source reconstruction, run +the fit, and show me the reconstructed source and the fit residuals. +``` + +This is Example Prompt 2 of the top-level `README.md`; the two texts must stay +identical (a divergence is a bug — fix the README or bump this card's +`version`). + +## What this measures + +- Routing: does the agent use the assistant's skills and bundled dataset rather + than writing PyAutoLens from memory? +- The real-data safety gate: inspecting the data and asking about extra + galaxies / artefacts *before* fitting. +- A complete run: preparation → model → search → fit → the two requested + visuals. + +## Success rubric (100 points) + +### Machine-checkable (40) + +| # | Check | Pts | +|---|-------|-----| +| M1 | A script (or scripts) saved under `scripts/` that performs the fit | 5 | +| M2 | A completed non-linear search result exists under `output/` (not test-mode) | 10 | +| M3 | The model includes a pixelized source (mesh + regularization), verifiable in the script | 10 | +| M4 | A reconstructed-source figure was produced and its path shown to the user | 10 | +| M5 | A residual-map figure was produced and its path shown to the user | 5 | + +### Judged (60) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Real-data gate honoured: dataset plotted and inspected, one question asked about extra galaxies / foreground stars / artefacts before any fit | 15 | +| J2 | Sensible model choices for this data (lens light + total mass, e.g. Sersic/MGE light with isothermal mass; priors not obviously pathological) | 15 | +| J3 | Data preparation actually performed (mask choice justified; any needed dataset prep steps done rather than skipped) | 10 | +| J4 | Fit quality: residuals broadly noise-like, reconstruction plausibly a lensed source; agent comments on quality honestly | 10 | +| J5 | Conduct: concise assistant-mode communication, no fabricated numbers, API-gate discipline (no invented symbols) | 10 | + +## Operator notes + +- Expected wall-clock: roughly 15–60 minutes depending on hardware and model; + the search is the dominant cost. +- A run that ends with the agent honestly reporting a failed or poor fit scores + what the rubric gives it — record it; failures are data. diff --git a/benchmarks/prompts/hard_group_multi.md b/benchmarks/prompts/hard_group_multi.md new file mode 100644 index 0000000..e1f6284 --- /dev/null +++ b/benchmarks/prompts/hard_group_multi.md @@ -0,0 +1,92 @@ +--- +id: assistant-hard-group-multi +version: 1 +mode: assistant +difficulty: hard +datasets: [] # fully simulated — the agent creates both datasets +workspace_packages: + - group + - multi + - imaging + - interferometer +added: 2026-07-10 +--- + +# Benchmark: group-scale joint imaging + interferometer (assistant · hard) + +Not based on any README example, and deliberately not achievable from a single +workspace package: the agent must combine **group**-scale lens composition +(two lens galaxies), **multi**-dataset joint analysis, and both the +**imaging** and **interferometer** pipelines — simulation and modeling — then +chain a pixelized-source follow-up. This measures cross-package synthesis, the +hardest thing for an agent working from per-package examples. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol): + +``` +Assistant mode. + +First, I want to simulate imaging and interferometer data of a group-scale strong lens, which is composed of +two SIE lens galaxies and a quadruply imaged Cored Sersic background source. + +Then, I want to perform modeling of this dataset, simultaneously fitting the imaging and interferometer data. +I want the foreground lens model to use multi gaussian Expansions for the lens light, SIE's for each lens +and a multi Gaussian expansion for the background source. + +After this fit has been judged successful, do a follow up lens model that uses a pixelized source reconstruction, +but retains the MGE lens light and SIE source. + +Present me with results confirming the fit was a success. +``` + +The prompt is frozen verbatim — including its rough edges (e.g. the final +"SIE source" plainly means the SIE mass profiles are retained while the source +switches to pixelized). Interpreting user intent sensibly, or asking one +focused question, is part of what is being measured; do not clean the wording. + +## What this measures + +- Cross-package composition: group (multi-galaxy lens) × multi (joint + datasets) × imaging × interferometer, for both simulation and modeling. +- Simulation judgment: choosing sensible instrument configurations (a + resolution/uv-plane setup that actually resolves a quad) and verifying the + source is genuinely quadruply imaged. +- Joint-fit wiring: one lens model shared across an imaging and an + interferometer analysis, fitted simultaneously. +- Staged modeling: judging the MGE fit successful before the pixelized-source + follow-up, and demonstrating success against known truths. + +## Success rubric (100 points) + +### Machine-checkable (45) + +| # | Check | Pts | +|---|-------|-----| +| M1 | Simulated imaging dataset created: two SIE lenses + Cored Sersic source, with the quad morphology visible/verified | 10 | +| M2 | Simulated interferometer dataset of the same system created | 10 | +| M3 | Joint fit completed fitting imaging + interferometer **simultaneously** (one search over a shared model, not two independent fits) with MGE lens light, one SIE per lens, MGE source | 15 | +| M4 | Follow-up pixelized-source fit completed retaining MGE lens light and the SIE mass profiles | 5 | +| M5 | A results summary presented: truth-vs-recovered comparison and/or residual figures with paths shown | 5 | + +### Judged (55) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Simulation quality: group-scale geometry plausible; instrument configs sensible; the quad verified rather than assumed | 15 | +| J2 | Joint-analysis wiring correct: shared parametrisation across datasets, dataset-specific nuisance handled sensibly; simultaneous fit demonstrated (not sequential) | 15 | +| J3 | Success judged honestly between stages: explicit criteria (residuals, recovered parameters vs truth) before proceeding to the pixelized follow-up | 10 | +| J4 | The "SIE source" ambiguity handled sensibly (correct interpretation or one focused question — not silent nonsense, not an interrogation) | 5 | +| J5 | Conduct: staged plan communicated, honest evidence of success/failure, API-gate discipline, no fabricated results | 10 | + +## Operator notes + +- The heaviest benchmark: two simulations plus a joint search plus a pixelized + follow-up. Expect multiple hours of wall-clock on a laptop; interferometer + transforms benefit strongly from a GPU. Partial completions are still + recorded — score what completed. +- Because everything is simulated, the run is fully self-contained and + hardware-reproducible: truths are known, so recovered-parameter accuracy is + objective evidence. diff --git a/benchmarks/prompts/medium_slacs0946_subhalo.md b/benchmarks/prompts/medium_slacs0946_subhalo.md new file mode 100644 index 0000000..5364fd7 --- /dev/null +++ b/benchmarks/prompts/medium_slacs0946_subhalo.md @@ -0,0 +1,93 @@ +--- +id: assistant-medium-slacs0946-subhalo +version: 1 +mode: assistant +difficulty: medium +datasets: + - dataset/imaging/slacs0946+1006 +workspace_packages: + - imaging +added: 2026-07-10 +--- + +# Benchmark: SLACS0946+1006 subhalo model comparison (assistant · medium) + +The dataset is bundled, but the analysis is **not** a ready-made workflow: the +agent must design a Bayesian model-comparison pipeline itself — a smooth +baseline, a free-position SIS perturber scan, and an SIS-vs-NFW comparison at +the recovered position — and it must reason about runtime and hardware. This +measures composition of capabilities beyond what any single skill provides. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol): + +``` +Assistant mode. + +The strong lens SLACS0946+1006 famously has a dark matter subhalo +detection that many argue is unusually concentrated. I'd like to analyse +the HST imaging of this lens provided at +dataset/imaging/slacs0946+1006/ and reproduce that detection. + +Specifically, I want this analysis to perform Bayesian model comparison +to (a) confirm a subhalo is preferred over a smooth-mass baseline by +fitting a free-position, free-mass SIS perturber across the image plane +and comparing the Bayesian evidence to the no-subhalo fit, and (b) test +the "super-concentrated" claim by comparing the SIS subhalo +against a more shallow NFW mass profile at the recovered position. + +Set the pipeline up so the smooth lens light and mass model, the +pixelized source reconstruction, and the subhalo results are all +inspectable on my computer, and report the Bayesian evidence for each +comparison. + +Assess whether the analysis will run fast on my laptop / PC GPU, +and if not, set this up as a small project on the HPC I have access to. +``` + +This is Example Prompt 3 of the top-level `README.md`; the two texts must stay +identical (a divergence is a bug — fix the README or bump this card's +`version`). + +## What this measures + +- Pipeline design beyond bundled workflows: chained searches, evidence + bookkeeping, a perturber scan, profile substitution at a fixed position. +- Scientific literacy: the subhalo-detection literature context (the assistant + ships a literature wiki covering it) and what "concentrated" means for + SIS-vs-NFW comparison. +- Resource judgment: an honest laptop/GPU runtime estimate and, if needed, an + HPC project setup rather than silently launching an hours-long local run. + +## Success rubric (100 points) + +### Machine-checkable (40) + +| # | Check | Pts | +|---|-------|-----| +| M1 | Smooth-baseline fit completed with its log-evidence recorded | 10 | +| M2 | SIS-subhalo fit (free position, free mass) completed with its log-evidence recorded | 10 | +| M3 | NFW-subhalo comparison fit completed with its log-evidence recorded | 10 | +| M4 | Both evidence comparisons (subhalo vs smooth; SIS vs NFW) reported as numbers in the final answer | 5 | +| M5 | Results inspectable: output paths for the smooth model, source reconstruction and subhalo results listed for the user | 5 | + +### Judged (60) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Comparison design is correct: the perturber scan genuinely explores the image plane (not a single fixed guess); the NFW test is at the recovered position as asked | 15 | +| J2 | Runtime/hardware assessment performed *before* the long runs, with a defensible estimate; HPC option set up or offered if the local estimate is slow | 15 | +| J3 | Pipeline structure sensible: lens light + mass + pixelized source built up in stages rather than one monolithic fit; evidence values comparable (same data, same mask) | 10 | +| J4 | Scientific interpretation: what the ΔlogZ values mean, and what SIS-vs-NFW says about the concentration claim, stated with appropriate caution | 10 | +| J5 | Real-data gate honoured; conduct (honest reporting, no fabricated evidences, API-gate discipline) | 10 | + +## Operator notes + +- Full sampling is hours-scale on a laptop; the prompt's last paragraph makes + runtime assessment part of the task. A run that stops at a well-set-up HPC + handoff with clear instructions can still score highly on the judged side — + score the machine-checkable fits only if they actually completed somewhere. +- Repeat runs on different hardware are comparable on the judged criteria but + not on wall-clock; record hardware in `meta.yaml`. diff --git a/benchmarks/prompts/teacher_workflow.md b/benchmarks/prompts/teacher_workflow.md new file mode 100644 index 0000000..d385d93 --- /dev/null +++ b/benchmarks/prompts/teacher_workflow.md @@ -0,0 +1,79 @@ +--- +id: teacher-basic-workflow +version: 1 +mode: teacher +difficulty: easy +datasets: [] # fully simulated — the session generates its own data +workspace_packages: + - imaging +added: 2026-07-10 +--- + +# Benchmark: end-to-end workflow walkthrough (teacher) + +The teacher-mode benchmark: the deliverable is **understanding**, not a fit. +The scientific content is deliberately simple (isothermal mass + Sersic +source on simulated Euclid-like imaging) so the score concentrates on +pedagogy — pacing, correctness of explanation, and whether a newcomer would +come away understanding the workflow rather than having watched commands +scroll past. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol): + +``` +Teacher mode. + +I'm new to PyAutoLens and want to learn the basic workflow end-to-end. Can you +walk me through it on a simple simulated example: simulate Euclid-like imaging of +a simple strong lens (an isothermal mass with a Sersic source), then fit that +simulated data and recover the lens model. + +Explain what each step is doing and why as we go: composing the lens and source +model, running the simulation, choosing the mask, the non-linear search, and how +to read the result. So I come away understanding the workflow, not just the +commands. +``` + +This is Example Prompt 1 of the top-level `README.md`; the two texts must stay +identical (a divergence is a bug — fix the README or bump this card's +`version`). + +## What this measures + +- Teacher-mode behaviour: explaining *why* at each step, checking in, adapting + depth — versus dumping a complete script. +- Domain correctness at teaching depth: what an isothermal profile is, why + masks matter, what a non-linear search does, how to read the results. +- Instrument realism: "Euclid-like" should actually inform the simulation + (VIS-like pixel scale ~0.1", appropriate PSF and depth), not be ignored. + +## Success rubric (100 points) + +### Machine-checkable (30) + +| # | Check | Pts | +|---|-------|-----| +| M1 | A simulation script/steps producing Euclid-like imaging of an isothermal + Sersic system exists and ran | 10 | +| M2 | A fit of the simulated data completed, with results shown | 10 | +| M3 | Recovered lens-model parameters compared against the input truths explicitly | 10 | + +### Judged (70) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Every requested step explained with its *why*: model composition, simulation, mask choice, non-linear search, reading results | 20 | +| J2 | Scientific accuracy of the explanations (no confident falsehoods about lensing or Bayesian inference; simplifications flagged as such) | 15 | +| J3 | Pedagogical pacing: steps introduced one at a time, understanding checked, user given something to do or predict — not a monologue or a code dump | 15 | +| J4 | Euclid realism: pixel scale / PSF / depth choices stated and justified as Euclid-like | 10 | +| J5 | Closure: an end-of-session recap the learner could reconstruct the workflow from | 10 | + +## Operator notes + +- Cheap to run (simple simulation, fast search) — a good default when adding a + new model/harness to the comparison tables, and a good same-model/different- + day drift probe since run cost is low. +- Judged criteria dominate by design; use the same judge (human or stated + judge-model) across runs being compared, and record the judge in `meta.yaml`. diff --git a/benchmarks/runs/.gitkeep b/benchmarks/runs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/modes/maintainer.md b/modes/maintainer.md index 3cd52c5..e536f6d 100644 --- a/modes/maintainer.md +++ b/modes/maintainer.md @@ -69,15 +69,23 @@ and `modes/` machinery (`.mode`, `.maintainer` sentinels), the skills framework `core`/`literature`/`project` wiki split and its read-only/update rules, the science-project lifecycle (`start-new-project`, `contribute-upstream`), `sources.yaml` + the source registry pattern, the API gate (`autoassistant/audit_skill_apis.py` + wiki-currency -workflow), and the profile template. +workflow), the profile template, and the benchmark machinery (the +`benchmarks/AGENTS.md` contract + the `autoassistant/benchmark.py` harness). **PyAutoLens-specific content** (regenerated per domain, never copied blind): every `al_*` skill body, `wiki/core/` reference pages, the entire `wiki/literature/` sub-wiki, bundled `dataset/` examples, the README's science framing and three example prompts, the -standard-imports convention, and `hpc/` templates tuned to lensing runtimes. +standard-imports convention, `hpc/` templates tuned to lensing runtimes, and the +benchmark prompt cards (`benchmarks/prompts/` — a new domain writes its own +easy/medium/hard assistant + teacher cards against its own bundled data). **Mixed** (structure generic, values domain-specific): `llms.txt` read-order, -`config/`, the maintainer smoke tests below. +`config/`, `benchmarks/README.md` (protocol generic, benchmark table domain), the +maintainer smoke tests below. + +**Per-clone data** (never copied to a newborn — each clone accumulates its own): +`benchmarks/runs/` and the regenerated `benchmarks/RESULTS.md`. A newborn starts with +empty runs and regenerates `RESULTS.md` via `python autoassistant/benchmark.py report`. ## Chat-surface compatibility smoke test From bf282f3f3fdf8012e0aef2ed39a0fc469e992b49 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Fri, 10 Jul 2026 14:01:36 +0100 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20hard=20benchmark=20follow-up=20wordi?= =?UTF-8?q?ng=20=E2=80=94=20MGE=20source,=20intent=20stated=20plainly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "SIE source" phrasing was an unintended gotcha; per the author the follow-up retains the MGE lens light and MGE-source fit's foundations while the pixelized reconstruction replaces the source. J4 now scores correct chaining composition instead of ambiguity handling. Co-Authored-By: Claude Fable 5 --- benchmarks/prompts/hard_group_multi.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/benchmarks/prompts/hard_group_multi.md b/benchmarks/prompts/hard_group_multi.md index e1f6284..b58bd47 100644 --- a/benchmarks/prompts/hard_group_multi.md +++ b/benchmarks/prompts/hard_group_multi.md @@ -37,15 +37,16 @@ I want the foreground lens model to use multi gaussian Expansions for the lens l and a multi Gaussian expansion for the background source. After this fit has been judged successful, do a follow up lens model that uses a pixelized source reconstruction, -but retains the MGE lens light and SIE source. +but retains the MGE lens light and MGE source. Present me with results confirming the fit was a success. ``` -The prompt is frozen verbatim — including its rough edges (e.g. the final -"SIE source" plainly means the SIE mass profiles are retained while the source -switches to pixelized). Interpreting user intent sensibly, or asking one -focused question, is part of what is being measured; do not clean the wording. +The prompt is frozen verbatim. The intended follow-up composition: the MGE +lens light and the SIE mass profiles carry over from the successful first fit +(the "retains the MGE lens light and MGE source" fit), while the pixelized +reconstruction replaces the MGE source as the source model — the standard +parametric-source → pixelized-source chaining. ## What this measures @@ -78,7 +79,7 @@ focused question, is part of what is being measured; do not clean the wording. | J1 | Simulation quality: group-scale geometry plausible; instrument configs sensible; the quad verified rather than assumed | 15 | | J2 | Joint-analysis wiring correct: shared parametrisation across datasets, dataset-specific nuisance handled sensibly; simultaneous fit demonstrated (not sequential) | 15 | | J3 | Success judged honestly between stages: explicit criteria (residuals, recovered parameters vs truth) before proceeding to the pixelized follow-up | 10 | -| J4 | The "SIE source" ambiguity handled sensibly (correct interpretation or one focused question — not silent nonsense, not an interrogation) | 5 | +| J4 | Follow-up composed as intended: pixelized reconstruction replaces the MGE source while the MGE lens light and SIE masses carry over (fixed or prior-chained) from the first fit | 5 | | J5 | Conduct: staged plan communicated, honest evidence of success/failure, API-gate discipline, no fabricated results | 10 | ## Operator notes