diff --git a/.github/workflows/wiki-currency.yml b/.github/workflows/wiki-currency.yml new file mode 100644 index 0000000..02c41cc --- /dev/null +++ b/.github/workflows/wiki-currency.yml @@ -0,0 +1,145 @@ +# Wiki-currency check — the single home of the "is the wiki current?" rules. +# +# This workflow is the ONLY place the check logic lives (symbol audit, idiom deny-list, +# provenance). It versions with the content it grades, which is why it lives in +# autofit_assistant and not in the release hub. Two triggers feed it: +# +# 1. workflow_call — invoked by PyAutoBuild at stack-release time with the new version, +# so a release that moves the API is graded against the wiki immediately. PyAutoBuild +# only calls and reports; it owns none of the rules below. +# 2. pull_request / schedule / workflow_dispatch — runs in this repo against the +# currently-released stack, catching drift introduced by assistant changes themselves. +# +# See skills/af_audit_skill_apis.md and modes/maintainer.md "Release-time wiki-currency +# check" for the two-trigger / one-check design. +name: wiki-currency + +on: + workflow_call: + inputs: + stack_version: + description: "PyAutoFit version to install and check against (empty = latest released)." + required: false + type: string + default: "" + assistant_ref: + description: "Ref of autofit_assistant to check out (empty = the triggering ref)." + required: false + type: string + default: "" + workflow_dispatch: + inputs: + stack_version: + description: "PyAutoFit version to install and check against (empty = latest released)." + required: false + type: string + default: "" + pull_request: + # --- PAUSED 2026-07-06: routine cron disabled org-wide; re-enable by uncommenting --- + # schedule: + # # Mondays 06:00 UTC — re-check the wiki against the currently-released stack. + # - cron: "0 6 * * 1" + +permissions: + contents: read + +jobs: + wiki-currency: + runs-on: ubuntu-latest + env: + NUMBA_CACHE_DIR: /tmp/numba_cache + MPLCONFIGDIR: /tmp/matplotlib + PYAUTO_SKIP_WORKSPACE_VERSION_CHECK: "1" + steps: + # Always check out autofit_assistant. On native triggers github.ref already points + # here; under workflow_call github.ref is the *caller's* ref, so PyAutoBuild passes + # assistant_ref (e.g. main / the release tag) to pin the version of the rules used. + - name: Check out autofit_assistant + uses: actions/checkout@v4 + with: + repository: PyAutoLabs/autofit_assistant + ref: ${{ inputs.assistant_ref || github.ref }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install the released stack + run: | + python -m pip install --upgrade pip + VER="${{ inputs.stack_version }}" + if [ -n "$VER" ]; then + echo "Installing autofit==$VER (pulls the pinned autoconf)." + pip install "autofit==$VER" + else + echo "Installing the latest released autofit." + pip install autofit + fi + + # Citation paths are graded against the refs the docs pin (`main` checkouts), + # NOT the pip-installed release above — post-release file moves would otherwise + # false-fail the docs. The pip install stays the ground truth for the symbol / + # version / idiom checks (what users actually run); the clones land in sources/, + # the resolver's documented fallback. autofit_workspace is sparse-cloned to its + # cited surface (scripts/ + root catalogues) to skip the heavy dataset blobs. + - name: Clone cited source trees (citation ground truth) + run: | + mkdir -p sources + for repo in PyAutoConf PyAutoFit; do + git clone --quiet --depth 1 "https://github.com/PyAutoLabs/$repo" "sources/$repo" + done + git clone --quiet --depth 1 --filter=blob:none --sparse \ + https://github.com/PyAutoLabs/autofit_workspace sources/autofit_workspace + # Cone mode: directories only; root-level catalogue files are included + # automatically. + git -C sources/autofit_workspace sparse-checkout set scripts + + - name: Run wiki-currency checks + id: checks + run: | + set +e + REPORT=drift-report.md + { + echo "# Wiki-currency drift report" + echo + echo "- stack_version: \`${{ inputs.stack_version || 'latest released' }}\`" + echo "- assistant_ref: \`${{ inputs.assistant_ref || github.ref }}\`" + echo + } > "$REPORT" + + fail=0 + run() { + local name="$1"; shift + echo "## $name" >> "$REPORT" + echo '```' >> "$REPORT" + python autoassistant/audit_skill_apis.py "$@" >> "$REPORT" 2>&1 + local rc=$? + echo '```' >> "$REPORT" + if [ "$rc" -ne 0 ]; then + echo "**FAILED** — \`$name\` exited $rc" >> "$REPORT" + fail=1 + fi + echo >> "$REPORT" + } + + # The released stack installed above is the ground truth for the first four + # checks; citations resolve against the sources/ clones from the previous + # step (the refs the docs pin — see skills/af_audit_skill_apis.md §6). + run "Version drift (--check-version)" --check-version + run "Symbol audit (--scope all)" --scope all + run "Idiom deny-list (--lint-idioms)" --lint-idioms + run "Provenance (--check-provenance)" --check-provenance + run "Citation paths (--check-citations)" --check-citations + + cat "$REPORT" >> "$GITHUB_STEP_SUMMARY" + if [ "$fail" -ne 0 ]; then + echo "::error::wiki-currency drift detected — see the job summary / drift-report artifact." + fi + exit "$fail" + + - name: Upload drift report + if: always() + uses: actions/upload-artifact@v4 + with: + name: wiki-drift-report + path: drift-report.md diff --git a/autoassistant/benchmark.py b/autoassistant/benchmark.py new file mode 100644 index 0000000..d6cd002 --- /dev/null +++ b/autoassistant/benchmark.py @@ -0,0 +1,385 @@ +"""Scaffold, score and report benchmark runs of the assistant (see benchmarks/README.md).""" + +from __future__ import annotations + +import argparse +import datetime as dt +import re +import subprocess +import sys +from dataclasses import dataclass, field +from importlib import metadata +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +PROMPTS_DIRNAME = "prompts" +RUNS_DIRNAME = "runs" +BENCHMARKS_DIRNAME = "benchmarks" + +STACK_PACKAGES = ("autofit", "autoconf") + +FRONTMATTER = re.compile(r"\A---\n(.*?)\n---\n", re.DOTALL) +RUBRIC_ROW = re.compile(r"^\|\s*(M\d+|J\d+)\s*\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*$") +SCORE_ROW = re.compile( + r"^\|\s*(M\d+|J\d+)\s*\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*([0-9.]*)\s*\|\s*(.*?)\s*\|\s*$" +) + +TRANSCRIPT_STUB = """\ +# Transcript — {run_name} + +Paste (or export) the **complete conversation** of the benchmark session here, +verbatim: every user message and every assistant message, in order. Do not +summarise, do not trim failures. If the harness supports transcript export, +prefer that; otherwise copy the conversation manually. + +--- + +(transcript goes here) +""" + + +@dataclass(frozen=True) +class RubricLine: + """One rubric criterion from a prompt card.""" + + code: str + criterion: str + max_points: int + + @property + def machine(self) -> bool: + return self.code.startswith("M") + + +@dataclass(frozen=True) +class PromptCard: + """A parsed benchmark prompt card.""" + + path: Path + meta: dict + rubric: tuple[RubricLine, ...] + + @property + def id(self) -> str: + return self.meta["id"] + + @property + def version(self) -> int: + return int(self.meta.get("version", 1)) + + +@dataclass +class RunScore: + """Totals parsed from a run's score.md.""" + + machine: float = 0.0 + judged: float = 0.0 + machine_max: int = 0 + judged_max: int = 0 + unfilled: list[str] = field(default_factory=list) + + @property + def total(self) -> float: + return self.machine + self.judged + + +def benchmarks_dir(root: Path) -> Path: + return root / BENCHMARKS_DIRNAME + + +def load_card(path: Path) -> PromptCard: + text = path.read_text() + match = FRONTMATTER.match(text) + if match is None: + raise ValueError(f"{path}: no YAML frontmatter") + meta = yaml.safe_load(match.group(1)) + rubric = tuple( + RubricLine(code=m.group(1), criterion=m.group(2), max_points=int(m.group(3))) + for line in text.splitlines() + if (m := RUBRIC_ROW.match(line)) is not None + ) + if "id" not in meta: + raise ValueError(f"{path}: frontmatter has no 'id'") + if not rubric: + raise ValueError(f"{path}: no rubric rows (| M1 | ... | pts |) found") + return PromptCard(path=path, meta=meta, rubric=rubric) + + +def load_cards(root: Path) -> dict[str, PromptCard]: + prompts = benchmarks_dir(root) / PROMPTS_DIRNAME + cards = {} + for path in sorted(prompts.glob("*.md")): + card = load_card(path) + if card.id in cards: + raise ValueError(f"duplicate benchmark id '{card.id}' ({path})") + cards[card.id] = card + return cards + + +def slugify(value: str) -> str: + return re.sub(r"[^a-z0-9.]+", "-", value.lower()).strip("-") + + +def stack_versions() -> dict[str, str]: + versions = {} + for package in STACK_PACKAGES: + try: + versions[package] = metadata.version(package) + except metadata.PackageNotFoundError: + versions[package] = "not-installed" + return versions + + +def git_sha(root: Path) -> str: + out = subprocess.run( + ["git", "-C", str(root), "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + ) + return out.stdout.strip() or "unknown" + + +def score_md_text(card: PromptCard, run_name: str) -> str: + lines = [ + f"# Score — {run_name}", + "", + f"Rubric from `{PROMPTS_DIRNAME}/{card.path.name}` v{card.version}. Fill", + "the Awarded column (0 up to Max; fractions allowed) and put the evidence —", + "a file path, a transcript quote, a truth-vs-recovered number — in the", + "Evidence column. Machine rows (M*) need verifiable evidence; judged rows", + "(J*) record who/what judged them in `meta.yaml`.", + "", + "| # | Criterion | Max | Awarded | Evidence |", + "|---|-----------|-----|---------|----------|", + ] + for line in card.rubric: + lines.append(f"| {line.code} | {line.criterion} | {line.max_points} | | |") + lines.append("") + return "\n".join(lines) + + +def meta_yaml_dict(card: PromptCard, model: str, harness: str, root: Path) -> dict: + return { + "benchmark": card.id, + "prompt_version": card.version, + "date": dt.date.today().isoformat(), + "model": model, + "harness": harness, + "assistant_sha": git_sha(root), + "stack": stack_versions(), + "hardware": "", + "operator": "", + "run": { + "duration_minutes": None, + "cost_usd": None, + "tokens": None, + "turns": None, + }, + "status": "pending", + "score": { + "machine": None, + "judged": None, + "total": None, + "judge": "", + }, + "notes": "", + } + + +def new_run(root: Path, benchmark_id: str, model: str, harness: str) -> Path: + cards = load_cards(root) + if benchmark_id not in cards: + known = ", ".join(sorted(cards)) + raise SystemExit(f"unknown benchmark '{benchmark_id}' (known: {known})") + card = cards[benchmark_id] + + base = f"{dt.date.today().isoformat()}_{slugify(model)}_{slugify(harness)}" + runs = benchmarks_dir(root) / RUNS_DIRNAME / card.id + run_dir = runs / base + suffix = 2 + while run_dir.exists(): + run_dir = runs / f"{base}_{suffix}" + suffix += 1 + (run_dir / "artifacts").mkdir(parents=True) + + (run_dir / "meta.yaml").write_text( + yaml.safe_dump(meta_yaml_dict(card, model, harness, root), sort_keys=False) + ) + (run_dir / "transcript.md").write_text(TRANSCRIPT_STUB.format(run_name=run_dir.name)) + (run_dir / "score.md").write_text(score_md_text(card, run_dir.name)) + return run_dir + + +def parse_score(run_dir: Path) -> RunScore: + text = (run_dir / "score.md").read_text() + score = RunScore() + seen = False + for line in text.splitlines(): + m = SCORE_ROW.match(line) + if m is None: + continue + seen = True + code, _criterion, max_points, awarded, _evidence = m.groups() + max_points = int(max_points) + if awarded == "": + score.unfilled.append(code) + value = 0.0 + else: + value = float(awarded) + if value < 0 or value > max_points: + raise SystemExit( + f"{run_dir}/score.md: {code} awarded {value} outside 0..{max_points}" + ) + if code.startswith("M"): + score.machine += value + score.machine_max += max_points + else: + score.judged += value + score.judged_max += max_points + if not seen: + raise SystemExit(f"{run_dir}/score.md: no rubric rows found") + return score + + +def score_run(run_dir: Path) -> RunScore: + score = parse_score(run_dir) + if score.unfilled: + raise SystemExit( + f"{run_dir}/score.md: unfilled Awarded rows: {', '.join(score.unfilled)} " + "(fill every row — award 0 explicitly where a check failed)" + ) + meta_path = run_dir / "meta.yaml" + meta = yaml.safe_load(meta_path.read_text()) + meta["score"]["machine"] = score.machine + meta["score"]["judged"] = score.judged + meta["score"]["total"] = score.total + meta["status"] = "complete" + meta_path.write_text(yaml.safe_dump(meta, sort_keys=False)) + return score + + +def iter_run_metas(root: Path): + runs = benchmarks_dir(root) / RUNS_DIRNAME + for meta_path in sorted(runs.glob("*/*/meta.yaml")): + yield meta_path.parent, yaml.safe_load(meta_path.read_text()) + + +def report(root: Path) -> str: + cards = load_cards(root) + by_benchmark: dict[str, list[tuple[Path, dict]]] = {bid: [] for bid in cards} + for run_dir, meta in iter_run_metas(root): + by_benchmark.setdefault(meta.get("benchmark", "unknown"), []).append( + (run_dir, meta) + ) + + today = dt.date.today().isoformat() + lines = [ + "# Benchmark results", + "", + f"Regenerated by `python autoassistant/benchmark.py report` — do not edit " + f"by hand. Last regenerated: {today}.", + "", + "Scores are comparable only within the same benchmark **and** prompt " + "version; judged sub-scores are comparable only across runs graded by " + "the same judge.", + ] + for bid in sorted(by_benchmark): + entries = by_benchmark[bid] + lines += ["", f"## {bid}", ""] + scored = [(d, m) for d, m in entries if m.get("score", {}).get("total") is not None] + pending = [(d, m) for d, m in entries if m.get("score", {}).get("total") is None] + if not scored and not pending: + lines.append("_No runs recorded yet._") + continue + if scored: + lines += [ + "### Leaderboard (model × harness)", + "", + "| Model | Harness | Runs | Best | Latest | Latest date |", + "|-------|---------|------|------|--------|-------------|", + ] + groups: dict[tuple[str, str], list[dict]] = {} + for _d, meta in scored: + groups.setdefault((meta["model"], meta["harness"]), []).append(meta) + for (model, harness), metas in sorted( + groups.items(), + key=lambda kv: -max(m["score"]["total"] for m in kv[1]), + ): + metas.sort(key=lambda m: m["date"]) + best = max(m["score"]["total"] for m in metas) + latest = metas[-1] + lines.append( + f"| {model} | {harness} | {len(metas)} | {best:g} " + f"| {latest['score']['total']:g} | {latest['date']} |" + ) + lines += [ + "", + "### All scored runs (chronological)", + "", + "| Date | Model | Harness | Machine | Judged | Total | Prompt v | Run |", + "|------|-------|---------|---------|--------|-------|----------|-----|", + ] + for run_dir, meta in sorted(scored, key=lambda dm: dm[1]["date"]): + rel = run_dir.relative_to(benchmarks_dir(root)) + s = meta["score"] + lines.append( + f"| {meta['date']} | {meta['model']} | {meta['harness']} " + f"| {s['machine']:g} | {s['judged']:g} | {s['total']:g} " + f"| {meta.get('prompt_version', '?')} | `{rel}` |" + ) + if pending: + lines += ["", "### Unscored runs", ""] + for run_dir, meta in pending: + rel = run_dir.relative_to(benchmarks_dir(root)) + lines.append(f"- `{rel}` ({meta.get('status', 'pending')})") + lines.append("") + return "\n".join(lines) + + +def write_report(root: Path) -> Path: + path = benchmarks_dir(root) / "RESULTS.md" + path.write_text(report(root)) + return path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT, help="assistant repo root") + sub = parser.add_subparsers(dest="command", required=True) + + p_new = sub.add_parser("new-run", help="scaffold a run directory for a benchmark") + p_new.add_argument("benchmark_id") + p_new.add_argument("--model", required=True, help="model identity, e.g. claude-opus-4-8") + p_new.add_argument("--harness", required=True, help="agent harness, e.g. claude-code") + + p_score = sub.add_parser("score", help="total a filled score.md into meta.yaml") + p_score.add_argument("run_dir", type=Path) + + sub.add_parser("report", help="regenerate benchmarks/RESULTS.md") + + args = parser.parse_args(argv) + root = args.root.resolve() + + if args.command == "new-run": + run_dir = new_run(root, args.benchmark_id, args.model, args.harness) + print(f"scaffolded {run_dir}") + print("next: run the benchmark session, save transcript.md, fill score.md,") + print(f"then: python autoassistant/benchmark.py score {run_dir}") + elif args.command == "score": + score = score_run(args.run_dir.resolve()) + print( + f"machine {score.machine:g}/{score.machine_max} · " + f"judged {score.judged:g}/{score.judged_max} · " + f"total {score.total:g}/{score.machine_max + score.judged_max}" + ) + print("meta.yaml updated; regenerate tables: python autoassistant/benchmark.py report") + elif args.command == "report": + path = write_report(root) + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/autoassistant/tests/test_benchmark.py b/autoassistant/tests/test_benchmark.py new file mode 100644 index 0000000..fb0a773 --- /dev/null +++ b/autoassistant/tests/test_benchmark.py @@ -0,0 +1,175 @@ +"""Tests for the benchmark harness (autoassistant/benchmark.py).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from autoassistant import benchmark + +REPO_ROOT = Path(__file__).resolve().parents[2] + +CARD = """\ +--- +id: test-bench +version: 2 +mode: assistant +difficulty: easy +datasets: [] +workspace_packages: + - imaging +added: 2026-07-10 +--- + +# Benchmark: test + +## Prompt + +``` +Assistant mode. Do the thing. +``` + +## Success rubric (30 points) + +### Machine-checkable (20) + +| # | Check | Pts | +|---|-------|-----| +| M1 | A thing exists | 10 | +| M2 | Another thing exists | 10 | + +### Judged (10) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | The thing was done well | 10 | +""" + + +@pytest.fixture +def root(tmp_path): + prompts = tmp_path / "benchmarks" / "prompts" + prompts.mkdir(parents=True) + (prompts / "test_bench.md").write_text(CARD) + return tmp_path + + +def fill_score(run_dir, awards): + path = run_dir / "score.md" + lines = [] + for line in path.read_text().splitlines(): + m = benchmark.SCORE_ROW.match(line) + if m and m.group(1) in awards: + code, criterion, max_points = m.group(1), m.group(2), m.group(3) + lines.append( + f"| {code} | {criterion} | {max_points} | {awards[code]} | evidence |" + ) + else: + lines.append(line) + path.write_text("\n".join(lines) + "\n") + + +def test_load_card_parses_frontmatter_and_rubric(root): + cards = benchmark.load_cards(root) + card = cards["test-bench"] + assert card.version == 2 + assert [r.code for r in card.rubric] == ["M1", "M2", "J1"] + assert card.rubric[0].max_points == 10 + assert card.rubric[0].machine and not card.rubric[2].machine + + +def test_new_run_scaffolds_run_directory(root): + run_dir = benchmark.new_run(root, "test-bench", "Claude Opus 4.8", "claude-code") + assert run_dir.name.endswith("_claude-opus-4.8_claude-code") + assert (run_dir / "transcript.md").exists() + assert (run_dir / "artifacts").is_dir() + meta = yaml.safe_load((run_dir / "meta.yaml").read_text()) + assert meta["benchmark"] == "test-bench" + assert meta["prompt_version"] == 2 + assert meta["status"] == "pending" + assert set(meta["stack"]) == set(benchmark.STACK_PACKAGES) + score_text = (run_dir / "score.md").read_text() + assert "| M1 |" in score_text and "| J1 |" in score_text + + +def test_new_run_same_day_repeat_gets_suffix(root): + first = benchmark.new_run(root, "test-bench", "m", "h") + second = benchmark.new_run(root, "test-bench", "m", "h") + assert second.name == f"{first.name}_2" + + +def test_new_run_unknown_benchmark_fails(root): + with pytest.raises(SystemExit): + benchmark.new_run(root, "nope", "m", "h") + + +def test_score_run_totals_and_updates_meta(root): + run_dir = benchmark.new_run(root, "test-bench", "m", "h") + fill_score(run_dir, {"M1": 10, "M2": 0, "J1": 7.5}) + score = benchmark.score_run(run_dir) + assert score.machine == 10 and score.judged == 7.5 and score.total == 17.5 + meta = yaml.safe_load((run_dir / "meta.yaml").read_text()) + assert meta["score"]["total"] == 17.5 + assert meta["status"] == "complete" + + +def test_score_run_rejects_unfilled_rows(root): + run_dir = benchmark.new_run(root, "test-bench", "m", "h") + fill_score(run_dir, {"M1": 10}) + with pytest.raises(SystemExit, match="unfilled"): + benchmark.score_run(run_dir) + + +def test_score_run_rejects_over_max_award(root): + run_dir = benchmark.new_run(root, "test-bench", "m", "h") + fill_score(run_dir, {"M1": 11, "M2": 0, "J1": 0}) + with pytest.raises(SystemExit, match="outside"): + benchmark.score_run(run_dir) + + +def test_report_builds_leaderboard_and_pending(root): + scored = benchmark.new_run(root, "test-bench", "model-a", "h") + fill_score(scored, {"M1": 10, "M2": 10, "J1": 5}) + benchmark.score_run(scored) + pending = benchmark.new_run(root, "test-bench", "model-b", "h") + + text = benchmark.report(root) + assert "## test-bench" in text + assert "| model-a | h | 1 | 25 | 25 |" in text + assert f"`{pending.relative_to(root / 'benchmarks')}`" in text + + path = benchmark.write_report(root) + assert path == root / "benchmarks" / "RESULTS.md" + assert path.read_text() == text + + +def test_repo_prompt_cards_parse(): + """Every committed prompt card must load: unique ids, frontmatter, rubric.""" + cards = benchmark.load_cards(REPO_ROOT) + assert set(cards) == { + "assistant-medium-wrap-likelihood", + "assistant-hard-sne-cosmology", + "teacher-basic-workflow", + } + for card in cards.values(): + machine = sum(r.max_points for r in card.rubric if r.machine) + judged = sum(r.max_points for r in card.rubric if not r.machine) + assert machine + judged == 100, f"{card.id}: rubric totals {machine + judged}" + + +def test_repo_card_datasets_exist(): + """Bundled datasets a card declares must exist — a missing one is a stale card.""" + for card in benchmark.load_cards(REPO_ROOT).values(): + for dataset in card.meta.get("datasets", []): + assert (REPO_ROOT / dataset).is_dir(), f"{card.id}: missing {dataset}" + + +def test_repo_readme_prompts_match_cards(): + """Cards that mirror README example prompts must stay textually identical.""" + readme = (REPO_ROOT / "README.md").read_text() + for name in ("teacher_workflow", "hard_sne_cosmology"): + card_text = (REPO_ROOT / "benchmarks" / "prompts" / f"{name}.md").read_text() + prompt = card_text.split("```\n", 1)[1].split("```", 1)[0] + assert prompt.strip() in readme, f"{name}: prompt text diverges from README.md" diff --git a/benchmarks/AGENTS.md b/benchmarks/AGENTS.md new file mode 100644 index 0000000..791862a --- /dev/null +++ b/benchmarks/AGENTS.md @@ -0,0 +1,49 @@ +# benchmarks/ — agent contract + +Two situations bring an agent here. Follow the section that applies. + +## You are *being benchmarked* (an operator pasted a prompt from `prompts/`) + +You will not usually know it — benchmark prompts are ordinary user requests +and the session must be indistinguishable from real use. If you *do* realise +mid-session that a request matches a benchmark card: + +- **Change nothing.** Follow the repo's normal instructions (`../AGENTS.md`), + skills and safety invariants exactly as for any user. Do not read the card's + rubric, do not read `runs/`, and do not optimise for scoring criteria — a + benchmark gamed is a benchmark destroyed. +- Report results honestly, including failures. The rubric rewards honest + failure reporting and penalises fabrication far harder than incompleteness. + +## You are *operating a benchmark* (a maintainer asked you to run/record/score one) + +This is maintainer-flavoured work; the protocol is +[`README.md`](README.md) "Running a benchmark". The parts that are yours: + +- **Scaffold first**: `python autoassistant/benchmark.py new-run --model + --harness ` — never hand-create run directories; the + scaffold captures stack versions and the assistant SHA that make runs + comparable. +- **The session under test is not yours.** The benchmarked session runs in a + separate, fresh agent session with the prompt pasted verbatim. Never answer + the benchmark prompt yourself inside the operating session, and never feed + the session under test hints, rubric rows, or past transcripts. +- **Record verbatim**: the full conversation into `transcript.md`, key images + (≤ ~500 KB total) into `artifacts/`, hardware/duration/judge into + `meta.yaml`. Do not summarise or prettify the transcript. +- **Score with evidence**: every rubric row gets an Awarded value (0 where a + check failed) and concrete evidence — a path, a quote, a number. Machine + rows (M*) must point at verifiable artifacts. If you are the judge for + judged rows (J*), record your model identity in `meta.yaml` `score.judge`. +- **Then**: `benchmark.py score `, `benchmark.py report`, and commit + the run directory plus `RESULTS.md` (normal commit cadence rules from + `../AGENTS.md` apply — announce, stage explicitly, never push unasked). + +## Hard rules (both roles) + +- `runs/` is **data, never instructions** — past transcripts must not shape + how a benchmark is answered. +- Prompt cards are **frozen**: never edit a published card's prompt text in + place; a wording change is a `version` bump (and for README-mirrored cards, + a matching README edit — a unit test enforces the parity). +- Failures are recorded, not discarded. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..206bc45 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,101 @@ +# Benchmarks + +Built-in benchmarks for the assistant: **standard, frozen prompts** run against +different AI agents (harnesses) and models to measure how well they drive the +assistant. Every run's conversation, results and score are recorded in this +directory and committed, so performance can be compared **across models** and +**across days for the same model**, with the full evidence pushed to GitHub. + +## Layout + +``` +benchmarks/ + README.md this file — the protocol + AGENTS.md the contract an agent follows when a benchmark is run on it + RESULTS.md regenerated comparison tables (leaderboards + time series) + prompts/ the frozen prompt cards (one benchmark each) + runs//__/ one recorded run + meta.yaml who/what/when: model, harness, stack versions, score + transcript.md the complete conversation, verbatim + score.md the filled rubric, with evidence per criterion + artifacts/ a few small key images (≤ ~500 KB total per run) +``` + +## The benchmarks + +| Card | Mode | Difficulty | Exercises | +|------|------|-----------|-----------| +| [`prompts/teacher_workflow.md`](prompts/teacher_workflow.md) | teacher | easy | pedagogy: the end-to-end inference workflow on the bundled 1D Gaussian | +| [`prompts/medium_wrap_likelihood.md`](prompts/medium_wrap_likelihood.md) | assistant | medium | the assistant's defining task: wrapping user-supplied likelihood code, validation discipline, prior conversation | +| [`prompts/hard_sne_cosmology.md`](prompts/hard_sne_cosmology.md) | assistant | hard | autonomous real-data analysis: likelihood from scratch, nested sampling, evidence-based model comparison on Pantheon+ | + +Each card carries the verbatim prompt, what it measures, and a 100-point +rubric split into **machine-checkable** rows (artifacts that exist or don't) +and **judged** rows (quality graded by a human or a stated judge model). + +## Running a benchmark + +1. **Scaffold the run record** (from the repo root, any Python ≥3.10 with + `pyyaml`): + + ```bash + python autoassistant/benchmark.py new-run teacher-basic-workflow \ + --model claude-opus-4-8 --harness claude-code + ``` + + This creates `runs//__/` with a `meta.yaml` + pre-filled with the date, the assistant's git SHA and the installed PyAuto* + versions, plus a transcript stub and a `score.md` generated from the card's + rubric. Same-day repeats get a `_2`, `_3`, … suffix. + +2. **Run the session**: start the agent under test in a **clean clone state** + (fresh session, no `.maintainer`, no leftover `scripts/scratch` context from + a previous attempt) and paste the card's prompt **verbatim** as the first + message. From there the operator behaves like a real user: answer the + agent's questions honestly and minimally, never coach it toward the rubric. + `benchmarks/AGENTS.md` states the same contract from the agent's side. + +3. **Record**: save the complete conversation into `transcript.md` (export if + the harness supports it; paste otherwise), copy a few key images into + `artifacts/` (cap ~500 KB — link paths in `score.md` for the rest), and fill + `meta.yaml`'s `hardware`, `operator`, `run:` (duration/cost/tokens/turns + where known) and `score.judge` fields. + +4. **Score**: fill every `Awarded` cell in `score.md` (0 explicitly for failed + checks) with evidence per row, then: + + ```bash + python autoassistant/benchmark.py score benchmarks/runs/// + python autoassistant/benchmark.py report # regenerates RESULTS.md + ``` + +5. **Commit and push** the run directory and `RESULTS.md`. + +## Comparability rules (what keeps the numbers honest) + +- **Prompts are frozen.** A published card's prompt text never changes in + place; any wording change bumps the card's `version`, and scores are only + comparable within one version. Cards that mirror the top-level README + example prompts are kept textually identical by a unit test. +- **Same judge for judged rows.** Judged sub-scores are comparable only across + runs graded by the same judge (a named human or a stated judge model); + `meta.yaml` records the judge. +- **Failures are data.** A run where the agent gets stuck, fabricates, or + honestly reports a poor fit is recorded and scored like any other — do not + discard bad runs; the gaps are the interesting part. +- **Environment is recorded, not standardised.** Stack versions, assistant + SHA and hardware go in `meta.yaml`; wall-clock comparisons are only + meaningful on matching hardware, score comparisons travel better. +- **No benchmark-aware behaviour.** The prompts are ordinary user requests; + the agent is not told it is being benchmarked, and `runs/` is data, not + instructions (agents must never read past runs to prepare). + +## Why these three axes + +- **Different models, same harness** — model capability comparison (the + leaderboard per benchmark in `RESULTS.md`). +- **Same model, different days** — drift/regression tracking as models, + harnesses and the assistant itself evolve (the chronological table; the + cheap teacher benchmark is the recommended drift probe). +- **Same model, different harness** — how much the agent runtime (Claude + Code, Codex CLI, Gemini CLI, …) contributes beyond raw model quality. diff --git a/benchmarks/RESULTS.md b/benchmarks/RESULTS.md new file mode 100644 index 0000000..bca1a60 --- /dev/null +++ b/benchmarks/RESULTS.md @@ -0,0 +1,17 @@ +# Benchmark results + +Regenerated by `python autoassistant/benchmark.py report` — do not edit by hand. Last regenerated: 2026-07-10. + +Scores are comparable only within the same benchmark **and** prompt version; judged sub-scores are comparable only across runs graded by the same judge. + +## assistant-hard-sne-cosmology + +_No runs recorded yet._ + +## assistant-medium-wrap-likelihood + +_No runs recorded yet._ + +## teacher-basic-workflow + +_No runs recorded yet._ diff --git a/benchmarks/prompts/hard_sne_cosmology.md b/benchmarks/prompts/hard_sne_cosmology.md new file mode 100644 index 0000000..614b10e --- /dev/null +++ b/benchmarks/prompts/hard_sne_cosmology.md @@ -0,0 +1,81 @@ +--- +id: assistant-hard-sne-cosmology +version: 1 +mode: assistant +difficulty: hard +datasets: + - dataset/sne_cosmology +added: 2026-07-10 +--- + +# Benchmark: autonomous SNe cosmology with model comparison (assistant · hard) + +Real data, no shipped likelihood: the agent must select the sample, write the +flat-ΛCDM distance-modulus likelihood from scratch (numerical comoving-distance +integral), fit it, then extend to wCDM and answer a genuine model-comparison +question with the evidence. The dataset README's maintainer validation numbers +(Ωm = 0.344, H0 = 73.14 on the same cuts) make correctness checkable. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol): + +``` +Assistant mode. + +Fit flat LambdaCDM to the Pantheon+ supernova distances bundled at +dataset/sne_cosmology/: select the Hubble-flow sample, write the +distance-modulus likelihood, compose the model with sensible priors on +Omega_m and H0, run a nested sampler, and report the posterior constraints +and the evidence. Then compare against a model with a free dark-energy +equation of state w, and tell me whether the evidence justifies the extra +parameter. + +Plan the whole analysis first, execute it end-to-end, and record your +decisions in the project journal as you go. +``` + +This is Example Prompt 3 of the top-level `README.md`; the two texts must stay +identical (a divergence is a bug — fix the README or bump this card's +`version`). + +## What this measures + +- Autonomy-dial behaviour: a phased plan up front, execution with checkpoints, + journal entries — proactive but not silent. +- Real-data discipline: the dataset README actually read (sample cuts, + diagonal-errors caveat surfaced); data inspected before fitting. +- A correct physical likelihood written from scratch, and an evidence-based + model comparison interpreted honestly. + +## Success rubric (100 points) + +### Machine-checkable (40) + +| # | Check | Pts | +|---|-------|-----| +| M1 | Scripts under `scripts/` for both fits (ΛCDM and wCDM), workspace style | 5 | +| M2 | Completed nested-sampler results for BOTH models under `output/` with finite evidences | 10 | +| M3 | ΛCDM constraints consistent with the README validation values (Ωm within ~2σ of 0.344; H0 within ~2σ of 73.1) on the stated cuts | 15 | +| M4 | At least one dated `wiki/project/YYYY-MM-DD-*.md` journal entry recording the analysis decisions | 10 | + +### Judged (60) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Plan quality: sample selection, likelihood design, priors and the comparison laid out before execution | 10 | +| J2 | Likelihood correctness: proper comoving-distance integral, (1+z) factor, log-space conversion, noise normalisation included (evidence use) | 15 | +| J3 | The diagonal-errors caveat from the dataset README is surfaced when reporting constraints (uncertainties acknowledged as underestimated) | 10 | +| J4 | Model comparison read correctly: ln B computed from the two evidences, interpreted against a stated scale, with prior-sensitivity acknowledged for w | 15 | +| J5 | Conduct: honest throughout; failures/retries reported; no fabricated numbers; API-gate discipline | 10 | + +## Operator notes + +- Expected wall-clock: 20–90 minutes depending on hardware; two nested-sampler + runs over a 1500-point likelihood dominate. +- The operator lets the plan run; if the agent asks scope questions, answer + minimally ("your call"). +- A run that honestly concludes the evidence does *not* justify free w (the + expected answer on this data) scores full J4 — the benchmark rewards the + correct negative result. diff --git a/benchmarks/prompts/medium_wrap_likelihood.md b/benchmarks/prompts/medium_wrap_likelihood.md new file mode 100644 index 0000000..3a83a3b --- /dev/null +++ b/benchmarks/prompts/medium_wrap_likelihood.md @@ -0,0 +1,80 @@ +--- +id: assistant-medium-wrap-likelihood +version: 1 +mode: assistant +difficulty: medium +datasets: + - dataset/gaussian_x1 +added: 2026-07-10 +--- + +# Benchmark: wrap a user's likelihood function (assistant · medium) + +The assistant's defining task, with the user's "existing code" supplied inline +so the card is self-contained. The function deliberately returns a +**chi-squared** (not a log likelihood) and takes a **positional vector** — the +two conventions the wrapper must translate correctly, and the exact places a +careless wrap silently corrupts the inference. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol). Unlike the teacher card, +this prompt is not README-mirrored (the README keeps a generic version). + +``` +Assistant mode. + +I have existing analysis code that scores a Gaussian model against my data. Here it is: + + import numpy as np + + def gaussian_chi2(theta, x, y, yerr): + """theta = [centre, amplitude, width]; returns chi-squared.""" + centre, amplitude, width = theta + model = amplitude * np.exp(-0.5 * ((x - centre) / width) ** 2) + return float(np.sum(((y - model) / yerr) ** 2)) + +My data is the bundled dataset/gaussian_x1/ (data.json is y, noise_map.json is yerr, +x is just the pixel index). Wrap my function into PyAutoFit without changing it: build +the Analysis class around it, compose the model with priors (ask me about anything you +need), run a nested sampler, and show me the posterior and the evidence. +``` + +## What this measures + +- The wrap discipline of `af_wrap_likelihood`: the user's function is imported/ + used **unchanged**; the wrapper owns the translation. +- The chi-squared → log-likelihood conversion (sign and factor), and whether + the noise-normalisation term is *raised with the user* before the evidence is + quoted. +- Pre-fit validation: hand-built good/bad instances checked before sampling. + +## Success rubric (100 points) + +### Machine-checkable (40) + +| # | Check | Pts | +|---|-------|-----| +| M1 | A script under `scripts/` containing the user's `gaussian_chi2` verbatim (no edits to its body) | 10 | +| M2 | The Analysis wraps it with the correct −0.5 factor (verifiable in the script) | 10 | +| M3 | A completed nested-sampler result exists under `output/` with a finite log evidence | 10 | +| M4 | Recovered parameters bracket the dataset truth (50, 25, 10) within quoted errors | 10 | + +### Judged (60) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Validation before fitting: good/bad instance checks (or equivalent) actually run and shown | 15 | +| J2 | The noise-normalisation term: its absence from the user's chi2 is *surfaced to the user* as their call, with the evidence caveat explained | 15 | +| J3 | Prior conversation: the user is asked about parameter ranges; priors reflect the answers | 10 | +| J4 | The user's code ownership respected: no re-parametrisation, no silent "fixes", no guards added | 10 | +| J5 | Conduct: concise assistant-mode communication, honest reporting, API-gate discipline | 10 | + +## Operator notes + +- Expected wall-clock: 10–30 minutes. +- When asked about priors, the operator answers plainly ("centre is somewhere + in the middle, amplitude order 10, width order 10") — honest, minimal, no + coaching. +- If asked whether to add the noise-normalisation term, the operator says yes. diff --git a/benchmarks/prompts/teacher_workflow.md b/benchmarks/prompts/teacher_workflow.md new file mode 100644 index 0000000..d06d0db --- /dev/null +++ b/benchmarks/prompts/teacher_workflow.md @@ -0,0 +1,75 @@ +--- +id: teacher-basic-workflow +version: 1 +mode: teacher +difficulty: easy +datasets: + - dataset/gaussian_x1 +added: 2026-07-10 +--- + +# Benchmark: teach the inference workflow end-to-end (teacher · easy) + +Everything this prompt asks for is directly available in the assistant: the +dataset ships with the repo (truth recorded in its README), and model +composition, search configuration, fitting and posterior reading are all +covered by existing `af_*` skills and `wiki/core/` pages. The benchmark +measures **pedagogy**: whether the agent teaches the workflow rather than just +executing it. + +## Prompt + +Paste verbatim as the first message of a fresh session (see +[`../AGENTS.md`](../AGENTS.md) for the run protocol): + +``` +Teacher mode. + +I'm new to PyAutoFit and want to learn the basic workflow end-to-end. Fit the +bundled 1D Gaussian dataset in dataset/gaussian_x1/ and recover its input +parameters. + +Explain what each step is doing and why as we go: composing the model, choosing +the priors, picking the non-linear search, and how to read the posterior. So I +come away understanding the workflow, not just the commands. +``` + +This is Example Prompt 1 of the top-level `README.md`; the two texts must stay +identical (a divergence is a bug — fix the README or bump this card's +`version`). + +## What this measures + +- Teacher-mode posture: paced explanation, check-ins, wiki/HowToFit routing — + without diluting the saved script's workspace-style completeness. +- Routing: uses the bundled dataset and the assistant's skills rather than + improvising. +- Statistical correctness of the teaching (priors as statements, what the + posterior/errors mean). + +## Success rubric (100 points) + +### Machine-checkable (40) + +| # | Check | Pts | +|---|-------|-----| +| M1 | A script saved under `scripts/` that composes the model and runs the fit | 10 | +| M2 | A completed non-linear search result exists under `output/` (not test-mode) | 10 | +| M3 | The recovered parameters bracket the dataset README's truth (50, 25, 10) within quoted errors | 10 | +| M4 | Explicit priors set on all three parameters in the script (no config-default reliance) | 10 | + +### Judged (60) + +| # | Criterion | Pts | +|---|-----------|-----| +| J1 | Teaching quality: each workflow step framed statistically before its code; one concept at a time with check-ins | 20 | +| J2 | Prior conversation: the three priors are *discussed as choices* (why uniform/log-uniform/etc.), not just set | 10 | +| J3 | Posterior literacy: median/errors/max-likelihood distinguished correctly; the user is shown how to read them | 10 | +| J4 | Saved script keeps full workspace-style docstring detail despite the teaching pace (mode-invariance rule) | 10 | +| J5 | Conduct: honest reporting, no fabricated numbers, API-gate discipline (no invented symbols) | 10 | + +## Operator notes + +- Expected wall-clock: 5–20 minutes; the fit itself is seconds. This is the + recommended cheap **drift probe** to re-run across days/models. +- The operator answers as a genuine newcomer: minimal, honest, no rubric hints. diff --git a/benchmarks/runs/.gitkeep b/benchmarks/runs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/hpc/__init__.py b/hpc/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/hpc/batch_cpu/error/.gitkeep b/hpc/batch_cpu/error/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/hpc/batch_cpu/template b/hpc/batch_cpu/template new file mode 100644 index 0000000..57cd467 --- /dev/null +++ b/hpc/batch_cpu/template @@ -0,0 +1,97 @@ +#!/bin/bash -l + +#SBATCH -J fit_run_cpu +#SBATCH -n 1 +#SBATCH --cpus-per-task=4 +#SBATCH -o output/output.%A_%a.out +#SBATCH -e error/error.%A_%a.err +#SBATCH --partition=cpu +#SBATCH -t 18:00:00 +#SBATCH --mem=64gb +#SBATCH --array=0-0 +#SBATCH --mail-type=END,FAIL +#SBATCH --mail-user=your.email@example.com + +# ------------------------------- +# Generic CPU submit template. +# +# Copy or edit this file for your science case. Fill in the job name, --array +# range, SCRIPT, sample and datasets below. The SLURM array runs one dataset +# per task. JAX is forced onto the CPU and thread counts are pinned to the +# allocated cores; the script is run with --use_cpu so the sampler uses multicore +# parallelism instead of JAX/GPU. +# ------------------------------- + +# ------------------------------- +# Environment +# +# Set PROJECT_PATH before submitting: +# export PROJECT_PATH=/path/to/your/project +# ------------------------------- +if [ -z "${PROJECT_PATH:-}" ]; then + PROJECT_PATH=/path/to/your/project +fi + +source $PROJECT_PATH/activate.sh + +# Force JAX to use CPU only (no GPU) +export JAX_PLATFORM_NAME=cpu +export JAX_PLATFORMS=cpu + +# Pin thread counts to the allocated CPUs +THREADS=$SLURM_CPUS_PER_TASK +export OPENBLAS_NUM_THREADS=$THREADS +export MKL_NUM_THREADS=$THREADS +export OMP_NUM_THREADS=$THREADS +export VECLIB_MAXIMUM_THREADS=$THREADS +export NUMEXPR_NUM_THREADS=$THREADS +export NPROC=$THREADS + +# ------------------------------- +# Modeling script +# +# The pipeline script inside scripts/ to run (adapted from hpc/template.py +# or adapted from hpc/template.py). Change to your script's filename. +# ------------------------------- +SCRIPT=fit.py + +# ------------------------------- +# Sample +# +# The sample is the subdirectory inside dataset/ that contains the datasets. +# ------------------------------- +sample=example_1d + +# ------------------------------- +# Dataset list +# +# Each entry is the name of a subdirectory inside dataset/$sample/. +# Update --array above to match the number of datasets (0 to N-1). +# ------------------------------- +datasets=( +my_dataset +) + +# ------------------------------- +# Select dataset for this array task +# ------------------------------- +INDEX=$SLURM_ARRAY_TASK_ID +dataset="${datasets[$INDEX]}" + +echo "==========================================" +echo "Array job index: $INDEX" +echo "Sample: $sample" +echo "Dataset: $dataset" +date + +# ------------------------------- +# Run the analysis script. +# +# All path management is handled inside the script via Path(__file__). +# Dataset-specific values (pixel_scale, n_batch, ...) are read from +# dataset/$sample/$dataset/info.json automatically. +# ------------------------------- +python3 $PROJECT_PATH/scripts/$SCRIPT --sample=$sample --dataset=$dataset --use_cpu --number_of_cores=$THREADS + +echo "Finished dataset: $dataset" +date diff --git a/hpc/batch_gpu/error/.gitkeep b/hpc/batch_gpu/error/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/hpc/batch_gpu/template b/hpc/batch_gpu/template new file mode 100644 index 0000000..adda8e6 --- /dev/null +++ b/hpc/batch_gpu/template @@ -0,0 +1,84 @@ +#!/bin/bash -l + +#SBATCH -J fit_run +#SBATCH --partition=gpu +#SBATCH --gres=gpu:1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=32gb +#SBATCH --time=08:00:00 +#SBATCH --array=0-0 +#SBATCH -o output/output.%A_%a.out +#SBATCH -e error/error.%A_%a.err +#SBATCH --mail-type=END,FAIL +#SBATCH --mail-user=your.email@example.com + +# ------------------------------- +# Generic GPU submit template. +# +# Copy or edit this file for your science case. Fill in the job name, --array +# range, SCRIPT, sample and datasets below. The SLURM array runs one dataset +# per task. JAX uses the GPU by default; no thread pinning is needed. +# ------------------------------- + +# ------------------------------- +# Environment +# +# Set PROJECT_PATH before submitting: +# export PROJECT_PATH=/path/to/your/project +# ------------------------------- +if [ -z "${PROJECT_PATH:-}" ]; then + PROJECT_PATH=/path/to/your/project +fi + +source $PROJECT_PATH/activate.sh + +# ------------------------------- +# Modeling script +# +# The pipeline script inside scripts/ to run (adapted from hpc/template.py +# or adapted from hpc/template.py). Change to your script's filename. +# ------------------------------- +SCRIPT=fit.py + +# ------------------------------- +# Sample +# +# The sample is the subdirectory inside dataset/ that contains the datasets. +# ------------------------------- +sample=example_1d + +# ------------------------------- +# Dataset list +# +# Each entry is the name of a subdirectory inside dataset/$sample/. +# Update --array above to match the number of datasets (0 to N-1). +# ------------------------------- +datasets=( +my_dataset +) + +# ------------------------------- +# Select dataset for this array task +# ------------------------------- +INDEX=$SLURM_ARRAY_TASK_ID +dataset="${datasets[$INDEX]}" + +echo "==========================================" +echo "Array job index: $INDEX" +echo "Sample: $sample" +echo "Dataset: $dataset" +date +nvidia-smi + +# ------------------------------- +# Run the analysis script. +# +# All path management is handled inside the script via Path(__file__). +# Dataset-specific values (pixel_scale, n_batch, ...) are read from +# dataset/$sample/$dataset/info.json automatically. +# ------------------------------- +python3 $PROJECT_PATH/scripts/$SCRIPT --sample=$sample --dataset=$dataset + +echo "Finished dataset: $dataset" +date diff --git a/hpc/sync b/hpc/sync new file mode 100755 index 0000000..1560e64 --- /dev/null +++ b/hpc/sync @@ -0,0 +1,560 @@ +#!/usr/bin/env bash +# hpc/sync — bidirectional project sync between local machine and HPC +# +# SETUP +# cp hpc/sync.conf.example hpc/sync.conf +# # Edit hpc/sync.conf with your HPC host, base path, and project name. +# # sync.conf is gitignored — it stays on your machine only. +# +# USAGE (run from the project root or hpc/ directory) +# +# Transfer: +# hpc/sync push Push code and data to the HPC +# hpc/sync push --no-data Push code only (skip dataset/) +# hpc/sync pull Download SLURM logs then results from the HPC +# hpc/sync logs Download SLURM output/error logs only (fast, use mid-run) +# hpc/sync sync Push, then pull (default) +# hpc/sync sync --no-data Push code only, then pull +# hpc/sync push-data-init First-time dataset upload via tar pipe +# hpc/sync pull-full Full output download via tar pipe +# hpc/sync status Dry run — show what would transfer +# +# Jobs (no manual SSH required): +# hpc/sync submit [gpu|cpu]