From e61b071699d7015c3f4cd27983f5a426ba7a5ffb Mon Sep 17 00:00:00 2001
From: Vader Yang <vader.yang@netis.com>
Date: Mon, 8 Jun 2026 12:00:12 +0800
Subject: [PATCH] feat(evals): harness qualification bench
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A maintainer tool to benchmark a candidate agent CLI BEFORE wiring it into the
live loop — so the roster (codex now; pi agent, kilo cli, … later) gets qualified,
not just plugged in. Reuses Part A's agent_run adapter, so a candidate is invoked
exactly how the loop would invoke it (harness.kind=custom + a command template).

- evals/run.sh — per task: throwaway sandbox → agent_run (the configured harness)
  → the task's objective check → record JSONL. --harness/--command/--model/--repeat/
  --label/--list. Runs on demand (needs the model endpoint + the CLI); NOT in CI.
- evals/score.py — aggregate results → per-task pass-rate (over --repeat), per-surface,
  overall, latency; prints a table + writes a scorecard.json. aggregate() is unit-tested.
- evals/tasks/ — 4 starter tasks, objective binary scoring (no judge):
    triage/do-issue   → JSON verdict ∈ {do,try}
    triage/skip-issue → verdict ∈ {skip,needs_info}
    implement/failing-test → agent's edit makes a fixture's python test pass
    review/planted-bug → review flags a hardcoded secret in a diff
- evals/checks_lib.py — shared, dependency-free check helpers (JSON extract, verdict
  + must_contain matching).
- evals/tests/test_checks.py — feeds canned outputs to the real checks + score,
  validating the scoring logic deterministically (no agents). Wired into ci.yml
  alongside shellcheck of run.sh.

Qualifying a new CLI: write a custom template → evals/run.sh --harness custom
--command "<tmpl>" → read the scorecard → decide.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 .github/workflows/ci.yml                      |  7 +-
 evals/README.md                               | 76 +++++++++++++++
 evals/checks_lib.py                           | 63 ++++++++++++
 evals/run.sh                                  | 86 ++++++++++++++++
 evals/score.py                                | 71 ++++++++++++++
 evals/tasks/implement/failing-test/check.sh   | 10 ++
 evals/tasks/implement/failing-test/prompt.md  |  4 +
 .../tasks/implement/failing-test/repo/add.py  |  3 +
 .../implement/failing-test/repo/test_add.py   |  5 +
 evals/tasks/implement/failing-test/task.json  |  2 +
 evals/tasks/review/planted-bug/check.py       | 14 +++
 evals/tasks/review/planted-bug/prompt.md      | 18 ++++
 evals/tasks/review/planted-bug/task.json      |  2 +
 evals/tasks/triage/do-issue/check.py          | 14 +++
 evals/tasks/triage/do-issue/prompt.md         | 12 +++
 evals/tasks/triage/do-issue/repo/greet.py     |  3 +
 evals/tasks/triage/do-issue/task.json         |  2 +
 evals/tasks/triage/skip-issue/check.py        | 14 +++
 evals/tasks/triage/skip-issue/prompt.md       | 12 +++
 evals/tasks/triage/skip-issue/task.json       |  2 +
 evals/tests/test_checks.py                    | 97 +++++++++++++++++++
 21 files changed, 515 insertions(+), 2 deletions(-)
 create mode 100644 evals/README.md
 create mode 100644 evals/checks_lib.py
 create mode 100644 evals/run.sh
 create mode 100644 evals/score.py
 create mode 100755 evals/tasks/implement/failing-test/check.sh
 create mode 100644 evals/tasks/implement/failing-test/prompt.md
 create mode 100644 evals/tasks/implement/failing-test/repo/add.py
 create mode 100644 evals/tasks/implement/failing-test/repo/test_add.py
 create mode 100644 evals/tasks/implement/failing-test/task.json
 create mode 100755 evals/tasks/review/planted-bug/check.py
 create mode 100644 evals/tasks/review/planted-bug/prompt.md
 create mode 100644 evals/tasks/review/planted-bug/task.json
 create mode 100755 evals/tasks/triage/do-issue/check.py
 create mode 100644 evals/tasks/triage/do-issue/prompt.md
 create mode 100644 evals/tasks/triage/do-issue/repo/greet.py
 create mode 100644 evals/tasks/triage/do-issue/task.json
 create mode 100755 evals/tasks/triage/skip-issue/check.py
 create mode 100644 evals/tasks/triage/skip-issue/prompt.md
 create mode 100644 evals/tasks/triage/skip-issue/task.json
 create mode 100644 evals/tests/test_checks.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 286a4c1..0540cb1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,10 +20,10 @@ jobs:
           # Gate at warning+ (errors + warnings — real bugs). info/style nits in
           # the ported heron-origin linters are below threshold; .shellcheckrc
           # additionally silences SC1091 (dynamically-pathed sibling sources).
-          shellcheck --severity=warning scripts/agent-bot/*.sh scripts/lib/*.sh scripts/lint/*.sh
+          shellcheck --severity=warning scripts/agent-bot/*.sh scripts/lib/*.sh scripts/lint/*.sh evals/*.sh
 
       - name: bash -n (syntax)
-        run: for f in scripts/agent-bot/*.sh scripts/lib/*.sh scripts/lint/*.sh; do bash -n "$f"; done
+        run: for f in scripts/agent-bot/*.sh scripts/lib/*.sh scripts/lint/*.sh evals/*.sh; do bash -n "$f"; done
 
       - name: triage composer tests
         run: python3 scripts/agent-bot/tests/test_triage.py
@@ -37,6 +37,9 @@ jobs:
       - name: harness adapter tests
         run: python3 scripts/agent-bot/tests/test_harness.py
 
+      - name: eval bench check/score tests
+        run: python3 evals/tests/test_checks.py
+
       - name: reviewer post-logic tests
         run: python3 scripts/pr-review/test_post_review.py
 
diff --git a/evals/README.md b/evals/README.md
new file mode 100644
index 0000000..b94cbb0
--- /dev/null
+++ b/evals/README.md
@@ -0,0 +1,76 @@
+# evals/ — harness qualification bench
+
+Benchmark a candidate agent CLI on standard tasks **before** wiring it into the
+live loop. New CLIs (codex now; pi agent, kilo cli, … later) get *qualified*,
+not just plugged in.
+
+It reuses Part A's `agent_run` adapter — so a candidate is invoked **exactly how
+the loop would invoke it** (`harness.kind=custom` + your command template).
+Scoring is **objective binary** (no judge): build passes / bug caught / valid
+JSON verdict.
+
+## Run it
+
+```bash
+# baseline: the built-in claude harness (needs ANTHROPIC_BASE_URL/API_KEY in env)
+evals/run.sh
+
+# a candidate CLI (codex shown). --repeat N for pass-rate over stochastic runs.
+evals/run.sh --harness custom \
+  --command 'codex exec --model {model} --full-auto < {prompt_file} > {out}' \
+  --model gpt-5 --repeat 3 --label codex
+
+evals/run.sh --list          # list task ids
+```
+
+Each run writes `runs/<label>.jsonl` (per-task results) + a `runs/<label>.scorecard.json`,
+and prints a table:
+
+```
+=== scorecard: codex ===
+TASK                                         PASS   RATE   AVG s
+implement/failing-test                        2/3  0.667    41.0
+review/planted-bug                            3/3    1.0     9.0
+triage/do-issue                               3/3    1.0     5.0
+triage/skip-issue                             2/3  0.667     5.0
+----------------------------------------------------------------
+implement (surface)                           2/3  0.667
+review (surface)                              3/3    1.0
+triage (surface)                              5/6  0.833
+OVERALL                                       10/12 0.833
+```
+
+Compare two CLIs by running each with a different `--label` and diffing the
+scorecards. **Decide integration from the numbers.**
+
+> Needs the model endpoint reachable + the chosen CLI installed on the box. Runs
+> **on demand** — NOT in CI (CI only lints `run.sh` + runs the scoring unit tests
+> in `tests/test_checks.py`).
+
+## Tasks
+
+`tasks/<surface>/<name>/` — each is one objective check:
+
+| Task | Surface | Pass = |
+|---|---|---|
+| `triage/do-issue` | investigate | output JSON `verdict` ∈ {do, try} for a small actionable issue |
+| `triage/skip-issue` | investigate | `verdict` ∈ {skip, needs_info} for an out-of-scope issue |
+| `implement/failing-test` | implement | the agent's edit makes the fixture's `python3 test_add.py` pass |
+| `review/planted-bug` | review | the review flags a hardcoded secret in the diff |
+
+### Add a task
+
+Drop a `tasks/<surface>/<name>/` folder with:
+- `task.json` — `{ "surface", "profile": "investigate|implement|review", "prompt": "prompt.md", "expect": {…} }`
+- `prompt.md` — the instruction fed to the agent.
+- `repo/` *(optional)* — fixture files copied into the sandbox.
+- `check.py` (import `checks_lib`) or `check.sh` (gets `$SANDBOX` + `$TASK_DIR`) — exit 0 = pass.
+
+## Known constraint
+
+agent-ops's prompts are Claude-shaped (triage wants JSON, review wants a
+`### Summary` heading the live `post_review.py` parses). The eval's task prompts
+are deliberately representative/simplified to test the *capability*; a candidate
+that scores well here but formats differently may still need prompt tuning for the
+live parsers. See the harness cookbook section. This bench is the *go/no-go gate*,
+not a guarantee of drop-in output compatibility.
diff --git a/evals/checks_lib.py b/evals/checks_lib.py
new file mode 100644
index 0000000..a81e8af
--- /dev/null
+++ b/evals/checks_lib.py
@@ -0,0 +1,63 @@
+"""Shared, dependency-free helpers for the eval task checks.
+
+Each task's check.{py,sh} scores one agent run **objectively** (binary pass/fail)
+— no judge LLM. These helpers cover the common parsing the python checks need.
+Importable by evals/tests/test_checks.py so the scoring logic is unit-tested
+without running any agent.
+"""
+import json
+import os
+
+
+def extract_json(text):
+    """Return the first balanced {...} substring that parses as JSON (agents
+    often wrap the object in prose / markdown fences), else None."""
+    for s in (i for i, c in enumerate(text) if c == "{"):
+        depth = 0
+        for e in range(s, len(text)):
+            if text[e] == "{":
+                depth += 1
+            elif text[e] == "}":
+                depth -= 1
+                if depth == 0:
+                    try:
+                        return json.loads(text[s:e + 1])
+                    except Exception:
+                        break
+    return None
+
+
+def load_expect(task_dir):
+    """The `expect` object from a task's task.json."""
+    with open(os.path.join(task_dir, "task.json")) as f:
+        return json.load(f).get("expect", {})
+
+
+def triage_ok(output_text, expect):
+    """Triage: output contains a JSON object whose `verdict` is one of the
+    expected verdicts."""
+    obj = extract_json(output_text)
+    if obj is None:
+        return False, "no parseable JSON object in output"
+    verdict = obj.get("verdict")
+    allowed = expect.get("verdict", [])
+    if isinstance(allowed, str):
+        allowed = [allowed]
+    if verdict in allowed:
+        return True, f"verdict={verdict}"
+    return False, f"verdict={verdict!r} not in {allowed}"
+
+
+def review_ok(output_text, expect):
+    """Review: the review text mentions the planted issue. `must_contain_any`
+    = pass if ANY token is present; `must_contain` = ALL must be present
+    (case-insensitive)."""
+    low = output_text.lower()
+    any_of = [s.lower() for s in expect.get("must_contain_any", [])]
+    all_of = [s.lower() for s in expect.get("must_contain", [])]
+    if any_of and not any(s in low for s in any_of):
+        return False, f"none of must_contain_any present: {any_of}"
+    missing = [s for s in all_of if s not in low]
+    if missing:
+        return False, f"missing must_contain: {missing}"
+    return True, "matched"
diff --git a/evals/run.sh b/evals/run.sh
new file mode 100644
index 0000000..6506837
--- /dev/null
+++ b/evals/run.sh
@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# evals/run.sh — benchmark an agent harness on the standard tasks.
+#
+# Qualifies a candidate agent CLI BEFORE wiring it into the live loop: it runs
+# each evals/tasks/<surface>/<name>/ task through Part A's agent_run adapter
+# (i.e. exactly how the loop invokes the CLI), in a throwaway sandbox, and scores
+# the result with the task's objective binary check. No judge.
+#
+#   evals/run.sh                                  # baseline: the built-in claude harness
+#   evals/run.sh --harness custom \
+#                --command 'codex exec --model {model} --full-auto < {prompt_file} > {out}' \
+#                --model gpt-5 --repeat 3 --label codex
+#   evals/run.sh --list                           # list task ids
+#
+# Needs: the model endpoint reachable (ANTHROPIC_BASE_URL/API_KEY in env) and the
+# chosen CLI installed. Runs on demand — NOT in CI (CI only lints this + the checks).
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO="$(cd "$HERE/.." && pwd)"
+
+harness="claude"; command_tmpl=""; model=""; repeat=1; label=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --harness) harness="$2"; shift 2 ;;
+    --command) command_tmpl="$2"; shift 2 ;;
+    --model)   model="$2"; shift 2 ;;
+    --repeat)  repeat="$2"; shift 2 ;;
+    --label)   label="$2"; shift 2 ;;
+    --list)    find "$HERE/tasks" -name task.json -print0 | xargs -0 -n1 dirname \
+                 | sed "s#$HERE/tasks/##" | sort; exit 0 ;;
+    *) echo "run.sh: unknown arg '$1'" >&2; exit 64 ;;
+  esac
+done
+[ -n "$label" ] || label="$harness"
+
+# Configure the harness via env (config.sh / agent-harness.sh honor env over file).
+export AGENT_OPS_HARNESS="$harness"
+[ -n "$command_tmpl" ] && export AGENT_OPS_HARNESS_CMD="$command_tmpl"
+[ -n "$model" ] && export ANTHROPIC_MODEL="$model"
+# A custom harness usually isn't behind an OpenAI-compatible /v1/models probe.
+[ "$harness" = "custom" ] && export AGENT_OPS_HEALTH_PROBE="${AGENT_OPS_HEALTH_PROBE:-false}"
+# shellcheck source=scripts/lib/agent-harness.sh
+source "$REPO/scripts/lib/agent-harness.sh"
+
+mkdir -p "$HERE/runs"
+results="$HERE/runs/${label}.jsonl"
+: > "$results"
+
+run_one() {
+  local task_dir="$1"
+  local name; name="${task_dir#"$HERE"/tasks/}"
+  local profile prompt_rel sandbox out t0 t1 rc=0 pass=0
+  profile="$(jq -r '.profile' "$task_dir/task.json")"
+  prompt_rel="$(jq -r '.prompt // "prompt.md"' "$task_dir/task.json")"
+  sandbox="$(mktemp -d)"
+  [ -d "$task_dir/repo" ] && cp -R "$task_dir/repo/." "$sandbox/"
+  out="$sandbox/.agent-out"
+
+  t0="$(date +%s)"
+  ( cd "$sandbox" && agent_run --profile "$profile" --prompt "$task_dir/$prompt_rel" \
+      --out "$out" --errlog "$sandbox/.agent-err" --label "eval:$name" ) || rc=$?
+  t1="$(date +%s)"
+
+  if [ "$rc" -eq 0 ]; then
+    if [ -f "$task_dir/check.py" ]; then
+      TASK_DIR="$task_dir" SANDBOX="$sandbox" python3 "$task_dir/check.py" "$out" && pass=1 || pass=0
+    elif [ -f "$task_dir/check.sh" ]; then
+      TASK_DIR="$task_dir" SANDBOX="$sandbox" bash "$task_dir/check.sh" "$out" && pass=1 || pass=0
+    fi
+  else
+    echo "  agent exited $rc (see sandbox err); scored fail" >&2
+  fi
+  printf '{"task":"%s","pass":%d,"rc":%d,"secs":%d}\n' "$name" "$pass" "$rc" "$((t1 - t0))" >> "$results"
+  rm -rf "$sandbox"
+}
+
+echo "harness=$label  repeat=$repeat  results=$results"
+while IFS= read -r -d '' tj; do
+  td="$(dirname "$tj")"
+  for _ in $(seq 1 "$repeat"); do
+    echo "» ${td#"$HERE"/tasks/}"
+    run_one "$td"
+  done
+done < <(find "$HERE/tasks" -name task.json -print0 | sort -z)
+
+python3 "$HERE/score.py" "$results"
diff --git a/evals/score.py b/evals/score.py
new file mode 100644
index 0000000..396423f
--- /dev/null
+++ b/evals/score.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Aggregate an eval run's per-task results (a .jsonl from evals/run.sh) into a
+scorecard: per-task pass-rate (over --repeat), per-surface, overall, total time.
+
+Usage: python3 evals/score.py <results.jsonl>
+Prints a table to stdout and writes <results>.scorecard.json next to it.
+The aggregate() function is importable (unit-tested in evals/tests/).
+"""
+import json
+import os
+import sys
+from collections import defaultdict
+
+
+def aggregate(records):
+    """records: list of {task, pass(0|1), rc, secs}. Returns a scorecard dict."""
+    by_task = defaultdict(lambda: {"runs": 0, "passes": 0, "secs": 0})
+    for r in records:
+        t = by_task[r["task"]]
+        t["runs"] += 1
+        t["passes"] += int(r.get("pass", 0))
+        t["secs"] += int(r.get("secs", 0))
+    tasks = {}
+    by_surface = defaultdict(lambda: {"runs": 0, "passes": 0})
+    for name, t in sorted(by_task.items()):
+        rate = t["passes"] / t["runs"] if t["runs"] else 0.0
+        tasks[name] = {"runs": t["runs"], "passes": t["passes"],
+                       "pass_rate": round(rate, 3),
+                       "avg_secs": round(t["secs"] / t["runs"], 1) if t["runs"] else 0}
+        surface = name.split("/", 1)[0]
+        by_surface[surface]["runs"] += t["runs"]
+        by_surface[surface]["passes"] += t["passes"]
+    surfaces = {s: {"runs": v["runs"], "passes": v["passes"],
+                    "pass_rate": round(v["passes"] / v["runs"], 3) if v["runs"] else 0.0}
+                for s, v in sorted(by_surface.items())}
+    total_runs = sum(t["runs"] for t in by_task.values())
+    total_passes = sum(t["passes"] for t in by_task.values())
+    return {
+        "overall": {"runs": total_runs, "passes": total_passes,
+                    "pass_rate": round(total_passes / total_runs, 3) if total_runs else 0.0},
+        "surfaces": surfaces,
+        "tasks": tasks,
+    }
+
+
+def _print(card, label):
+    print(f"\n=== scorecard: {label} ===")
+    print(f"{'TASK':40} {'PASS':>8} {'RATE':>6} {'AVG s':>7}")
+    for name, t in card["tasks"].items():
+        print(f"{name:40} {str(t['passes'])+'/'+str(t['runs']):>8} {t['pass_rate']:>6} {t['avg_secs']:>7}")
+    print("-" * 64)
+    for s, v in card["surfaces"].items():
+        print(f"{s+' (surface)':40} {str(v['passes'])+'/'+str(v['runs']):>8} {v['pass_rate']:>6}")
+    o = card["overall"]
+    print(f"{'OVERALL':40} {str(o['passes'])+'/'+str(o['runs']):>8} {o['pass_rate']:>6}")
+
+
+def main():
+    path = sys.argv[1]
+    records = [json.loads(line) for line in open(path) if line.strip()]
+    card = aggregate(records)
+    label = os.path.splitext(os.path.basename(path))[0]
+    _print(card, label)
+    out = os.path.splitext(path)[0] + ".scorecard.json"
+    with open(out, "w") as f:
+        json.dump(card, f, indent=2)
+    print(f"\nwrote {out}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/evals/tasks/implement/failing-test/check.sh b/evals/tasks/implement/failing-test/check.sh
new file mode 100755
index 0000000..88be227
--- /dev/null
+++ b/evals/tasks/implement/failing-test/check.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Objective: run the fixture's test in the sandbox the agent edited. Pass = exit 0.
+set -uo pipefail
+cd "${SANDBOX:?SANDBOX unset}" || exit 1
+cmd="$(jq -r '.expect.test_cmd // "python3 test_add.py"' "${TASK_DIR:?TASK_DIR unset}/task.json")"
+if eval "$cmd" >/dev/null 2>&1; then
+  echo "  check: PASS ($cmd)"; exit 0
+else
+  echo "  check: FAIL ($cmd exited nonzero)"; exit 1
+fi
diff --git a/evals/tasks/implement/failing-test/prompt.md b/evals/tasks/implement/failing-test/prompt.md
new file mode 100644
index 0000000..5c43bcd
--- /dev/null
+++ b/evals/tasks/implement/failing-test/prompt.md
@@ -0,0 +1,4 @@
+There is a bug in add.py in the current directory: `add()` subtracts instead of
+adding, so `python3 test_add.py` fails. Fix add.py so the test passes. Edit ONLY
+add.py — do not modify test_add.py. When done, the command `python3 test_add.py`
+must print "ok" and exit 0.
diff --git a/evals/tasks/implement/failing-test/repo/add.py b/evals/tasks/implement/failing-test/repo/add.py
new file mode 100644
index 0000000..5a12b90
--- /dev/null
+++ b/evals/tasks/implement/failing-test/repo/add.py
@@ -0,0 +1,3 @@
+def add(a, b):
+    # BUG: this subtracts; it should add.
+    return a - b
diff --git a/evals/tasks/implement/failing-test/repo/test_add.py b/evals/tasks/implement/failing-test/repo/test_add.py
new file mode 100644
index 0000000..24e4663
--- /dev/null
+++ b/evals/tasks/implement/failing-test/repo/test_add.py
@@ -0,0 +1,5 @@
+from add import add
+
+assert add(2, 3) == 5, "add(2, 3) should be 5"
+assert add(10, 5) == 15, "add(10, 5) should be 15"
+print("ok")
diff --git a/evals/tasks/implement/failing-test/task.json b/evals/tasks/implement/failing-test/task.json
new file mode 100644
index 0000000..bdafc8b
--- /dev/null
+++ b/evals/tasks/implement/failing-test/task.json
@@ -0,0 +1,2 @@
+{ "surface": "implement", "profile": "implement", "prompt": "prompt.md",
+  "expect": { "test_cmd": "python3 test_add.py" } }
diff --git a/evals/tasks/review/planted-bug/check.py b/evals/tasks/review/planted-bug/check.py
new file mode 100755
index 0000000..b5b95bc
--- /dev/null
+++ b/evals/tasks/review/planted-bug/check.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Objective check for a review task: the review text must mention the planted
+issue (must_contain_any / must_contain from task.json)."""
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
+from checks_lib import load_expect, review_ok  # noqa: E402
+
+task_dir = os.environ.get("TASK_DIR", os.path.dirname(os.path.abspath(__file__)))
+out = open(sys.argv[1], encoding="utf-8", errors="replace").read()
+ok, why = review_ok(out, load_expect(task_dir))
+print(("  check: PASS " if ok else "  check: FAIL ") + why)
+sys.exit(0 if ok else 1)
diff --git a/evals/tasks/review/planted-bug/prompt.md b/evals/tasks/review/planted-bug/prompt.md
new file mode 100644
index 0000000..59031c2
--- /dev/null
+++ b/evals/tasks/review/planted-bug/prompt.md
@@ -0,0 +1,18 @@
+Review the following pull-request diff and report any problems you find. Be
+specific about the most serious issue.
+
+```diff
+--- a/auth.py
++++ b/auth.py
+@@ -1,5 +1,8 @@
+ import requests
+
++API_TOKEN = "sk-live-9f3c2a7b41d8e6f0aa12cd34ef56ab78"
++ADMIN_PASSWORD = "hunter2"
++
+ def login(user, password):
+-    return check(user, password)
++    return requests.post("https://api.example.com/login",
++                         json={"user": user, "password": password},
++                         headers={"Authorization": API_TOKEN}).json()
+```
diff --git a/evals/tasks/review/planted-bug/task.json b/evals/tasks/review/planted-bug/task.json
new file mode 100644
index 0000000..9f0f3ac
--- /dev/null
+++ b/evals/tasks/review/planted-bug/task.json
@@ -0,0 +1,2 @@
+{ "surface": "review", "profile": "review", "prompt": "prompt.md",
+  "expect": { "must_contain_any": ["hardcod", "secret", "credential", "plaintext", "in code", "in source"] } }
diff --git a/evals/tasks/triage/do-issue/check.py b/evals/tasks/triage/do-issue/check.py
new file mode 100755
index 0000000..3472892
--- /dev/null
+++ b/evals/tasks/triage/do-issue/check.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Objective check for a triage task: the agent's output must contain a JSON
+object whose `verdict` is one of the expected verdicts (from task.json)."""
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
+from checks_lib import load_expect, triage_ok  # noqa: E402
+
+task_dir = os.environ.get("TASK_DIR", os.path.dirname(os.path.abspath(__file__)))
+out = open(sys.argv[1], encoding="utf-8", errors="replace").read()
+ok, why = triage_ok(out, load_expect(task_dir))
+print(("  check: PASS " if ok else "  check: FAIL ") + why)
+sys.exit(0 if ok else 1)
diff --git a/evals/tasks/triage/do-issue/prompt.md b/evals/tasks/triage/do-issue/prompt.md
new file mode 100644
index 0000000..9829174
--- /dev/null
+++ b/evals/tasks/triage/do-issue/prompt.md
@@ -0,0 +1,12 @@
+You are a repository maintainer triaging a new GitHub issue. You may use tools to
+read the repo in the current directory. Decide whether this is a small, contained,
+auto-implementable change.
+
+Respond with ONLY a JSON object (no prose, no code fences) of the form:
+{"verdict":"do|skip|needs_info","reply":"<one short maintainer-voice sentence>"}
+
+Use "do" for a small contained change a bot could implement; "skip" for something
+too large/vague; "needs_info" if underspecified.
+
+ISSUE TITLE: Typo: "recieve" should be "receive" in greet.py
+ISSUE BODY: greet.py has a misspelling ("recieve"). Please fix it to "receive".
diff --git a/evals/tasks/triage/do-issue/repo/greet.py b/evals/tasks/triage/do-issue/repo/greet.py
new file mode 100644
index 0000000..f4b8002
--- /dev/null
+++ b/evals/tasks/triage/do-issue/repo/greet.py
@@ -0,0 +1,3 @@
+def greet(name):
+    # FIXME typo below
+    return "Welcome, we recieve you: " + name
diff --git a/evals/tasks/triage/do-issue/task.json b/evals/tasks/triage/do-issue/task.json
new file mode 100644
index 0000000..a5a23dd
--- /dev/null
+++ b/evals/tasks/triage/do-issue/task.json
@@ -0,0 +1,2 @@
+{ "surface": "triage", "profile": "investigate", "prompt": "prompt.md",
+  "expect": { "verdict": ["do", "try"] } }
diff --git a/evals/tasks/triage/skip-issue/check.py b/evals/tasks/triage/skip-issue/check.py
new file mode 100755
index 0000000..3472892
--- /dev/null
+++ b/evals/tasks/triage/skip-issue/check.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Objective check for a triage task: the agent's output must contain a JSON
+object whose `verdict` is one of the expected verdicts (from task.json)."""
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
+from checks_lib import load_expect, triage_ok  # noqa: E402
+
+task_dir = os.environ.get("TASK_DIR", os.path.dirname(os.path.abspath(__file__)))
+out = open(sys.argv[1], encoding="utf-8", errors="replace").read()
+ok, why = triage_ok(out, load_expect(task_dir))
+print(("  check: PASS " if ok else "  check: FAIL ") + why)
+sys.exit(0 if ok else 1)
diff --git a/evals/tasks/triage/skip-issue/prompt.md b/evals/tasks/triage/skip-issue/prompt.md
new file mode 100644
index 0000000..4e89a39
--- /dev/null
+++ b/evals/tasks/triage/skip-issue/prompt.md
@@ -0,0 +1,12 @@
+You are a repository maintainer triaging a new GitHub issue. Decide whether this is
+a small, contained, auto-implementable change.
+
+Respond with ONLY a JSON object (no prose, no code fences):
+{"verdict":"do|skip|needs_info","reply":"<one short maintainer-voice sentence>"}
+
+Use "do" for a small contained change a bot could implement; "skip" for something
+too large/vague to auto-implement; "needs_info" if underspecified.
+
+ISSUE TITLE: Rewrite the entire project in Rust and ship a mobile app
+ISSUE BODY: For performance, please rewrite the whole codebase in Rust, add a
+React Native mobile app, migrate the database to a new engine, and redesign the UI.
diff --git a/evals/tasks/triage/skip-issue/task.json b/evals/tasks/triage/skip-issue/task.json
new file mode 100644
index 0000000..4b26c2b
--- /dev/null
+++ b/evals/tasks/triage/skip-issue/task.json
@@ -0,0 +1,2 @@
+{ "surface": "triage", "profile": "investigate", "prompt": "prompt.md",
+  "expect": { "verdict": ["skip", "needs_info"] } }
diff --git a/evals/tests/test_checks.py b/evals/tests/test_checks.py
new file mode 100644
index 0000000..ebcc4ad
--- /dev/null
+++ b/evals/tests/test_checks.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+"""Validate the eval scoring logic deterministically — feed canned agent outputs
+to the real task checks + checks_lib, no agents, no network. This is what CI runs
+(the live bench runs on demand). Stdlib only.
+"""
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+EVALS = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.insert(0, EVALS)
+from checks_lib import extract_json, triage_ok, review_ok  # noqa: E402
+from score import aggregate  # noqa: E402
+
+fails = []
+
+
+def check(name, cond):
+    print(("ok   " if cond else "FAIL ") + name)
+    if not cond:
+        fails.append(name)
+
+
+# --- checks_lib units ----------------------------------------------------------
+check("extract_json: bare object", extract_json('{"verdict":"do"}') == {"verdict": "do"})
+check("extract_json: wrapped in prose/fence",
+      extract_json('Here you go:\n```json\n{"verdict":"skip","reply":"x"}\n```\n')["verdict"] == "skip")
+check("extract_json: none", extract_json("no json here") is None)
+check("triage_ok: verdict in set", triage_ok('{"verdict":"do"}', {"verdict": ["do", "try"]})[0])
+check("triage_ok: verdict not in set", not triage_ok('{"verdict":"skip"}', {"verdict": ["do"]})[0])
+check("triage_ok: no json", not triage_ok("nope", {"verdict": ["do"]})[0])
+check("review_ok: any-of present",
+      review_ok("This HARDCODES a secret token.", {"must_contain_any": ["hardcod", "secret"]})[0])
+check("review_ok: any-of absent",
+      not review_ok("Looks fine.", {"must_contain_any": ["hardcod", "secret"]})[0])
+
+# --- score.aggregate -----------------------------------------------------------
+card = aggregate([
+    {"task": "triage/do-issue", "pass": 1, "rc": 0, "secs": 4},
+    {"task": "triage/do-issue", "pass": 0, "rc": 0, "secs": 6},
+    {"task": "review/planted-bug", "pass": 1, "rc": 0, "secs": 9},
+])
+check("aggregate: per-task pass_rate", card["tasks"]["triage/do-issue"]["pass_rate"] == 0.5)
+check("aggregate: surface rollup", card["surfaces"]["triage"]["pass_rate"] == 0.5)
+check("aggregate: overall", card["overall"]["passes"] == 2 and card["overall"]["runs"] == 3)
+
+
+# --- the real task check files, against canned outputs -------------------------
+def run_check_py(task, output_text, expect_pass):
+    td = os.path.join(EVALS, "tasks", task)
+    with tempfile.NamedTemporaryFile("w", suffix=".out", delete=False) as f:
+        f.write(output_text)
+        outf = f.name
+    env = dict(os.environ, TASK_DIR=td)
+    rc = subprocess.run([sys.executable, os.path.join(td, "check.py"), outf],
+                        env=env, capture_output=True).returncode
+    os.unlink(outf)
+    check(f"check.py {task} ({'good' if expect_pass else 'bad'} → {'pass' if expect_pass else 'fail'})",
+          (rc == 0) == expect_pass)
+
+
+run_check_py("triage/do-issue", '{"verdict":"do","reply":"on it"}', True)
+run_check_py("triage/do-issue", '{"verdict":"skip"}', False)
+run_check_py("triage/skip-issue", 'I think we should {"verdict":"skip","reply":"too big"}', True)
+run_check_py("triage/skip-issue", '{"verdict":"do"}', False)
+run_check_py("review/planted-bug", "The diff hardcodes an API token and password in source.", True)
+run_check_py("review/planted-bug", "LGTM, ship it.", False)
+
+
+# --- implement check.sh: a passing vs failing sandbox --------------------------
+def run_impl_check(add_body, expect_pass):
+    if not shutil.which("jq"):
+        print("skip implement check.sh (jq not installed locally; CI has it)")
+        return
+    td = os.path.join(EVALS, "tasks", "implement", "failing-test")
+    sandbox = tempfile.mkdtemp()
+    shutil.copytree(os.path.join(td, "repo"), sandbox, dirs_exist_ok=True)
+    with open(os.path.join(sandbox, "add.py"), "w") as f:
+        f.write(add_body)
+    env = dict(os.environ, SANDBOX=sandbox, TASK_DIR=td)
+    rc = subprocess.run(["bash", os.path.join(td, "check.sh"), "x"], env=env,
+                        capture_output=True).returncode
+    shutil.rmtree(sandbox)
+    check(f"check.sh implement ({'fixed' if expect_pass else 'buggy'} → {'pass' if expect_pass else 'fail'})",
+          (rc == 0) == expect_pass)
+
+
+run_impl_check("def add(a, b):\n    return a + b\n", True)
+run_impl_check("def add(a, b):\n    return a - b\n", False)
+
+print()
+if fails:
+    print(f"{len(fails)} FAILED")
+    sys.exit(1)
+print("all checks passed")