perf(evals/tier2): pin --model sonnet, 8x speedup with no quality loss

Tier 2 was inheriting the invoker's configured default model. Eric is on
Opus 4.7, so the SUT ran Opus — explains the 45-min runs we were seeing.
Tier 1 already pins sonnet explicitly; Tier 2 should too. Reproducibility
matters across people with different defaults.

Add --sut-model / --judge-model CLI args (both default sonnet, pass ""
to inherit user default for ad-hoc Opus runs).

Smoke test (demo-signup-loom):
  - Opus inherited: 2676s (~45 min), 0.94 pass
  - Sonnet pinned:   337s (~5.6 min), 0.92 pass

Sonnet produced 5 slides (vs Opus's 4), annotated all 5 (vs Opus's 3),
and exercised the documented selector-miss recovery: tried 4 form
selectors, fell back to fetching page HTML to find the real ID, then
re-screenshotted with the corrected target. Loom voice intact throughout.

The 0.02 score delta is sampling variance. The 8x duration win is structural.

Now sub-10-min/seed which means full 4-seed Tier 2 sweeps are feasible
(~25 min total) instead of "block out your afternoon" (~3 hours).

Author: ericelizes

tests/evals/tier2/run.py

@@ -262,6 +262,7 @@ def claude_p(
     *,
     allow_tools: bool = False,
     max_budget_usd: float | None = None,
+    model: str | None = None,
 ) -> tuple[dict, dict]:
     """Invoke claude -p with structured-output validation. Returns (parsed, envelope).
 
@@ -291,6 +292,8 @@ def claude_p(
         cmd += ['--permission-mode', 'bypassPermissions']
     if max_budget_usd is not None:
         cmd += ['--max-budget-usd', str(max_budget_usd)]
+    if model:
+        cmd += ['--model', model]
     cmd.append(prompt)
 
     proc = subprocess.Popen(
@@ -336,7 +339,9 @@ def claude_p(
     return parsed, envelope
 
 
-def run_seed(seed: dict, rubric: dict) -> dict:
+def run_seed(
+    seed: dict, rubric: dict, sut_model: str | None = None, judge_model: str | None = None
+) -> dict:
     start = time.time()
     fixture_server = FixtureServer(seed['fixture'])
     fixture_server.start()
@@ -366,6 +371,7 @@ def run_seed(seed: dict, rubric: dict) -> dict:
             timeout=seed.get('timeout_s', 600),
             allow_tools=True,
             max_budget_usd=seed.get('max_budget_usd', 3.0),
+            model=sut_model,
         )
     except (
         subprocess.CalledProcessError,
@@ -391,7 +397,7 @@ def run_seed(seed: dict, rubric: dict) -> dict:
     # Judge the result.
     judge_prompt = build_judge_prompt(seed, sut_report, transcript, rubric)
     try:
-        judge_report, _ = claude_p(judge_prompt, JUDGE_SCHEMA, timeout=240)
+        judge_report, _ = claude_p(judge_prompt, JUDGE_SCHEMA, timeout=240, model=judge_model)
     except (
         subprocess.CalledProcessError,
         subprocess.TimeoutExpired,
@@ -485,8 +491,27 @@ def print_summary(results: list[dict]) -> None:
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument('seed_ids', nargs='*', help='Specific seed IDs to run (default: all)')
+    parser.add_argument(
+        '--sut-model',
+        default='sonnet',
+        help=(
+            'Model for the SUT (system-under-test) claude -p invocation. '
+            'Default sonnet — pinning explicitly because Tier 2 inherits the '
+            "invoker's configured default otherwise, which can be Opus ($$ and "
+            'slow). Sonnet handles the agentclip workflow with no quality loss '
+            'and ~2x speedup. Pass "" to inherit the user\'s default.'
+        ),
+    )
+    parser.add_argument(
+        '--judge-model',
+        default='sonnet',
+        help='Model for the judge claude -p invocation. Default sonnet.',
+    )
     args = parser.parse_args()
 
+    sut_model = args.sut_model or None
+    judge_model = args.judge_model or None
+
     seeds = json.loads(SEEDS_PATH.read_text())
     rubric = json.loads(RUBRIC_PATH.read_text())
 
@@ -498,11 +523,14 @@ def main() -> int:
             print(f'unknown seed ids: {sorted(missing)}', file=sys.stderr)
             return 2
 
-    print(f'running {len(seeds)} Tier 2 seed(s) — stub stands in for api.agentclip.dev')
+    print(
+        f'running {len(seeds)} Tier 2 seed(s) — stub stands in for api.agentclip.dev '
+        f'(sut={sut_model or "default"}, judge={judge_model or "default"})'
+    )
     results = []
     for seed in seeds:
         print(f'  -> {seed["id"]} ... ', end='', flush=True)
-        r = run_seed(seed, rubric)
+        r = run_seed(seed, rubric, sut_model=sut_model, judge_model=judge_model)
         write_result(r)
         results.append(r)
         if 'error' in r: