Netis · vaderyang · Jun 8, 2026 · Jun 8, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,6 +34,9 @@ jobs:
       - name: config loader tests
         run: python3 scripts/agent-bot/tests/test_config.py
 
+      - name: harness adapter tests
+        run: python3 scripts/agent-bot/tests/test_harness.py
+
       - name: reviewer post-logic tests
         run: python3 scripts/pr-review/test_post_review.py
 

diff --git a/docs/config-reference.md b/docs/config-reference.md
@@ -60,7 +60,22 @@ Precedence for any value: an exported **env var** > the **config file** > the
 ## `model`
 | Field | Default | Meaning |
 |---|---|---|
-| `model` | `claude-3-5-sonnet-20241022` | Model name sent to the gateway. |
+| `model` | `claude-3-5-sonnet-20241022` | Model name sent to the gateway. Overridden by `harness.model` if set. |
+
+## `harness`
+Which agent CLI drives the surfaces. **Omit the whole block for the built-in
+`claude` (Claude Code) harness** — that's back-compatible with every existing
+consumer.
+
+| Field | Default | Meaning |
+|---|---|---|
+| `harness.kind` | `claude` | `claude` = built-in Claude Code. `custom` = run `harness.command`. |
+| `harness.command` | — | **custom only.** A shell command template run per agent invocation. Placeholders: `{model}` `{prompt_file}` `{out}` `{tools}` `{write}` `{max_turns}`. e.g. `codex exec --model {model} --full-auto < {prompt_file} > {out}`. |
+| `harness.model` | — | Model id for this harness; overrides `.model`. For `custom`, the `{model}` value. |
+| `harness.health_probe` | `true` | Poll the gateway's OpenAI-compatible `/v1/models` before each run (and to detect mid-run outages). Set `false` if your harness's endpoint isn't OpenAI-compatible. |
+
+See [docs/cookbooks](cookbooks/README.md#swapping-the-agent-cli-harness) for a
+full `custom` (codex) walkthrough + the prompt-shape constraint.
 
 ## Example
 

diff --git a/docs/cookbooks/README.md b/docs/cookbooks/README.md
@@ -90,6 +90,44 @@ the official LiteLLM docs since exact flags evolve.
 
 ---
 
+## Swapping the agent CLI (harness)
+
+The section above changes the **model** behind Claude Code. This changes the
+**agent CLI itself** — run `codex`, `aider`, or any wrapper instead of `claude`.
+Set the `harness` block in `.agent-ops.json`:
+
+```jsonc
+{
+  "harness": {
+    "kind": "custom",
+    "command": "codex exec --model {model} --full-auto < {prompt_file} > {out}",
+    "model": "gpt-5",
+    "health_probe": false
+  }
+}
+```
+
+agent-ops fills the placeholders per run: `{model}` `{prompt_file}` (the agent's
+instructions) `{out}` (where to write the agent's output) `{tools}` (the
+allow-listed tools for that surface) `{write}` (`true` only for the implement
+surface — your harness should refuse edits otherwise) `{max_turns}`. Set
+`health_probe: false` unless your harness talks to an OpenAI-compatible
+`/v1/models` endpoint. The runner must have the CLI installed + its auth in env
+(e.g. `OPENAI_API_KEY` for codex — add it as a repo secret / set it on the box).
+
+Omit the `harness` block entirely to use the built-in **`claude`** harness
+(the default; identical to every existing consumer).
+
+> ⚠️ **Known constraint — prompt shape.** agent-ops's prompts are written for
+> Claude Code's behaviour: **triage** expects the agent to emit a JSON object
+> (`verdict` / `reply` / …) and **review** expects a `### Summary` heading that
+> `post_review.py` parses. A non-claude harness may format its output
+> differently and need prompt tuning to satisfy those parsers — agent-ops does
+> **not** normalise output across harnesses. **Qualify a candidate CLI with the
+> `evals/` bench (the harness qualification suite) before wiring it into the live loop.**
+
+---
+
 ## Sizing
 
 | Workload | Suggested box |

diff --git a/examples/consumer/.agent-ops.json b/examples/consumer/.agent-ops.json
@@ -36,5 +36,9 @@
       "expect": "ok"
     }
   },
-  "model": "claude-3-5-sonnet-20241022"
+  "model": "claude-3-5-sonnet-20241022",
+
+  "harness": {
+    "kind": "claude"
+  }
 }
diff --git a/schema/agent-ops.schema.json b/schema/agent-ops.schema.json
@@ -76,6 +76,17 @@
         }
       }
     },
-    "model": { "type": "string", "default": "claude-3-5-sonnet-20241022", "description": "Model name sent to the LiteLLM-style gateway." }
+    "model": { "type": "string", "default": "claude-3-5-sonnet-20241022", "description": "Model name sent to the LiteLLM-style gateway." },
+    "harness": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Which agent CLI drives the surfaces. Omit for the built-in claude (Claude Code) harness — that's back-compatible with every existing consumer.",
+      "properties": {
+        "kind": { "type": "string", "enum": ["claude", "custom"], "default": "claude", "description": "claude = built-in Claude Code. custom = run the 'command' template." },
+        "command": { "type": "string", "description": "custom only: a shell command template invoked per agent run. Placeholders: {model} {prompt_file} {out} {tools} {write} {max_turns}. e.g. 'codex exec --model {model} --full-auto < {prompt_file} > {out}'." },
+        "model": { "type": "string", "description": "Model id for this harness; overrides top-level .model. For a custom harness this is the {model} placeholder value." },
+        "health_probe": { "type": "boolean", "default": true, "description": "Poll the gateway's OpenAI-compatible /v1/models before each run (and to detect mid-run outages). Set false for harnesses whose endpoint isn't OpenAI-compatible." }
+      }
+    }
   }
 }
diff --git a/scripts/agent-bot/run_revise.sh b/scripts/agent-bot/run_revise.sh
@@ -79,37 +79,14 @@ ${INLINE:-none}
 ${DIFF}
 EOF
 
-# Wait for the model gateway before launching claude — a transient backend
-# restart during the runner queue wait would otherwise kill the run.
-LITELLM_WAIT="$(cd "$HERE/../lib" && pwd)/litellm-wait.sh"
-# shellcheck source=../lib/litellm-wait.sh
-source "$LITELLM_WAIT"
-wait_for_litellm || exit $?
-
-# Stream claude to both the workflow log and a file; retry on transient
-# backend-down failures (same policy as run_wiwi.sh).
-CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}"
-attempt=0
+# Run the configured agent harness (default: claude) on the revise prompt,
+# streaming to both the workflow log and a file. agent-harness.sh owns the
+# gateway pre-flight wait + retry-on-gateway-down loop (same policy as run_wiwi.sh);
+# which CLI runs is .agent-ops.json's harness.kind.
+# shellcheck source=../lib/agent-harness.sh
+source "$(cd "$HERE/../lib" && pwd)/agent-harness.sh"
 claude_exit=0
-while true; do
-    set +e
-    stdbuf -oL claude --print \
-      --allowed-tools Bash Read Write Edit Grep Glob \
-      --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \
-      < "$PROMPT" 2>&1 | stdbuf -oL tee /tmp/wiwi-revise-run.log
-    claude_exit=${PIPESTATUS[0]}
-    set -e
-
-    [ "$claude_exit" -eq 0 ] && break
-    if ! litellm_appears_down; then break; fi
-    if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then
-        echo "::error::wiwi revise: claude died $((attempt+1))× with gateway down; giving up" >&2
-        break
-    fi
-    attempt=$((attempt + 1))
-    echo "::warning::wiwi revise: claude exited $claude_exit, gateway down; waiting + retrying ($((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2
-    wait_for_litellm || break
-done
+agent_run --profile implement --prompt "$PROMPT" --stream /tmp/wiwi-revise-run.log --label "wiwi revise" || claude_exit=$?
 
 if [ "$claude_exit" != "0" ]; then
   echo "wiwi revise failed (claude exit=$claude_exit; see /tmp/wiwi-revise-run.log)" >&2

diff --git a/scripts/agent-bot/run_triage.sh b/scripts/agent-bot/run_triage.sh
@@ -200,50 +200,16 @@ Issue title: ${ISSUE_TITLE}
 Author: ${ISSUE_AUTHOR}
 EOF
 
-# Wait for LiteLLM to be reachable before burning a runner slot
-# inside claude --print. Caps at 30 min by default; configurable
-# via MAX_LITELLM_WAIT_SECONDS. See scripts/lib/litellm-wait.sh.
-LITELLM_WAIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/litellm-wait.sh"
-# shellcheck source=../lib/litellm-wait.sh
-source "$LITELLM_WAIT"
-wait_for_litellm || exit $?
-
-# Run claude in print mode against our LiteLLM-style endpoint.
-# Wrap in a retry loop that catches the specific case "claude
-# crashed because LiteLLM died mid-stream". Up to CLAUDE_RETRY_MAX
-# retries (default 2 → 3 attempts total). Triage is idempotent —
-# the input is just the issue body — so restart from scratch is
-# safe.
-CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}"
-attempt=0
+# Run the configured agent harness (default: claude) on the triage prompt.
+# agent-harness.sh sources litellm-wait.sh and owns the gateway pre-flight wait
+# + the retry-on-gateway-down loop; which CLI runs is .agent-ops.json's
+# harness.kind. Triage is idempotent (input = the issue body) so a from-scratch
+# retry is safe.
+# shellcheck source=../lib/agent-harness.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/agent-harness.sh"
 claude_rc=0
-while true; do
-    set +e
-    claude --print \
-      --allowed-tools Bash Read Grep Glob WebFetch \
-      --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \
-      < "$PROMPT" > "$OUT" 2> /tmp/triage-stderr.log
-    claude_rc=$?
-    set -e
-
-    [ "$claude_rc" -eq 0 ] && break
-
-    # claude failed. Was it because LiteLLM is down right now?
-    if ! litellm_appears_down; then
-        # LiteLLM is up; the failure is something else (rate limit,
-        # token budget, claude bug). Don't retry.
-        break
-    fi
-
-    if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then
-        echo "::error::triage claude died $((attempt+1)) times with LiteLLM down each time; giving up" >&2
-        break
-    fi
-
-    attempt=$((attempt + 1))
-    echo "::warning::triage claude exited $claude_rc with LiteLLM down; waiting + retrying (attempt $((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2
-    wait_for_litellm || break
-done
+agent_run --profile investigate --prompt "$PROMPT" --out "$OUT" \
+  --errlog /tmp/triage-stderr.log --label triage || claude_rc=$?
 
 if [ "$claude_rc" -ne 0 ]; then
     echo "triage agent failed (see workflow log)" >&2

diff --git a/scripts/agent-bot/run_wiwi.sh b/scripts/agent-bot/run_wiwi.sh
@@ -75,67 +75,18 @@ the PR body. End it with the literal line:
 Issue title: ${ISSUE_TITLE}
 EOF
 
-# Wait for LiteLLM to be reachable before launching claude. Without
-# this, a transient model backend / LiteLLM restart that happens during the
-# (often-15+ min) self-hosted runner queue wait kills the run
-# immediately and wastes the queue wait + branch-prep work. Caps at
-# 30 min by default; configurable via MAX_LITELLM_WAIT_SECONDS.
-LITELLM_WAIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/litellm-wait.sh"
-# shellcheck source=../lib/litellm-wait.sh
-source "$LITELLM_WAIT"
-wait_for_litellm || exit $?
-
-# Stream claude's output to BOTH the workflow log (stdout) and a
-# preserved file. Previously this was `> /tmp/wiwi-run.log 2>&1`,
-# which silenced the GH Actions log entirely for the run — you
-# couldn't tell from outside whether wiwi was making progress,
-# looping, or hung. Streaming is essential when the run can take an
-# hour: a developer watching `gh run watch` should see each tool
-# call land in near-real-time.
-#
-# `stdbuf -oL` forces line-buffered stdout from claude and tee
-# (otherwise the 4KB pipe buffer can hide progress for minutes when
-# claude emits small lines slowly).
-#
-# `tee` always exits 0, so we briefly drop `set -e` and capture
-# claude's actual exit via `${PIPESTATUS[0]}`.
-#
-# Retry on transient LiteLLM mid-stream failures: if claude exits
-# non-zero AND the backend now looks down (5xx / connect-refused /
-# timeout — not 4xx, which means LiteLLM is choosing to reject and
-# waiting won't help), wait for LiteLLM to come back and re-run
-# claude from scratch. The prompt above tells claude how to resume
-# from the partial state on disk. Up to CLAUDE_RETRY_MAX retries
-# (default 2 → 3 attempts total).
-CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}"
-attempt=0
+# Run the configured agent harness (default: claude) on the implement prompt,
+# STREAMING its output to BOTH the workflow log (stdout) and /tmp/wiwi-run.log —
+# so a developer watching `gh run watch` sees each tool call land in near
+# real-time (a silent `> log 2>&1` once hid whether wiwi was progressing or
+# hung). agent-harness.sh sources litellm-wait.sh and owns the gateway pre-flight
+# wait + retry-on-gateway-down loop (it re-runs from scratch; the prompt above
+# tells the agent how to resume from the partial on-disk state). Which CLI runs
+# is .agent-ops.json's harness.kind.
+# shellcheck source=../lib/agent-harness.sh
+source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/agent-harness.sh"
 claude_exit=0
-while true; do
-    set +e
-    stdbuf -oL claude --print \
-      --allowed-tools Bash Read Write Edit Grep Glob \
-      --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \
-      < "$PROMPT" 2>&1 | stdbuf -oL tee /tmp/wiwi-run.log
-    claude_exit=${PIPESTATUS[0]}
-    set -e
-
-    [ "$claude_exit" -eq 0 ] && break
-
-    if ! litellm_appears_down; then
-        # LiteLLM is up — failure is something else (claude
-        # internal error, prompt issue, token budget). Don't retry.
-        break
-    fi
-
-    if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then
-        echo "::error::wiwi claude died $((attempt+1)) times with LiteLLM down; giving up" >&2
-        break
-    fi
-
-    attempt=$((attempt + 1))
-    echo "::warning::wiwi claude exited $claude_exit, LiteLLM down; waiting + retrying (attempt $((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2
-    wait_for_litellm || break
-done
+agent_run --profile implement --prompt "$PROMPT" --stream /tmp/wiwi-run.log --label wiwi || claude_exit=$?
 
 if [ "$claude_exit" != "0" ]; then
   echo "wiwi run failed (claude exit=$claude_exit; see /tmp/wiwi-run.log)" >&2

diff --git a/scripts/agent-bot/tests/test_harness.py b/scripts/agent-bot/tests/test_harness.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Unit test for scripts/lib/agent-harness.sh's agent_run, via its dry-run shim
+(AGENT_HARNESS_DRYRUN=1 prints the command it WOULD run, no exec / no gateway).
+
+Two things it locks down:
+  1. Back-compat: with NO config, the `claude` harness builds the same flags the
+     four inline call sites used before the refactor (per profile).
+  2. The `custom` harness substitutes the command-template placeholders.
+
+Stdlib only; no network. Run: python3 scripts/agent-bot/tests/test_harness.py
+"""
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+REPO = os.path.abspath(os.path.join(HERE, "..", "..", ".."))
+HARNESS = os.path.join(REPO, "scripts", "lib", "agent-harness.sh")
+CONFIG = os.path.join(REPO, "scripts", "lib", "config.sh")
+
+failures = []
+
+
+def check(name, cond):
+    print(("ok   " if cond else "FAIL ") + name)
+    if not cond:
+        failures.append(name)
+
+
+def dry_run(args, config=None):
+    env = dict(os.environ)
+    env["AGENT_HARNESS_DRYRUN"] = "1"
+    pre = ""
+    if config:
+        env["AGENT_OPS_CONFIG"] = config
+        pre = f'source "{CONFIG}"; agent_ops_load_config 2>/dev/null; '
+    script = pre + f'source "{HARNESS}"; agent_run ' + args
+    r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env)
+    return r.stdout.strip()
+
+
+# --- 1. claude back-compat, per profile (no config => default claude) ----------
+out = dry_run('--profile investigate --prompt /tmp/p --out /tmp/o --errlog /tmp/e')
+check("investigate starts with 'claude --print'", out.startswith("claude --print"))
+check("investigate default model", "--model claude-3-5-sonnet-20241022" in out)
+check("investigate tools = read-only + WebFetch",
+      "--allowed-tools Bash Read Grep Glob WebFetch" in out)
+
+out = dry_run('--profile implement --prompt /tmp/p --stream /tmp/s')
+check("implement tools = read+write set",
+      "--allowed-tools Bash Read Write Edit Grep Glob" in out)
+
+out = dry_run('--profile review --prompt /tmp/p --out /tmp/o --tools "Bash,Read,Grep" '
+              '--max-turns 60 --timeout 7200 --output-format text --permission-mode acceptEdits')
+check("review tools override (comma list)", "--allowed-tools Bash,Read,Grep" in out)
+check("review --max-turns 60", "--max-turns 60" in out)
+check("review --output-format text", "--output-format text" in out)
+check("review --permission-mode acceptEdits", "--permission-mode acceptEdits" in out)
+
+# --- 2. custom harness: template substitution ---------------------------------
+with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
+    json.dump({"harness": {
+        "kind": "custom",
+        "command": "codex exec --model {model} < {prompt_file} > {out}  # w={write} t={tools} m={max_turns}",
+        "model": "gpt-5",
+    }}, f)
+    cfg = f.name
+out = dry_run('--profile implement --prompt /tmp/pp --out /tmp/oo --max-turns 40', config=cfg)
+check("custom substitutes {model}", "--model gpt-5" in out)
+check("custom substitutes {prompt_file}", "< /tmp/pp" in out)
+check("custom substitutes {out}", "> /tmp/oo" in out)
+check("custom {write}=true for implement", "w=true" in out)
+check("custom substitutes {tools}", "t=Bash Read Write Edit Grep Glob" in out)
+check("custom substitutes {max_turns}", "m=40" in out)
+os.unlink(cfg)
+
+# {write} is false for a non-implement profile
+with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
+    json.dump({"harness": {"kind": "custom", "command": "x --w {write}"}}, f)
+    cfg = f.name
+out = dry_run('--profile review --prompt /tmp/p --out /tmp/o', config=cfg)
+check("custom {write}=false for review", "--w false" in out)
+os.unlink(cfg)
+
+# --- 3. model resolution from top-level .model --------------------------------
+with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
+    json.dump({"model": "claude-test-model"}, f)
+    cfg = f.name
+out = dry_run('--profile investigate --prompt /tmp/p --out /tmp/o', config=cfg)
+check("model from .model", "--model claude-test-model" in out)
+os.unlink(cfg)
+
+print()
+if failures:
+    print(f"{len(failures)} check(s) FAILED")
+    sys.exit(1)
+print("all checks passed")