diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ae185..286a4c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: - name: config loader tests run: python3 scripts/agent-bot/tests/test_config.py + - name: harness adapter tests + run: python3 scripts/agent-bot/tests/test_harness.py + - name: reviewer post-logic tests run: python3 scripts/pr-review/test_post_review.py diff --git a/docs/config-reference.md b/docs/config-reference.md index a118689..f44a1b4 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -60,7 +60,22 @@ Precedence for any value: an exported **env var** > the **config file** > the ## `model` | Field | Default | Meaning | |---|---|---| -| `model` | `claude-3-5-sonnet-20241022` | Model name sent to the gateway. | +| `model` | `claude-3-5-sonnet-20241022` | Model name sent to the gateway. Overridden by `harness.model` if set. | + +## `harness` +Which agent CLI drives the surfaces. **Omit the whole block for the built-in +`claude` (Claude Code) harness** — that's back-compatible with every existing +consumer. + +| Field | Default | Meaning | +|---|---|---| +| `harness.kind` | `claude` | `claude` = built-in Claude Code. `custom` = run `harness.command`. | +| `harness.command` | — | **custom only.** A shell command template run per agent invocation. Placeholders: `{model}` `{prompt_file}` `{out}` `{tools}` `{write}` `{max_turns}`. e.g. `codex exec --model {model} --full-auto < {prompt_file} > {out}`. | +| `harness.model` | — | Model id for this harness; overrides `.model`. For `custom`, the `{model}` value. | +| `harness.health_probe` | `true` | Poll the gateway's OpenAI-compatible `/v1/models` before each run (and to detect mid-run outages). Set `false` if your harness's endpoint isn't OpenAI-compatible. | + +See [docs/cookbooks](cookbooks/README.md#swapping-the-agent-cli-harness) for a +full `custom` (codex) walkthrough + the prompt-shape constraint. ## Example diff --git a/docs/cookbooks/README.md b/docs/cookbooks/README.md index c80cb86..f5c118c 100644 --- a/docs/cookbooks/README.md +++ b/docs/cookbooks/README.md @@ -90,6 +90,44 @@ the official LiteLLM docs since exact flags evolve. --- +## Swapping the agent CLI (harness) + +The section above changes the **model** behind Claude Code. This changes the +**agent CLI itself** — run `codex`, `aider`, or any wrapper instead of `claude`. +Set the `harness` block in `.agent-ops.json`: + +```jsonc +{ + "harness": { + "kind": "custom", + "command": "codex exec --model {model} --full-auto < {prompt_file} > {out}", + "model": "gpt-5", + "health_probe": false + } +} +``` + +agent-ops fills the placeholders per run: `{model}` `{prompt_file}` (the agent's +instructions) `{out}` (where to write the agent's output) `{tools}` (the +allow-listed tools for that surface) `{write}` (`true` only for the implement +surface — your harness should refuse edits otherwise) `{max_turns}`. Set +`health_probe: false` unless your harness talks to an OpenAI-compatible +`/v1/models` endpoint. The runner must have the CLI installed + its auth in env +(e.g. `OPENAI_API_KEY` for codex — add it as a repo secret / set it on the box). + +Omit the `harness` block entirely to use the built-in **`claude`** harness +(the default; identical to every existing consumer). + +> ⚠️ **Known constraint — prompt shape.** agent-ops's prompts are written for +> Claude Code's behaviour: **triage** expects the agent to emit a JSON object +> (`verdict` / `reply` / …) and **review** expects a `### Summary` heading that +> `post_review.py` parses. A non-claude harness may format its output +> differently and need prompt tuning to satisfy those parsers — agent-ops does +> **not** normalise output across harnesses. **Qualify a candidate CLI with the +> `evals/` bench (the harness qualification suite) before wiring it into the live loop.** + +--- + ## Sizing | Workload | Suggested box | diff --git a/examples/consumer/.agent-ops.json b/examples/consumer/.agent-ops.json index f932d6e..03a7a45 100644 --- a/examples/consumer/.agent-ops.json +++ b/examples/consumer/.agent-ops.json @@ -36,5 +36,9 @@ "expect": "ok" } }, - "model": "claude-3-5-sonnet-20241022" + "model": "claude-3-5-sonnet-20241022", + + "harness": { + "kind": "claude" + } } diff --git a/schema/agent-ops.schema.json b/schema/agent-ops.schema.json index e1b4a87..f08883d 100644 --- a/schema/agent-ops.schema.json +++ b/schema/agent-ops.schema.json @@ -76,6 +76,17 @@ } } }, - "model": { "type": "string", "default": "claude-3-5-sonnet-20241022", "description": "Model name sent to the LiteLLM-style gateway." } + "model": { "type": "string", "default": "claude-3-5-sonnet-20241022", "description": "Model name sent to the LiteLLM-style gateway." }, + "harness": { + "type": "object", + "additionalProperties": false, + "description": "Which agent CLI drives the surfaces. Omit for the built-in claude (Claude Code) harness — that's back-compatible with every existing consumer.", + "properties": { + "kind": { "type": "string", "enum": ["claude", "custom"], "default": "claude", "description": "claude = built-in Claude Code. custom = run the 'command' template." }, + "command": { "type": "string", "description": "custom only: a shell command template invoked per agent run. Placeholders: {model} {prompt_file} {out} {tools} {write} {max_turns}. e.g. 'codex exec --model {model} --full-auto < {prompt_file} > {out}'." }, + "model": { "type": "string", "description": "Model id for this harness; overrides top-level .model. For a custom harness this is the {model} placeholder value." }, + "health_probe": { "type": "boolean", "default": true, "description": "Poll the gateway's OpenAI-compatible /v1/models before each run (and to detect mid-run outages). Set false for harnesses whose endpoint isn't OpenAI-compatible." } + } + } } } diff --git a/scripts/agent-bot/run_revise.sh b/scripts/agent-bot/run_revise.sh index bd20db0..c3eb275 100755 --- a/scripts/agent-bot/run_revise.sh +++ b/scripts/agent-bot/run_revise.sh @@ -79,37 +79,14 @@ ${INLINE:-none} ${DIFF} EOF -# Wait for the model gateway before launching claude — a transient backend -# restart during the runner queue wait would otherwise kill the run. -LITELLM_WAIT="$(cd "$HERE/../lib" && pwd)/litellm-wait.sh" -# shellcheck source=../lib/litellm-wait.sh -source "$LITELLM_WAIT" -wait_for_litellm || exit $? - -# Stream claude to both the workflow log and a file; retry on transient -# backend-down failures (same policy as run_wiwi.sh). -CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}" -attempt=0 +# Run the configured agent harness (default: claude) on the revise prompt, +# streaming to both the workflow log and a file. agent-harness.sh owns the +# gateway pre-flight wait + retry-on-gateway-down loop (same policy as run_wiwi.sh); +# which CLI runs is .agent-ops.json's harness.kind. +# shellcheck source=../lib/agent-harness.sh +source "$(cd "$HERE/../lib" && pwd)/agent-harness.sh" claude_exit=0 -while true; do - set +e - stdbuf -oL claude --print \ - --allowed-tools Bash Read Write Edit Grep Glob \ - --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \ - < "$PROMPT" 2>&1 | stdbuf -oL tee /tmp/wiwi-revise-run.log - claude_exit=${PIPESTATUS[0]} - set -e - - [ "$claude_exit" -eq 0 ] && break - if ! litellm_appears_down; then break; fi - if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then - echo "::error::wiwi revise: claude died $((attempt+1))× with gateway down; giving up" >&2 - break - fi - attempt=$((attempt + 1)) - echo "::warning::wiwi revise: claude exited $claude_exit, gateway down; waiting + retrying ($((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2 - wait_for_litellm || break -done +agent_run --profile implement --prompt "$PROMPT" --stream /tmp/wiwi-revise-run.log --label "wiwi revise" || claude_exit=$? if [ "$claude_exit" != "0" ]; then echo "wiwi revise failed (claude exit=$claude_exit; see /tmp/wiwi-revise-run.log)" >&2 diff --git a/scripts/agent-bot/run_triage.sh b/scripts/agent-bot/run_triage.sh index c6601bd..9f353e6 100755 --- a/scripts/agent-bot/run_triage.sh +++ b/scripts/agent-bot/run_triage.sh @@ -200,50 +200,16 @@ Issue title: ${ISSUE_TITLE} Author: ${ISSUE_AUTHOR} EOF -# Wait for LiteLLM to be reachable before burning a runner slot -# inside claude --print. Caps at 30 min by default; configurable -# via MAX_LITELLM_WAIT_SECONDS. See scripts/lib/litellm-wait.sh. -LITELLM_WAIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/litellm-wait.sh" -# shellcheck source=../lib/litellm-wait.sh -source "$LITELLM_WAIT" -wait_for_litellm || exit $? - -# Run claude in print mode against our LiteLLM-style endpoint. -# Wrap in a retry loop that catches the specific case "claude -# crashed because LiteLLM died mid-stream". Up to CLAUDE_RETRY_MAX -# retries (default 2 → 3 attempts total). Triage is idempotent — -# the input is just the issue body — so restart from scratch is -# safe. -CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}" -attempt=0 +# Run the configured agent harness (default: claude) on the triage prompt. +# agent-harness.sh sources litellm-wait.sh and owns the gateway pre-flight wait +# + the retry-on-gateway-down loop; which CLI runs is .agent-ops.json's +# harness.kind. Triage is idempotent (input = the issue body) so a from-scratch +# retry is safe. +# shellcheck source=../lib/agent-harness.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/agent-harness.sh" claude_rc=0 -while true; do - set +e - claude --print \ - --allowed-tools Bash Read Grep Glob WebFetch \ - --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \ - < "$PROMPT" > "$OUT" 2> /tmp/triage-stderr.log - claude_rc=$? - set -e - - [ "$claude_rc" -eq 0 ] && break - - # claude failed. Was it because LiteLLM is down right now? - if ! litellm_appears_down; then - # LiteLLM is up; the failure is something else (rate limit, - # token budget, claude bug). Don't retry. - break - fi - - if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then - echo "::error::triage claude died $((attempt+1)) times with LiteLLM down each time; giving up" >&2 - break - fi - - attempt=$((attempt + 1)) - echo "::warning::triage claude exited $claude_rc with LiteLLM down; waiting + retrying (attempt $((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2 - wait_for_litellm || break -done +agent_run --profile investigate --prompt "$PROMPT" --out "$OUT" \ + --errlog /tmp/triage-stderr.log --label triage || claude_rc=$? if [ "$claude_rc" -ne 0 ]; then echo "triage agent failed (see workflow log)" >&2 diff --git a/scripts/agent-bot/run_wiwi.sh b/scripts/agent-bot/run_wiwi.sh index 7452521..1feb81f 100755 --- a/scripts/agent-bot/run_wiwi.sh +++ b/scripts/agent-bot/run_wiwi.sh @@ -75,67 +75,18 @@ the PR body. End it with the literal line: Issue title: ${ISSUE_TITLE} EOF -# Wait for LiteLLM to be reachable before launching claude. Without -# this, a transient model backend / LiteLLM restart that happens during the -# (often-15+ min) self-hosted runner queue wait kills the run -# immediately and wastes the queue wait + branch-prep work. Caps at -# 30 min by default; configurable via MAX_LITELLM_WAIT_SECONDS. -LITELLM_WAIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/litellm-wait.sh" -# shellcheck source=../lib/litellm-wait.sh -source "$LITELLM_WAIT" -wait_for_litellm || exit $? - -# Stream claude's output to BOTH the workflow log (stdout) and a -# preserved file. Previously this was `> /tmp/wiwi-run.log 2>&1`, -# which silenced the GH Actions log entirely for the run — you -# couldn't tell from outside whether wiwi was making progress, -# looping, or hung. Streaming is essential when the run can take an -# hour: a developer watching `gh run watch` should see each tool -# call land in near-real-time. -# -# `stdbuf -oL` forces line-buffered stdout from claude and tee -# (otherwise the 4KB pipe buffer can hide progress for minutes when -# claude emits small lines slowly). -# -# `tee` always exits 0, so we briefly drop `set -e` and capture -# claude's actual exit via `${PIPESTATUS[0]}`. -# -# Retry on transient LiteLLM mid-stream failures: if claude exits -# non-zero AND the backend now looks down (5xx / connect-refused / -# timeout — not 4xx, which means LiteLLM is choosing to reject and -# waiting won't help), wait for LiteLLM to come back and re-run -# claude from scratch. The prompt above tells claude how to resume -# from the partial state on disk. Up to CLAUDE_RETRY_MAX retries -# (default 2 → 3 attempts total). -CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}" -attempt=0 +# Run the configured agent harness (default: claude) on the implement prompt, +# STREAMING its output to BOTH the workflow log (stdout) and /tmp/wiwi-run.log — +# so a developer watching `gh run watch` sees each tool call land in near +# real-time (a silent `> log 2>&1` once hid whether wiwi was progressing or +# hung). agent-harness.sh sources litellm-wait.sh and owns the gateway pre-flight +# wait + retry-on-gateway-down loop (it re-runs from scratch; the prompt above +# tells the agent how to resume from the partial on-disk state). Which CLI runs +# is .agent-ops.json's harness.kind. +# shellcheck source=../lib/agent-harness.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/agent-harness.sh" claude_exit=0 -while true; do - set +e - stdbuf -oL claude --print \ - --allowed-tools Bash Read Write Edit Grep Glob \ - --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \ - < "$PROMPT" 2>&1 | stdbuf -oL tee /tmp/wiwi-run.log - claude_exit=${PIPESTATUS[0]} - set -e - - [ "$claude_exit" -eq 0 ] && break - - if ! litellm_appears_down; then - # LiteLLM is up — failure is something else (claude - # internal error, prompt issue, token budget). Don't retry. - break - fi - - if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then - echo "::error::wiwi claude died $((attempt+1)) times with LiteLLM down; giving up" >&2 - break - fi - - attempt=$((attempt + 1)) - echo "::warning::wiwi claude exited $claude_exit, LiteLLM down; waiting + retrying (attempt $((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2 - wait_for_litellm || break -done +agent_run --profile implement --prompt "$PROMPT" --stream /tmp/wiwi-run.log --label wiwi || claude_exit=$? if [ "$claude_exit" != "0" ]; then echo "wiwi run failed (claude exit=$claude_exit; see /tmp/wiwi-run.log)" >&2 diff --git a/scripts/agent-bot/tests/test_harness.py b/scripts/agent-bot/tests/test_harness.py new file mode 100644 index 0000000..b3bcabd --- /dev/null +++ b/scripts/agent-bot/tests/test_harness.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Unit test for scripts/lib/agent-harness.sh's agent_run, via its dry-run shim +(AGENT_HARNESS_DRYRUN=1 prints the command it WOULD run, no exec / no gateway). + +Two things it locks down: + 1. Back-compat: with NO config, the `claude` harness builds the same flags the + four inline call sites used before the refactor (per profile). + 2. The `custom` harness substitutes the command-template placeholders. + +Stdlib only; no network. Run: python3 scripts/agent-bot/tests/test_harness.py +""" +import json +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..")) +HARNESS = os.path.join(REPO, "scripts", "lib", "agent-harness.sh") +CONFIG = os.path.join(REPO, "scripts", "lib", "config.sh") + +failures = [] + + +def check(name, cond): + print(("ok " if cond else "FAIL ") + name) + if not cond: + failures.append(name) + + +def dry_run(args, config=None): + env = dict(os.environ) + env["AGENT_HARNESS_DRYRUN"] = "1" + pre = "" + if config: + env["AGENT_OPS_CONFIG"] = config + pre = f'source "{CONFIG}"; agent_ops_load_config 2>/dev/null; ' + script = pre + f'source "{HARNESS}"; agent_run ' + args + r = subprocess.run(["bash", "-c", script], capture_output=True, text=True, env=env) + return r.stdout.strip() + + +# --- 1. claude back-compat, per profile (no config => default claude) ---------- +out = dry_run('--profile investigate --prompt /tmp/p --out /tmp/o --errlog /tmp/e') +check("investigate starts with 'claude --print'", out.startswith("claude --print")) +check("investigate default model", "--model claude-3-5-sonnet-20241022" in out) +check("investigate tools = read-only + WebFetch", + "--allowed-tools Bash Read Grep Glob WebFetch" in out) + +out = dry_run('--profile implement --prompt /tmp/p --stream /tmp/s') +check("implement tools = read+write set", + "--allowed-tools Bash Read Write Edit Grep Glob" in out) + +out = dry_run('--profile review --prompt /tmp/p --out /tmp/o --tools "Bash,Read,Grep" ' + '--max-turns 60 --timeout 7200 --output-format text --permission-mode acceptEdits') +check("review tools override (comma list)", "--allowed-tools Bash,Read,Grep" in out) +check("review --max-turns 60", "--max-turns 60" in out) +check("review --output-format text", "--output-format text" in out) +check("review --permission-mode acceptEdits", "--permission-mode acceptEdits" in out) + +# --- 2. custom harness: template substitution --------------------------------- +with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump({"harness": { + "kind": "custom", + "command": "codex exec --model {model} < {prompt_file} > {out} # w={write} t={tools} m={max_turns}", + "model": "gpt-5", + }}, f) + cfg = f.name +out = dry_run('--profile implement --prompt /tmp/pp --out /tmp/oo --max-turns 40', config=cfg) +check("custom substitutes {model}", "--model gpt-5" in out) +check("custom substitutes {prompt_file}", "< /tmp/pp" in out) +check("custom substitutes {out}", "> /tmp/oo" in out) +check("custom {write}=true for implement", "w=true" in out) +check("custom substitutes {tools}", "t=Bash Read Write Edit Grep Glob" in out) +check("custom substitutes {max_turns}", "m=40" in out) +os.unlink(cfg) + +# {write} is false for a non-implement profile +with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump({"harness": {"kind": "custom", "command": "x --w {write}"}}, f) + cfg = f.name +out = dry_run('--profile review --prompt /tmp/p --out /tmp/o', config=cfg) +check("custom {write}=false for review", "--w false" in out) +os.unlink(cfg) + +# --- 3. model resolution from top-level .model -------------------------------- +with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump({"model": "claude-test-model"}, f) + cfg = f.name +out = dry_run('--profile investigate --prompt /tmp/p --out /tmp/o', config=cfg) +check("model from .model", "--model claude-test-model" in out) +os.unlink(cfg) + +print() +if failures: + print(f"{len(failures)} check(s) FAILED") + sys.exit(1) +print("all checks passed") diff --git a/scripts/lib/agent-harness.sh b/scripts/lib/agent-harness.sh new file mode 100644 index 0000000..47ca078 --- /dev/null +++ b/scripts/lib/agent-harness.sh @@ -0,0 +1,156 @@ +# shellcheck shell=bash +# agent-harness.sh — the configurable agent-CLI adapter. +# +# Every agent surface (triage / implement / review / revise) used to invoke the +# `claude` CLI inline, each with its own copy of the "wait for the gateway, run, +# retry if the gateway died mid-stream" loop. This centralises that into ONE +# function so the harness CLI becomes a `.agent-ops.json` choice: +# +# harness.kind = "claude" (default — byte-compatible with the old inline calls) +# | "custom" (run AGENT_OPS_HARNESS_CMD, a command template) +# +# Public entry point: +# agent_run --profile {investigate|implement|review} --prompt \ +# [--out ] [--errlog ] [--stream ] \ +# [--tools ""] [--max-turns N] [--timeout S] \ +# [--output-format F] [--permission-mode M] [--label NAME] +# +# Returns the harness exit code. Encapsulates: the pre-flight gateway wait + the +# retry-on-gateway-down loop (both no-ops when AGENT_OPS_HEALTH_PROBE != "true"). +# +# Config consumed (exported by config.sh; safe defaults if unset): +# AGENT_OPS_HARNESS claude | custom (default claude) +# AGENT_OPS_HARNESS_CMD custom command template (custom only) +# AGENT_OPS_HEALTH_PROBE true | false (default true) +# ANTHROPIC_MODEL model id for the claude harness / {model} placeholder +# +# Dry run: AGENT_HARNESS_DRYRUN=1 prints the command that WOULD run (one line) +# and returns 0 — no pre-flight, no exec. Used by test_harness.py + eval self-check. +# +# The retry loop needs wait_for_litellm + litellm_appears_down from litellm-wait.sh; +# source it here (idempotent) so callers don't have to. +if ! declare -F wait_for_litellm >/dev/null 2>&1; then + # shellcheck source=scripts/lib/litellm-wait.sh + source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/litellm-wait.sh" +fi + +# Profile → the claude --allowed-tools set, and whether the agent may write. +_agent_profile_tools() { + case "$1" in + investigate) printf '%s' 'Bash Read Grep Glob WebFetch' ;; + implement) printf '%s' 'Bash Read Write Edit Grep Glob' ;; + review) printf '%s' 'Bash Read Grep Glob' ;; # default; review usually passes --tools + *) printf '%s' 'Bash Read Grep Glob' ;; + esac +} +_agent_profile_write() { # "true" if the profile is allowed to edit files + if [ "$1" = "implement" ]; then printf 'true'; else printf 'false'; fi +} + +agent_run() { + local profile="" prompt="" out="" errlog="/dev/null" streamlog="" tools="" \ + max_turns="" timeout_s="" output_format="" permission_mode="" label="" + while [ $# -gt 0 ]; do + case "$1" in + --profile) profile="$2"; shift 2 ;; + --prompt) prompt="$2"; shift 2 ;; + --out) out="$2"; shift 2 ;; + --errlog) errlog="$2"; shift 2 ;; + --stream) streamlog="$2"; shift 2 ;; + --tools) tools="$2"; shift 2 ;; + --max-turns) max_turns="$2"; shift 2 ;; + --timeout) timeout_s="$2"; shift 2 ;; + --output-format) output_format="$2"; shift 2 ;; + --permission-mode) permission_mode="$2"; shift 2 ;; + --label) label="$2"; shift 2 ;; + *) echo "agent_run: unknown arg '$1'" >&2; return 64 ;; + esac + done + [ -n "$profile" ] && [ -n "$prompt" ] || { echo "agent_run: --profile and --prompt are required" >&2; return 64; } + [ -n "$label" ] || label="$profile" + + local model="${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" + local tool_set; tool_set="${tools:-$(_agent_profile_tools "$profile")}" + local write; write="$(_agent_profile_write "$profile")" + local kind="${AGENT_OPS_HARNESS:-claude}" + + # --- build the harness command -------------------------------------------- + # For the claude harness we keep a real argv array (no eval). For custom we + # build a single shell string from the template so consumers can use pipes. + local -a claude_cmd=() + local custom_cmd="" + if [ "$kind" = "custom" ]; then + [ -n "${AGENT_OPS_HARNESS_CMD:-}" ] || { echo "agent_run: harness.kind=custom but AGENT_OPS_HARNESS_CMD is empty" >&2; return 78; } + # custom always writes the agent's output to a file we control; for stream + # profiles that's a temp the loop tees to stdout afterwards. + local cust_out="${out:-${streamlog:-/dev/stdout}}" + custom_cmd="$AGENT_OPS_HARNESS_CMD" + custom_cmd="${custom_cmd//\{model\}/$model}" + custom_cmd="${custom_cmd//\{prompt_file\}/$prompt}" + custom_cmd="${custom_cmd//\{out\}/$cust_out}" + custom_cmd="${custom_cmd//\{tools\}/$tool_set}" + custom_cmd="${custom_cmd//\{write\}/$write}" + custom_cmd="${custom_cmd//\{max_turns\}/${max_turns:-0}}" + else + claude_cmd=(claude --print --model "$model") + [ -n "$output_format" ] && claude_cmd+=(--output-format "$output_format") + [ -n "$max_turns" ] && claude_cmd+=(--max-turns "$max_turns") + [ -n "$permission_mode" ] && claude_cmd+=(--permission-mode "$permission_mode") + if [ -n "$tools" ]; then + claude_cmd+=(--allowed-tools "$tools") # single (comma-sep) override + else + # shellcheck disable=SC2206 # deliberate word-split of the tool set + claude_cmd+=(--allowed-tools $tool_set) + fi + fi + + # --- dry run: print the command and stop ---------------------------------- + if [ "${AGENT_HARNESS_DRYRUN:-}" = "1" ]; then + if [ "$kind" = "custom" ]; then printf '%s\n' "$custom_cmd"; + else printf '%s\n' "${claude_cmd[*]}"; fi + return 0 + fi + + # --- pre-flight: gateway reachable? --------------------------------------- + if [ "${AGENT_OPS_HEALTH_PROBE:-true}" = "true" ]; then + wait_for_litellm || return $? + fi + + # --- run with retry-on-gateway-down --------------------------------------- + # Optional `timeout S` prefix as a quoted array (empty array → no prefix). + local -a runner=() + [ -n "$timeout_s" ] && runner=(timeout "$timeout_s") + local retry_max="${CLAUDE_RETRY_MAX:-2}" attempt=0 rc=0 + while true; do + set +e + if [ -n "$streamlog" ]; then + if [ "$kind" = "custom" ]; then + stdbuf -oL bash -c "$custom_cmd" 2>&1 | stdbuf -oL tee "$streamlog" + else + stdbuf -oL "${claude_cmd[@]}" < "$prompt" 2>&1 | stdbuf -oL tee "$streamlog" + fi + rc=${PIPESTATUS[0]} + else + if [ "$kind" = "custom" ]; then + "${runner[@]}" bash -c "$custom_cmd" > "${out:-/dev/stdout}" 2> "$errlog" + else + "${runner[@]}" "${claude_cmd[@]}" < "$prompt" > "${out:-/dev/stdout}" 2> "$errlog" + fi + rc=$? + fi + set -e + + [ "$rc" -eq 0 ] && break + if [ "${AGENT_OPS_HEALTH_PROBE:-true}" != "true" ] || ! litellm_appears_down; then + break # failure is not a gateway outage — don't retry + fi + if [ "$attempt" -ge "$retry_max" ]; then + echo "::error::${label} agent died $((attempt+1))x with the gateway down; giving up" >&2 + break + fi + attempt=$((attempt + 1)) + echo "::warning::${label} agent exited $rc, gateway down; waiting + retrying ($((attempt+1))/$((retry_max+1)))" >&2 + wait_for_litellm || break + done + return "$rc" +} diff --git a/scripts/lib/config.sh b/scripts/lib/config.sh index a4e4ff7..d5ef9c9 100644 --- a/scripts/lib/config.sh +++ b/scripts/lib/config.sh @@ -69,8 +69,20 @@ agent_ops_load_config() { # --- implement (dev agent) --- export AGENT_OPS_BUILD_CMD="$(_aoc_get '.implement.build_cmd' '' AGENT_OPS_BUILD_CMD)" - # --- model --- - export ANTHROPIC_MODEL="$(_aoc_get '.model' "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" ANTHROPIC_MODEL)" + # --- model (harness.model wins over top-level .model) --- + export ANTHROPIC_MODEL="$(_aoc_get '(.harness.model // .model)' "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" ANTHROPIC_MODEL)" + + # --- harness (which agent CLI drives the surfaces; default claude) --- + # Read by scripts/lib/agent-harness.sh's agent_run. kind=custom runs the + # command template AGENT_OPS_HARNESS_CMD ({model}/{prompt_file}/{out}/{tools}/ + # {write}/{max_turns} placeholders). health_probe=false skips the gateway + # /v1/models wait (for harnesses whose endpoint isn't OpenAI-compatible). + export AGENT_OPS_HARNESS="$(_aoc_get '.harness.kind' 'claude' AGENT_OPS_HARNESS)" + export AGENT_OPS_HARNESS_CMD="$(_aoc_get '.harness.command' '' AGENT_OPS_HARNESS_CMD)" + # jq's `// empty` (in _aoc_get) treats boolean false as empty, so a literal + # `health_probe: false` would be lost — normalise to the string "false"/"true" + # inside the filter so only an explicit false disables the probe. + export AGENT_OPS_HEALTH_PROBE="$(_aoc_get '(.harness.health_probe | if . == false then "false" else "true" end)' 'true' AGENT_OPS_HEALTH_PROBE)" # --- observer (mara): map .observer.* onto the MARA_* env mara.sh reads, # unless the env already carries them (workflow/unit override wins). --- diff --git a/scripts/pr-review/run_review.sh b/scripts/pr-review/run_review.sh index 7431e12..f223444 100755 --- a/scripts/pr-review/run_review.sh +++ b/scripts/pr-review/run_review.sh @@ -15,6 +15,12 @@ set -euo pipefail PR_NUMBER="${1:?usage: $0 }" WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Load .agent-ops.json so the configured harness (harness.kind) + model land in +# the env before the agent runs. No config ⇒ claude (back-compatible). +# shellcheck source=../lib/config.sh +source "$(cd "$WORKDIR/../lib" && pwd)/config.sh" +agent_ops_load_config + OUT="/tmp/pr-review-${PR_NUMBER}-out.md" LOG="/tmp/pr-review-${PR_NUMBER}-agent.log" PROMPT="/tmp/pr-review-${PR_NUMBER}-prompt.md" @@ -29,19 +35,11 @@ BASE_REF="$(gh pr view "$PR_NUMBER" --json baseRefName --jq .baseRefName)" export PR_NUMBER HEAD_SHA BASE_REF envsubst < "$WORKDIR/prompt.md" > "$PROMPT" -# Pre-flight: wait until LiteLLM is reachable AND our API key is -# accepted. The wait covers the common case where model backend / LiteLLM -# is restarting when this workflow fires; if it's still down after -# 30 min (MAX_LITELLM_WAIT_SECONDS) the function returns 2 and we -# pass through with the same diagnostic shape as before. Shared -# helper in scripts/lib/litellm-wait.sh. -LITELLM_WAIT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/litellm-wait.sh" -# shellcheck source=../lib/litellm-wait.sh -source "$LITELLM_WAIT" -if ! wait_for_litellm; then - echo "ERROR: agent unavailable (pre-flight)" > "$OUT" - exit 2 -fi +# Source the harness adapter — it sources litellm-wait.sh and owns the gateway +# pre-flight wait + the retry-on-gateway-down loop. The CLI is the consumer's +# .agent-ops.json harness.kind (default: claude). +# shellcheck source=../lib/agent-harness.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")/../lib" && pwd)/agent-harness.sh" # Build the tool allowlist as a single comma-separated string (the # format `claude --allowed-tools` accepts). @@ -49,57 +47,17 @@ ALLOWED_TOOLS="$(grep -v '^#' "$WORKDIR/allowed_tools.txt" \ | grep -v '^[[:space:]]*$' \ | paste -sd, -)" -# Headless agent run. The 7200 s outer cap is a hard fence — if the -# model loops we'd rather post a "review timed out" than wedge the -# workflow. 7200 s lets large-diff reviews (200+ files, rebrand-class -# refactors) complete instead of timing out mid-read. -# -# Feed the prompt over stdin instead of as a trailing positional arg — -# `--allowed-tools` is variadic (), so a positional prompt -# right after it gets consumed as an extra tool name and claude then -# errors with "Input must be provided either through stdin or as a -# prompt argument when using --print". -# -# Retry on transient LiteLLM mid-stream failures: if claude exits -# non-zero AND the backend looks down right now (5xx / connect- -# refused / timeout — NOT 4xx), wait for LiteLLM and re-run. Review -# is idempotent (the OUT file just gets overwritten); restart from -# scratch is safe. Up to CLAUDE_RETRY_MAX retries (default 2). -CLAUDE_RETRY_MAX="${CLAUDE_RETRY_MAX:-2}" -attempt=0 +# Headless agent run via the configured harness (default: claude). The 7200 s +# outer timeout is a hard fence — large-diff reviews (200+ files) can run long, +# but we'd rather post "review timed out" than wedge the workflow. agent-harness.sh +# owns the gateway wait + retry-on-gateway-down loop; review is idempotent ($OUT is +# overwritten) so a from-scratch retry is safe. The prompt is fed on stdin because +# claude's --allowed-tools is variadic (a positional prompt would be swallowed as a +# tool name). claude_rc=0 -while true; do - set +e - timeout 7200 claude \ - --print \ - --model "${ANTHROPIC_MODEL:-claude-3-5-sonnet-20241022}" \ - --max-turns 60 \ - --output-format text \ - --permission-mode acceptEdits \ - --allowed-tools "$ALLOWED_TOOLS" \ - < "$PROMPT" \ - > "$OUT" \ - 2> "$LOG" - claude_rc=$? - set -e - - [ "$claude_rc" -eq 0 ] && break - - if ! litellm_appears_down; then - # LiteLLM is up — failure is real (timeout, claude crash, - # rate limit, etc). Don't retry. - break - fi - - if [ "$attempt" -ge "$CLAUDE_RETRY_MAX" ]; then - echo "::error::vivi claude died $((attempt+1)) times with LiteLLM down; giving up" >&2 - break - fi - - attempt=$((attempt + 1)) - echo "::warning::vivi claude exited $claude_rc, LiteLLM down; waiting + retrying (attempt $((attempt+1))/$((CLAUDE_RETRY_MAX+1)))" >&2 - wait_for_litellm || break -done +agent_run --profile review --prompt "$PROMPT" --out "$OUT" --errlog "$LOG" \ + --tools "$ALLOWED_TOOLS" --max-turns 60 --timeout 7200 \ + --output-format text --permission-mode acceptEdits --label vivi || claude_rc=$? if [ "$claude_rc" -ne 0 ]; then echo "ERROR: agent exited with code $claude_rc" >> "$LOG"