From c000d597309ab9b4420ea8fba0a03a110ffa7277 Mon Sep 17 00:00:00 2001 From: Wanli-Lee <1181451942@qq.com> Date: Tue, 30 Jun 2026 14:43:17 +0800 Subject: [PATCH] fix: enforce max_steps on codex/claudecode/hermes harnesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the openclaw harness actually enforced max_steps (via a step-cap watchdog). codex/claudecode/hermes accepted the parameter but never used it: codex's "translated to model_max_turns" comment was never implemented, hermes's "--effort" path doesn't exist in 0.14.0, and claudecode hardcoded MAX_STEPS_REACHED=0. As a result those three runtimes were bounded only by the wall-clock timeout, breaking cross-harness comparability. - claudecode: pass the native `--max-turns N` flag on both the first-turn and resume invocations (verified accepted by claude 2.1.76); detect the error_max_turns result subtype to emit a real MAX_STEPS_REACHED marker. - codex: add a background step-cap watchdog (parity with openclaw) that counts "type":"function_call" events across the rollout.jsonl files and pkills codex once the count reaches max_steps * (1 + max_refusal_retries). - hermes: same watchdog, counting "role":"assistant" occurrences in the incrementally-written session_*.json — the same unit openclaw counts (verified on a real VM: hermes writes session_*.json, not request_dump_*.json). All three guard the retry loop with `[ -f $STEPS_CAPPED ] && break` so a watchdog kill is not misread as a transient failure and retried. A step-capped task keeps its deliverables and is still judged (parity with openclaw): hitting the cap is a real "model looping" signal, not a fake zero. Co-Authored-By: Claude --- tests/test_harness_max_steps.py | 112 ++++++++++++++++++++++++++ weavebench/agents/claudecode_agent.py | 25 ++++++ weavebench/agents/codex_agent.py | 39 +++++++++ weavebench/agents/hermes_agent.py | 40 +++++++++ 4 files changed, 216 insertions(+) create mode 100644 tests/test_harness_max_steps.py diff --git a/tests/test_harness_max_steps.py b/tests/test_harness_max_steps.py new file mode 100644 index 0000000..a586554 --- /dev/null +++ b/tests/test_harness_max_steps.py @@ -0,0 +1,112 @@ +"""Tests for the four-harness max_steps alignment. + +openclaw already had a real step-cap watchdog; codex/claudecode/hermes used to +accept max_steps but never enforce it. These tests assert each harness now +renders a runner script that actually limits steps: + - claudecode: native `--max-turns N` flag on both call branches + - codex/hermes: a background watchdog that counts steps and kills the CLI + once it reaches max_steps * (1 + max_refusal_retries) +""" +from __future__ import annotations + +from weavebench.agents.codex_agent import CodexAgent +from weavebench.agents.claudecode_agent import ClaudeCodeAgent +from weavebench.agents.hermes_agent import HermesAgent + + +def _kw(): + return dict(model="gpt-5.5", litellm_base_url="http://x/v1", + litellm_api_key="k") + + +# --- claudecode: native --max-turns ----------------------------------------- + +def test_claudecode_injects_max_turns_both_branches(): + a = ClaudeCodeAgent(max_steps=7, **_kw()) + s = a._render_run_script(None) + # one for the first call, one for the resume branch + assert s.count("--max-turns 7") == 2 + + +def test_claudecode_writes_real_step_marker(): + a = ClaudeCodeAgent(max_steps=7, **_kw()) + s = a._render_run_script(None) + # runner emits a real MAX_STEPS_REACHED marker derived from claude's result + assert "MAX_STEPS_REACHED=$STEPS_CAPPED" in s + assert "error_max_turns" in s + assert "turns >= 7" in s + + +# --- codex: watchdog --------------------------------------------------------- + +def test_codex_watchdog_cap_formula(): + # default max_refusal_retries=3 -> cap = 5 * (1+3) = 20 + a = CodexAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert "WATCHDOG_CAP=20" in s + + +def test_codex_counts_function_calls(): + a = CodexAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert '"type":"function_call"' in s + assert "/root/.codex/sessions" in s + + +def test_codex_kills_codex_and_breaks_on_cap(): + a = CodexAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert "@openai/codex" in s # pkill target + # must stop the retry loop when the watchdog tripped (else OTHER_FAIL re-runs) + assert '[ -f "$STEPS_CAPPED" ] && break' in s + assert "kill $WATCH_PID" in s + + +def test_codex_emits_step_marker_to_log(): + a = CodexAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert "MAX_STEPS_REACHED=1 cap=" in s + assert "MAX_STEPS_REACHED=0 cap=" in s + + +# --- hermes: watchdog counting request_dump files --------------------------- + +def test_hermes_watchdog_cap_formula(): + a = HermesAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert "WATCHDOG_CAP=20" in s + + +def test_hermes_counts_assistant_in_session_json(): + a = HermesAgent(max_steps=5, **_kw()) + s = a._render_run_script() + # counts "role":"assistant" in the incrementally-written session_*.json — + # same unit openclaw counts (verified on a real VM: hermes produces + # session_*.json, NOT request_dump_*.json). + assert '"role": *"assistant"' in s + assert "session_*.json" in s + assert "request_dump" not in s # the disproven assumption must be gone + + +def test_hermes_kills_hermes_and_breaks_on_cap(): + a = HermesAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert "/opt/hermes/.venv/bin/hermes" in s # pkill target + assert '[ -f "$STEPS_CAPPED" ] && break' in s + assert "kill $WATCH_PID" in s + + +def test_hermes_emits_step_marker_to_log(): + a = HermesAgent(max_steps=5, **_kw()) + s = a._render_run_script() + assert "MAX_STEPS_REACHED=1 cap=" in s + + +# --- cross-harness: cap scales with max_steps ------------------------------- + +def test_cap_scales_with_max_steps(): + for C, render in ((CodexAgent, "_render_run_script"), + (HermesAgent, "_render_run_script")): + a = C(max_steps=10, **_kw()) # default retries=3 -> cap=40 + s = getattr(a, render)() + assert "WATCHDOG_CAP=40" in s diff --git a/weavebench/agents/claudecode_agent.py b/weavebench/agents/claudecode_agent.py index 5f697de..d310721 100644 --- a/weavebench/agents/claudecode_agent.py +++ b/weavebench/agents/claudecode_agent.py @@ -512,6 +512,10 @@ def run(self, env, instruction: str, output_dir: Path, fh.write(f"\n[claudecode_agent] AGENT_EXIT=" f"{exit_code if exit_code is not None else -1}\n") fh.write(f"[claudecode_agent] RETRIES_USED={retries_used}\n") + # The runner already wrote a real MAX_STEPS_REACHED=N marker + # (from claude's --max-turns result subtype) into retry.log. Only + # synthesize one here if the run timed out before the runner + # could emit it (no done-file => stream-json result never came). if not done: fh.write(f"[claudecode_agent] MAX_STEPS_REACHED=0 " f"timed_out_after={elapsed:.0f}s\n") @@ -559,6 +563,7 @@ def _render_run_script(self, system_override: Optional[str]) -> str: """ model_arg = shlex.quote(self.model) max_retries = self.max_refusal_retries + max_turns = self.max_steps recovery_msg = ( "The previous turn ended with a transient upstream provider error " "(silent fail / response.failed / connection error). This was an " @@ -622,6 +627,7 @@ def _render_run_script(self, system_override: Optional[str]) -> str: echo "$RECOVERY_MSG" | {CLAUDE_BIN_IN_VM} --print --output-format stream-json \\ --verbose --dangerously-skip-permissions --setting-sources user \\ --effort high --model {model_arg} --resume "$SESSION_ID" \\ + --max-turns {max_turns} \\ --disallowedTools "$DISALLOWED_TOOLS" \\ >> "$RUN_LOG" 2>> "$RETRY_LOG" else @@ -629,6 +635,7 @@ def _render_run_script(self, system_override: Optional[str]) -> str: {CLAUDE_BIN_IN_VM} --print --output-format stream-json \\ --verbose --dangerously-skip-permissions --setting-sources user \\ --effort high --model {model_arg} \\ + --max-turns {max_turns} \\ --disallowedTools "$DISALLOWED_TOOLS" \\ < "$PROMPT_FILE" >> "$RUN_LOG" 2>> "$RETRY_LOG" fi @@ -701,6 +708,24 @@ def _render_run_script(self, system_override: Optional[str]) -> str: echo "RETRIES_USED=$RETRIES_USED" >> "$RETRY_LOG" echo "AGENT_EXIT=$RC" >> "$RETRY_LOG" + +# Step-cap detection: claude --max-turns ends the final turn with a result +# event whose subtype is "error_max_turns" (and num_turns >= the cap). Surface +# it as a tail marker so run() / classify_trajectories.py can tell a step-capped +# run apart from a clean finish. Parity with openclaw's MAX_STEPS_REACHED. +LAST_RES=$(tac "$RUN_LOG" | grep -m1 '"type":"result"' || true) +STEPS_CAPPED=$(echo "$LAST_RES" | python3 -c ' +import sys, json +try: + r = json.loads(sys.stdin.read()) + sub = r.get("subtype", "") + turns = r.get("num_turns", 0) or 0 + print("1" if (sub == "error_max_turns" or turns >= {max_turns}) else "0") +except Exception: + print("0") +' 2>/dev/null || echo 0) +echo "MAX_STEPS_REACHED=$STEPS_CAPPED max_turns={max_turns}" >> "$RETRY_LOG" + echo $RC > "$DONE_FILE" exit $RC ''' diff --git a/weavebench/agents/codex_agent.py b/weavebench/agents/codex_agent.py index edff7bf..31874db 100644 --- a/weavebench/agents/codex_agent.py +++ b/weavebench/agents/codex_agent.py @@ -650,6 +650,9 @@ def run(self, env, instruction: str, output_dir: Path, with (output_dir / "agent.log").open("a", encoding="utf-8") as fh: fh.write(f"\n[codex_agent] AGENT_EXIT={exit_code if exit_code is not None else -1}\n") fh.write(f"[codex_agent] RETRIES_USED={retries_used}\n") + # The runner already wrote a real MAX_STEPS_REACHED=N marker (from + # the step-cap watchdog) into run.log => agent.log. Only synthesize + # a fallback when the run timed out before the runner finished. if not done: fh.write(f"[codex_agent] MAX_STEPS_REACHED=0 timed_out_after={elapsed:.0f}s\n") except OSError: @@ -698,6 +701,11 @@ def _render_run_script(self) -> str: """ display_export = 'export DISPLAY=:0' if self.gui else 'unset DISPLAY' max_retries = self.max_refusal_retries + # Step-cap budget: parity with openclaw. Each refusal-retry re-runs the + # whole codex exec from scratch (no resume), so each retry can spend up + # to max_steps more tool calls — bump the cap by (1 + max_retries) so the + # watchdog doesn't kill a legitimate retry sequence early. + watchdog_cap = self.max_steps * (1 + self.max_refusal_retries) return f'''#!/usr/bin/env bash # wcb codex runner v0.2 — retry-hardened (parity with openclaw Hook A) set -u @@ -710,6 +718,29 @@ def _render_run_script(self) -> str: RUN_LOG={CODEX_RUN_LOG} DONE_FILE={CODEX_RUN_DONE} MAX_RETRIES={max_retries} +STEPS_CAPPED={CODEX_RUN_DONE}.steps_capped +rm -f "$STEPS_CAPPED" + +# Step-cap watchdog (parity with openclaw): codex exec is a oneshot autonomous +# loop with no native max-turns flag, so we count tool calls across every +# rollout.jsonl (retries spawn new uuid dirs) and kill codex once the count +# reaches watchdog_cap = max_steps * (1 + max_refusal_retries). Each +# "type":"function_call" event in rollout.jsonl is one step. +WATCHDOG_CAP={watchdog_cap} +( + while : ; do + sleep 5 + n=$(find {CODEX_SESSIONS_DIR} -name '*.jsonl' -exec grep -o '"type":"function_call"' {{}} + 2>/dev/null | wc -l) + if [ "$n" -ge "$WATCHDOG_CAP" ]; then + echo "MAX_STEPS_REACHED ($n function_calls >= $WATCHDOG_CAP) — killing codex" >> "$RUN_LOG" + echo MAX_STEPS_REACHED > "$STEPS_CAPPED" + pkill -TERM -f '@openai/codex' 2>/dev/null || true + pkill -TERM -f '/usr/local/bin/codex' 2>/dev/null || true + break + fi + done +) & +WATCH_PID=$! TRY=0 RC=0 @@ -720,6 +751,8 @@ def _render_run_script(self) -> str: < "$PROMPT_FILE" >> "$RUN_LOG" 2>&1 RC=$? echo "AGENT_TURN_EXIT=$RC try=$TRY" >> "$RUN_LOG" + # Watchdog killed codex for hitting the step cap — stop, don't retry. + [ -f "$STEPS_CAPPED" ] && break [ "$TRY" -ge "$MAX_RETRIES" ] && break # Detect transient upstream errors in the LAST ~200 lines of run log. @@ -761,6 +794,12 @@ def _render_run_script(self) -> str: echo "RETRIES_USED=$RETRIES_USED" >> "$RUN_LOG" echo "AGENT_EXIT=$RC" >> "$RUN_LOG" +kill $WATCH_PID 2>/dev/null || true +if [ -f "$STEPS_CAPPED" ]; then + echo "MAX_STEPS_REACHED=1 cap=$WATCHDOG_CAP" >> "$RUN_LOG" +else + echo "MAX_STEPS_REACHED=0 cap=$WATCHDOG_CAP" >> "$RUN_LOG" +fi echo $RC > "$DONE_FILE" exit $RC ''' diff --git a/weavebench/agents/hermes_agent.py b/weavebench/agents/hermes_agent.py index 2f29a34..fcca765 100644 --- a/weavebench/agents/hermes_agent.py +++ b/weavebench/agents/hermes_agent.py @@ -867,6 +867,9 @@ def run(self, env, instruction: str, output_dir: Path, with (output_dir / "agent.log").open("a", encoding="utf-8") as fh: fh.write(f"\n[hermes_agent] AGENT_EXIT={exit_code if exit_code is not None else -1}\n") fh.write(f"[hermes_agent] RETRIES_USED={retries_used}\n") + # The runner already wrote a real MAX_STEPS_REACHED=N marker (from + # the step-cap watchdog) into run.log => agent.log. Only synthesize + # a fallback when the run timed out before the runner finished. if not done: fh.write(f"[hermes_agent] MAX_STEPS_REACHED=0 timed_out_after={elapsed:.0f}s\n") except OSError: @@ -939,6 +942,10 @@ def _render_run_script(self) -> str: """ display_export = "export DISPLAY=:0" if self.gui else "unset DISPLAY" max_retries = self.max_refusal_retries + # Step-cap budget: parity with openclaw. Each refusal-retry re-runs the + # whole hermes -z oneshot from scratch, so bump the cap by + # (1 + max_refusal_retries) so the watchdog tolerates retry sequences. + watchdog_cap = self.max_steps * (1 + self.max_refusal_retries) return f'''#!/usr/bin/env bash # wcb hermes runner v0.2 — retry-hardened (parity with openclaw Hook A) set -u @@ -970,6 +977,31 @@ def _render_run_script(self) -> str: RUN_LOG={HERMES_RUN_LOG} DONE_FILE={HERMES_RUN_DONE} MAX_RETRIES={max_retries} +STEPS_CAPPED={HERMES_RUN_DONE}.steps_capped +rm -f "$STEPS_CAPPED" + +# Step-cap watchdog (parity with openclaw): hermes -z is a oneshot autonomous +# loop with no native max-turns flag. hermes writes its transcript incrementally +# to a single session_*.json (.messages array); we count "role":"assistant" +# occurrences in it — the SAME unit openclaw counts in chat.jsonl. We grep the +# raw bytes rather than parse JSON so a half-written file just under-counts by +# one and recovers next poll (no crash). Kill hermes once the count reaches +# watchdog_cap = max_steps * (1 + max_refusal_retries). +WATCHDOG_CAP={watchdog_cap} +( + while : ; do + sleep 5 + n=$(grep -o '"role": *"assistant"' {HERMES_SESSIONS_DIR}/session_*.json 2>/dev/null | wc -l) + if [ "$n" -ge "$WATCHDOG_CAP" ]; then + echo "MAX_STEPS_REACHED ($n assistant replies >= $WATCHDOG_CAP) — killing hermes" >> "$RUN_LOG" + echo MAX_STEPS_REACHED > "$STEPS_CAPPED" + pkill -TERM -f '/opt/hermes/.venv/bin/hermes' 2>/dev/null || true + pkill -TERM -f 'hermes_cli' 2>/dev/null || true + break + fi + done +) & +WATCH_PID=$! TRY=0 RC=0 @@ -983,6 +1015,8 @@ def _render_run_script(self) -> str: >> "$RUN_LOG" 2>&1 RC=$? echo "AGENT_TURN_EXIT=$RC try=$TRY" >> "$RUN_LOG" + # Watchdog killed hermes for hitting the step cap — stop, don't retry. + [ -f "$STEPS_CAPPED" ] && break [ "$TRY" -ge "$MAX_RETRIES" ] && break TAIL=$(tail -200 "$RUN_LOG" 2>/dev/null) @@ -1025,6 +1059,12 @@ def _render_run_script(self) -> str: echo "RETRIES_USED=$RETRIES_USED" >> "$RUN_LOG" echo "AGENT_EXIT=$RC" >> "$RUN_LOG" +kill $WATCH_PID 2>/dev/null || true +if [ -f "$STEPS_CAPPED" ]; then + echo "MAX_STEPS_REACHED=1 cap=$WATCHDOG_CAP" >> "$RUN_LOG" +else + echo "MAX_STEPS_REACHED=0 cap=$WATCHDOG_CAP" >> "$RUN_LOG" +fi echo $RC > "$DONE_FILE" exit $RC '''