weavebench · Wanli-Lee · Jul 1, 2026 · Jun 30, 2026
diff --git a/tests/test_harness_max_steps.py b/tests/test_harness_max_steps.py
@@ -0,0 +1,112 @@
+"""Tests for the four-harness max_steps alignment.
+
+openclaw already had a real step-cap watchdog; codex/claudecode/hermes used to
+accept max_steps but never enforce it. These tests assert each harness now
+renders a runner script that actually limits steps:
+  - claudecode: native `--max-turns N` flag on both call branches
+  - codex/hermes: a background watchdog that counts steps and kills the CLI
+    once it reaches max_steps * (1 + max_refusal_retries)
+"""
+from __future__ import annotations
+
+from weavebench.agents.codex_agent import CodexAgent
+from weavebench.agents.claudecode_agent import ClaudeCodeAgent
+from weavebench.agents.hermes_agent import HermesAgent
+
+
+def _kw():
+    return dict(model="gpt-5.5", litellm_base_url="http://x/v1",
+                litellm_api_key="k")
+
+
+# --- claudecode: native --max-turns -----------------------------------------
+
+def test_claudecode_injects_max_turns_both_branches():
+    a = ClaudeCodeAgent(max_steps=7, **_kw())
+    s = a._render_run_script(None)
+    # one for the first call, one for the resume branch
+    assert s.count("--max-turns 7") == 2
+
+
+def test_claudecode_writes_real_step_marker():
+    a = ClaudeCodeAgent(max_steps=7, **_kw())
+    s = a._render_run_script(None)
+    # runner emits a real MAX_STEPS_REACHED marker derived from claude's result
+    assert "MAX_STEPS_REACHED=$STEPS_CAPPED" in s
+    assert "error_max_turns" in s
+    assert "turns >= 7" in s
+
+
+# --- codex: watchdog ---------------------------------------------------------
+
+def test_codex_watchdog_cap_formula():
+    # default max_refusal_retries=3 -> cap = 5 * (1+3) = 20
+    a = CodexAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert "WATCHDOG_CAP=20" in s
+
+
+def test_codex_counts_function_calls():
+    a = CodexAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert '"type":"function_call"' in s
+    assert "/root/.codex/sessions" in s
+
+
+def test_codex_kills_codex_and_breaks_on_cap():
+    a = CodexAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert "@openai/codex" in s          # pkill target
+    # must stop the retry loop when the watchdog tripped (else OTHER_FAIL re-runs)
+    assert '[ -f "$STEPS_CAPPED" ] && break' in s
+    assert "kill $WATCH_PID" in s
+
+
+def test_codex_emits_step_marker_to_log():
+    a = CodexAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert "MAX_STEPS_REACHED=1 cap=" in s
+    assert "MAX_STEPS_REACHED=0 cap=" in s
+
+
+# --- hermes: watchdog counting request_dump files ---------------------------
+
+def test_hermes_watchdog_cap_formula():
+    a = HermesAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert "WATCHDOG_CAP=20" in s
+
+
+def test_hermes_counts_assistant_in_session_json():
+    a = HermesAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    # counts "role":"assistant" in the incrementally-written session_*.json —
+    # same unit openclaw counts (verified on a real VM: hermes produces
+    # session_*.json, NOT request_dump_*.json).
+    assert '"role": *"assistant"' in s
+    assert "session_*.json" in s
+    assert "request_dump" not in s  # the disproven assumption must be gone
+
+
+def test_hermes_kills_hermes_and_breaks_on_cap():
+    a = HermesAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert "/opt/hermes/.venv/bin/hermes" in s   # pkill target
+    assert '[ -f "$STEPS_CAPPED" ] && break' in s
+    assert "kill $WATCH_PID" in s
+
+
+def test_hermes_emits_step_marker_to_log():
+    a = HermesAgent(max_steps=5, **_kw())
+    s = a._render_run_script()
+    assert "MAX_STEPS_REACHED=1 cap=" in s
+
+
+# --- cross-harness: cap scales with max_steps -------------------------------
+
+def test_cap_scales_with_max_steps():
+    for C, render in ((CodexAgent, "_render_run_script"),
+                      (HermesAgent, "_render_run_script")):
+        a = C(max_steps=10, **_kw())  # default retries=3 -> cap=40
+        s = getattr(a, render)()
+        assert "WATCHDOG_CAP=40" in s
diff --git a/weavebench/agents/claudecode_agent.py b/weavebench/agents/claudecode_agent.py
@@ -512,6 +512,10 @@ def run(self, env, instruction: str, output_dir: Path,
                 fh.write(f"\n[claudecode_agent] AGENT_EXIT="
                          f"{exit_code if exit_code is not None else -1}\n")
                 fh.write(f"[claudecode_agent] RETRIES_USED={retries_used}\n")
+                # The runner already wrote a real MAX_STEPS_REACHED=N marker
+                # (from claude's --max-turns result subtype) into retry.log. Only
+                # synthesize one here if the run timed out before the runner
+                # could emit it (no done-file => stream-json result never came).
                 if not done:
                     fh.write(f"[claudecode_agent] MAX_STEPS_REACHED=0 "
                              f"timed_out_after={elapsed:.0f}s\n")
@@ -559,6 +563,7 @@ def _render_run_script(self, system_override: Optional[str]) -> str:
         """
         model_arg = shlex.quote(self.model)
         max_retries = self.max_refusal_retries
+        max_turns = self.max_steps
         recovery_msg = (
             "The previous turn ended with a transient upstream provider error "
             "(silent fail / response.failed / connection error). This was an "
@@ -622,13 +627,15 @@ def _render_run_script(self, system_override: Optional[str]) -> str:
     echo "$RECOVERY_MSG" | {CLAUDE_BIN_IN_VM} --print --output-format stream-json \\
       --verbose --dangerously-skip-permissions --setting-sources user \\
       --effort high --model {model_arg} --resume "$SESSION_ID" \\
+      --max-turns {max_turns} \\
       --disallowedTools "$DISALLOWED_TOOLS" \\
       >> "$RUN_LOG" 2>> "$RETRY_LOG"
   else
     echo "=== claudecode turn (try=0, first) ===" >> "$RETRY_LOG"
     {CLAUDE_BIN_IN_VM} --print --output-format stream-json \\
       --verbose --dangerously-skip-permissions --setting-sources user \\
       --effort high --model {model_arg} \\
+      --max-turns {max_turns} \\
       --disallowedTools "$DISALLOWED_TOOLS" \\
       < "$PROMPT_FILE" >> "$RUN_LOG" 2>> "$RETRY_LOG"
   fi
@@ -701,6 +708,24 @@ def _render_run_script(self, system_override: Optional[str]) -> str:
 
 echo "RETRIES_USED=$RETRIES_USED" >> "$RETRY_LOG"
 echo "AGENT_EXIT=$RC" >> "$RETRY_LOG"
+
+# Step-cap detection: claude --max-turns ends the final turn with a result
+# event whose subtype is "error_max_turns" (and num_turns >= the cap). Surface
+# it as a tail marker so run() / classify_trajectories.py can tell a step-capped
+# run apart from a clean finish. Parity with openclaw's MAX_STEPS_REACHED.
+LAST_RES=$(tac "$RUN_LOG" | grep -m1 '"type":"result"' || true)
+STEPS_CAPPED=$(echo "$LAST_RES" | python3 -c '
+import sys, json
+try:
+    r = json.loads(sys.stdin.read())
+    sub = r.get("subtype", "")
+    turns = r.get("num_turns", 0) or 0
+    print("1" if (sub == "error_max_turns" or turns >= {max_turns}) else "0")
+except Exception:
+    print("0")
+' 2>/dev/null || echo 0)
+echo "MAX_STEPS_REACHED=$STEPS_CAPPED max_turns={max_turns}" >> "$RETRY_LOG"
+
 echo $RC > "$DONE_FILE"
 exit $RC
 '''

diff --git a/weavebench/agents/codex_agent.py b/weavebench/agents/codex_agent.py
@@ -650,6 +650,9 @@ def run(self, env, instruction: str, output_dir: Path,
             with (output_dir / "agent.log").open("a", encoding="utf-8") as fh:
                 fh.write(f"\n[codex_agent] AGENT_EXIT={exit_code if exit_code is not None else -1}\n")
                 fh.write(f"[codex_agent] RETRIES_USED={retries_used}\n")
+                # The runner already wrote a real MAX_STEPS_REACHED=N marker (from
+                # the step-cap watchdog) into run.log => agent.log. Only synthesize
+                # a fallback when the run timed out before the runner finished.
                 if not done:
                     fh.write(f"[codex_agent] MAX_STEPS_REACHED=0 timed_out_after={elapsed:.0f}s\n")
         except OSError:
@@ -698,6 +701,11 @@ def _render_run_script(self) -> str:
         """
         display_export = 'export DISPLAY=:0' if self.gui else 'unset DISPLAY'
         max_retries = self.max_refusal_retries
+        # Step-cap budget: parity with openclaw. Each refusal-retry re-runs the
+        # whole codex exec from scratch (no resume), so each retry can spend up
+        # to max_steps more tool calls — bump the cap by (1 + max_retries) so the
+        # watchdog doesn't kill a legitimate retry sequence early.
+        watchdog_cap = self.max_steps * (1 + self.max_refusal_retries)
         return f'''#!/usr/bin/env bash
 # wcb codex runner v0.2 — retry-hardened (parity with openclaw Hook A)
 set -u
@@ -710,6 +718,29 @@ def _render_run_script(self) -> str:
 RUN_LOG={CODEX_RUN_LOG}
 DONE_FILE={CODEX_RUN_DONE}
 MAX_RETRIES={max_retries}
+STEPS_CAPPED={CODEX_RUN_DONE}.steps_capped
+rm -f "$STEPS_CAPPED"
+
+# Step-cap watchdog (parity with openclaw): codex exec is a oneshot autonomous
+# loop with no native max-turns flag, so we count tool calls across every
+# rollout.jsonl (retries spawn new uuid dirs) and kill codex once the count
+# reaches watchdog_cap = max_steps * (1 + max_refusal_retries). Each
+# "type":"function_call" event in rollout.jsonl is one step.
+WATCHDOG_CAP={watchdog_cap}
+(
+  while : ; do
+    sleep 5
+    n=$(find {CODEX_SESSIONS_DIR} -name '*.jsonl' -exec grep -o '"type":"function_call"' {{}} + 2>/dev/null | wc -l)
+    if [ "$n" -ge "$WATCHDOG_CAP" ]; then
+      echo "MAX_STEPS_REACHED ($n function_calls >= $WATCHDOG_CAP) — killing codex" >> "$RUN_LOG"
+      echo MAX_STEPS_REACHED > "$STEPS_CAPPED"
+      pkill -TERM -f '@openai/codex' 2>/dev/null || true
+      pkill -TERM -f '/usr/local/bin/codex' 2>/dev/null || true
+      break
+    fi
+  done
+) &
+WATCH_PID=$!
 
 TRY=0
 RC=0
@@ -720,6 +751,8 @@ def _render_run_script(self) -> str:
     < "$PROMPT_FILE" >> "$RUN_LOG" 2>&1
   RC=$?
   echo "AGENT_TURN_EXIT=$RC try=$TRY" >> "$RUN_LOG"
+  # Watchdog killed codex for hitting the step cap — stop, don't retry.
+  [ -f "$STEPS_CAPPED" ] && break
   [ "$TRY" -ge "$MAX_RETRIES" ] && break
 
   # Detect transient upstream errors in the LAST ~200 lines of run log.
@@ -761,6 +794,12 @@ def _render_run_script(self) -> str:
 
 echo "RETRIES_USED=$RETRIES_USED" >> "$RUN_LOG"
 echo "AGENT_EXIT=$RC" >> "$RUN_LOG"
+kill $WATCH_PID 2>/dev/null || true
+if [ -f "$STEPS_CAPPED" ]; then
+  echo "MAX_STEPS_REACHED=1 cap=$WATCHDOG_CAP" >> "$RUN_LOG"
+else
+  echo "MAX_STEPS_REACHED=0 cap=$WATCHDOG_CAP" >> "$RUN_LOG"
+fi
 echo $RC > "$DONE_FILE"
 exit $RC
 '''

diff --git a/weavebench/agents/hermes_agent.py b/weavebench/agents/hermes_agent.py
@@ -867,6 +867,9 @@ def run(self, env, instruction: str, output_dir: Path,
             with (output_dir / "agent.log").open("a", encoding="utf-8") as fh:
                 fh.write(f"\n[hermes_agent] AGENT_EXIT={exit_code if exit_code is not None else -1}\n")
                 fh.write(f"[hermes_agent] RETRIES_USED={retries_used}\n")
+                # The runner already wrote a real MAX_STEPS_REACHED=N marker (from
+                # the step-cap watchdog) into run.log => agent.log. Only synthesize
+                # a fallback when the run timed out before the runner finished.
                 if not done:
                     fh.write(f"[hermes_agent] MAX_STEPS_REACHED=0 timed_out_after={elapsed:.0f}s\n")
         except OSError:
@@ -939,6 +942,10 @@ def _render_run_script(self) -> str:
         """
         display_export = "export DISPLAY=:0" if self.gui else "unset DISPLAY"
         max_retries = self.max_refusal_retries
+        # Step-cap budget: parity with openclaw. Each refusal-retry re-runs the
+        # whole hermes -z oneshot from scratch, so bump the cap by
+        # (1 + max_refusal_retries) so the watchdog tolerates retry sequences.
+        watchdog_cap = self.max_steps * (1 + self.max_refusal_retries)
         return f'''#!/usr/bin/env bash
 # wcb hermes runner v0.2 — retry-hardened (parity with openclaw Hook A)
 set -u
@@ -970,6 +977,31 @@ def _render_run_script(self) -> str:
 RUN_LOG={HERMES_RUN_LOG}
 DONE_FILE={HERMES_RUN_DONE}
 MAX_RETRIES={max_retries}
+STEPS_CAPPED={HERMES_RUN_DONE}.steps_capped
+rm -f "$STEPS_CAPPED"
+
+# Step-cap watchdog (parity with openclaw): hermes -z is a oneshot autonomous
+# loop with no native max-turns flag. hermes writes its transcript incrementally
+# to a single session_*.json (.messages array); we count "role":"assistant"
+# occurrences in it — the SAME unit openclaw counts in chat.jsonl. We grep the
+# raw bytes rather than parse JSON so a half-written file just under-counts by
+# one and recovers next poll (no crash). Kill hermes once the count reaches
+# watchdog_cap = max_steps * (1 + max_refusal_retries).
+WATCHDOG_CAP={watchdog_cap}
+(
+  while : ; do
+    sleep 5
+    n=$(grep -o '"role": *"assistant"' {HERMES_SESSIONS_DIR}/session_*.json 2>/dev/null | wc -l)
+    if [ "$n" -ge "$WATCHDOG_CAP" ]; then
+      echo "MAX_STEPS_REACHED ($n assistant replies >= $WATCHDOG_CAP) — killing hermes" >> "$RUN_LOG"
+      echo MAX_STEPS_REACHED > "$STEPS_CAPPED"
+      pkill -TERM -f '/opt/hermes/.venv/bin/hermes' 2>/dev/null || true
+      pkill -TERM -f 'hermes_cli' 2>/dev/null || true
+      break
+    fi
+  done
+) &
+WATCH_PID=$!
 
 TRY=0
 RC=0
@@ -983,6 +1015,8 @@ def _render_run_script(self) -> str:
     >> "$RUN_LOG" 2>&1
   RC=$?
   echo "AGENT_TURN_EXIT=$RC try=$TRY" >> "$RUN_LOG"
+  # Watchdog killed hermes for hitting the step cap — stop, don't retry.
+  [ -f "$STEPS_CAPPED" ] && break
   [ "$TRY" -ge "$MAX_RETRIES" ] && break
 
   TAIL=$(tail -200 "$RUN_LOG" 2>/dev/null)
@@ -1025,6 +1059,12 @@ def _render_run_script(self) -> str:
 
 echo "RETRIES_USED=$RETRIES_USED" >> "$RUN_LOG"
 echo "AGENT_EXIT=$RC" >> "$RUN_LOG"
+kill $WATCH_PID 2>/dev/null || true
+if [ -f "$STEPS_CAPPED" ]; then
+  echo "MAX_STEPS_REACHED=1 cap=$WATCHDOG_CAP" >> "$RUN_LOG"
+else
+  echo "MAX_STEPS_REACHED=0 cap=$WATCHDOG_CAP" >> "$RUN_LOG"
+fi
 echo $RC > "$DONE_FILE"
 exit $RC
 '''