From 1564713670454b381f56cc2071d6b08fdc10fb23 Mon Sep 17 00:00:00 2001 From: Wanli-Lee <1181451942@qq.com> Date: Wed, 1 Jul 2026 12:30:16 +0800 Subject: [PATCH] fix: don't flag codex/hermes runs as empty_run on unparseable token stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _classify_agent_status called a run "empty_run" whenever aggregate_chat_jsonl reported n_calls=0 / output=0. But aggregate_chat_jsonl only understands openclaw's message.usage records — on codex/hermes (different chat.jsonl schema) it returns zero counts even for a full multi-MB transcript. This mislabeled real, high-scoring codex runs (e.g. a task that scored 0.984) as empty_run, polluting the agent_status / error fields with false positives. Guard the empty_run verdict with the chat.jsonl size: a transcript at or above NONEMPTY_CHAT_BYTES (4 KB) proves the agent ran, so we never call it empty on the strength of an uncountable token stat alone. Genuine empty/failed runs leave a tiny or absent chat.jsonl and are still caught. Only the agent_status / error labels were affected — scores were never touched. Co-Authored-By: Claude --- tests/test_run_one_aj.py | 25 ++++++++++++++++++----- weavebench/eval/agent_judge/run_one_aj.py | 23 +++++++++++++++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/tests/test_run_one_aj.py b/tests/test_run_one_aj.py index aec102e..75b46b3 100644 --- a/tests/test_run_one_aj.py +++ b/tests/test_run_one_aj.py @@ -106,9 +106,11 @@ def test_classify_crashed_on_nonzero_exit(): def test_classify_empty_run_retry_exhausted(): """THE bug: clean process signals (done True, exit 0) but no model output — - retry-exhausted upstream failure must be caught as empty_run, not ok.""" + retry-exhausted upstream failure must be caught as empty_run, not ok. + chat_bytes=0 because a genuine empty run leaves a tiny/absent chat.jsonl.""" v = _classify_agent_status( - True, {"agent_exit": 0, "retries_used": 3}, _tu(n_calls=0, output=0)) + True, {"agent_exit": 0, "retries_used": 3}, _tu(n_calls=0, output=0), + chat_bytes=0) assert v["status"] == "empty_run" assert "retries_used=3" in v["reason"] @@ -116,14 +118,27 @@ def test_classify_empty_run_retry_exhausted(): def test_classify_empty_run_bare_refusal(): """A bare refusal emits a few dozen tokens but no real work → still empty.""" v = _classify_agent_status( - True, {"agent_exit": 0, "retries_used": 0}, _tu(n_calls=1, output=5)) + True, {"agent_exit": 0, "retries_used": 0}, _tu(n_calls=1, output=5), + chat_bytes=200) assert v["status"] == "empty_run" +def test_classify_not_empty_when_chat_is_large(): + """Regression: aggregate_chat_jsonl only parses openclaw's message.usage, so + on codex/hermes it reports n_calls=0 even for a rich multi-MB transcript. + A large chat.jsonl must veto the empty_run verdict — the agent clearly ran + (verified: a codex task scored 0.984 while token stats showed n_calls=0).""" + v = _classify_agent_status( + True, {"agent_exit": 0, "retries_used": 0}, _tu(n_calls=0, output=0), + chat_bytes=4_887_014) # real size of the misclassified codex chat.jsonl + assert v["status"] == "ok" + + def test_classify_crashed_takes_precedence_over_empty(): """Non-zero exit is reported as crashed even when tokens are also ~0.""" v = _classify_agent_status( - True, {"agent_exit": 1, "retries_used": 2}, _tu(n_calls=0, output=0)) + True, {"agent_exit": 1, "retries_used": 2}, _tu(n_calls=0, output=0), + chat_bytes=0) assert v["status"] == "crashed" @@ -132,5 +147,5 @@ def test_classify_ok_when_token_usage_unavailable(): clean-exit run is NOT downgraded to empty_run on missing evidence.""" v = _classify_agent_status( True, {"agent_exit": 0, "retries_used": 0}, - {"ok": False, "error": "chat.jsonl missing"}) + {"ok": False, "error": "chat.jsonl missing"}, chat_bytes=0) assert v["status"] == "ok" diff --git a/weavebench/eval/agent_judge/run_one_aj.py b/weavebench/eval/agent_judge/run_one_aj.py index 5dd5aa5..028e9f5 100644 --- a/weavebench/eval/agent_judge/run_one_aj.py +++ b/weavebench/eval/agent_judge/run_one_aj.py @@ -99,8 +99,17 @@ def _parse_agent_log_markers(agent_log: Path) -> dict: # because a bare refusal ("I'm sorry, ...") still emits a few dozen tokens. EMPTY_RUN_OUTPUT_TOKENS = 8 +# aggregate_chat_jsonl only understands openclaw's message.usage records, so on +# codex/hermes (whose chat.jsonl uses a different record schema) it reports +# n_calls=0 / output=0 even for a rich, multi-MB transcript. A chat.jsonl this +# large therefore proves the agent DID run — we must not call it an empty_run on +# the strength of an unparseable token count alone. Empty/failed runs leave a +# tiny (or absent) chat.jsonl, well under this bound. +NONEMPTY_CHAT_BYTES = 4096 -def _classify_agent_status(agent_done, markers: dict, token_usage: dict) -> dict: + +def _classify_agent_status(agent_done, markers: dict, token_usage: dict, + chat_bytes: int = 0) -> dict: """Unified post-run health verdict for ALL four harnesses. Returns a dict {status, reason, agent_exit, retries_used} where status is: @@ -134,11 +143,15 @@ def _classify_agent_status(agent_done, markers: dict, token_usage: dict) -> dict return verdict # agent_done True (or unknown) and no non-zero exit: did the model actually - # produce anything? Use the token aggregation we already computed. + # produce anything? Use the token aggregation we already computed. But guard + # against aggregate_chat_jsonl's openclaw-only parsing: on codex/hermes it + # returns n_calls=0 even for a full transcript, so a large chat.jsonl vetoes + # the empty_run verdict (the agent clearly ran; we just can't count tokens). tu_ok = bool(token_usage.get("ok")) n_calls = token_usage.get("n_calls") or 0 output = token_usage.get("output") or 0 - if tu_ok and (n_calls == 0 or output <= EMPTY_RUN_OUTPUT_TOKENS): + if (tu_ok and (n_calls == 0 or output <= EMPTY_RUN_OUTPUT_TOKENS) + and chat_bytes < NONEMPTY_CHAT_BYTES): verdict["status"] = "empty_run" rsuffix = f", retries_used={retries_used}" if retries_used else "" verdict["reason"] = (f"clean exit but no model output " @@ -211,9 +224,11 @@ def run_one_aj(env, agent: "OpenClawAgent", task: dict, mode: str, # agent_done=True + AGENT_EXIT=0 alone cannot distinguish. agent_log = output_dir / "agent.log" markers = _parse_agent_log_markers(agent_log) + chat_path = output_dir / "chat.jsonl" + chat_bytes = chat_path.stat().st_size if chat_path.exists() else 0 verdict = _classify_agent_status( record.get("agent_done"), markers, - record.get("agent_token_usage") or {}) + record.get("agent_token_usage") or {}, chat_bytes) record["agent_status"] = verdict["status"] record["retries_used"] = verdict["retries_used"] record["agent_exit"] = verdict["agent_exit"]