From e446cecf2cd76af07485c63daf8bbda63ca8750c Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sat, 15 Aug 2026 02:36:52 -0400 Subject: [PATCH 1/3] fix(recipe): strip the carriage return a Windows jq emits jq's native Windows build opens stdout in text mode, so every `\n` it writes arrives as `\r\n`. Both shipped recipes read that output as paths, and neither reader drops the CR: `read -r` keeps it by definition, and `tr '\n' '\0'` converts only the newline. The carriage return then rides on the end of every path. The judge recipe reports `0/N verdicts present` because `[ -s "$response_path" ]` matches nothing, and the parallel-dispatch recipe hands every task a corrupted `eval_root`, `dispatch_prompt_path`, and `outputs_dir`. Git Bash with `jq` is the setup the tooling requirement documents, so this is the supported Windows path rather than an exotic one. Pipe each jq call through `tr -d '\r'`: a no-op wherever jq already writes LF, and `tr` is a declared requirement alongside `jq` itself. The regression test defines a CRLF-emitting `jq` as a shell function in front of the recipe rather than shimming `PATH`, so the failure mode is covered on every host instead of only where a Windows jq is installed. Co-Authored-By: Claude Opus 5 --- src/adapters/cli_command.rs | 91 ++++++++++++++++++- src/adapters/descriptor_adapter.rs | 4 +- .../golden/claude-code/judge-recipe.golden.md | 4 +- .../claude-code/manifest-nomodel.golden.md | 1 + tests/golden/claude-code/manifest.golden.md | 1 + tests/golden/claude-code/runbook.golden.md | 4 +- tests/golden/cline/judge-recipe.golden.md | 4 +- tests/golden/cline/manifest.golden.md | 1 + tests/golden/cline/runbook.golden.md | 4 +- .../codex/judge-recipe-noguard.golden.md | 4 +- tests/golden/codex/judge-recipe.golden.md | 4 +- tests/golden/codex/manifest-noguard.golden.md | 1 + tests/golden/codex/manifest.golden.md | 1 + tests/golden/codex/runbook.golden.md | 4 +- tests/golden/opencode/judge-recipe.golden.md | 4 +- tests/golden/opencode/manifest.golden.md | 1 + tests/golden/opencode/runbook.golden.md | 4 +- 17 files changed, 125 insertions(+), 12 deletions(-) diff --git a/src/adapters/cli_command.rs b/src/adapters/cli_command.rs index 6fcf39a..41e99d6 100644 --- a/src/adapters/cli_command.rs +++ b/src/adapters/cli_command.rs @@ -70,6 +70,10 @@ pub(crate) fn render_cli_model_arg(flag: Option<&str>, model: Option<&str>) -> S /// instead, where they do not survive argument passing — jq then emits no /// separators and `xargs -0` collapses every field into one bogus dispatch /// that exits 0. Paths containing a newline remain unsupported, as before. +/// +/// `tr -d '\r'` runs before that: jq's native Windows build writes CRLF, and +/// `tr '\n' '\0'` converts only the newline, so without it every path reaches +/// the dispatch with a carriage return on the end. pub(crate) fn render_parallel_dispatch_recipe( command_block: &str, one_shot_only: bool, @@ -85,6 +89,7 @@ pub(crate) fn render_parallel_dispatch_recipe( format!( "jq -r '{tasks} | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \\" ), + " | tr -d '\\r' \\".to_string(), " | tr '\\n' '\\0' \\".to_string(), " | xargs -0 -P \"$JOBS\" -n 3 sh -c '".to_string(), " eval_root=\"$1\"".to_string(), @@ -109,6 +114,11 @@ pub(crate) fn render_parallel_dispatch_recipe( /// model, ` ` otherwise) and end with ` \`; `model_flag` fills the /// `model_arg` assignment; `capture_prefix` names the per-task /// `$response_base.-events.jsonl` / `.-stderr.log` captures. +/// +/// Every jq call is piped through `tr -d '\r'`: jq's native Windows build +/// writes CRLF, and none of the three readers here drop it — `read -r` keeps a +/// carriage return by definition, `tr '\n' '\0'` converts only the newline, and +/// `[ "$judge_present" -eq "$judge_total" ]` needs a bare integer. pub(crate) fn render_judge_dispatch_recipe( command_line: &str, model_flag: &str, @@ -124,6 +134,7 @@ pub(crate) fn render_judge_dispatch_recipe( "```bash".to_string(), "JOBS=${JOBS:-4}".to_string(), "jq -r '.tasks[] | .dispatch_prompt_path, .response_path, (\"model=\" + (.model // \"\"))' judge-tasks.json \\".to_string(), + " | tr -d '\\r' \\".to_string(), " | tr '\\n' '\\0' \\".to_string(), " | xargs -0 -P \"$JOBS\" -n 3 sh -c '".to_string(), " prompt_path=\"$1\"".to_string(), @@ -140,9 +151,10 @@ pub(crate) fn render_judge_dispatch_recipe( format!(" 2> \"$response_base.{capture_prefix}-stderr.log\""), " ' sh".to_string(), "judge_dispatch_status=$?".to_string(), - "judge_total=$(jq '.tasks | length' judge-tasks.json)".to_string(), + "judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\\r')".to_string(), "judge_present=$(".to_string(), " jq -r '.tasks[].response_path' judge-tasks.json \\".to_string(), + " | tr -d '\\r' \\".to_string(), " | while IFS= read -r response_path; do".to_string(), " if [ -s \"$response_path\" ]; then printf '%s\\n' \"$response_path\"; fi" .to_string(), @@ -208,7 +220,26 @@ mod tests { } } + /// A `jq` that ends every line with CRLF, the way jq's native Windows build + /// does with stdout in text mode. A shell function rather than a `PATH` + /// shim: nothing has to be marked executable, so it reads and behaves the + /// same on every host, and only the outer pipeline calls jq — the `xargs` + /// child never does, so it does not need the definition. + const CRLF_JQ: &str = "jq() { command jq \"$@\" | tr -d '\\r' \ + | while IFS= read -r line; do printf '%s\\r\\n' \"$line\"; done; }\n"; + fn run_judge_recipe(shell: &Path, cwd: &Path, command_line: &str) -> Output { + run_judge_recipe_prefixed(shell, cwd, command_line, "") + } + + /// Run the rendered recipe with `preamble` in front of it, so a test can + /// replace a tool the recipe shells out to. + fn run_judge_recipe_prefixed( + shell: &Path, + cwd: &Path, + command_line: &str, + preamble: &str, + ) -> Output { let recipe = render_judge_dispatch_recipe(command_line, "--model", "judge"); let program = recipe .split_once("```bash\n") @@ -218,7 +249,7 @@ mod tests { .unwrap(); Command::new(shell) .arg("-c") - .arg(program) + .arg(format!("{preamble}{program}")) .current_dir(cwd) .env("JOBS", "1") .output() @@ -373,6 +404,62 @@ mod tests { ); } + /// jq's native Windows build opens stdout in text mode, so every `\n` it + /// writes arrives as `\r\n`. The recipes read that output as paths, and + /// neither reader drops the CR: `read -r` keeps it by definition, and + /// `tr '\n' '\0'` converts only the newline. The carriage return then rides + /// on the end of every path — `[ -s "$response_path" ]` matches nothing, the + /// summary reports zero verdicts present, and each dispatched task gets a + /// corrupted `$eval_root`. Git Bash with `jq` installed is the documented + /// Windows setup, so jq's output has to be normalised before anything reads + /// it. Same expectations as the plain-jq partial-completion test above; only + /// jq's line endings differ. + #[test] + fn judge_recipe_counts_verdicts_when_jq_emits_crlf() { + let Some(shell) = recipe_shell("judge_recipe_counts_verdicts_when_jq_emits_crlf") else { + return; + }; + let tmp = tempfile::TempDir::new().unwrap(); + let responses_dir = tmp.path().join("judge responses"); + fs::create_dir_all(&responses_dir).unwrap(); + let existing_response = responses_dir.join("existing.json"); + let missing_response = responses_dir.join("missing.json"); + fs::write(&existing_response, "{}\n").unwrap(); + write_judge_tasks(tmp.path(), &[&existing_response, &missing_response]); + + let output = + run_judge_recipe_prefixed(shell, tmp.path(), " true $model_arg \\", CRLF_JQ); + + assert!(!output.status.success(), "{output:?}"); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "1/2 verdicts present\n", + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + /// Every recipe that reads jq's output has to strip the carriage return + /// jq's Windows build adds, and it has to do so once per call: the judge + /// recipe pipes jq into `xargs`, into `read`, and into an arithmetic + /// comparison, and a CR left on any one of them breaks that stage alone. + #[test] + fn recipes_strip_the_carriage_return_a_windows_jq_emits() { + for recipe in [ + render_parallel_dispatch_recipe(" run \"$eval_root\"", false, &BTreeMap::new()), + render_parallel_dispatch_recipe(" run \"$eval_root\"", true, &BTreeMap::new()), + render_judge_dispatch_recipe(" judge $model_arg \\", "--model", "judge"), + ] { + let calls = recipe.lines().filter(|line| line.contains("jq ")).count(); + assert!(calls > 0, "{recipe}"); + assert_eq!( + calls, + recipe.matches("tr -d '\\r'").count(), + "every jq call needs its own CR strip\n{recipe}" + ); + } + } + #[test] fn judge_recipe_reports_complete_resumed_batch_and_exits_zero() { let Some(shell) = diff --git a/src/adapters/descriptor_adapter.rs b/src/adapters/descriptor_adapter.rs index 811939f..4399499 100644 --- a/src/adapters/descriptor_adapter.rs +++ b/src/adapters/descriptor_adapter.rs @@ -799,6 +799,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -815,9 +816,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.claude-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/claude-code/judge-recipe.golden.md b/tests/golden/claude-code/judge-recipe.golden.md index 7719cd2..c4f66ad 100644 --- a/tests/golden/claude-code/judge-recipe.golden.md +++ b/tests/golden/claude-code/judge-recipe.golden.md @@ -5,6 +5,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -21,9 +22,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.claude-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/claude-code/manifest-nomodel.golden.md b/tests/golden/claude-code/manifest-nomodel.golden.md index ee467b2..9d30538 100644 --- a/tests/golden/claude-code/manifest-nomodel.golden.md +++ b/tests/golden/claude-code/manifest-nomodel.golden.md @@ -28,6 +28,7 @@ Parallel dispatch from this iteration directory: ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' eval_root="$1" diff --git a/tests/golden/claude-code/manifest.golden.md b/tests/golden/claude-code/manifest.golden.md index 867ae65..a72f47f 100644 --- a/tests/golden/claude-code/manifest.golden.md +++ b/tests/golden/claude-code/manifest.golden.md @@ -28,6 +28,7 @@ Parallel dispatch from this iteration directory: ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' eval_root="$1" diff --git a/tests/golden/claude-code/runbook.golden.md b/tests/golden/claude-code/runbook.golden.md index a6a0cdf..265e33a 100644 --- a/tests/golden/claude-code/runbook.golden.md +++ b/tests/golden/claude-code/runbook.golden.md @@ -34,6 +34,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -50,9 +51,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.claude-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/cline/judge-recipe.golden.md b/tests/golden/cline/judge-recipe.golden.md index 85af0b8..355fb2d 100644 --- a/tests/golden/cline/judge-recipe.golden.md +++ b/tests/golden/cline/judge-recipe.golden.md @@ -5,6 +5,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -21,9 +22,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.cline-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/cline/manifest.golden.md b/tests/golden/cline/manifest.golden.md index 95c8bf5..df07631 100644 --- a/tests/golden/cline/manifest.golden.md +++ b/tests/golden/cline/manifest.golden.md @@ -30,6 +30,7 @@ Parallel dispatch from this iteration directory: ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' eval_root="$1" diff --git a/tests/golden/cline/runbook.golden.md b/tests/golden/cline/runbook.golden.md index 54a4c0d..f858583 100644 --- a/tests/golden/cline/runbook.golden.md +++ b/tests/golden/cline/runbook.golden.md @@ -36,6 +36,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -52,9 +53,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.cline-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/codex/judge-recipe-noguard.golden.md b/tests/golden/codex/judge-recipe-noguard.golden.md index 0293a75..ee5233c 100644 --- a/tests/golden/codex/judge-recipe-noguard.golden.md +++ b/tests/golden/codex/judge-recipe-noguard.golden.md @@ -5,6 +5,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -21,9 +22,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.codex-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/codex/judge-recipe.golden.md b/tests/golden/codex/judge-recipe.golden.md index 0293a75..ee5233c 100644 --- a/tests/golden/codex/judge-recipe.golden.md +++ b/tests/golden/codex/judge-recipe.golden.md @@ -5,6 +5,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -21,9 +22,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.codex-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/codex/manifest-noguard.golden.md b/tests/golden/codex/manifest-noguard.golden.md index 010061d..117a33c 100644 --- a/tests/golden/codex/manifest-noguard.golden.md +++ b/tests/golden/codex/manifest-noguard.golden.md @@ -29,6 +29,7 @@ Parallel dispatch from this iteration directory: ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' eval_root="$1" diff --git a/tests/golden/codex/manifest.golden.md b/tests/golden/codex/manifest.golden.md index 8ca9b68..7c802ff 100644 --- a/tests/golden/codex/manifest.golden.md +++ b/tests/golden/codex/manifest.golden.md @@ -29,6 +29,7 @@ Parallel dispatch from this iteration directory: ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' eval_root="$1" diff --git a/tests/golden/codex/runbook.golden.md b/tests/golden/codex/runbook.golden.md index 91a128f..959daf3 100644 --- a/tests/golden/codex/runbook.golden.md +++ b/tests/golden/codex/runbook.golden.md @@ -35,6 +35,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -51,9 +52,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.codex-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/opencode/judge-recipe.golden.md b/tests/golden/opencode/judge-recipe.golden.md index 5faa659..d7ce316 100644 --- a/tests/golden/opencode/judge-recipe.golden.md +++ b/tests/golden/opencode/judge-recipe.golden.md @@ -5,6 +5,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -21,9 +22,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.opencode-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ diff --git a/tests/golden/opencode/manifest.golden.md b/tests/golden/opencode/manifest.golden.md index 10b81fd..a25dd60 100644 --- a/tests/golden/opencode/manifest.golden.md +++ b/tests/golden/opencode/manifest.golden.md @@ -28,6 +28,7 @@ Parallel dispatch from this iteration directory: ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .eval_root, .dispatch_prompt_path, .outputs_dir' dispatch.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' eval_root="$1" diff --git a/tests/golden/opencode/runbook.golden.md b/tests/golden/opencode/runbook.golden.md index 23a7c1b..23efe8a 100644 --- a/tests/golden/opencode/runbook.golden.md +++ b/tests/golden/opencode/runbook.golden.md @@ -34,6 +34,7 @@ The final `N/M verdicts present` summary exits nonzero until every task has one. ```bash JOBS=${JOBS:-4} jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // ""))' judge-tasks.json \ + | tr -d '\r' \ | tr '\n' '\0' \ | xargs -0 -P "$JOBS" -n 3 sh -c ' prompt_path="$1" @@ -50,9 +51,10 @@ jq -r '.tasks[] | .dispatch_prompt_path, .response_path, ("model=" + (.model // 2> "$response_base.opencode-stderr.log" ' sh judge_dispatch_status=$? -judge_total=$(jq '.tasks | length' judge-tasks.json) +judge_total=$(jq '.tasks | length' judge-tasks.json | tr -d '\r') judge_present=$( jq -r '.tasks[].response_path' judge-tasks.json \ + | tr -d '\r' \ | while IFS= read -r response_path; do if [ -s "$response_path" ]; then printf '%s\n' "$response_path"; fi done \ From 191e231e61a0811c3ffa0a9e65037379979ffcc8 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sat, 15 Aug 2026 02:37:03 -0400 Subject: [PATCH 2/3] ci: run the suite on windows-latest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releases attach a Windows binary and the README documents a PowerShell installer, but both jobs ran on ubuntu-latest, so every Windows fix so far was found by hand rather than by the pipeline. Matrix the test job across ubuntu-latest and windows-latest with `fail-fast: false`, since a Linux failure cancelling the Windows leg would lose the result exactly when it is worth having. Clippy earns its place on both: the `#[cfg(windows)]` arms in `core::fs` and `command_check` are lint-invisible on Ubuntu. `EVAL_MAGIC_REQUIRE_POSIX_TOOLS` applies to both runners, so the Windows host is provisioned for the gated capabilities rather than exempted from them: `jq`, which Git for Windows does not bundle, and Developer Mode for symlink creation. Long paths need nothing — task repositories carry their own `core.longpaths`, and `LongPathsEnabled` stays unset deliberately so those tests keep proving what a default box does. `EVAL_MAGIC_SH` stays unset for the same reason: discovering the shell from the Git install root is what a Windows user hits. The guard test pins all of it together, because a matrix entry whose enforcement variable went missing would report green while covering six fewer tests than it appears to. Also corrects both contributor docs, which still described two gated capabilities before long-path staging became the third. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++----- AGENTS.md | 16 +++++++++------- docs/developer_overview.md | 5 +++-- tests/cli/package.rs | 22 ++++++++++++++++++++++ 4 files changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 952e98c..191bf69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,13 @@ env: jobs: test: name: Test suite - runs-on: ubuntu-latest + strategy: + # Windows is the platform this matrix exists to cover, so a Linux failure + # must not cancel it — that is exactly when its result is worth having. + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Install Rust toolchain @@ -25,15 +31,33 @@ jobs: components: rustfmt, clippy - name: Cache cargo build uses: Swatinem/rust-cache@v2 + - name: Install jq + # Git for Windows supplies sh, xargs, tr, and wc, but not jq, and the + # judge-recipe tests execute the shipped pipeline text rather than a + # stand-in for it. + if: runner.os == 'Windows' + run: choco install jq --yes --no-progress + - name: Permit symlink creation + # Windows creates symlinks only under Developer Mode or elevation, and + # the core::fs round-trips need one. Asking for it explicitly beats + # depending on how the runner's token happens to be built. + if: runner.os == 'Windows' + run: > + reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" + /t REG_DWORD /f /v AllowDevelopmentWithoutDevLicense /d 1 - name: Format check run: cargo fmt --all -- --check - name: Clippy run: cargo clippy --all-targets --all-features -- -D warnings - name: Test - # Capability-gated tests (the POSIX recipe pipelines, symlink round-trips) - # skip with a printed reason on a host that lacks the tool. This turns - # every such skip into a failure, so CI can never quietly stop covering - # them. Ubuntu runners ship jq, xargs, tr, and wc. + # Capability-gated tests (the POSIX recipe pipelines, symlink + # round-trips, long-path staging) skip with a printed reason on a host + # that lacks the capability. This turns every such skip into a failure, + # so neither runner can quietly stop covering them. Ubuntu ships the + # recipe tools; the steps above provide them on Windows. Long paths need + # no provisioning — the runner passes core.longpaths to git itself. + # EVAL_MAGIC_SH stays unset deliberately: discovering the shell from the + # Git install root is what a Windows user hits, so CI should run it too. env: EVAL_MAGIC_REQUIRE_POSIX_TOOLS: 1 run: cargo test --all-targets diff --git a/AGENTS.md b/AGENTS.md index 9b61d91..4dd5acb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,13 +63,15 @@ binary, `cargo test --lib` alone does not build it — run `cargo test`, or `car compilation and clippy on the other host and hides the coverage gap. Instead, probe for what the test actually needs and call `report_skip` (`src/core/runtime.rs`), which prints the reason and returns `true`. Setting `EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns every skip into a failure; CI sets -it, so a runner cannot quietly stop covering something. Two capabilities are gated today: the recipe -tools beyond the shell itself (`require_posix_toolchain` — in practice `jq`), and symlink creation, -which Windows allows only under Developer Mode. The shell is not one of them; it is a hard -requirement, per the section below. `require_posix_toolchain` is not test-only either — the `run` -preflight uses it to warn about the same gap. Where a genuine per-OS difference is the behavior under -test — signals, path separators — branch on `cfg!(windows)` at runtime so both arms still compile -everywhere. +it on both runners, so neither can quietly stop covering something. Three capabilities are gated +today: the recipe tools beyond the shell itself (`require_posix_toolchain` — in practice `jq`), +symlink creation, which Windows allows only under Developer Mode, and creating a path past +Windows' 259-character limit (`deep_task_root`, `src/cli/run/orchestrate/git.rs`). The Windows +runner is provisioned for those rather than exempted from them, so a skip there is a red build. The +shell is not one of them; it is a hard requirement, per the section below. +`require_posix_toolchain` is not test-only either — the `run` preflight uses it to warn about the +same gap. Where a genuine per-OS difference is the behavior under test — signals, path separators — +branch on `cfg!(windows)` at runtime so both arms still compile everywhere. **A POSIX shell is required, for use and for development.** Harness `exec_template`s are POSIX command lines, so the dispatch and probe paths spawn `sh` via `posix_shell()` diff --git a/docs/developer_overview.md b/docs/developer_overview.md index 8b57d73..0ecc36c 100644 --- a/docs/developer_overview.md +++ b/docs/developer_overview.md @@ -75,8 +75,9 @@ boundaries unless the evidence requires a named harness capability. Development carries the host requirement the tool itself declares: a POSIX shell with `jq`. The scripted-turn tests spawn `#!/bin/sh` harness stubs through the resolved shell and do not skip, so -the suite cannot pass without one. Tests needing `jq` or symlink creation report a skip instead; -`EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns those skips into failures, as CI sets it to do. +the suite cannot pass without one. Tests needing `jq`, symlink creation, or a path past Windows' +259-character limit report a skip instead; `EVAL_MAGIC_REQUIRE_POSIX_TOOLS=1` turns those skips into +failures, as CI sets it to do on both its Ubuntu and its Windows runner. Before handing work off, run: diff --git a/tests/cli/package.rs b/tests/cli/package.rs index 089be8b..b5149e7 100644 --- a/tests/cli/package.rs +++ b/tests/cli/package.rs @@ -70,6 +70,28 @@ fn ci_publishes_default_branch_coverage_for_readme_badge() { } } +/// Releases attach a Windows binary, so the suite has to run on Windows — but +/// the matrix entry alone proves nothing. Six of those tests are gated on +/// capabilities the runner has to be handed: `jq` for the judge recipes, and +/// symlink creation for the `core::fs` round-trips. Without the enforcement +/// variable they skip in silence, and the job reports green while covering +/// strictly less than it looks like it is. Every string below is load-bearing, +/// which is why they are pinned together rather than one standing for the rest. +#[test] +fn ci_runs_the_suite_on_windows_with_capability_skips_enforced() { + let workflow = read_repo_file(".github/workflows/ci.yml"); + + for expected in [ + "os: [ubuntu-latest, windows-latest]", + "fail-fast: false", + "EVAL_MAGIC_REQUIRE_POSIX_TOOLS: 1", + "choco install jq", + "AllowDevelopmentWithoutDevLicense", + ] { + assert!(workflow.contains(expected), "CI is missing {expected}"); + } +} + #[test] fn cargo_package_excludes_repo_local_authoring_files() { let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); From 5bb6d275b7bee907c5f8e888c3e508cf74cd7155 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sat, 15 Aug 2026 02:52:11 -0400 Subject: [PATCH 3/3] test(run): measure deep task roots the way git measures them The long-path tests padded from `std::env::temp_dir()`, which can hand back an 8.3 short name: a GitHub runner's `%TEMP%` lives under `RUNNER~1`, while git expands that to `runneradmin` before it measures. Three characters no length computed here could see. That was enough to move the deepest test from one side of a ceiling to the other. It passed locally with `.git/config` at 257 and failed on the runner with the same target at 260, which reads as a Windows-only product failure and is nothing of the sort. Canonicalise the base first, so the padding measures the spelling git will report. Then step the root back to 244 and assert the window it has to sit in, because git's long-path awareness turns out to be per-operation: creating `.git/objects/pack` survives well past the budget, `git init` writing `.git/config` stops exactly at it, and the `git config --local` that follows gives up two characters earlier still. The old literal sat one character inside the tightest of those, with nothing recording that it did. Co-Authored-By: Claude Opus 5 --- src/cli/run/orchestrate/git.rs | 42 ++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/cli/run/orchestrate/git.rs b/src/cli/run/orchestrate/git.rs index 6155c81..e6e2084 100644 --- a/src/cli/run/orchestrate/git.rs +++ b/src/cli/run/orchestrate/git.rs @@ -333,11 +333,28 @@ mod tests { root } + /// `base` spelled the way git will report it. + /// + /// `std::env::temp_dir()` can hand back an 8.3 short name — `RUNNER~1` for + /// `runneradmin` on a GitHub runner — which git expands before it measures. + /// Those three characters are invisible to a length computed from the short + /// spelling, and three is enough to push `.git/config` past the limit on a + /// host where the same target fits locally. Measure what git measures. + fn long_form(base: &Path) -> PathBuf { + let Ok(canonical) = base.canonicalize() else { + return base.to_path_buf(); + }; + let text = canonical.to_string_lossy().into_owned(); + // Canonicalising on Windows yields a `\\?\` verbatim path; git reports + // the plain spelling, so drop the prefix to keep the two comparable. + PathBuf::from(text.strip_prefix(r"\\?\").unwrap_or(&text)) + } + /// A `target`-character task root holding a staged `SKILL.md`, or `None` /// when this host cannot write that deep. The probe is the same `std::fs` /// write staging performs, so the gate is the capability, not the OS. fn deep_task_root(base: &Path, target: usize, test: &str) -> Option { - let root = padded_to(base, target); + let root = padded_to(&long_form(base), target); let staged = root.join(STAGED_SKILL); let written = fs::create_dir_all(staged.parent().expect("the staged path has a parent")) .and_then(|()| fs::write(&staged, "---\nname: widget-skill\n---\n\nbody\n")); @@ -421,14 +438,35 @@ mod tests { /// A root deep enough that `.git/objects/pack` crosses the budget: `git /// init` creates it before any repository-local configuration exists, so the /// lift has to reach that invocation too. + /// + /// 244 is not arbitrary and not the maximum. Git's long-path awareness is + /// per-operation: creating `.git/objects/pack` survives well past the + /// budget, `git init` writing `.git/config` stops at exactly + /// `WINDOWS_USABLE_PATH`, and the `git config --local` that follows gives up + /// two characters earlier still. 244 puts `pack` at 262 — past the budget, + /// which is the point — while leaving `.git/config` at 256, a deliberate + /// three inside the tightest of those ceilings. The assertions below pin the + /// window so a future edit cannot silently slide the root out of it. #[test] fn task_repository_initializes_when_its_git_directory_exceeds_the_windows_path_limit() { + const GIT_CONFIG: &str = ".git/config"; + const GIT_PACK: &str = ".git/objects/pack"; let test = "task_repository_initializes_when_its_git_directory_exceeds_the_windows_path_limit"; let tmp = tempfile::TempDir::new().unwrap(); - let Some(root) = deep_task_root(tmp.path(), 245, test) else { + let Some(root) = deep_task_root(tmp.path(), 244, test) else { return; }; + + let length = root.as_os_str().len(); + assert!( + length + 1 + GIT_PACK.len() > WINDOWS_USABLE_PATH, + "{length}-character root leaves `.git/objects/pack` inside the budget, testing nothing" + ); + assert!( + length + 1 + GIT_CONFIG.len() + 3 <= WINDOWS_USABLE_PATH, + "{length}-character root leaves `.git/config` no margin below the budget" + ); initialize_task_repository(&root) .expect("a task root deeper than `.git` needs initializes"); }