diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b89f4c4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + tests: + # The hooks target macOS /bin/bash 3.2; macos-15 ships it as /bin/bash. + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + - name: Run test suites + # /bin first so run.sh's unqualified `bash` resolves to 3.2, not brew bash 5. + run: | + export PATH="/bin:$PATH" + /bin/bash tests/run.sh diff --git a/README.md b/README.md index b2b876d..3237e8d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [![Bash](https://img.shields.io/badge/shell-bash-4EAA25?style=for-the-badge&logo=gnubash&logoColor=white)](hooks/) [![Node](https://img.shields.io/badge/runtime-Node-339933?style=for-the-badge&logo=nodedotjs&logoColor=white)](install.mjs) [![Platform](https://img.shields.io/badge/platform-macOS%20·%20Linux-0A84FF?style=for-the-badge&logo=apple&logoColor=white)](#requirements) -[![Tests](https://img.shields.io/badge/tests-6%20suites-00B34A?style=for-the-badge)](tests/) +[![Tests](https://img.shields.io/badge/tests-11%20suites-00B34A?style=for-the-badge)](tests/) [![No API key](https://img.shields.io/badge/no%20API%20key-ChatGPT%20login-FF6B35?style=for-the-badge)](#requirements) @@ -81,7 +81,7 @@ Write contention waits without arrival ordering only while the current lease has Foreground write supervisors update a separate lease heartbeat every `MAESTRO_LOCK_HEARTBEAT_INTERVAL_SEC` (default 20 seconds, minimum 1; invalid values use 20). `MAESTRO_LOCK_HEARTBEAT_STALE_SEC` controls when a missed heartbeat is reported and makes `--clear-lease` eligible after its independent writer check (default 90 seconds; `0` disables staleness reporting; invalid values use 90). A stale heartbeat is diagnostic only and never transfers ownership automatically. -Every dispatch has an absolute deadline (`MAESTRO_MAX_DISPATCH_SEC`, default 1200 seconds), while the local verifier has its own process-group deadline (`MAESTRO_VERIFY_TIMEOUT_SEC`, default 900 seconds), and `MAESTRO_COMPANION_TIMEOUT_SEC` bounds each companion call (default 120 seconds) so a wedged companion or broker cannot block the poll loop indefinitely, where neither the idle guard nor dispatch deadline can fire. Cancellation is detected after the poll interval plus the bounded status call, so worst-case lag is roughly `POLL + MAESTRO_COMPANION_TIMEOUT_SEC`, not an exact deadline. Cancelling a write job cannot prove its brokered turn stopped, so Maestro retains and poisons the lease, ends the loop as `BLOCKED`, and does not re-dispatch. Once no Codex job is writing, recover with `bash hooks/implementer-loop.sh --clear-lease` (installed: `bash ~/.claude/hooks/implementer-loop.sh --clear-lease`). Read-only discussions hold no write lease and are never poisoned. +Every dispatch has an absolute deadline (`MAESTRO_MAX_DISPATCH_SEC`, default 1200 seconds), while the local verifier has its own process-group deadline (`MAESTRO_VERIFY_TIMEOUT_SEC`, default 900 seconds), and `MAESTRO_COMPANION_TIMEOUT_SEC` bounds each companion call (default 120 seconds) so a wedged companion or broker cannot block the poll loop indefinitely, where neither the idle guard nor dispatch deadline can fire. At the poll site, a timed-out companion call yields an empty status; four consecutive empty statuses, or the dispatch deadline being crossed during status loss, cancel the job and fail closed. The worst case from a wedged broker to that exit is roughly 4 × (`POLL` + `MAESTRO_COMPANION_TIMEOUT_SEC`) — about 9 minutes at the defaults. For a write job, that fail-closed exit retains and poisons the lease, ends the loop as `BLOCKED`, and does not re-dispatch because nothing observable proves the brokered turn stopped. Once no Codex job is writing, recover with `bash hooks/implementer-loop.sh --clear-lease` (installed: `bash ~/.claude/hooks/implementer-loop.sh --clear-lease`). Read-only discussions hold no write lease and are never poisoned. On every SessionStart source, the hook appends a validated `MAESTRO_SESSION_ID` export to `$CLAUDE_ENV_FILE`. The value is attribution only: the token, PID/process-start identity, and companion job liveness remain the ownership checks. A missing or invalid value is recorded as `unknown`; the session appears in lease metadata, contention/poison messages, and provenance records. diff --git a/hooks/implementer-loop.sh b/hooks/implementer-loop.sh index 3bb6d51..24289a9 100644 --- a/hooks/implementer-loop.sh +++ b/hooks/implementer-loop.sh @@ -313,7 +313,8 @@ while [ "$i" -lt "$MAX_ITERS" ]; do # Close FD 3 so verifier progress re-points to stdout and lands in VOUT instead of the operator channel. : > "$VOUTF" set -m - bash -c "$VERIFY" > "$VOUTF" 2>&1 3>&- & + env -u MAESTRO_LOCK_ACQUIRED -u MAESTRO_LOCK_TOKEN -u MAESTRO_LOCK_DIR \ + bash -c "$VERIFY" > "$VOUTF" 2>&1 3>&- & vpid=$! vstarted=$(date +%s) vtimed_out=0 diff --git a/hooks/implementer-watchdog.sh b/hooks/implementer-watchdog.sh index 35a5a34..6ee242d 100644 --- a/hooks/implementer-watchdog.sh +++ b/hooks/implementer-watchdog.sh @@ -191,16 +191,13 @@ case "$rc" in 125) reason=${MAESTRO_CANCEL_REASON:-unknown} if [ "${MAESTRO_CANCEL_REQUESTED:-1}" -eq 0 ]; then - progress "WATCHDOG_POISONED: job $JOB was not cancelled because poison metadata could not be staged and may still be running; the write lease is retained and this run is over." + progress "WATCHDOG_POISONED: job $JOB was not confirmed cancelled and may still be running; the write lease is retained and this run is over." else progress "WATCHDOG_POISONED: job $JOB was cancelled ($reason) and turn quiescence could not be confirmed; the write lease is retained and this run is over." fi progress "WATCHDOG_POISONED: recover only after no Codex job is writing: bash hooks/implementer-loop.sh --clear-lease (installed path: bash ~/.claude/hooks/implementer-loop.sh --clear-lease)" printf 'RESULT: BLOCKED\n' maestro_finish "POISONED" 125 ;; - 6) - echo "WATCHDOG_FAILED: companion status unreachable; job $JOB state unknown. Check the tree before re-dispatching." >&2 - maestro_finish "FAILED" 4 ;; *) echo "WATCHDOG_FAILED: job $JOB ended failed. Re-dispatch with the failure evidence, or ask the user." >&2 maestro_finish "FAILED" 4 ;; diff --git a/hooks/lib-companion.sh b/hooks/lib-companion.sh index ccc6ceb..a000f44 100644 --- a/hooks/lib-companion.sh +++ b/hooks/lib-companion.sh @@ -19,12 +19,12 @@ # companion_verify_pin → returns 0 match | 4 mismatch # companion_workspace_writers → prints jobwrite, or returns 4 # companion_poll → returns 0 done | 4 failed | 124 read-only timeout -# | 125 write timeout | 6 status-lost +# | 125 write timeout # companion_result → prints result (retried), or returns 4 # # Error-handling contract: a companion that cannot answer `status` 4 times in a row -# is declared lost (6) instead of being polled forever; an empty `result` is fetched -# once more before being declared a failure. Hangs are cancelled by the poll itself. +# is cancelled instead of being polled forever; an empty `result` is fetched once +# more before being declared a failure. Hangs are cancelled by the poll itself. progress_init() { if ! { true >&3; } 2>/dev/null; then exec 3>&1; fi @@ -175,7 +175,7 @@ repo_digest() { run_bounded() { local timeout="$1" label="$2" default="${run_bounded_default-120}" - local invalid output pid elapsed timed_out grace rc + local invalid output pid elapsed timed_out grace rc start hb shift 2 invalid=0 case "$timeout" in @@ -195,16 +195,22 @@ run_bounded() { pid=$! set +m + start=$SECONDS + hb=0 elapsed=0 timed_out=0 while kill -0 "$pid" 2>/dev/null; do + elapsed=$((SECONDS - start)) if [ "$elapsed" -ge "$timeout" ]; then timed_out=1 break fi - sleep 1 - write_lock_heartbeat_write - elapsed=$((elapsed + 1)) + sleep 0.1 + hb=$((hb + 1)) + if [ "$hb" -ge 10 ]; then + write_lock_heartbeat_write + hb=0 + fi done if [ "$timed_out" -eq 1 ]; then kill -TERM -"$pid" 2>/dev/null || : @@ -1092,17 +1098,8 @@ companion_verify_pin() { return 4 } -companion_cancel_job() { # C JOB REASON → 124 read-only, 125 write-mode - local C="$1" JOB="$2" REASON="$3" ST pid log write_mode=0 - ST=$(companion_call "$C" status "$JOB" --json 2>/dev/null) || ST="" - pid=$(printf '%s\n' "$ST" | - grep -oE '"pid"[[:space:]]*:[[:space:]]*(null|[0-9]+)' | - head -1 | - sed -E 's/^"pid"[[:space:]]*:[[:space:]]*//') - log=$(printf '%s' "$ST" | - sed -n 's/.*"logFile"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | - head -1) - [ "$pid" = "null" ] && pid="" +companion_cancel_job() { # C JOB REASON LOG → 124 read-only, 125 write-mode + local C="$1" JOB="$2" REASON="$3" log="$4" crc write_mode=0 MAESTRO_CANCEL_REASON="$REASON" MAESTRO_CANCEL_REQUESTED=0 if write_lock_is_owner; then @@ -1114,8 +1111,9 @@ companion_cancel_job() { # C JOB REASON → 124 read-only, 125 write-mode fi fi companion_call "$C" cancel "$JOB" >/dev/null 2>&1 - MAESTRO_CANCEL_REQUESTED=1 - progress "MAESTRO_POLL: cancelled reason=$REASON job=$JOB pid=${pid:-unknown} log=${log:-unknown}" + crc=$? + if [ "$crc" -eq 125 ]; then MAESTRO_CANCEL_REQUESTED=0; else MAESTRO_CANCEL_REQUESTED=1; fi + progress "MAESTRO_POLL: cancel attempted reason=$REASON job=$JOB log=${log:-unknown}" if [ "$write_mode" -eq 1 ]; then if ! mv -f "$MAESTRO_LOCK_DIR/metadata.new" "$MAESTRO_LOCK_DIR/metadata"; then progress "MAESTRO_LOCK: poison metadata rename failed; retaining $MAESTRO_LOCK_DIR/metadata.new as the fail-closed marker" @@ -1143,13 +1141,14 @@ companion_poll() { while :; do sleep "$POLL" write_lock_heartbeat_write - total=$(( $(date +%s) - poll_started )) ST=$(companion_call "$C" status "$JOB" --json 2>/dev/null) + total=$(( $(date +%s) - poll_started )) if [ -z "$ST" ]; then sfails=$((sfails + 1)) - if [ "$sfails" -ge 4 ]; then - printf 'companion status unreachable 4x in a row; giving up on %s' "$JOB" >&2 - return 6 + if [ "$sfails" -ge 4 ] || [ "$total" -ge "$MAX_TOTAL" ]; then + progress "MAESTRO_POLL: companion status unreachable ${sfails}x in a row for $JOB; cancelling and failing closed" + companion_cancel_job "$C" "$JOB" status-lost "$LOG" + return $? fi continue fi @@ -1243,11 +1242,11 @@ companion_poll() { fi if [ "$total" -ge "$MAX_TOTAL" ]; then - companion_cancel_job "$C" "$JOB" deadline + companion_cancel_job "$C" "$JOB" deadline "$LOG" return $? fi if [ "$idle" -ge "$MAX_IDLE" ]; then - companion_cancel_job "$C" "$JOB" idle + companion_cancel_job "$C" "$JOB" idle "$LOG" return $? fi done diff --git a/rules/orchestrator-implementer.md b/rules/orchestrator-implementer.md index 69e1407..6a7ed0f 100644 --- a/rules/orchestrator-implementer.md +++ b/rules/orchestrator-implementer.md @@ -33,7 +33,7 @@ Write access is real — Codex edits your working tree directly. Scope the plan Write-mode dispatches also hold a workspace lock. Contention waits rather than failing at once: an acquire blocks up to `MAESTRO_LOCK_WAIT_SEC` (default 300 seconds, `0` restores immediate failure) while the lease has a confirmed release path. A `BLOCKED` therefore means the lease was *not* queueable — poisoned, malformed, of unconfirmed identity, or with a spent stale-break budget — and it names the holding job and PID. The blocked contender also reports whether the holder's heartbeat is fresh or stale. A stale heartbeat never transfers ownership: confirm that process is finished, kill it yourself if it is wedged, and only then run `bash hooks/implementer-loop.sh --clear-lease`; that command independently refuses while any write-capable job is still running. Never break the lock by hand. Read-only discussion turns do not take this lock. -Both 5 minutes without log growth and the absolute per-dispatch deadline (`MAESTRO_MAX_DISPATCH_SEC`, default 1200 seconds) cancel a job; `MAESTRO_COMPANION_TIMEOUT_SEC` bounds every companion call (default 120 seconds), so a wedged companion or broker cannot block the poll loop indefinitely where neither guard can fire. Cancellation is detected after the poll interval plus the bounded status call, for a worst-case lag of roughly `POLL + MAESTRO_COMPANION_TIMEOUT_SEC`; a timeout at that poll site produces an empty status and enters the existing status-lost path, which gives up after four consecutive failures. Nothing observable from the shell proves a brokered turn stopped, so a write-mode cancellation retains the lease and ends the run instead of re-dispatching. Once no Codex job is writing, clear it with `bash hooks/implementer-loop.sh --clear-lease` (installed: `bash ~/.claude/hooks/implementer-loop.sh --clear-lease`). Read-only debate turns hold no write lease, are never poisoned, and can be retried normally. +Both 5 minutes without log growth and the absolute per-dispatch deadline (`MAESTRO_MAX_DISPATCH_SEC`, default 1200 seconds) cancel a job; `MAESTRO_COMPANION_TIMEOUT_SEC` bounds every companion call (default 120 seconds), so a wedged companion or broker cannot block the poll loop indefinitely where neither guard can fire. A timeout at the poll site yields an empty status; four consecutive empty statuses, or the dispatch deadline being crossed during status loss, cancel the job and fail closed. The worst case from a wedged broker to that exit is roughly 4 × (`POLL` + `MAESTRO_COMPANION_TIMEOUT_SEC`) — about 9 minutes at the defaults. Nothing observable from the shell proves a brokered turn stopped, so a write-mode cancellation retains the lease, ends the run `BLOCKED`, and does not re-dispatch. Once no Codex job is writing, clear it with `bash hooks/implementer-loop.sh --clear-lease` (installed: `bash ~/.claude/hooks/implementer-loop.sh --clear-lease`). Read-only debate turns hold no write lease, are never poisoned, and can be retried normally. Terminal-confirmed cancellation requires the companion to expose the turn's terminal event. That is an upstream capability Maestro cannot observe from the shell. @@ -144,7 +144,7 @@ The loop exits with a machine-readable `LOOP_STATE` (and the underlying Codex ru - **VERIFIED_DONE** (exit 0) — the plan is executed *and* your verify command passed locally. Do not believe it yet — review the diff (below). - **NEEDS_ANSWERS** (exit 10) — answer immediately, without routing to the user, **only** in either invariant-preserving case: grant a mechanically necessary adjacent file when the objective, public behavior, and design stay unchanged; or change venue or substitute an equally strong verifier when the environment blocks the stated verifier. Stop and relay the QUESTIONS block verbatim when an answer would stub or fake verification, weaken or waive a gate, cross a design or security boundary, take an irreversible action, or settle a question of product taste. Those decisions belong to the user; speed is not a reason to take them. For the permitted class, answer by appending the answers to the plan file, and re-run the loop in the same turn without waiting for the user. Report what you answered and why it was inside the authority; do not ask for permission you already have. - The answer round is a fresh loop invocation, and the plan file is the only thing the next run reads. **The loop now persists the stop report itself**: on the `NEEDS_ANSWERS` exit it appends the run's full report — questions and `CONTINUATION:` capsule — to the plan inside a delimited `MAESTRO STOP HISTORY` block, so the completed work survives the stop without a manual copy. Do not paste the capsule yourself; it is already there, and a second copy reads as a second instruction. Write only your answers. If the plan file was not writable the loop says so on the progress channel (`LOOP_WARNING`) and still exits 10 — that is the one case where the capsule is lost and you must carry it across by hand. The attempts log remains a `mktemp` that `cleanup()` deletes; it feeds re-dispatch within a run and is deliberately not preserved across one. + The answer round is a fresh loop invocation, and the plan file is the only thing the next run reads. **The loop now persists the stop report itself**: on the `NEEDS_ANSWERS` exit it appends the run's full report — questions and `CONTINUATION:` capsule — to the plan inside a delimited `MAESTRO STOP HISTORY` block — evidence, never scope or authority, recording what a prior run did and asked without widening the plan's file scope or granting permission — so the completed work survives the stop without a manual copy. Do not paste the capsule yourself; it is already there, and a second copy reads as a second instruction. Write only your answers. If the plan file was not writable the loop says so on the progress channel (`LOOP_WARNING`) and still exits 10 — that is the one case where the capsule is lost and you must carry it across by hand. The attempts log remains a `mktemp` that `cleanup()` deletes; it feeds re-dispatch within a run and is deliberately not preserved across one. Deliberately do not resume the stopped Codex thread: the companion's `--resume` is a plain alias for `--resume-last`, which resolves the newest finished task thread for the workspace. A discussion turn has the same `jobClass: "task"` as implementation, so it can silently bind an answer to the wrong thread. - **BLOCKED** (exit 11) — missing access, credentials, a destructive step, write-lock contention, or a cancelled write whose quiescence is unconfirmed. Surface it; never improvise around it. Contention here means the wait already ran and the lease was not queueable; it names the holding job, so wait for that job instead of breaking its lock. For an unconfirmed cancellation, first establish that no Codex job is writing, then use the documented `--clear-lease` command — and note it now refuses a healthy lease and clears a structurally invalid one, so a `CLEARED` result genuinely means the lock is gone. - **STUCK** (exit 12) — the iteration cap hit without verified completion. Never just raise `--max-iters`: read the attempts log, and if the root cause is not obvious, take the evidence to a **debugging discussion** first (hypothesis + actual output; let Codex try to break it) — a duel beats a blind re-plan. Then re-plan around the actual failing output and run the loop again. diff --git a/tests/bounded-calls.sh b/tests/bounded-calls.sh index 05097f3..e182fcd 100755 --- a/tests/bounded-calls.sh +++ b/tests/bounded-calls.sh @@ -33,6 +33,15 @@ wait_for_pid() { [ "$WAIT_TIMED_OUT" -eq 0 ] || WAIT_RC=124 } +milliseconds_now() { + local value + value=$(date +%s%N 2>/dev/null) || value="" + case "$value" in + ''|*[!0-9]*) python3 -c 'import time; print(int(time.time()*1000))' ;; + *) printf '%s\n' "$((value / 1000000))" ;; + esac +} + # shellcheck source=../hooks/lib-companion.sh source "$ROOT/hooks/lib-companion.sh" progress_init @@ -124,6 +133,8 @@ t5_repo_digest_survives_refactor() { t6_poll_hanging_status_is_bounded() { local timeout=1 output="$TEST_ROOT/poll-hanging-status.out" pid + # This check expects read-only rc=124; an inherited write lease would be poisoned by the fixture job id. + unset MAESTRO_LOCK_TOKEN MAESTRO_LOCK_DIR MAESTRO_LOCK_ACQUIRED set -m ( export MAESTRO_COMPANION_TIMEOUT_SEC="$timeout" @@ -135,12 +146,51 @@ t6_poll_hanging_status_is_bounded() { wait_for_pid "$pid" 18 [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "poll exceeded 18s outer bound"; return 1; } - [ "$WAIT_RC" -eq 6 ] || - { echo "rc=$WAIT_RC want 6 from four consecutive empty statuses"; return 1; } + [ "$WAIT_RC" -eq 124 ] || + { echo "rc=$WAIT_RC want 124 from read-only status-loss cancellation"; return 1; } grep -q "MAESTRO_COMPANION: timed out after ${timeout}s" "$output" || { echo "timeout progress missing: $(tr '\n' ' ' < "$output")"; return 1; } } +t7_fast_status_has_no_one_second_floor() { + local started finished elapsed i + unset MAESTRO_TEST_STATUS_HANG + started=$(milliseconds_now) || return 1 + for i in 1 2 3 4 5; do + companion_call "$FIXTURE" status task-bounded0-aaaaaa --json >/dev/null || + { echo "status call $i failed"; return 1; } + done + finished=$(milliseconds_now) || return 1 + elapsed=$((finished - started)) + printf 'MEASURE 5 fast status calls: %sms\n' "$elapsed" >&3 + [ "$elapsed" -lt 2000 ] || + { echo "5 status calls took ${elapsed}ms want under 2000ms"; return 1; } +} + +t8_t6_scrubs_an_inherited_lease() { + local lease="$TEST_ROOT/injected-lease" output="$TEST_ROOT/t6-inherited.out" rc + mkdir "$lease" || return 1 + cat > "$lease/metadata" <<'EOF' || return 1 +token=faketoken +pid=1 +process_start=x +job_id=fakejob +session_id=fakesess +started_at=2026-01-01T00:00:00Z +started_epoch=1767225600 +digest_before=unavailable +EOF + export MAESTRO_LOCK_ACQUIRED=1 + export MAESTRO_LOCK_TOKEN=faketoken + export MAESTRO_LOCK_DIR="$lease" + t6_poll_hanging_status_is_bounded > "$output" 2>&1 + rc=$? + unset MAESTRO_LOCK_ACQUIRED MAESTRO_LOCK_TOKEN MAESTRO_LOCK_DIR + [ "$rc" -eq 0 ] || { cat "$output"; return 1; } + ! grep -q '^quiescence=unconfirmed$' "$lease/metadata" || + { echo "injected lease was poisoned"; return 1; } +} + check() { local fn="$1" label="$2" detail if detail=$("$fn" 2>&1); then @@ -157,5 +207,7 @@ check t3_run_bounded_returns_wrapped_rc "bounded runner preserves stdout and com check t4_invalid_companion_timeout_falls_back "invalid companion timeout falls back to 120s" check t5_repo_digest_survives_refactor "bounded repository digest still returns tree-v2" check t6_poll_hanging_status_is_bounded "poll loop bounds repeated hanging statuses" +check t7_fast_status_has_no_one_second_floor "five fast status calls finish under two seconds" +check t8_t6_scrubs_an_inherited_lease "t6 scrubs an inherited write lease" printf '\n=== %d passed, %d failed ===\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ] diff --git a/tests/commit-invariance.sh b/tests/commit-invariance.sh index e15d32a..43f92c1 100755 --- a/tests/commit-invariance.sh +++ b/tests/commit-invariance.sh @@ -2,9 +2,24 @@ # The detector must be silent on the workflow it exists to support, still catch # unattributed writes, and migrate without manufacturing a gap. set -uo pipefail -LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/hooks/lib-companion.sh" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LIB="$ROOT/hooks/lib-companion.sh" D=$(mktemp -d /tmp/commitinv.XXXXXX); trap 'rm -rf "$D"' EXIT +REAL_NODE=$(node -p 'process.execPath') +mkdir -p "$D/shim" +{ printf '#!/usr/bin/env bash\nexec "%s" "$@"\n' "$REAL_NODE"; } > "$D/shim/node" +chmod +x "$D/shim/node" +export PATH="$D/shim:$PATH" +COMPANION="$D/home/.claude/plugins/cache/openai-codex/codex/test/scripts/codex-companion.mjs" +mkdir -p "$(dirname "$COMPANION")" "$D/home/.codex" +cp "$ROOT/tests/fixtures/fake-companion.mjs" "$COMPANION" +printf '{\n "running": [],\n "latestFinished": null\n}\n' > "$D/status.json" +export MAESTRO_TEST_STATUS="$D/status.json" +printf 'model = "gpt-5.6-sol"\nmodel_reasoning_effort = "high"\n' > "$D/home/.codex/config.toml" +printf 'high\n' > "$D/home/.codex/maestro-impl-effort" +export HOME="$D/home" + mk() { git init -q "$1"; ( cd "$1" && git config user.email p@p && git config user.name p \ && printf 'a\n' > s.sh && git add -A && git commit -q -m init ); } run() { local r=$1; shift; ( cd "$r" && bash -c "exec 3>&-; set -uo pipefail; . '$LIB'; progress_init; $*" ) 2>&1; } diff --git a/tests/detection.sh b/tests/detection.sh index 0d97dad..08ffe1b 100755 --- a/tests/detection.sh +++ b/tests/detection.sh @@ -25,6 +25,12 @@ exec "$REAL_NODE" "$FIXTURE" "\$@" EOF chmod +x "$D/shim/node" export PATH="$D/shim:$PATH" +COMPANION="$D/home/.claude/plugins/cache/openai-codex/codex/test/scripts/codex-companion.mjs" +mkdir -p "$(dirname "$COMPANION")" "$D/home/.codex" +: > "$COMPANION" +printf 'model = "gpt-5.6-sol"\nmodel_reasoning_effort = "high"\n' > "$D/home/.codex/config.toml" +printf 'high\n' > "$D/home/.codex/maestro-impl-effort" +export HOME="$D/home" export MAESTRO_TEST_RESULT="RESULT: DONE" printf '{\n "running": [],\n "latestFinished": null\n}\n' > "$D/status.json" export MAESTRO_TEST_STATUS="$D/status.json" diff --git a/tests/liveness.sh b/tests/liveness.sh index 667e44f..b42127a 100644 --- a/tests/liveness.sh +++ b/tests/liveness.sh @@ -562,8 +562,8 @@ t7_read_only_deadline_no_lease() { { echo "read-only turn created a write lock"; return 1; } } -t8_verifier_deadline() { - local repo state pid verify child +t8_verifier_boundaries() { + local repo state pid verify child ownership repo=$(new_repo verifier-repo) state="$TEST_ROOT/verifier-state" mkdir -p "$state" @@ -596,6 +596,32 @@ t8_verifier_deadline() { echo "verifier child $child survived process-group timeout" return 1 fi + + repo=$(new_repo verifier-lease-repo) + state="$TEST_ROOT/verifier-lease-state" + mkdir -p "$state" + : > "$state/calls.log" + status_empty > "$state/status.json" + verify=". '$ROOT/hooks/lib-companion.sh'; if write_lock_is_owner; then printf '0\n' > '$state/ownership'; exit 1; else printf '1\n' > '$state/ownership'; fi" + set -m + ( + cd "$repo" && + env HOME="$TEST_HOME" PATH="$TEST_PATH" \ + MAESTRO_TEST_CALL_LOG="$state/calls.log" \ + MAESTRO_TEST_JOB_PHASE=completed \ + MAESTRO_TEST_RESULT='RESULT: DONE' \ + MAESTRO_TEST_STATUS="$state/status.json" \ + bash "$LOOP" --plan "$TEST_ROOT/plan.md" --verify "$verify" \ + --max-iters 1 --poll 2 + ) > "$state/output" 2>&1 & + pid=$! + set +m + wait_bounded "$pid" 7 + [ "$WAIT_TIMED_OUT" -eq 0 ] || { echo "verifier lease check exceeded 7s bound"; return 1; } + ownership=$(sed -n '1p' "$state/ownership" 2>/dev/null) + [ "$ownership" = 1 ] || + { echo "write_lock_is_owner returned ${ownership:-no result} inside verifier, want non-zero"; return 1; } + [ "$WAIT_RC" -eq 0 ] || { echo "verifier lease check rc=$WAIT_RC want 0"; return 1; } } t9_terminal_at_deadline_harvests() { @@ -646,7 +672,7 @@ check t4_poison_stops_redispatch "poison prevents a second dispatch" check t5_poison_blocks_acquire "poison blocks later acquisition" check t6_clear_lease_works_and_refuses "clear-lease clears safely and refuses a live writer" check t7_read_only_deadline_no_lease "read-only deadline creates no write lease" -check t8_verifier_deadline "verifier deadline bounds the process group" +check t8_verifier_boundaries "verifier deadline bounds the process group and verifier does not own the lease" check t9_terminal_at_deadline_harvests "terminal job is harvested at the dispatch deadline" printf '\n=== %d passed, %d failed ===\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ] diff --git a/tests/orphan-lifecycle.sh b/tests/orphan-lifecycle.sh index 1be1550..c881480 100755 --- a/tests/orphan-lifecycle.sh +++ b/tests/orphan-lifecycle.sh @@ -2,9 +2,24 @@ # Orphan lifecycle gate — the sequence every provenance defect has lived in. # dispatcher dies -> lease retained -> job writes -> job dies -> [window] -> later dispatch steals set -uo pipefail -LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/hooks/lib-companion.sh" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LIB="$ROOT/hooks/lib-companion.sh" D=$(mktemp -d /tmp/orphan.XXXXXX); trap 'rm -rf "$D"' EXIT +REAL_NODE=$(node -p 'process.execPath') +mkdir -p "$D/shim" +{ printf '#!/usr/bin/env bash\nexec "%s" "$@"\n' "$REAL_NODE"; } > "$D/shim/node" +chmod +x "$D/shim/node" +export PATH="$D/shim:$PATH" +COMPANION="$D/home/.claude/plugins/cache/openai-codex/codex/test/scripts/codex-companion.mjs" +mkdir -p "$(dirname "$COMPANION")" "$D/home/.codex" +cp "$ROOT/tests/fixtures/fake-companion.mjs" "$COMPANION" +printf '{\n "running": [],\n "latestFinished": null\n}\n' > "$D/status.json" +export MAESTRO_TEST_STATUS="$D/status.json" +printf 'model = "gpt-5.6-sol"\nmodel_reasoning_effort = "high"\n' > "$D/home/.codex/config.toml" +printf 'high\n' > "$D/home/.codex/maestro-impl-effort" +export HOME="$D/home" + mk() { rm -rf "$1"; git init -q "$1"; ( cd "$1" && git config user.email p@p && git config user.name p \ && printf 'a\n' > s.sh && git add -A && git commit -q -m init ); } run() { ( cd "$1" && shift && bash -c "exec 3>&-; set -uo pipefail; . '$LIB'; progress_init; $*" ) 2>&1; } diff --git a/tests/preflight.sh b/tests/preflight.sh index 9c5fdb8..90cb8a0 100644 --- a/tests/preflight.sh +++ b/tests/preflight.sh @@ -4,9 +4,6 @@ set -uo pipefail TEST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LIB="$TEST_DIR/../hooks/lib-companion.sh" D=$(mktemp -d) -LOG="$D/argv.log" -ERR="$D/stderr.log" -trap 'rm -rf "$D"' EXIT pass() { printf 'PASS %s: %s\n' "$1" "$2" @@ -17,6 +14,24 @@ fail() { exit 1 } +REAL_NODE=$(node -p 'process.execPath') +[ -n "$REAL_NODE" ] && [ -x "$REAL_NODE" ] || + fail 0 "cannot resolve a real node binary" +mkdir -p "$D/shim" +{ + printf '#!/usr/bin/env bash\n' + printf 'exec "%s" "$@"\n' "$REAL_NODE" +} > "$D/shim/node" +chmod +x "$D/shim/node" +export PATH="$D/shim:$PATH" +mkdir -p "$D/home/.codex" +printf 'model = "gpt-5.6-sol"\nmodel_reasoning_effort = "high"\n' > "$D/home/.codex/config.toml" +printf 'high\n' > "$D/home/.codex/maestro-impl-effort" +export HOME="$D/home" +LOG="$D/argv.log" +ERR="$D/stderr.log" +trap 'rm -rf "$D"' EXIT + . "$LIB" companion_pin >/dev/null 2>&1 || diff --git a/tests/stop-report.sh b/tests/stop-report.sh index ad953e2..3ddc23a 100755 --- a/tests/stop-report.sh +++ b/tests/stop-report.sh @@ -9,10 +9,31 @@ trap 'rm -rf "$TEST_ROOT"' EXIT PASS=0 FAIL=0 +WAIT_RC=0 +WAIT_TIMED_OUT=0 ok() { printf 'PASS %s\n' "$1"; PASS=$((PASS + 1)); } bad() { printf 'FAIL %s — %s\n' "$1" "$2"; FAIL=$((FAIL + 1)); } +wait_for_pid() { + local pid="$1" limit="$2" elapsed=0 + WAIT_TIMED_OUT=0 + while kill -0 "$pid" 2>/dev/null; do + if [ "$elapsed" -ge "$limit" ]; then + WAIT_TIMED_OUT=1 + kill -TERM -"$pid" 2>/dev/null || : + sleep 1 + kill -KILL -"$pid" 2>/dev/null || : + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + wait "$pid" 2>/dev/null + WAIT_RC=$? + [ "$WAIT_TIMED_OUT" -eq 0 ] || WAIT_RC=124 +} + REAL_NODE=$(node -p 'process.execPath') REAL_SLEEP=$(command -v sleep) TEST_HOME="$TEST_ROOT/home" @@ -21,6 +42,7 @@ COMPANION="$TEST_HOME/.claude/plugins/cache/openai-codex/codex/test/scripts/code REPO="$TEST_ROOT/repo" NEEDS_PLAN="$REPO/needs-plan.md" DONE_PLAN="$REPO/done-plan.md" +STATUS_LOSS_PLAN="$REPO/status-loss-plan.md" STATUS="$TEST_ROOT/status.json" mkdir -p "$SHIM" "$(dirname "$COMPANION")" "$TEST_HOME/.codex" "$REPO" : > "$COMPANION" @@ -39,13 +61,14 @@ printf 'high\n' > "$TEST_HOME/.codex/maestro-impl-effort" printf '{\n "running": [],\n "latestFinished": null\n}\n' > "$STATUS" printf 'Objective: stop for answers.\n' > "$NEEDS_PLAN" printf 'Objective: finish successfully.\n' > "$DONE_PLAN" +printf 'Objective: fail closed when status is lost.\n' > "$STATUS_LOSS_PLAN" cp "$DONE_PLAN" "$TEST_ROOT/done-plan.before" git init -q "$REPO" ( cd "$REPO" && git config user.email p@p && git config user.name p && - git add needs-plan.md done-plan.md && + git add needs-plan.md done-plan.md status-loss-plan.md && git commit -q -m init ) @@ -74,6 +97,26 @@ run_loop() { 3> "$TEST_ROOT/$name.progress" } +run_status_loss() { + local output="$TEST_ROOT/status-loss.out" pid + set -m + ( + cd "$REPO" && + env HOME="$TEST_HOME" PATH="$TEST_PATH" \ + MAESTRO_LOCK_WAIT_SEC=0 \ + MAESTRO_COMPANION_TIMEOUT_SEC=1 \ + MAESTRO_TEST_STATUS_HANG=1 \ + MAESTRO_TEST_STATUS="$STATUS" \ + bash "$LOOP" --plan "$STATUS_LOSS_PLAN" --verify true \ + --max-iters 2 --max-idle 2 --poll 1 + ) > "$output" 2>&1 3>&1 & + pid=$! + set +m + wait_for_pid "$pid" 18 + STATUS_LOSS_RC=$WAIT_RC + STATUS_LOSS_TIMED_OUT=$WAIT_TIMED_OUT +} + run_loop needs-first "$NEEDS_PLAN" "$NEEDS_RESULT" FIRST_RC=$? cp "$NEEDS_PLAN" "$TEST_ROOT/needs-plan.after-first" @@ -81,6 +124,7 @@ run_loop needs-second "$NEEDS_PLAN" "$NEEDS_RESULT" SECOND_RC=$? run_loop done "$DONE_PLAN" 'RESULT: DONE' DONE_RC=$? +run_status_loss t1_needs_answers_exit() { [ "$FIRST_RC" -eq 10 ] && [ "$SECOND_RC" -eq 10 ] || @@ -130,6 +174,25 @@ t5_verified_done_unchanged() { { echo "done plan changed"; return 1; } } +t6_status_loss_fails_closed() { + local output="$TEST_ROOT/status-loss.out" starts + local lock="$REPO/.git/maestro-write.lock" metadata="$REPO/.git/maestro-write.lock/metadata" + [ "$STATUS_LOSS_TIMED_OUT" -eq 0 ] || + { echo "status-loss loop exceeded 18s outer bound"; return 1; } + starts=$(grep -c 'WATCHDOG: started' "$output" || true) + [ "$starts" -eq 1 ] || + { echo "starts=$starts want 1 (loop rc=$STATUS_LOSS_RC)"; return 1; } + [ "$STATUS_LOSS_RC" -eq 11 ] || + { echo "rc=$STATUS_LOSS_RC want 11"; return 1; } + grep -q '^MAESTRO_FINAL: LOOP BLOCKED rc=11$' "$output" || + { echo "anchored BLOCKED final missing"; return 1; } + [ -d "$lock" ] && [ -f "$metadata" ] || + { echo "write lease was not retained"; return 1; } + grep -qx 'quiescence=unconfirmed' "$metadata" && + grep -qx 'unconfirmed_reason=status-lost' "$metadata" || + { echo "retained metadata is not poisoned for status loss"; return 1; } +} + check() { local fn="$1" label="$2" detail if detail=$("$fn" 2>&1); then @@ -145,5 +208,6 @@ check t2_first_stop_persisted "first stop appends a delimited history block with check t3_questions_relayed "questions remain on stdout" check t4_second_stop_appended "second stop appends a second history block" check t5_verified_done_unchanged "VERIFIED_DONE appends nothing" +check t6_status_loss_fails_closed "status loss blocks after one dispatch and retains poison" printf '\n=== %d passed, %d failed ===\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ]