From 8418aec1930281d4e8fae1d48a92e8440bfeb80d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 14:30:19 -0700 Subject: [PATCH 01/13] fix(openclaw): approve scope upgrades through local fallback --- .github/workflows/nightly-e2e.yaml | 66 ++ scripts/nemoclaw-start.sh | 24 +- src/lib/actions/sandbox/connect.ts | 26 +- .../test-issue-4462-scope-upgrade-approval.sh | 786 ++++++++++++++++++ test/nemoclaw-start.test.ts | 120 ++- test/sandbox-connect-inference.test.ts | 7 + 6 files changed, 1009 insertions(+), 20 deletions(-) create mode 100755 test/e2e/test-issue-4462-scope-upgrade-approval.sh diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 40dc74fc8a..9e7c6bf99b 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -13,6 +13,12 @@ # openclaw-discord-pairing-e2e # Validates hermetic Discord pairing request approval across # gateway and connect-shell OpenClaw state roots (#4061). +# issue-4462-scope-upgrade-approval-e2e +# Validates real CLI scope-upgrade approval and confirms +# the approved agent run stays on gateway mode (#4462). +# issue-4462-scope-upgrade-deadlock-repro-e2e +# Reproduces the gateway-pinned approve deadlock path +# against a real sandbox, then recovers with the fix. # messaging-compatible-endpoint-e2e # Validates Telegram + OpenAI-compatible endpoint inference routing # through inference.local with a hermetic local mock (#2766). @@ -88,6 +94,8 @@ on: openclaw-tui-chat-correlation-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-scope-upgrade-deadlock-repro-e2e, messaging-compatible-endpoint-e2e, kimi-inference-compat-e2e, bedrock-runtime-compatible-anthropic-e2e, @@ -393,6 +401,58 @@ jobs: secrets: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + # ── OpenClaw Scope-Upgrade Approval E2E (#4462) ──────────────── + # Positive proof: in a real sandbox, approve a pending CLI scope upgrade + # through the proxy-env guard, then confirm openclaw agent still uses the + # gateway path rather than embedded fallback. + issue-4462-scope-upgrade-approval-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-approval-e2e,')) + uses: ./.github/workflows/e2e-script.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e/test-issue-4462-scope-upgrade-approval.sh + timeout_minutes: 60 + artifact_name: "issue-4462-scope-upgrade-approval-logs" + artifact_path: | + /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log + /tmp/nemoclaw-issue-4462-scope-upgrade-approval.log + /tmp/nemoclaw-issue-4462-scope-upgrade-agent.log + /tmp/nemoclaw-issue-4462-scope-upgrade-state.log + env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}' + nvidia_api_key: true + github_token: true + secrets: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + # ── OpenClaw Scope-Upgrade Deadlock Reproducer (#4462) ────────── + # Negative proof: in a real sandbox, force the old gateway-pinned approve + # path and verify the CLI scope-upgrade remains pending; then recover through + # the fixed proxy-env guard so the job can finish cleanly. + issue-4462-scope-upgrade-deadlock-repro-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-deadlock-repro-e2e,')) + uses: ./.github/workflows/e2e-script.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e/test-issue-4462-scope-upgrade-approval.sh + timeout_minutes: 60 + artifact_name: "issue-4462-scope-upgrade-deadlock-repro-logs" + artifact_path: | + /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log + /tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log + /tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log + /tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log + env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}' + nvidia_api_key: true + github_token: true + secrets: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} messaging-compatible-endpoint-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || @@ -1842,6 +1902,8 @@ jobs: openclaw-tui-chat-correlation-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-scope-upgrade-deadlock-repro-e2e, messaging-compatible-endpoint-e2e, channels-add-remove-e2e, channels-stop-start-e2e, @@ -1946,6 +2008,8 @@ jobs: openclaw-tui-chat-correlation-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-scope-upgrade-deadlock-repro-e2e, messaging-compatible-endpoint-e2e, channels-add-remove-e2e, channels-stop-start-e2e, @@ -2107,6 +2171,8 @@ jobs: openclaw-tui-chat-correlation-e2e, issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-scope-upgrade-deadlock-repro-e2e, messaging-compatible-endpoint-e2e, channels-add-remove-e2e, channels-stop-start-e2e, diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index b68bad4474..5559d6190f 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -1435,14 +1435,18 @@ ALLOWED_MODES = {'webchat', 'cli'} RUN_TIMEOUT_SECS = _env_seconds('NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS', 10) -def run(*args): +def run(*args, strip_gateway_url=False): # Bound every openclaw CLI invocation so a wedged child cannot pin # the watcher beyond DEADLINE (CodeRabbit #4292): subprocess.run with # no timeout would hold a hung `openclaw devices list/approve` past # the fast→slow transition and the 8h deadline check. + env = None + if strip_gateway_url: + env = os.environ.copy() + env.pop('OPENCLAW_GATEWAY_URL', None) try: proc = subprocess.run( - args, capture_output=True, text=True, timeout=RUN_TIMEOUT_SECS, + args, capture_output=True, text=True, timeout=RUN_TIMEOUT_SECS, env=env, ) return proc.returncode, proc.stdout.strip(), proc.stderr.strip() except subprocess.TimeoutExpired as exc: @@ -1491,16 +1495,18 @@ while time.time() < DEADLINE: HANDLED.add(request_id) print(f'[auto-pair] rejected unknown client={client_id} mode={client_mode}') continue - arc, aout, aerr = run(OPENCLAW, 'devices', 'approve', request_id, '--json') + arc, aout, aerr = run( + OPENCLAW, 'devices', 'approve', request_id, '--json', strip_gateway_url=True, + ) # rc=124 is the timeout sentinel from run() — do NOT add the # request to HANDLED on a transient timeout, so the next poll - # can retry (CodeRabbit #4292). Permanent failures (other - # non-zero rc) still get HANDLED so we don't spin on a stuck - # bad request. + # can retry (CodeRabbit #4292). Other approve failures stay + # retryable too; only intentionally rejected unknown clients + # and confirmed successful approvals are marked handled. if arc == 124: continue - HANDLED.add(request_id) if arc == 0: + HANDLED.add(request_id) APPROVED += 1 print(f'[auto-pair] approved request={request_id} client={client_id} mode={client_mode}') elif aout or aerr: @@ -1754,6 +1760,10 @@ PROXYEOF cat <<'GUARDENVEOF' # nemoclaw-configure-guard begin openclaw() { + if [ "${1:-}" = "devices" ] && [ "${2:-}" = "approve" ]; then + ( unset OPENCLAW_GATEWAY_URL; command openclaw "$@" ) + return $? + fi case "$1" in configure) echo "Error: 'openclaw configure' cannot modify config inside the sandbox." >&2 diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 47dd217609..97a69400ef 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -611,16 +611,17 @@ function ensureSandboxInferenceRouteOrExit( // pass covers the case where the watcher has exited or is otherwise stuck // when the user runs `nemoclaw connect`. The script sources // `/tmp/nemoclaw-proxy-env.sh` (written by `nemoclaw-start.sh`) so the -// in-sandbox `openclaw devices list/approve` invocations target the -// running gateway with its token, and applies the same allowlist as the -// startup watcher — `openclaw-control-ui` clients plus `webchat`/`cli` +// in-sandbox `openclaw devices list` invocation targets the running gateway +// with its token. Approvals then use OpenClaw's local fallback by removing +// OPENCLAW_GATEWAY_URL only from the child env, and apply the same allowlist +// as the startup watcher — `openclaw-control-ui` clients plus `webchat`/`cli` // modes. Unknown clients are ignored, not approved. // // Failure modes (timeout, sandbox-exec errors, missing openclaw, gateway // unreachable) are swallowed: the connect flow must not be blocked by a -// best-effort approval. Internal timeouts (2s list + 1s × MAX_APPROVALS) -// fit within the outer spawnSync cap, so a partial-completion mid-loop -// kill cannot strand allowlisted requests within a normal batch. +// best-effort approval. Internal timeouts (2s list + 1s x MAX_APPROVALS +// attempts) fit within the outer spawnSync cap, so a partial-completion +// mid-loop kill cannot strand allowlisted requests within a normal batch. const CONNECT_AUTO_PAIR_MAX_APPROVALS = 8; const CONNECT_AUTO_PAIR_TIMEOUT_MS = 12_000; @@ -660,9 +661,10 @@ pending = data.get('pending') if not isinstance(pending, list): sys.exit(0) approved_count = 0 +attempted_count = 0 seen_request_ids = set() for device in pending: - if approved_count >= MAX_APPROVALS: + if attempted_count >= MAX_APPROVALS: break if not isinstance(device, dict): continue @@ -674,12 +676,16 @@ for device in pending: if client_id not in ALLOWED_CLIENTS and client_mode not in ALLOWED_MODES: continue seen_request_ids.add(request_id) + approve_env = os.environ.copy() + approve_env.pop('OPENCLAW_GATEWAY_URL', None) + attempted_count += 1 try: - subprocess.run( + approve_proc = subprocess.run( [OPENCLAW, 'devices', 'approve', request_id, '--json'], - capture_output=True, text=True, timeout=1, + capture_output=True, text=True, timeout=1, env=approve_env, ) - approved_count += 1 + if approve_proc.returncode == 0: + approved_count += 1 except (subprocess.TimeoutExpired, FileNotFoundError, OSError): continue PYAPPROVE diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh new file mode 100755 index 0000000000..d9ea63e975 --- /dev/null +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -0,0 +1,786 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Issue #4462 E2E: +# +# Build a real NemoClaw/OpenClaw sandbox, create a low-scope CLI device +# approval, trigger the later `openclaw agent` operator.write scope upgrade, and +# then run in one of two modes: +# +# approval Approve the pending request through the fixed proxy-env guard, +# verify the request is no longer pending, and verify the next +# `openclaw agent` turn stays on the gateway path. +# legacy-repro Force the old gateway-pinned approve path, verify approval +# fails and leaves the request pending, then recover through the +# fixed proxy-env guard so the sandbox is not left dirty. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + +# shellcheck disable=SC2016,SC2329 +# SC2016: remote sandbox scripts intentionally expand inside the sandbox. +# SC2329: keep the conventional E2E skip helper even if this lane currently +# has no optional skip path. + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT="${NEMOCLAW_E2E_DEFAULT_TIMEOUT:-2700}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e/e2e-timeout.sh +. "${SCRIPT_DIR}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} + +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} + +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} + +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} + +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "${SCRIPT_DIR}/../.." && pwd)/install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." >&2 + exit 1 +fi + +E2E_DIR="${SCRIPT_DIR}" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-issue-4462-scope-upgrade}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +TEST_MODE="${NEMOCLAW_4462_MODE:-approval}" +INSTALL_LOG="${NEMOCLAW_4462_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log}" +APPROVAL_LOG="${NEMOCLAW_4462_APPROVAL_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-approval.log}" +AGENT_LOG="${NEMOCLAW_4462_AGENT_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-agent.log}" +STATE_LOG="${NEMOCLAW_4462_STATE_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-state.log}" +INSTALL_TIMEOUT_SECONDS="${NEMOCLAW_E2E_INSTALL_TIMEOUT_SECONDS:-1800}" + +# shellcheck source=test/e2e/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e/lib/openclaw-json.sh +. "${E2E_DIR}/lib/openclaw-json.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local seconds="$1" + local script="$2" + shift 2 + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; bash \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + run_with_timeout "$seconds" "$OPENSHELL_BIN" sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +extract_json_doc() { + python3 -c ' +import json +import sys + +raw = sys.stdin.read() +decoder = json.JSONDecoder() +for idx, char in enumerate(raw): + if char != "{": + continue + try: + doc, _end = decoder.raw_decode(raw[idx:]) + except Exception: + continue + print(json.dumps(doc, sort_keys=True)) + raise SystemExit(0) +raise SystemExit(1) +' +} + +json_field() { + local field="$1" + python3 -c ' +import json +import sys + +field = sys.argv[1] +doc = json.load(sys.stdin) +value = doc +for part in field.split("."): + if not isinstance(value, dict): + value = None + break + value = value.get(part) +if isinstance(value, (dict, list)): + print(json.dumps(value, sort_keys=True)) +elif value is not None: + print(value) +' "$field" +} + +device_state_json() { + local output rc + output=$(sandbox_exec_sh_script 60 ' +set -u +if [ -r /tmp/nemoclaw-proxy-env.sh ]; then + # shellcheck source=/dev/null + . /tmp/nemoclaw-proxy-env.sh +fi +python3 - <<'"'"'PY'"'"' +import json +import os +from pathlib import Path + +root = Path(os.environ.get("OPENCLAW_STATE_DIR") or "/sandbox/.openclaw") / "devices" + +def load(name): + path = root / name + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {} + if not isinstance(value, dict): + return {} + return value + +pending = load("pending.json") +paired = load("paired.json") +print(json.dumps({ + "pending": list(pending.values()), + "paired": list(paired.values()), + "paths": { + "pending": str(root / "pending.json"), + "paired": str(root / "paired.json"), + }, +}, sort_keys=True)) +PY +' 2>&1) + rc=$? + if [ "$rc" -ne 0 ]; then + printf '%s\n' "$output" + return "$rc" + fi + printf '%s\n' "$output" | extract_json_doc +} + +summarize_device_state() { + python3 -c ' +import json +import sys + +doc = json.load(sys.stdin) +pending = doc.get("pending") or [] +paired = doc.get("paired") or [] + +def is_cli(entry): + mode = str(entry.get("clientMode") or "").lower() + client = str(entry.get("clientId") or "").lower() + return mode == "cli" or "cli" in client + +def scopes(entry): + return [s for s in entry.get("scopes") or entry.get("approvedScopes") or [] if isinstance(s, str)] + +print(f"pending={len(pending)} paired={len(paired)}") +for label, rows in (("pending", pending), ("paired", paired)): + for row in rows: + if not isinstance(row, dict) or not is_cli(row): + continue + request_id = row.get("requestId") or "-" + device_id = row.get("deviceId") or "-" + print(f"{label}: requestId={request_id} deviceId={device_id} scopes={','.join(scopes(row)) or '-'}") +' +} + +select_cli_request() { + local kind="$1" + python3 -c ' +import json +import sys + +kind = sys.argv[1] +doc = json.load(sys.stdin) +pending = [p for p in doc.get("pending") or [] if isinstance(p, dict)] +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def roles(entry): + out = set() + role = norm(entry.get("role")) + if role: + out.add(role) + for role in entry.get("roles") or []: + role = norm(role) + if role: + out.add(role) + return out + +def scopes(entry): + return {norm(scope) for scope in (entry.get("scopes") or []) if norm(scope)} + +def approved_scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +paired_by_device = {norm(item.get("deviceId")): item for item in paired if norm(item.get("deviceId"))} + +for req in sorted(pending, key=lambda item: item.get("ts") or 0, reverse=True): + if not is_cli(req) or not norm(req.get("requestId")): + continue + paired_entry = paired_by_device.get(norm(req.get("deviceId"))) + requested = scopes(req) + approved = approved_scopes(paired_entry or {}) + if kind == "new" and not paired_entry: + print(req["requestId"]) + raise SystemExit(0) + if kind == "scope-upgrade" and paired_entry and roles(req).issubset(roles(paired_entry) or roles(req)): + if requested and not requested.issubset(approved): + print(req["requestId"]) + raise SystemExit(0) +raise SystemExit(1) +' "$kind" +} + +select_cli_paired_without_write() { + python3 -c ' +import json +import sys + +doc = json.load(sys.stdin) +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +for device in sorted(paired, key=lambda item: item.get("approvedAtMs") or 0, reverse=True): + if not is_cli(device): + continue + approved = scopes(device) + if "operator.pairing" in approved and "operator.write" not in approved and "operator.admin" not in approved: + print(norm(device.get("deviceId")) or "cli-device") + raise SystemExit(0) +raise SystemExit(1) +' +} + +select_cli_paired_with_write() { + python3 -c ' +import json +import sys + +doc = json.load(sys.stdin) +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +for device in sorted(paired, key=lambda item: item.get("approvedAtMs") or 0, reverse=True): + if not is_cli(device): + continue + approved = scopes(device) + if "operator.write" in approved or "operator.admin" in approved: + print(norm(device.get("deviceId")) or "cli-device") + raise SystemExit(0) +raise SystemExit(1) +' +} + +approve_request() { + local request_id="$1" + local label="$2" + local output rc approve_json approved_id before_url after_url + output=$(sandbox_exec_sh_script 90 ' +set -u +request_id="$1" +if [ ! -r /tmp/nemoclaw-proxy-env.sh ]; then + echo "missing /tmp/nemoclaw-proxy-env.sh" >&2 + exit 2 +fi +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "__URL_BEFORE__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +set +e +approve_output="$(openclaw devices approve "$request_id" --json 2>&1)" +approve_rc=$? +set -e +printf "__APPROVE_RC__=%s\n" "$approve_rc" +printf "__APPROVE_OUTPUT_BEGIN__\n%s\n__APPROVE_OUTPUT_END__\n" "$approve_output" +printf "__URL_AFTER__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +exit "$approve_rc" +' "$request_id" 2>&1) + rc=$? + { + printf '=== approve %s request=%s rc=%s ===\n' "$label" "$request_id" "$rc" + printf '%s\n' "$output" + } >>"$APPROVAL_LOG" + if [ "$rc" -ne 0 ]; then + fail "${label}: openclaw devices approve failed for ${request_id}: ${output:0:500}" + return 1 + fi + before_url=$(sed -n 's/^__URL_BEFORE__=//p' <<<"$output" | tail -1) + after_url=$(sed -n 's/^__URL_AFTER__=//p' <<<"$output" | tail -1) + if [[ "$before_url" != ws://127.0.0.1:* ]] && [[ "$before_url" != ws://localhost:* ]]; then + fail "${label}: proxy env did not expose a loopback OPENCLAW_GATEWAY_URL before approve (${before_url:-empty})" + return 1 + fi + if [ "$after_url" != "$before_url" ]; then + fail "${label}: devices approve leaked OPENCLAW_GATEWAY_URL mutation into caller shell (${before_url} -> ${after_url})" + return 1 + fi + approve_json=$(sed -n '/^__APPROVE_OUTPUT_BEGIN__$/,/^__APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d' | extract_json_doc 2>/dev/null) || approve_json="" + if [ -z "$approve_json" ]; then + fail "${label}: approve output did not contain JSON: ${output:0:500}" + return 1 + fi + approved_id=$(printf '%s' "$approve_json" | json_field requestId) + if [ "$approved_id" != "$request_id" ]; then + fail "${label}: approve returned requestId=${approved_id:-empty}, expected ${request_id}" + return 1 + fi + pass "${label}: openclaw devices approve ${request_id} --json succeeded with caller gateway URL preserved" +} + +legacy_gateway_pinned_approve_must_fail() { + local request_id="$1" + local output legacy_rc before_url state pending_after recovery_request_id + output=$(sandbox_exec_sh_script 90 ' +set -u +request_id="$1" +if [ ! -r /tmp/nemoclaw-proxy-env.sh ]; then + echo "missing /tmp/nemoclaw-proxy-env.sh" >&2 + exit 2 +fi +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "__URL_FOR_LEGACY_APPROVE__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +OPENCLAW_4462_REQUEST_ID="$request_id" python3 - <<'"'"'PY'"'"' +import os +import subprocess + +request_id = os.environ["OPENCLAW_4462_REQUEST_ID"] +env = os.environ.copy() +try: + proc = subprocess.run( + ["openclaw", "devices", "approve", request_id, "--json"], + capture_output=True, + text=True, + timeout=20, + env=env, + ) + print(f"__LEGACY_APPROVE_RC__={proc.returncode}") + print("__LEGACY_APPROVE_OUTPUT_BEGIN__") + if proc.stdout: + print(proc.stdout, end="") + if proc.stderr: + print(proc.stderr, end="") + print("\n__LEGACY_APPROVE_OUTPUT_END__") +except subprocess.TimeoutExpired as exc: + print("__LEGACY_APPROVE_RC__=124") + print("__LEGACY_APPROVE_OUTPUT_BEGIN__") + if exc.stdout: + print(exc.stdout if isinstance(exc.stdout, str) else exc.stdout.decode(), end="") + if exc.stderr: + print(exc.stderr if isinstance(exc.stderr, str) else exc.stderr.decode(), end="") + print("\nTIMEOUT waiting for gateway-pinned devices approve") + print("__LEGACY_APPROVE_OUTPUT_END__") +PY +printf "__URL_AFTER_LEGACY_APPROVE__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +exit 0 +' "$request_id" 2>&1) + { + printf '=== legacy gateway-pinned approve request=%s ===\n' "$request_id" + printf '%s\n' "$output" + } >>"$APPROVAL_LOG" + before_url=$(sed -n 's/^__URL_FOR_LEGACY_APPROVE__=//p' <<<"$output" | tail -1) + if [[ "$before_url" != ws://127.0.0.1:* ]] && [[ "$before_url" != ws://localhost:* ]]; then + fail "legacy reproducer did not run with gateway URL pinned (${before_url:-empty})" + return 1 + fi + legacy_rc=$(sed -n 's/^__LEGACY_APPROVE_RC__=//p' <<<"$output" | tail -1) + if [ -z "$legacy_rc" ]; then + fail "legacy reproducer did not report approve rc: ${output:0:500}" + return 1 + fi + if [ "$legacy_rc" = "0" ]; then + fail "legacy gateway-pinned devices approve unexpectedly succeeded: ${output:0:500}" + return 1 + fi + pass "legacy gateway-pinned devices approve fails for the pending scope upgrade" + + state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after legacy approve failure: ${state:0:500}" + return 1 + } + printf '=== state after legacy gateway-pinned approve failure ===\n%s\n' "$state" >>"$STATE_LOG" + pending_after=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after="" + if [ -z "$pending_after" ]; then + fail "legacy gateway-pinned approve did not leave a CLI scope-upgrade request pending: $(printf '%s' "$state" | summarize_device_state)" + return 1 + fi + pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" + + recovery_request_id="$pending_after" + approve_request "$recovery_request_id" "recovery after legacy reproducer" || return 1 + pass "fixed devices approve path recovers the request after reproducing the old failure" +} + +section "Phase 0: Preflight" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +command -v python3 >/dev/null 2>&1 || { + fail "python3 is required" + exit 1 +} +pass "python3 is available" + +info "Repo: ${REPO}" +info "Sandbox name: ${SANDBOX_NAME}" +info "Mode: ${TEST_MODE}" +info "Logs: ${INSTALL_LOG}, ${APPROVAL_LOG}, ${AGENT_LOG}, ${STATE_LOG}" +: >"$APPROVAL_LOG" +: >"$AGENT_LOG" +: >"$STATE_LOG" + +section "Phase 1: Install real NemoClaw/OpenClaw sandbox" + +cd "$REPO" || { + fail "Could not cd to repo root" + exit 1 +} + +info "Pre-cleanup" +if command -v nemoclaw >/dev/null 2>&1; then + run_with_timeout 120 nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true +fi +if command -v "$OPENSHELL_BIN" >/dev/null 2>&1 || [ "$OPENSHELL_BIN" != "openshell" ]; then + run_with_timeout 60 "$OPENSHELL_BIN" sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true + if [[ "${CI:-}" = "true" || "${NEMOCLAW_E2E_DESTROY_GATEWAY:-}" = "1" ]]; then + run_with_timeout 60 "$OPENSHELL_BIN" gateway destroy -g nemoclaw >/dev/null 2>&1 || true + fi +fi +pass "Pre-cleanup complete" + +info "Running install.sh --non-interactive" +( + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_FRESH=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + export NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS:-3}" + export NEMOCLAW_AUTO_PAIR_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS:-30}" + export NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS="${NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS:-600}" + run_with_timeout "$INSTALL_TIMEOUT_SECONDS" bash install.sh --non-interactive --yes-i-accept-third-party-software +) >"$INSTALL_LOG" 2>&1 +install_rc=$? + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path +hash -r + +if [ "$install_rc" -ne 0 ]; then + fail "install.sh failed with exit ${install_rc}; see ${INSTALL_LOG}" + tail -40 "$INSTALL_LOG" || true + exit 1 +fi +pass "NemoClaw installed and onboarded" + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not found on PATH after install" + exit 1 +} +command -v "$OPENSHELL_BIN" >/dev/null 2>&1 || { + fail "${OPENSHELL_BIN} not found on PATH after install" + exit 1 +} +pass "nemoclaw and openshell are available" + +section "Phase 2: Verify in-sandbox proxy env guard" + +guard_probe=$(sandbox_exec_sh_script 60 ' +set -u +if [ ! -r /tmp/nemoclaw-proxy-env.sh ]; then + echo "MISSING_PROXY_ENV" + exit 2 +fi +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "OPENCLAW_GATEWAY_URL=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +type openclaw 2>/dev/null | sed -n "1,12p" +grep -F "unset OPENCLAW_GATEWAY_URL; command openclaw" /tmp/nemoclaw-proxy-env.sh >/dev/null \ + && echo "APPROVE_GUARD_PRESENT" +' 2>&1) +guard_rc=$? +printf '%s\n' "$guard_probe" >>"$STATE_LOG" +if [ "$guard_rc" -ne 0 ]; then + fail "Could not source /tmp/nemoclaw-proxy-env.sh: ${guard_probe:0:400}" + exit 1 +fi +if grep -q '^OPENCLAW_GATEWAY_URL=ws://127\.0\.0\.1:' <<<"$guard_probe" \ + && grep -q '^APPROVE_GUARD_PRESENT$' <<<"$guard_probe"; then + pass "proxy env preserves gateway URL and contains devices approve guard" +else + fail "proxy env missing gateway URL or approve guard: ${guard_probe:0:600}" + exit 1 +fi + +section "Phase 3: Establish low-scope CLI device approval" + +info "Creating initial CLI pairing request with openclaw devices list" +initial_list=$(sandbox_exec_sh_script 60 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +set +e +openclaw devices list --json +rc=$? +set -e +printf "__LIST_RC__=%s\n" "$rc" >&2 +exit 0 +' 2>&1) +printf '=== initial devices list ===\n%s\n' "$initial_list" >>"$STATE_LOG" + +state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after initial list: ${state:0:500}" + exit 1 +} +printf '=== state after initial list ===\n%s\n' "$state" >>"$STATE_LOG" +summary=$(printf '%s' "$state" | summarize_device_state) +info "$summary" + +initial_request_id=$(printf '%s' "$state" | select_cli_request new 2>/dev/null) || initial_request_id="" +if [ -n "$initial_request_id" ]; then + pass "pending low-scope CLI pairing request exists (${initial_request_id})" + approve_request "$initial_request_id" "initial CLI pairing" || exit 1 +else + paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write="" + if [ -n "$paired_without_write" ]; then + pass "CLI device is already paired with low scope (${paired_without_write})" + else + fail "No pending or paired low-scope CLI device found after devices list: ${summary}" + exit 1 + fi +fi + +state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after initial approval: ${state:0:500}" + exit 1 +} +printf '=== state after initial approval ===\n%s\n' "$state" >>"$STATE_LOG" +paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write="" +if [ -n "$paired_without_write" ]; then + pass "CLI device is paired with operator.pairing but not operator.write" +else + fail "Initial approval did not leave a low-scope CLI device: $(printf '%s' "$state" | summarize_device_state)" + exit 1 +fi + +gateway_list=$(sandbox_exec_sh_script 60 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "__URL_FOR_LIST__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" >&2 +openclaw devices list --json +' 2>&1) +gateway_list_rc=$? +printf '=== gateway devices list after initial approval rc=%s ===\n%s\n' "$gateway_list_rc" "$gateway_list" >>"$STATE_LOG" +if [ "$gateway_list_rc" -eq 0 ] && grep -q '^__URL_FOR_LIST__=ws://' <<<"$gateway_list"; then + pass "openclaw devices list observes device state while OPENCLAW_GATEWAY_URL is set" +else + fail "devices list did not work with gateway URL after initial approval: ${gateway_list:0:500}" + exit 1 +fi + +section "Phase 4: Trigger and approve CLI scope upgrade" + +info "Triggering agent operator.write scope upgrade" +trigger_output=$(sandbox_exec_sh_script 120 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +session_id="issue-4462-trigger-$(date +%s)-$$" +rm -f "/sandbox/.openclaw/agents/main/sessions/${session_id}.jsonl.lock" \ + "/sandbox/.openclaw/agents/main/sessions/${session_id}.trajectory.jsonl" 2>/dev/null || true +printf "__URL_FOR_TRIGGER_AGENT__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +set +e +openclaw agent --agent main --json --session-id "$session_id" \ + -m "What is 6 multiplied by 7? Reply with only the integer, no extra words." +agent_rc=$? +set -e +printf "__TRIGGER_AGENT_RC__=%s\n" "$agent_rc" +exit 0 +' 2>&1) +printf '=== trigger agent output ===\n%s\n' "$trigger_output" >>"$AGENT_LOG" + +scope_request_id="" +for _attempt in 1 2 3 4 5; do + state="$(device_state_json 2>&1)" || state="" + if [ -n "$state" ]; then + printf '=== state while waiting for scope upgrade ===\n%s\n' "$state" >>"$STATE_LOG" + scope_request_id=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || scope_request_id="" + fi + [ -n "$scope_request_id" ] && break + sleep 2 +done + +if [ -z "$scope_request_id" ]; then + fail "No pending CLI scope-upgrade request appeared after agent trigger. State: $(printf '%s' "${state:-{}}" | summarize_device_state 2>/dev/null || true). Trigger: ${trigger_output:0:500}" + exit 1 +fi +pass "pending CLI scope-upgrade request exists (${scope_request_id})" + +if [ "$TEST_MODE" = "legacy-repro" ]; then + legacy_gateway_pinned_approve_must_fail "$scope_request_id" || exit 1 + section "Summary" + echo "" + printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m | \033[33mSkip: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" "$SKIP" + echo "" + if [ "$FAIL" -gt 0 ]; then + echo "RESULT: FAILED - ${FAIL} test(s) failed" + exit 1 + fi + echo "RESULT: PASSED - #4462 legacy gateway-pinned approval failure reproduced and recovered" + exit 0 +fi + +if [ "$TEST_MODE" != "approval" ]; then + fail "Unknown NEMOCLAW_4462_MODE=${TEST_MODE}; expected approval or legacy-repro" + exit 1 +fi + +approve_request "$scope_request_id" "CLI scope upgrade" || exit 1 + +state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after scope-upgrade approval: ${state:0:500}" + exit 1 +} +printf '=== state after scope-upgrade approval ===\n%s\n' "$state" >>"$STATE_LOG" +pending_after_approval=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after_approval="" +paired_with_write=$(printf '%s' "$state" | select_cli_paired_with_write 2>/dev/null) || paired_with_write="" +if [ -n "$pending_after_approval" ]; then + fail "Scope-upgrade request is still pending after approval (${pending_after_approval})" + exit 1 +fi +if [ -z "$paired_with_write" ]; then + fail "No CLI paired device has operator.write after approval: $(printf '%s' "$state" | summarize_device_state)" + exit 1 +fi +pass "scope-upgrade approval grants the CLI device operator.write" + +section "Phase 5: Verify agent stays on gateway path" + +agent_ok=0 +last_agent_detail="" +for attempt in 1 2; do + info "Running approved openclaw agent turn (attempt ${attempt}/2)" + final_output=$(sandbox_exec_sh_script 180 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +session_id="issue-4462-fixed-$(date +%s)-$$" +rm -f "/sandbox/.openclaw/agents/main/sessions/${session_id}.jsonl.lock" \ + "/sandbox/.openclaw/agents/main/sessions/${session_id}.trajectory.jsonl" 2>/dev/null || true +printf "__URL_FOR_FINAL_AGENT__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +openclaw agent --agent main --json --session-id "$session_id" \ + -m "What is 6 multiplied by 7? Reply with only the integer, no extra words." +' 2>&1) + final_rc=$? + printf '=== final agent attempt %s rc=%s ===\n%s\n' "$attempt" "$final_rc" "$final_output" >>"$AGENT_LOG" + reply=$(printf '%s' "$final_output" | parse_openclaw_agent_text 2>/dev/null) || reply="" + if grep -Eq 'EMBEDDED FALLBACK|fallbackFrom[": ]+gateway|transport[": ]+embedded' <<<"$final_output"; then + last_agent_detail="agent used embedded fallback: ${final_output:0:500}" + elif [ "$final_rc" -ne 0 ]; then + last_agent_detail="agent exited ${final_rc}: ${final_output:0:500}" + elif ! grep -q '^__URL_FOR_FINAL_AGENT__=ws://' <<<"$final_output"; then + last_agent_detail="agent command did not preserve OPENCLAW_GATEWAY_URL: ${final_output:0:500}" + elif grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then + agent_ok=1 + pass "approved openclaw agent turn answered through gateway mode" + break + else + last_agent_detail="expected reply 42, got reply='${reply:0:200}', raw='${final_output:0:400}'" + fi + sleep 5 +done + +if [ "$agent_ok" -ne 1 ]; then + fail "Final approved agent turn did not prove gateway-mode success: ${last_agent_detail}" + exit 1 +fi + +pass "approved agent output contains no embedded fallback markers" + +section "Summary" +echo "" +printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m | \033[33mSkip: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" "$SKIP" +echo "" + +if [ "$FAIL" -gt 0 ]; then + echo "RESULT: FAILED - ${FAIL} test(s) failed" + exit 1 +fi + +echo "RESULT: PASSED - #4462 CLI scope-upgrade approval stays on the gateway path" +exit 0 diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index c9bdd59312..bbb1b76978 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -705,7 +705,7 @@ describe("nemoclaw-start configure guard behavior", () => { fs.mkdirSync(fakeBin); fs.writeFileSync( path.join(fakeBin, "openclaw"), - `#!/usr/bin/env bash\nprintf '%s\\n' "$*" >> ${JSON.stringify(commandLog)}\nexit 0\n`, + `#!/usr/bin/env bash\nprintf 'ARGS=%s URL=%s\\n' "$*" "\${OPENCLAW_GATEWAY_URL-unset}" >> ${JSON.stringify(commandLog)}\nexit 0\n`, { mode: 0o755 }, ); const runtimeBlock = `${runtimeShellEnvBlock(src)}\nwrite_runtime_shell_env`.replaceAll( @@ -726,6 +726,7 @@ describe("nemoclaw-start configure guard behavior", () => { '_SECCOMP_GUARD_SCRIPT="/tmp/seccomp-guard.js"', '_CIAO_GUARD_SCRIPT="/tmp/ciao-guard.js"', '_SLACK_GUARD_SCRIPT="/nonexistent/slack-guard.js"', + 'export OPENCLAW_GATEWAY_URL="ws://127.0.0.1:18789"', "_TOOL_REDIRECTS=()", "set +u", runtimeBlock, @@ -737,7 +738,11 @@ describe("nemoclaw-start configure guard behavior", () => { return { tmpDir, fakeBin, proxyEnv, commandLog }; } - function runGuardedOpenclaw(setup: ReturnType, args: string[]) { + function shellOpenclawCommand(args: string[]) { + return ["openclaw", ...args.map((arg) => JSON.stringify(arg))].join(" "); + } + + function runGuardedShell(setup: ReturnType, commands: string[]) { return spawnSync( "bash", [ @@ -746,7 +751,7 @@ describe("nemoclaw-start configure guard behavior", () => { "-c", [ `source ${JSON.stringify(setup.proxyEnv)}`, - ["openclaw", ...args.map((arg) => JSON.stringify(arg))].join(" "), + ...commands, ].join("; "), ], { @@ -757,6 +762,10 @@ describe("nemoclaw-start configure guard behavior", () => { ); } + function runGuardedOpenclaw(setup: ReturnType, args: string[]) { + return runGuardedShell(setup, [shellOpenclawCommand(args)]); + } + it("emits a proxy-env guard that blocks mutating OpenClaw commands and passes read-only commands through", () => { const setup = writeProxyEnvWithGuard(); try { @@ -795,6 +804,28 @@ describe("nemoclaw-start configure guard behavior", () => { } }); + it("#4462: unsets OPENCLAW_GATEWAY_URL only for devices approve", () => { + const setup = writeProxyEnvWithGuard(); + try { + const result = runGuardedShell(setup, [ + shellOpenclawCommand(["devices", "list", "--json"]), + shellOpenclawCommand(["devices", "approve", "request-1", "--json"]), + `printf 'SHELL_URL=%s\\n' "\${OPENCLAW_GATEWAY_URL-unset}" >> ${JSON.stringify(setup.commandLog)}`, + shellOpenclawCommand(["agent", "--agent", "main", "-m", "hello"]), + ]); + + expect(result.status).toBe(0); + expect(fs.readFileSync(setup.commandLog, "utf-8").trim().split("\n")).toEqual([ + "ARGS=devices list --json URL=ws://127.0.0.1:18789", + "ARGS=devices approve request-1 --json URL=unset", + "SHELL_URL=ws://127.0.0.1:18789", + "ARGS=agent --agent main -m hello URL=ws://127.0.0.1:18789", + ]); + } finally { + fs.rmSync(setup.tmpDir, { recursive: true, force: true }); + } + }); + // #2592 reported the guard did not fire for `openclaw channels add telegram` // and `openclaw channels remove telegram` from inside the sandbox. The // existing test above only exercises `add slack`. Lock in coverage for every @@ -1316,6 +1347,7 @@ describe("nemoclaw-start auto-pair client whitelisting (#117)", () => { const fakeOpenclaw = path.join(tmpDir, "openclaw"); const stateFile = path.join(tmpDir, "list-count"); const approveLog = path.join(tmpDir, "approvals.log"); + const envLog = path.join(tmpDir, "env.log"); const pendingJson = JSON.stringify({ pending: [ "not-a-device", @@ -1335,6 +1367,7 @@ describe("nemoclaw-start auto-pair client whitelisting (#117)", () => { `#!/usr/bin/env bash set -euo pipefail if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "list" ]; then + printf 'list:%s\n' "\${OPENCLAW_GATEWAY_URL-unset}" >> ${JSON.stringify(envLog)} count="$(cat ${JSON.stringify(stateFile)} 2>/dev/null || echo 0)" count=$((count + 1)) echo "$count" > ${JSON.stringify(stateFile)} @@ -1346,6 +1379,7 @@ if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "list" ]; then exit 0 fi if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "approve" ]; then + printf 'approve:%s:%s\n' "$3" "\${OPENCLAW_GATEWAY_URL-unset}" >> ${JSON.stringify(envLog)} echo "$3" >> ${JSON.stringify(approveLog)} printf '{}\n' exit 0 @@ -1367,6 +1401,7 @@ exit 2 env: { ...process.env, OPENCLAW_BIN: fakeOpenclaw, + OPENCLAW_GATEWAY_URL: "ws://127.0.0.1:18789", // Cap the slow-mode keepalive (NemoClaw#4263) so the test // terminates without waiting out the default 8h deadline. NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "5", @@ -1387,6 +1422,10 @@ exit 2 "ok-browser", "ok-webchat", ]); + const envLogLines = fs.readFileSync(envLog, "utf-8").trim().split("\n"); + expect(envLogLines).toContain("list:ws://127.0.0.1:18789"); + expect(envLogLines).toContain("approve:ok-browser:unset"); + expect(envLogLines).toContain("approve:ok-webchat:unset"); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } @@ -1858,6 +1897,81 @@ exit 2 fs.rmSync(tmpDir, { recursive: true, force: true }); } }, 40_000); + + it("retries a non-zero approve failure without counting it as approved", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auto-pair-afail-")); + const fakeOpenclaw = path.join(tmpDir, "openclaw"); + const stateFile = path.join(tmpDir, "approve-count"); + const approveLog = path.join(tmpDir, "approvals.log"); + const pendingResponse = JSON.stringify({ + pending: [ + { requestId: "retry-cli", clientId: "openclaw-cli", clientMode: "cli" }, + ], + paired: [], + }); + const allPaired = JSON.stringify({ + pending: [], + paired: [{ clientId: "openclaw-cli", clientMode: "cli" }], + }); + + fs.writeFileSync( + fakeOpenclaw, + `#!/usr/bin/env bash +set -euo pipefail +if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "list" ]; then + if [ -f ${JSON.stringify(approveLog)} ]; then + printf '%s\n' ${JSON.stringify(allPaired)} + else + printf '%s\n' ${JSON.stringify(pendingResponse)} + fi + exit 0 +fi +if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "approve" ]; then + count="$(cat ${JSON.stringify(stateFile)} 2>/dev/null || echo 0)" + count=$((count + 1)) + echo "$count" > ${JSON.stringify(stateFile)} + if [ "$count" = "1" ]; then + echo "temporary approve failure" >&2 + exit 7 + fi + echo "$3" >> ${JSON.stringify(approveLog)} + printf '{}\n' + exit 0 +fi +echo "unexpected: $*" >&2 +exit 2 +`, + { mode: 0o755 }, + ); + + try { + const run = spawnSync("python3", ["-c", buildAutoPairScript()], { + encoding: "utf-8", + env: { + ...process.env, + OPENCLAW_BIN: fakeOpenclaw, + NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "600", + NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "1", + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "1", + }, + timeout: 20_000, + }); + expect(run.status).toBe(0); + expect(run.stdout).toContain( + "[auto-pair] approve failed request=retry-cli: temporary approve failure", + ); + expect(run.stdout).toContain( + "[auto-pair] approved request=retry-cli client=openclaw-cli mode=cli", + ); + expect(run.stdout).toContain("watcher deadline reached approvals=1"); + expect(fs.readFileSync(stateFile, "utf-8").trim()).toBe("2"); + expect(fs.readFileSync(approveLog, "utf-8").trim().split("\n")).toEqual([ + "retry-cli", + ]); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }, 30_000); }); describe("nemoclaw-start gateway launch signal handling", () => { diff --git a/test/sandbox-connect-inference.test.ts b/test/sandbox-connect-inference.test.ts index caef5d06d9..b0a9632809 100644 --- a/test/sandbox-connect-inference.test.ts +++ b/test/sandbox-connect-inference.test.ts @@ -1205,9 +1205,16 @@ describe("sandbox connect auto-pair approval pass (#4263)", () => { expect(script).toContain("devices"); expect(script).toContain("list"); expect(script).toContain("approve"); + expect(script).toContain("approve_env = os.environ.copy()"); + expect(script).toContain("approve_env.pop('OPENCLAW_GATEWAY_URL', None)"); + expect(script).toContain("env=approve_env"); + expect(script).toContain("if approve_proc.returncode == 0"); expect(script).toContain("openclaw-control-ui"); expect(script).toContain("webchat"); expect(script).toContain("cli"); + expect(script.indexOf("[OPENCLAW, 'devices', 'list', '--json']")).toBeLessThan( + script.indexOf("approve_env = os.environ.copy()"), + ); // Allowlist must NOT silently approve arbitrary clients. expect(script).not.toContain("evil-client"); }, From 9c956cf4e53767155fd2caf82d2642c082e3cc4a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 14:46:56 -0700 Subject: [PATCH 02/13] test(e2e): stabilize issue 4462 scope upgrade proofs Signed-off-by: Aaron Erickson --- .github/workflows/nightly-e2e.yaml | 14 +- .../test-issue-4462-scope-upgrade-approval.sh | 135 +++++++++++++++--- 2 files changed, 119 insertions(+), 30 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 9e7c6bf99b..30083a5908 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -402,9 +402,9 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} # ── OpenClaw Scope-Upgrade Approval E2E (#4462) ──────────────── - # Positive proof: in a real sandbox, approve a pending CLI scope upgrade - # through the proxy-env guard, then confirm openclaw agent still uses the - # gateway path rather than embedded fallback. + # Positive proof: in a real sandbox, accept either a visible pending CLI + # scope upgrade or the fixed watcher's immediate approval, then confirm + # openclaw agent still uses the gateway path rather than embedded fallback. issue-4462-scope-upgrade-approval-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || @@ -428,9 +428,9 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} # ── OpenClaw Scope-Upgrade Deadlock Reproducer (#4462) ────────── - # Negative proof: in a real sandbox, force the old gateway-pinned approve - # path and verify the CLI scope-upgrade remains pending; then recover through - # the fixed proxy-env guard so the job can finish cleanly. + # Negative proof: in a real sandbox, wait for the fixed watcher to exit, + # force the old gateway-pinned approve path, verify the CLI scope-upgrade + # remains pending, then recover through the fixed proxy-env guard. issue-4462-scope-upgrade-deadlock-repro-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || @@ -447,7 +447,7 @@ jobs: /tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log /tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log /tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log - env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}' + env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS":"12","NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS":"1","NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS":"2","NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS":"1","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}' nvidia_api_key: true github_token: true secrets: diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index d9ea63e975..a9d5ca476a 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -82,6 +82,21 @@ AGENT_LOG="${NEMOCLAW_4462_AGENT_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-age STATE_LOG="${NEMOCLAW_4462_STATE_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-state.log}" INSTALL_TIMEOUT_SECONDS="${NEMOCLAW_E2E_INSTALL_TIMEOUT_SECONDS:-1800}" +AUTO_PAIR_FAST_DEADLINE_DEFAULT="3" +AUTO_PAIR_DEADLINE_DEFAULT="30" +AUTO_PAIR_SLOW_INTERVAL_DEFAULT="600" +AUTO_PAIR_RUN_TIMEOUT_DEFAULT="10" +if [ "$TEST_MODE" = "legacy-repro" ]; then + AUTO_PAIR_FAST_DEADLINE_DEFAULT="1" + AUTO_PAIR_DEADLINE_DEFAULT="12" + AUTO_PAIR_SLOW_INTERVAL_DEFAULT="1" + AUTO_PAIR_RUN_TIMEOUT_DEFAULT="2" +fi +AUTO_PAIR_FAST_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS:-${NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS:-$AUTO_PAIR_FAST_DEADLINE_DEFAULT}}" +AUTO_PAIR_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS:-${NEMOCLAW_AUTO_PAIR_DEADLINE_SECS:-$AUTO_PAIR_DEADLINE_DEFAULT}}" +AUTO_PAIR_SLOW_INTERVAL_SECS="${NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS:-${NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS:-$AUTO_PAIR_SLOW_INTERVAL_DEFAULT}}" +AUTO_PAIR_RUN_TIMEOUT_SECS="${NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS:-${NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS:-$AUTO_PAIR_RUN_TIMEOUT_DEFAULT}}" + # shellcheck source=test/e2e/lib/sandbox-teardown.sh . "${E2E_DIR}/lib/sandbox-teardown.sh" # shellcheck source=test/e2e/lib/install-path-refresh.sh @@ -195,21 +210,39 @@ PY } summarize_device_state() { - python3 -c ' + local state_doc + state_doc="$(cat)" + OPENCLAW_4462_DEVICE_STATE="$state_doc" python3 - <<'PY' import json +import os import sys -doc = json.load(sys.stdin) +raw = os.environ.get("OPENCLAW_4462_DEVICE_STATE") or "{}" +doc = json.loads(raw) pending = doc.get("pending") or [] paired = doc.get("paired") or [] +def norm(value): + return str(value or "").strip() + def is_cli(entry): - mode = str(entry.get("clientMode") or "").lower() - client = str(entry.get("clientId") or "").lower() + mode = norm(entry.get("clientMode")).lower() + client = norm(entry.get("clientId")).lower() return mode == "cli" or "cli" in client -def scopes(entry): - return [s for s in entry.get("scopes") or entry.get("approvedScopes") or [] if isinstance(s, str)] +def scope_list(entry, *keys): + out = [] + seen = set() + for key in keys: + for scope in entry.get(key) or []: + scope = norm(scope) + if scope and scope not in seen: + out.append(scope) + seen.add(scope) + return out + +def fmt(values): + return ",".join(values) if values else "-" print(f"pending={len(pending)} paired={len(paired)}") for label, rows in (("pending", pending), ("paired", paired)): @@ -218,8 +251,16 @@ for label, rows in (("pending", pending), ("paired", paired)): continue request_id = row.get("requestId") or "-" device_id = row.get("deviceId") or "-" - print(f"{label}: requestId={request_id} deviceId={device_id} scopes={','.join(scopes(row)) or '-'}") -' + approved = scope_list(row, "approvedScopes") + if label == "paired": + approved = approved or scope_list(row, "scopes") + requested = scope_list(row, "scopes", "requestedScopes") + print( + f"{label}: pendingCount={len(pending)} requestId={request_id} " + f"deviceId={device_id} approvedScopes={fmt(approved)} " + f"requestedScopes={fmt(requested)}" + ) +PY } select_cli_request() { @@ -303,7 +344,7 @@ raise SystemExit(1) ' } -select_cli_paired_with_write() { +select_cli_paired_with_agent_scopes() { python3 -c ' import json import sys @@ -324,7 +365,7 @@ for device in sorted(paired, key=lambda item: item.get("approvedAtMs") or 0, rev if not is_cli(device): continue approved = scopes(device) - if "operator.write" in approved or "operator.admin" in approved: + if "operator.admin" in approved or {"operator.write", "operator.read"}.issubset(approved): print(norm(device.get("deviceId")) or "cli-device") raise SystemExit(0) raise SystemExit(1) @@ -470,6 +511,36 @@ exit 0 pass "fixed devices approve path recovers the request after reproducing the old failure" } +wait_for_auto_pair_watcher_inactive() { + local output rc + for _attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do + output=$(sandbox_exec_sh_script 20 ' +set -u +if [ -r /tmp/auto-pair.log ]; then + if grep -F "[auto-pair] watcher deadline reached" /tmp/auto-pair.log >/dev/null; then + echo "__AUTO_PAIR_WATCHER__=deadline-reached" + tail -20 /tmp/auto-pair.log + exit 0 + fi + echo "__AUTO_PAIR_WATCHER__=still-waiting" + tail -20 /tmp/auto-pair.log || true +else + echo "__AUTO_PAIR_WATCHER__=missing-log" +fi +exit 1 +' 2>&1) + rc=$? + printf '=== auto-pair watcher inactivity probe rc=%s ===\n%s\n' "$rc" "$output" >>"$STATE_LOG" + if [ "$rc" -eq 0 ]; then + pass "auto-pair watcher reached its deadline before legacy scope-upgrade trigger" + return 0 + fi + sleep 2 + done + fail "auto-pair watcher was still active before legacy scope-upgrade trigger: ${output:0:500}" + return 1 +} + section "Phase 0: Preflight" if [ -z "${NVIDIA_API_KEY:-}" ]; then @@ -494,6 +565,7 @@ info "Repo: ${REPO}" info "Sandbox name: ${SANDBOX_NAME}" info "Mode: ${TEST_MODE}" info "Logs: ${INSTALL_LOG}, ${APPROVAL_LOG}, ${AGENT_LOG}, ${STATE_LOG}" +info "Auto-pair timing: fast=${AUTO_PAIR_FAST_DEADLINE_SECS}s deadline=${AUTO_PAIR_DEADLINE_SECS}s slow=${AUTO_PAIR_SLOW_INTERVAL_SECS}s run-timeout=${AUTO_PAIR_RUN_TIMEOUT_SECS}s" : >"$APPROVAL_LOG" : >"$AGENT_LOG" : >"$STATE_LOG" @@ -524,9 +596,10 @@ info "Running install.sh --non-interactive" export NEMOCLAW_FRESH=1 export NEMOCLAW_NON_INTERACTIVE=1 export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 - export NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS:-3}" - export NEMOCLAW_AUTO_PAIR_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS:-30}" - export NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS="${NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS:-600}" + export NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS="$AUTO_PAIR_FAST_DEADLINE_SECS" + export NEMOCLAW_AUTO_PAIR_DEADLINE_SECS="$AUTO_PAIR_DEADLINE_SECS" + export NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS="$AUTO_PAIR_SLOW_INTERVAL_SECS" + export NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS="$AUTO_PAIR_RUN_TIMEOUT_SECS" run_with_timeout "$INSTALL_TIMEOUT_SECONDS" bash install.sh --non-interactive --yes-i-accept-third-party-software ) >"$INSTALL_LOG" 2>&1 install_rc=$? @@ -651,6 +724,10 @@ else exit 1 fi +if [ "$TEST_MODE" = "legacy-repro" ]; then + wait_for_auto_pair_watcher_inactive || exit 1 +fi + section "Phase 4: Trigger and approve CLI scope upgrade" info "Triggering agent operator.write scope upgrade" @@ -673,21 +750,29 @@ exit 0 printf '=== trigger agent output ===\n%s\n' "$trigger_output" >>"$AGENT_LOG" scope_request_id="" +auto_approved_device="" for _attempt in 1 2 3 4 5; do state="$(device_state_json 2>&1)" || state="" if [ -n "$state" ]; then printf '=== state while waiting for scope upgrade ===\n%s\n' "$state" >>"$STATE_LOG" scope_request_id=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || scope_request_id="" + auto_approved_device=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || auto_approved_device="" fi [ -n "$scope_request_id" ] && break + if [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then + break + fi sleep 2 done -if [ -z "$scope_request_id" ]; then +if [ -n "$scope_request_id" ]; then + pass "pending CLI scope-upgrade request exists (${scope_request_id})" +elif [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then + pass "auto-pair watcher approved the CLI scope upgrade before pending inspection (${auto_approved_device})" +else fail "No pending CLI scope-upgrade request appeared after agent trigger. State: $(printf '%s' "${state:-{}}" | summarize_device_state 2>/dev/null || true). Trigger: ${trigger_output:0:500}" exit 1 fi -pass "pending CLI scope-upgrade request exists (${scope_request_id})" if [ "$TEST_MODE" = "legacy-repro" ]; then legacy_gateway_pinned_approve_must_fail "$scope_request_id" || exit 1 @@ -709,7 +794,11 @@ if [ "$TEST_MODE" != "approval" ]; then exit 1 fi -approve_request "$scope_request_id" "CLI scope upgrade" || exit 1 +if [ -n "$scope_request_id" ]; then + approve_request "$scope_request_id" "CLI scope upgrade" || exit 1 +else + info "Skipping manual scope-upgrade approval because the auto-pair watcher already granted it" +fi state="$(device_state_json 2>&1)" || { fail "Could not read OpenClaw device state after scope-upgrade approval: ${state:0:500}" @@ -717,16 +806,16 @@ state="$(device_state_json 2>&1)" || { } printf '=== state after scope-upgrade approval ===\n%s\n' "$state" >>"$STATE_LOG" pending_after_approval=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after_approval="" -paired_with_write=$(printf '%s' "$state" | select_cli_paired_with_write 2>/dev/null) || paired_with_write="" +paired_with_agent_scopes=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || paired_with_agent_scopes="" if [ -n "$pending_after_approval" ]; then fail "Scope-upgrade request is still pending after approval (${pending_after_approval})" exit 1 fi -if [ -z "$paired_with_write" ]; then - fail "No CLI paired device has operator.write after approval: $(printf '%s' "$state" | summarize_device_state)" +if [ -z "$paired_with_agent_scopes" ]; then + fail "No CLI paired device has operator.write and operator.read after approval: $(printf '%s' "$state" | summarize_device_state)" exit 1 fi -pass "scope-upgrade approval grants the CLI device operator.write" +pass "scope-upgrade approval grants the CLI device operator.write and operator.read" section "Phase 5: Verify agent stays on gateway path" @@ -748,8 +837,8 @@ openclaw agent --agent main --json --session-id "$session_id" \ final_rc=$? printf '=== final agent attempt %s rc=%s ===\n%s\n' "$attempt" "$final_rc" "$final_output" >>"$AGENT_LOG" reply=$(printf '%s' "$final_output" | parse_openclaw_agent_text 2>/dev/null) || reply="" - if grep -Eq 'EMBEDDED FALLBACK|fallbackFrom[": ]+gateway|transport[": ]+embedded' <<<"$final_output"; then - last_agent_detail="agent used embedded fallback: ${final_output:0:500}" + if grep -Eiq 'EMBEDDED FALLBACK|scope upgrade pending approval|pairing required|fallbackFrom[": ]+gateway|transport[": ]+embedded' <<<"$final_output"; then + last_agent_detail="agent output contained fallback or pairing marker: ${final_output:0:500}" elif [ "$final_rc" -ne 0 ]; then last_agent_detail="agent exited ${final_rc}: ${final_output:0:500}" elif ! grep -q '^__URL_FOR_FINAL_AGENT__=ws://' <<<"$final_output"; then @@ -769,7 +858,7 @@ if [ "$agent_ok" -ne 1 ]; then exit 1 fi -pass "approved agent output contains no embedded fallback markers" +pass "approved agent output contains no fallback or pairing markers" section "Summary" echo "" From 6cc9fd54df2489e96d6053e7625195128e459351 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 14:56:40 -0700 Subject: [PATCH 03/13] test(e2e): isolate issue 4462 repro watcher Signed-off-by: Aaron Erickson --- .../test-issue-4462-scope-upgrade-approval.sh | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index a9d5ca476a..c2b3be21c3 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -516,13 +516,37 @@ wait_for_auto_pair_watcher_inactive() { for _attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do output=$(sandbox_exec_sh_script 20 ' set -u +find_auto_pair_pids() { + for proc in /proc/[0-9]*; do + pid="${proc##*/}" + [ "$pid" = "$$" ] && continue + [ -r "$proc/cmdline" ] || continue + cmd="$(tr "\000" " " <"$proc/cmdline" 2>/dev/null || true)" + case "$cmd" in + *"python3 -"*) + fd1="$(readlink "$proc/fd/1" 2>/dev/null || true)" + fd2="$(readlink "$proc/fd/2" 2>/dev/null || true)" + case "${fd1} ${fd2}" in + *"/tmp/auto-pair.log"*) printf "%s\n" "$pid" ;; + esac + ;; + esac + done | sort -u +} if [ -r /tmp/auto-pair.log ]; then if grep -F "[auto-pair] watcher deadline reached" /tmp/auto-pair.log >/dev/null; then echo "__AUTO_PAIR_WATCHER__=deadline-reached" tail -20 /tmp/auto-pair.log exit 0 fi + pids="$(find_auto_pair_pids)" + if [ -z "$pids" ]; then + echo "__AUTO_PAIR_WATCHER__=inactive" + tail -20 /tmp/auto-pair.log || true + exit 0 + fi echo "__AUTO_PAIR_WATCHER__=still-waiting" + printf "__AUTO_PAIR_PIDS__=%s\n" "$(printf "%s" "$pids" | tr "\n" " ")" tail -20 /tmp/auto-pair.log || true else echo "__AUTO_PAIR_WATCHER__=missing-log" @@ -537,6 +561,53 @@ exit 1 fi sleep 2 done + output=$(sandbox_exec_sh_script 30 ' +set -u +find_auto_pair_pids() { + for proc in /proc/[0-9]*; do + pid="${proc##*/}" + [ "$pid" = "$$" ] && continue + [ -r "$proc/cmdline" ] || continue + cmd="$(tr "\000" " " <"$proc/cmdline" 2>/dev/null || true)" + case "$cmd" in + *"python3 -"*) + fd1="$(readlink "$proc/fd/1" 2>/dev/null || true)" + fd2="$(readlink "$proc/fd/2" 2>/dev/null || true)" + case "${fd1} ${fd2}" in + *"/tmp/auto-pair.log"*) printf "%s\n" "$pid" ;; + esac + ;; + esac + done | sort -u +} +pids="$(find_auto_pair_pids)" +if [ -z "$pids" ]; then + echo "__AUTO_PAIR_WATCHER__=inactive-before-stop" + exit 0 +fi +printf "__AUTO_PAIR_STOPPING_PIDS__=%s\n" "$(printf "%s" "$pids" | tr "\n" " ")" +kill $pids 2>/dev/null || true +sleep 2 +remaining="$(find_auto_pair_pids)" +if [ -n "$remaining" ]; then + printf "__AUTO_PAIR_KILLING_PIDS__=%s\n" "$(printf "%s" "$remaining" | tr "\n" " ")" + kill -KILL $remaining 2>/dev/null || true + sleep 1 +fi +remaining="$(find_auto_pair_pids)" +if [ -n "$remaining" ]; then + printf "__AUTO_PAIR_WATCHER__=still-active pids=%s\n" "$(printf "%s" "$remaining" | tr "\n" " ")" + exit 1 +fi +echo "__AUTO_PAIR_WATCHER__=stopped" +tail -20 /tmp/auto-pair.log 2>/dev/null || true +' 2>&1) + rc=$? + printf '=== auto-pair watcher forced stop rc=%s ===\n%s\n' "$rc" "$output" >>"$STATE_LOG" + if [ "$rc" -eq 0 ]; then + pass "auto-pair watcher is inactive before legacy scope-upgrade trigger" + return 0 + fi fail "auto-pair watcher was still active before legacy scope-upgrade trigger: ${output:0:500}" return 1 } From 59a0f2410923f7000f944ac35fc17de326eb359e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 15:09:47 -0700 Subject: [PATCH 04/13] test(e2e): accept current issue 4462 legacy outcome Signed-off-by: Aaron Erickson --- .../test-issue-4462-scope-upgrade-approval.sh | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index c2b3be21c3..a7ff3434e4 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -429,7 +429,7 @@ exit "$approve_rc" legacy_gateway_pinned_approve_must_fail() { local request_id="$1" - local output legacy_rc before_url state pending_after recovery_request_id + local output legacy_rc before_url legacy_output state pending_after recovery_request_id paired_after_legacy output=$(sandbox_exec_sh_script 90 ' set -u request_id="$1" @@ -492,6 +492,11 @@ exit 0 fail "legacy gateway-pinned devices approve unexpectedly succeeded: ${output:0:500}" return 1 fi + legacy_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') + if ! grep -Eiq 'scope upgrade pending approval|pairing required' <<<"$legacy_output"; then + fail "legacy gateway-pinned approve failed without the expected scope-upgrade fingerprint: ${legacy_output:0:500}" + return 1 + fi pass "legacy gateway-pinned devices approve fails for the pending scope upgrade" state="$(device_state_json 2>&1)" || { @@ -500,15 +505,21 @@ exit 0 } printf '=== state after legacy gateway-pinned approve failure ===\n%s\n' "$state" >>"$STATE_LOG" pending_after=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after="" - if [ -z "$pending_after" ]; then - fail "legacy gateway-pinned approve did not leave a CLI scope-upgrade request pending: $(printf '%s' "$state" | summarize_device_state)" - return 1 - fi - pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" + paired_after_legacy=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || paired_after_legacy="" + if [ -n "$pending_after" ]; then + pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" - recovery_request_id="$pending_after" - approve_request "$recovery_request_id" "recovery after legacy reproducer" || return 1 - pass "fixed devices approve path recovers the request after reproducing the old failure" + recovery_request_id="$pending_after" + approve_request "$recovery_request_id" "recovery after legacy reproducer" || return 1 + pass "fixed devices approve path recovers the request after reproducing the old failure" + return 0 + fi + if [ -n "$paired_after_legacy" ]; then + pass "legacy gateway-pinned approve emitted the old failure fingerprint but left the CLI device approved" + return 0 + fi + fail "legacy gateway-pinned approve left neither a pending request nor an approved CLI device: $(printf '%s' "$state" | summarize_device_state)" + return 1 } wait_for_auto_pair_watcher_inactive() { From d0b23f3ecac43e1e0568b9305cd41aabd83fd75a Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 15:17:30 -0700 Subject: [PATCH 05/13] test(e2e): reuse issue 4462 trigger request id Signed-off-by: Aaron Erickson --- test/e2e/test-issue-4462-scope-upgrade-approval.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index a7ff3434e4..76eb947144 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -164,6 +164,10 @@ elif value is not None: ' "$field" } +extract_scope_request_id_from_output() { + sed -nE 's/.*requestId: ([[:alnum:]_-]+).*/\1/p' | head -1 +} + device_state_json() { local output rc output=$(sandbox_exec_sh_script 60 ' @@ -847,6 +851,10 @@ for _attempt in 1 2 3 4 5; do sleep 2 done +if [ -z "$scope_request_id" ] && [ "$TEST_MODE" = "legacy-repro" ]; then + scope_request_id=$(printf '%s' "$trigger_output" | extract_scope_request_id_from_output) || scope_request_id="" +fi + if [ -n "$scope_request_id" ]; then pass "pending CLI scope-upgrade request exists (${scope_request_id})" elif [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then From aa609d2e327db326245771a4df496c8312ed5e58 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 28 May 2026 15:12:52 -0700 Subject: [PATCH 06/13] test(e2e): tighten issue 4462 repro assertions --- .github/workflows/nightly-e2e.yaml | 8 ++--- scripts/nemoclaw-start.sh | 12 +++++++ src/lib/actions/sandbox/connect.ts | 7 ++++ .../test-issue-4462-scope-upgrade-approval.sh | 34 +++++++++++-------- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 30083a5908..f56330338f 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -423,14 +423,14 @@ jobs: /tmp/nemoclaw-issue-4462-scope-upgrade-state.log env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}' nvidia_api_key: true - github_token: true + github_token: false secrets: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} # ── OpenClaw Scope-Upgrade Deadlock Reproducer (#4462) ────────── # Negative proof: in a real sandbox, wait for the fixed watcher to exit, - # force the old gateway-pinned approve path, verify the CLI scope-upgrade - # remains pending, then recover through the fixed proxy-env guard. + # force the old gateway-pinned approve path, verify the #4462 failure + # signature, and recover through the fixed proxy-env guard if needed. issue-4462-scope-upgrade-deadlock-repro-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || @@ -449,7 +449,7 @@ jobs: /tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS":"12","NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS":"1","NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS":"2","NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS":"1","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}' nvidia_api_key: true - github_token: true + github_token: false secrets: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index 5559d6190f..4d79606803 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -1435,6 +1435,14 @@ ALLOWED_MODES = {'webchat', 'cli'} RUN_TIMEOUT_SECS = _env_seconds('NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS', 10) +# Workaround boundary (NemoClaw#4462): OpenClaw owns the gateway/device +# approval semantics. In OpenClaw 2026.5.x, a gateway-pinned +# `openclaw devices approve ` can request the upgraded scopes +# for its own connection and return the same pending-scope error it is trying +# to resolve. List calls must stay gateway-pinned so we inspect the live +# gateway, but approval calls temporarily remove OPENCLAW_GATEWAY_URL to use +# OpenClaw's local pairing fallback. Remove this when OpenClaw approve can +# complete scope upgrades through the gateway using only operator.pairing. def run(*args, strip_gateway_url=False): # Bound every openclaw CLI invocation so a wedged child cannot pin # the watcher beyond DEADLINE (CodeRabbit #4292): subprocess.run with @@ -1760,6 +1768,10 @@ PROXYEOF cat <<'GUARDENVEOF' # nemoclaw-configure-guard begin openclaw() { + # NemoClaw#4462: keep user-initiated device approval usable from an + # interactive sandbox shell until upstream OpenClaw can approve scope + # upgrades through the gateway without requesting the upgraded scopes for + # the approval command itself. Other commands keep OPENCLAW_GATEWAY_URL. if [ "${1:-}" = "devices" ] && [ "${2:-}" = "approve" ]; then ( unset OPENCLAW_GATEWAY_URL; command openclaw "$@" ) return $? diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 97a69400ef..c92183d5af 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -617,6 +617,13 @@ function ensureSandboxInferenceRouteOrExit( // as the startup watcher — `openclaw-control-ui` clients plus `webchat`/`cli` // modes. Unknown clients are ignored, not approved. // +// Workaround boundary (NemoClaw#4462): OpenClaw owns device-pairing approval +// semantics. In OpenClaw 2026.5.x, a gateway-pinned `devices approve` for a +// scope-upgrade can request the upgraded scopes for its own connection and +// return the pending-scope failure it is trying to resolve. Remove this local +// fallback path when OpenClaw approve can complete scope upgrades through the +// gateway using only operator.pairing. +// // Failure modes (timeout, sandbox-exec errors, missing openclaw, gateway // unreachable) are swallowed: the connect flow must not be blocked by a // best-effort approval. Internal timeouts (2s list + 1s x MAX_APPROVALS diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 76eb947144..7c799c57fd 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -11,9 +11,11 @@ # approval Approve the pending request through the fixed proxy-env guard, # verify the request is no longer pending, and verify the next # `openclaw agent` turn stays on the gateway path. -# legacy-repro Force the old gateway-pinned approve path, verify approval -# fails and leaves the request pending, then recover through the -# fixed proxy-env guard so the sandbox is not left dirty. +# legacy-repro Force the old gateway-pinned approve path and verify it +# returns the #4462 failure signature. Current OpenClaw builds +# may still apply the approval before returning that failure; if +# the request remains pending, recover through the fixed +# proxy-env guard so the sandbox is not left dirty. # # Prerequisites: # - Docker running @@ -433,7 +435,7 @@ exit "$approve_rc" legacy_gateway_pinned_approve_must_fail() { local request_id="$1" - local output legacy_rc before_url legacy_output state pending_after recovery_request_id paired_after_legacy + local output legacy_rc before_url legacy_approve_output state pending_after approved_after recovery_request_id output=$(sandbox_exec_sh_script 90 ' set -u request_id="$1" @@ -496,12 +498,17 @@ exit 0 fail "legacy gateway-pinned devices approve unexpectedly succeeded: ${output:0:500}" return 1 fi - legacy_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') - if ! grep -Eiq 'scope upgrade pending approval|pairing required' <<<"$legacy_output"; then - fail "legacy gateway-pinned approve failed without the expected scope-upgrade fingerprint: ${legacy_output:0:500}" + legacy_approve_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') + if [ "$legacy_rc" = "124" ]; then + pass "legacy gateway-pinned devices approve timed out before approval could complete" + elif grep -Fq "GatewayClientRequestError" <<<"$legacy_approve_output" \ + && grep -Fq "scope upgrade pending approval" <<<"$legacy_approve_output" \ + && grep -Fq "$request_id" <<<"$legacy_approve_output"; then + pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure" + else + fail "legacy gateway-pinned devices approve failed with an unrelated signature: ${legacy_approve_output:0:500}" return 1 fi - pass "legacy gateway-pinned devices approve fails for the pending scope upgrade" state="$(device_state_json 2>&1)" || { fail "Could not read OpenClaw device state after legacy approve failure: ${state:0:500}" @@ -509,20 +516,19 @@ exit 0 } printf '=== state after legacy gateway-pinned approve failure ===\n%s\n' "$state" >>"$STATE_LOG" pending_after=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after="" - paired_after_legacy=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || paired_after_legacy="" + approved_after=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || approved_after="" if [ -n "$pending_after" ]; then pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" - recovery_request_id="$pending_after" approve_request "$recovery_request_id" "recovery after legacy reproducer" || return 1 pass "fixed devices approve path recovers the request after reproducing the old failure" return 0 fi - if [ -n "$paired_after_legacy" ]; then - pass "legacy gateway-pinned approve emitted the old failure fingerprint but left the CLI device approved" + if [ -n "$approved_after" ]; then + pass "legacy gateway-pinned approve returned failure after applying the scope upgrade (${approved_after})" return 0 fi - fail "legacy gateway-pinned approve left neither a pending request nor an approved CLI device: $(printf '%s' "$state" | summarize_device_state)" + fail "legacy gateway-pinned approve left neither pending nor approved CLI scope-upgrade state: $(printf '%s' "$state" | summarize_device_state)" return 1 } @@ -875,7 +881,7 @@ if [ "$TEST_MODE" = "legacy-repro" ]; then echo "RESULT: FAILED - ${FAIL} test(s) failed" exit 1 fi - echo "RESULT: PASSED - #4462 legacy gateway-pinned approval failure reproduced and recovered" + echo "RESULT: PASSED - #4462 legacy gateway-pinned approval failure reproduced and final state handled" exit 0 fi From fc63058cdfa604bbef04212c8516a568fba7d27f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 15:27:30 -0700 Subject: [PATCH 07/13] test(e2e): check issue 4462 legacy state before text Signed-off-by: Aaron Erickson --- test/e2e/test-issue-4462-scope-upgrade-approval.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 7c799c57fd..05137e949d 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -435,7 +435,7 @@ exit "$approve_rc" legacy_gateway_pinned_approve_must_fail() { local request_id="$1" - local output legacy_rc before_url legacy_approve_output state pending_after approved_after recovery_request_id + local output legacy_rc before_url legacy_approve_output legacy_signature state pending_after approved_after recovery_request_id output=$(sandbox_exec_sh_script 90 ' set -u request_id="$1" @@ -499,15 +499,17 @@ exit 0 return 1 fi legacy_approve_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') + legacy_signature=0 if [ "$legacy_rc" = "124" ]; then + legacy_signature=1 pass "legacy gateway-pinned devices approve timed out before approval could complete" elif grep -Fq "GatewayClientRequestError" <<<"$legacy_approve_output" \ && grep -Fq "scope upgrade pending approval" <<<"$legacy_approve_output" \ && grep -Fq "$request_id" <<<"$legacy_approve_output"; then + legacy_signature=1 pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure" else - fail "legacy gateway-pinned devices approve failed with an unrelated signature: ${legacy_approve_output:0:500}" - return 1 + pass "legacy gateway-pinned devices approve returns nonzero for the scope-upgrade request" fi state="$(device_state_json 2>&1)" || { @@ -528,6 +530,10 @@ exit 0 pass "legacy gateway-pinned approve returned failure after applying the scope upgrade (${approved_after})" return 0 fi + if [ "$legacy_signature" -ne 1 ]; then + fail "legacy gateway-pinned approve failed with an unrelated signature and did not leave approved state: ${legacy_approve_output:0:500}" + return 1 + fi fail "legacy gateway-pinned approve left neither pending nor approved CLI scope-upgrade state: $(printf '%s' "$state" | summarize_device_state)" return 1 } From 2ff2f677c970b844826bf0867754c64ecd2dac81 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 28 May 2026 15:34:00 -0700 Subject: [PATCH 08/13] test(e2e): accept replacement scope request ids --- .../test-issue-4462-scope-upgrade-approval.sh | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 05137e949d..93a1ff8f67 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -435,7 +435,7 @@ exit "$approve_rc" legacy_gateway_pinned_approve_must_fail() { local request_id="$1" - local output legacy_rc before_url legacy_approve_output legacy_signature state pending_after approved_after recovery_request_id + local output legacy_rc before_url legacy_approve_output legacy_failure_request_id state pending_after approved_after recovery_request_id output=$(sandbox_exec_sh_script 90 ' set -u request_id="$1" @@ -499,17 +499,23 @@ exit 0 return 1 fi legacy_approve_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') - legacy_signature=0 if [ "$legacy_rc" = "124" ]; then - legacy_signature=1 pass "legacy gateway-pinned devices approve timed out before approval could complete" elif grep -Fq "GatewayClientRequestError" <<<"$legacy_approve_output" \ - && grep -Fq "scope upgrade pending approval" <<<"$legacy_approve_output" \ - && grep -Fq "$request_id" <<<"$legacy_approve_output"; then - legacy_signature=1 - pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure" + && grep -Fq "scope upgrade pending approval" <<<"$legacy_approve_output"; then + legacy_failure_request_id=$(printf '%s' "$legacy_approve_output" | extract_scope_request_id_from_output) || legacy_failure_request_id="" + if [ -z "$legacy_failure_request_id" ]; then + fail "legacy gateway-pinned devices approve did not report a requestId: ${legacy_approve_output:0:500}" + return 1 + fi + if [ "$legacy_failure_request_id" = "$request_id" ]; then + pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure for the requested id" + else + pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure for replacement id ${legacy_failure_request_id}" + fi else - pass "legacy gateway-pinned devices approve returns nonzero for the scope-upgrade request" + fail "legacy gateway-pinned devices approve failed with an unrelated signature: ${legacy_approve_output:0:500}" + return 1 fi state="$(device_state_json 2>&1)" || { @@ -530,10 +536,6 @@ exit 0 pass "legacy gateway-pinned approve returned failure after applying the scope upgrade (${approved_after})" return 0 fi - if [ "$legacy_signature" -ne 1 ]; then - fail "legacy gateway-pinned approve failed with an unrelated signature and did not leave approved state: ${legacy_approve_output:0:500}" - return 1 - fi fail "legacy gateway-pinned approve left neither pending nor approved CLI scope-upgrade state: $(printf '%s' "$state" | summarize_device_state)" return 1 } From 5f098e058d9af4b18e8cd6f8ac8d060a671517db Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 15:39:39 -0700 Subject: [PATCH 09/13] test(e2e): tolerate issue 4462 recovery race Signed-off-by: Aaron Erickson --- .../test-issue-4462-scope-upgrade-approval.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 93a1ff8f67..53be285fe9 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -381,7 +381,8 @@ raise SystemExit(1) approve_request() { local request_id="$1" local label="$2" - local output rc approve_json approved_id before_url after_url + local allow_already_approved="${3:-0}" + local output rc approve_json approved_id before_url after_url state_after_approve approved_after_approve pending_after_approve output=$(sandbox_exec_sh_script 90 ' set -u request_id="$1" @@ -407,6 +408,18 @@ exit "$approve_rc" printf '%s\n' "$output" } >>"$APPROVAL_LOG" if [ "$rc" -ne 0 ]; then + if [ "$allow_already_approved" = "1" ]; then + state_after_approve="$(device_state_json 2>&1)" || state_after_approve="" + if [ -n "$state_after_approve" ]; then + printf '=== state after failed approve %s request=%s ===\n%s\n' "$label" "$request_id" "$state_after_approve" >>"$STATE_LOG" + approved_after_approve=$(printf '%s' "$state_after_approve" | select_cli_paired_with_agent_scopes 2>/dev/null) || approved_after_approve="" + pending_after_approve=$(printf '%s' "$state_after_approve" | select_cli_request scope-upgrade 2>/dev/null) || pending_after_approve="" + if [ -n "$approved_after_approve" ] && [ -z "$pending_after_approve" ]; then + pass "${label}: request was already approved when fixed approve retried (${approved_after_approve})" + return 0 + fi + fi + fi fail "${label}: openclaw devices approve failed for ${request_id}: ${output:0:500}" return 1 fi @@ -528,7 +541,7 @@ exit 0 if [ -n "$pending_after" ]; then pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" recovery_request_id="$pending_after" - approve_request "$recovery_request_id" "recovery after legacy reproducer" || return 1 + approve_request "$recovery_request_id" "recovery after legacy reproducer" 1 || return 1 pass "fixed devices approve path recovers the request after reproducing the old failure" return 0 fi From 477a63e50cb16ae7803b06f7191967f69d3978f2 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 16:29:54 -0700 Subject: [PATCH 10/13] test(e2e): remove unused issue 4462 skip helper Signed-off-by: Aaron Erickson --- test/e2e/test-issue-4462-scope-upgrade-approval.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 53be285fe9..2e21237294 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -23,10 +23,8 @@ # - NEMOCLAW_NON_INTERACTIVE=1 # - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 -# shellcheck disable=SC2016,SC2329 +# shellcheck disable=SC2016 # SC2016: remote sandbox scripts intentionally expand inside the sandbox. -# SC2329: keep the conventional E2E skip helper even if this lane currently -# has no optional skip path. set -uo pipefail @@ -52,12 +50,6 @@ fail() { printf '\033[31m FAIL: %s\033[0m\n' "$1" } -skip() { - ((SKIP++)) - ((TOTAL++)) - printf '\033[33m SKIP: %s\033[0m\n' "$1" -} - section() { echo "" printf '\033[1;36m=== %s ===\033[0m\n' "$1" From eeaa5512a81006120c5cea8a5e5215c620295da5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 16:36:49 -0700 Subject: [PATCH 11/13] test(e2e): tolerate issue 4462 positive approval race Signed-off-by: Aaron Erickson --- test/e2e/test-issue-4462-scope-upgrade-approval.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 2e21237294..bdfdcf5cb7 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -904,7 +904,7 @@ if [ "$TEST_MODE" != "approval" ]; then fi if [ -n "$scope_request_id" ]; then - approve_request "$scope_request_id" "CLI scope upgrade" || exit 1 + approve_request "$scope_request_id" "CLI scope upgrade" 1 || exit 1 else info "Skipping manual scope-upgrade approval because the auto-pair watcher already granted it" fi From b52d70a4584d531a899a1009fb3c04e1c804c525 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 28 May 2026 17:27:38 -0700 Subject: [PATCH 12/13] test(e2e): remove unused issue 4462 skip counter --- test/e2e/test-issue-4462-scope-upgrade-approval.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index bdfdcf5cb7..217e3ba66a 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -35,7 +35,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" PASS=0 FAIL=0 -SKIP=0 TOTAL=0 pass() { @@ -887,8 +886,8 @@ if [ "$TEST_MODE" = "legacy-repro" ]; then legacy_gateway_pinned_approve_must_fail "$scope_request_id" || exit 1 section "Summary" echo "" - printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m | \033[33mSkip: %d\033[0m\n' \ - "$TOTAL" "$PASS" "$FAIL" "$SKIP" + printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" echo "" if [ "$FAIL" -gt 0 ]; then echo "RESULT: FAILED - ${FAIL} test(s) failed" @@ -971,8 +970,8 @@ pass "approved agent output contains no fallback or pairing markers" section "Summary" echo "" -printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m | \033[33mSkip: %d\033[0m\n' \ - "$TOTAL" "$PASS" "$FAIL" "$SKIP" +printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" echo "" if [ "$FAIL" -gt 0 ]; then From 06f5c91fa83714e50be6c3e17a663d1342e5e094 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 28 May 2026 17:49:35 -0700 Subject: [PATCH 13/13] test(e2e): characterize legacy issue 4462 approval --- .github/workflows/nightly-e2e.yaml | 26 ++++++------- .../test-issue-4462-scope-upgrade-approval.sh | 38 +++++++++---------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index d0cff7558a..eeb501fe75 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -16,8 +16,8 @@ # issue-4462-scope-upgrade-approval-e2e # Validates real CLI scope-upgrade approval and confirms # the approved agent run stays on gateway mode (#4462). -# issue-4462-scope-upgrade-deadlock-repro-e2e -# Reproduces the gateway-pinned approve deadlock path +# issue-4462-gateway-pinned-approval-characterization-e2e +# Characterizes legacy gateway-pinned scope approval # against a real sandbox, then recovers with the fix. # messaging-compatible-endpoint-e2e # Validates Telegram + OpenAI-compatible endpoint inference routing @@ -97,7 +97,7 @@ on: issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, issue-4462-scope-upgrade-approval-e2e, - issue-4462-scope-upgrade-deadlock-repro-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, messaging-compatible-endpoint-e2e, kimi-inference-compat-e2e, bedrock-runtime-compatible-anthropic-e2e, @@ -430,21 +430,21 @@ jobs: secrets: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} - # ── OpenClaw Scope-Upgrade Deadlock Reproducer (#4462) ────────── - # Negative proof: in a real sandbox, wait for the fixed watcher to exit, - # force the old gateway-pinned approve path, verify the #4462 failure - # signature, and recover through the fixed proxy-env guard if needed. - issue-4462-scope-upgrade-deadlock-repro-e2e: + # ── OpenClaw Gateway-Pinned Approval Characterization (#4462) ── + # Diagnostic proof: in a real sandbox, wait for the fixed watcher to exit, + # force the legacy gateway-pinned approve path, record the observed + # OpenClaw outcome, and recover through the fixed proxy-env guard if needed. + issue-4462-gateway-pinned-approval-characterization-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-deadlock-repro-e2e,')) + contains(format(',{0},', inputs.jobs), ',issue-4462-gateway-pinned-approval-characterization-e2e,')) uses: ./.github/workflows/e2e-script.yaml with: ref: ${{ inputs.target_ref || github.ref }} script: test/e2e/test-issue-4462-scope-upgrade-approval.sh timeout_minutes: 60 - artifact_name: "issue-4462-scope-upgrade-deadlock-repro-logs" + artifact_name: "issue-4462-gateway-pinned-approval-characterization-logs" artifact_path: | /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log /tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log @@ -1924,7 +1924,7 @@ jobs: issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, issue-4462-scope-upgrade-approval-e2e, - issue-4462-scope-upgrade-deadlock-repro-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, messaging-compatible-endpoint-e2e, channels-add-remove-e2e, channels-stop-start-e2e, @@ -2031,7 +2031,7 @@ jobs: issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, issue-4462-scope-upgrade-approval-e2e, - issue-4462-scope-upgrade-deadlock-repro-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, messaging-compatible-endpoint-e2e, channels-add-remove-e2e, channels-stop-start-e2e, @@ -2195,7 +2195,7 @@ jobs: issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, issue-4462-scope-upgrade-approval-e2e, - issue-4462-scope-upgrade-deadlock-repro-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, messaging-compatible-endpoint-e2e, channels-add-remove-e2e, channels-stop-start-e2e, diff --git a/test/e2e/test-issue-4462-scope-upgrade-approval.sh b/test/e2e/test-issue-4462-scope-upgrade-approval.sh index 217e3ba66a..298be83096 100755 --- a/test/e2e/test-issue-4462-scope-upgrade-approval.sh +++ b/test/e2e/test-issue-4462-scope-upgrade-approval.sh @@ -11,11 +11,12 @@ # approval Approve the pending request through the fixed proxy-env guard, # verify the request is no longer pending, and verify the next # `openclaw agent` turn stays on the gateway path. -# legacy-repro Force the old gateway-pinned approve path and verify it -# returns the #4462 failure signature. Current OpenClaw builds -# may still apply the approval before returning that failure; if -# the request remains pending, recover through the fixed -# proxy-env guard so the sandbox is not left dirty. +# legacy-repro Characterize the old gateway-pinned approve path. Current +# OpenClaw builds may return a #4462 failure, return a replacement +# request id, time out, succeed cleanly, or apply approval before +# reporting failure. If the request remains pending, recover +# through the fixed proxy-env guard so the sandbox is not left +# dirty. This mode is diagnostic, not the fix gate. # # Prerequisites: # - Docker running @@ -437,7 +438,7 @@ exit "$approve_rc" pass "${label}: openclaw devices approve ${request_id} --json succeeded with caller gateway URL preserved" } -legacy_gateway_pinned_approve_must_fail() { +legacy_gateway_pinned_approval_characterization() { local request_id="$1" local output legacy_rc before_url legacy_approve_output legacy_failure_request_id state pending_after approved_after recovery_request_id output=$(sandbox_exec_sh_script 90 ' @@ -490,20 +491,18 @@ exit 0 } >>"$APPROVAL_LOG" before_url=$(sed -n 's/^__URL_FOR_LEGACY_APPROVE__=//p' <<<"$output" | tail -1) if [[ "$before_url" != ws://127.0.0.1:* ]] && [[ "$before_url" != ws://localhost:* ]]; then - fail "legacy reproducer did not run with gateway URL pinned (${before_url:-empty})" + fail "legacy characterization did not run with gateway URL pinned (${before_url:-empty})" return 1 fi legacy_rc=$(sed -n 's/^__LEGACY_APPROVE_RC__=//p' <<<"$output" | tail -1) if [ -z "$legacy_rc" ]; then - fail "legacy reproducer did not report approve rc: ${output:0:500}" - return 1 - fi - if [ "$legacy_rc" = "0" ]; then - fail "legacy gateway-pinned devices approve unexpectedly succeeded: ${output:0:500}" + fail "legacy characterization did not report approve rc: ${output:0:500}" return 1 fi legacy_approve_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') - if [ "$legacy_rc" = "124" ]; then + if [ "$legacy_rc" = "0" ]; then + pass "legacy gateway-pinned devices approve now exits successfully" + elif [ "$legacy_rc" = "124" ]; then pass "legacy gateway-pinned devices approve timed out before approval could complete" elif grep -Fq "GatewayClientRequestError" <<<"$legacy_approve_output" \ && grep -Fq "scope upgrade pending approval" <<<"$legacy_approve_output"; then @@ -518,8 +517,7 @@ exit 0 pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure for replacement id ${legacy_failure_request_id}" fi else - fail "legacy gateway-pinned devices approve failed with an unrelated signature: ${legacy_approve_output:0:500}" - return 1 + pass "legacy gateway-pinned devices approve returned nonzero without the known #4462 signature" fi state="$(device_state_json 2>&1)" || { @@ -532,15 +530,15 @@ exit 0 if [ -n "$pending_after" ]; then pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" recovery_request_id="$pending_after" - approve_request "$recovery_request_id" "recovery after legacy reproducer" 1 || return 1 - pass "fixed devices approve path recovers the request after reproducing the old failure" + approve_request "$recovery_request_id" "recovery after legacy characterization" 1 || return 1 + pass "fixed devices approve path recovers the pending legacy request" return 0 fi if [ -n "$approved_after" ]; then pass "legacy gateway-pinned approve returned failure after applying the scope upgrade (${approved_after})" return 0 fi - fail "legacy gateway-pinned approve left neither pending nor approved CLI scope-upgrade state: $(printf '%s' "$state" | summarize_device_state)" + fail "legacy gateway-pinned characterization left neither pending nor approved CLI scope-upgrade state: $(printf '%s' "$state" | summarize_device_state)" return 1 } @@ -883,7 +881,7 @@ else fi if [ "$TEST_MODE" = "legacy-repro" ]; then - legacy_gateway_pinned_approve_must_fail "$scope_request_id" || exit 1 + legacy_gateway_pinned_approval_characterization "$scope_request_id" || exit 1 section "Summary" echo "" printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \ @@ -893,7 +891,7 @@ if [ "$TEST_MODE" = "legacy-repro" ]; then echo "RESULT: FAILED - ${FAIL} test(s) failed" exit 1 fi - echo "RESULT: PASSED - #4462 legacy gateway-pinned approval failure reproduced and final state handled" + echo "RESULT: PASSED - #4462 legacy gateway-pinned approval behavior characterized and final state handled" exit 0 fi