From 381ada4d2ee917c38288dc037849492b347528a2 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 11 Jun 2026 16:01:06 +0530 Subject: [PATCH 1/2] fix(e2e): shard channel stop-start nightly Signed-off-by: San Dang --- .github/workflows/nightly-e2e.yaml | 45 +++++++++++++++++++++------- test/e2e/test-channels-stop-start.sh | 41 ++++++++++++++++++++----- test/e2e/test-overlayfs-autofix.sh | 2 +- 3 files changed, 68 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 9aab20e6da..a996ee2607 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -127,7 +127,8 @@ on: issue-4462-scope-upgrade-approval-e2e, issue-4462-gateway-pinned-approval-characterization-e2e, messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e, - channels-add-remove-e2e, channels-stop-start-e2e, brave-search-e2e, + channels-add-remove-e2e, channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, brave-search-e2e, common-egress-agent-e2e, kimi-inference-compat-e2e, bedrock-runtime-compatible-anthropic-e2e, token-rotation-e2e, sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, @@ -750,24 +751,43 @@ jobs: secrets: *nightly-e2e-default-secrets # ── Channels stop/start lifecycle E2E (#3462) ─────────────────────── # Regression coverage for #3453 (stop must disable across rebuild), #3381 - # (start must re-attach from cached credentials). - # Exercises OpenClaw and Hermes across telegram, discord, wechat, slack, and whatsapp. - channels-stop-start-e2e: + # (start must re-attach from cached credentials). Sharded by agent so the + # OpenClaw and Hermes coverage can run in parallel. + channels-stop-start-openclaw-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || inputs.jobs == '' || - contains(format(',{0},', inputs.jobs), ',channels-stop-start-e2e,')) + contains(format(',{0},', inputs.jobs), ',channels-stop-start-openclaw-e2e,')) uses: ./.github/workflows/e2e-script.yaml with: ref: ${{ inputs.target_ref || github.ref }} script: test/e2e/test-channels-stop-start.sh - timeout_minutes: 120 - artifact_name: "install-log-channels-stop-start" + timeout_minutes: 75 + artifact_name: "install-log-channels-stop-start-openclaw" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + /tmp/nemoclaw-e2e-channels-*-install.log + /tmp/nc-channels-*.log + env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"openclaw","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-default-secrets + channels-stop-start-hermes-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',channels-stop-start-hermes-e2e,')) + uses: ./.github/workflows/e2e-script.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e/test-channels-stop-start.sh + timeout_minutes: 75 + artifact_name: "install-log-channels-stop-start-hermes" artifact_path: | /tmp/nemoclaw-e2e-install.log /tmp/nemoclaw-e2e-channels-*-install.log /tmp/nc-channels-*.log - env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}' + env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}' nvidia_api_key: true github_token: true secrets: *nightly-e2e-default-secrets @@ -2259,7 +2279,8 @@ jobs: messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e, channels-add-remove-e2e, - channels-stop-start-e2e, + channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, brave-search-e2e, common-egress-agent-e2e, kimi-inference-compat-e2e, @@ -2376,7 +2397,8 @@ jobs: messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e, channels-add-remove-e2e, - channels-stop-start-e2e, + channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, brave-search-e2e, common-egress-agent-e2e, kimi-inference-compat-e2e, @@ -2552,7 +2574,8 @@ jobs: messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e, channels-add-remove-e2e, - channels-stop-start-e2e, + channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, brave-search-e2e, common-egress-agent-e2e, kimi-inference-compat-e2e, diff --git a/test/e2e/test-channels-stop-start.sh b/test/e2e/test-channels-stop-start.sh index 7d86bd2c4a..dc98bf8a09 100755 --- a/test/e2e/test-channels-stop-start.sh +++ b/test/e2e/test-channels-stop-start.sh @@ -5,9 +5,10 @@ # Channel stop/start lifecycle E2E test. # # Covers Test 1 from issue #3462 ("onboard telegram -> channels stop -> channels start"). -# The regression surface is intentionally exercised for both supported agents -# (OpenClaw and Hermes) and every messaging channel (telegram, discord, wechat, -# slack, whatsapp). +# The regression surface is exercised for both supported agents (OpenClaw and +# Hermes) and every messaging channel (telegram, discord, wechat, slack, +# whatsapp). Set NEMOCLAW_CHANNELS_STOP_START_AGENT=openclaw or hermes to run a +# single-agent shard. # # Regression coverage: # - #3453: `channels stop ` + rebuild must actually remove the channel @@ -96,10 +97,29 @@ fi BASE_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-channels-stop-start}" OPENCLAW_SANDBOX_NAME="${NEMOCLAW_CHANNELS_OPENCLAW_SANDBOX_NAME:-${BASE_SANDBOX_NAME}-openclaw}" HERMES_SANDBOX_NAME="${NEMOCLAW_CHANNELS_HERMES_SANDBOX_NAME:-${BASE_SANDBOX_NAME}-hermes}" +REQUESTED_AGENT="${NEMOCLAW_CHANNELS_STOP_START_AGENT:-all}" REGISTRY="$HOME/.nemoclaw/sandboxes.json" OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" CHANNELS=(telegram discord wechat slack whatsapp) TOKENLESS_CHANNELS=(whatsapp) +SELECTED_AGENT_SCENARIOS=() + +case "$REQUESTED_AGENT" in + all) + SELECTED_AGENT_SCENARIOS=("openclaw:${OPENCLAW_SANDBOX_NAME}" "hermes:${HERMES_SANDBOX_NAME}") + ;; + openclaw) + SELECTED_AGENT_SCENARIOS=("openclaw:${OPENCLAW_SANDBOX_NAME}") + ;; + hermes) + SELECTED_AGENT_SCENARIOS=("hermes:${HERMES_SANDBOX_NAME}") + ;; + *) + section "Phase 0: Prerequisites" + fail_msg "C0: NEMOCLAW_CHANNELS_STOP_START_AGENT must be all, openclaw, or hermes (got ${REQUESTED_AGENT})" + print_summary + ;; +esac ACTIVE_AGENT="" ACTIVE_SANDBOX="" @@ -132,8 +152,9 @@ openshell() { # shellcheck source=test/e2e/lib/sandbox-teardown.sh . "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" -register_sandbox_for_teardown "$OPENCLAW_SANDBOX_NAME" -register_sandbox_for_teardown "$HERMES_SANDBOX_NAME" +for scenario in "${SELECTED_AGENT_SCENARIOS[@]}"; do + register_sandbox_for_teardown "${scenario#*:}" +done refresh_path() { if [ -f "$HOME/.bashrc" ]; then @@ -806,8 +827,12 @@ fi refresh_path -run_agent_scenario "openclaw" "$OPENCLAW_SANDBOX_NAME" -destroy_completed_sandbox "$OPENCLAW_SANDBOX_NAME" -run_agent_scenario "hermes" "$HERMES_SANDBOX_NAME" +for index in "${!SELECTED_AGENT_SCENARIOS[@]}"; do + scenario="${SELECTED_AGENT_SCENARIOS[$index]}" + run_agent_scenario "${scenario%%:*}" "${scenario#*:}" + if [ "$index" -lt "$((${#SELECTED_AGENT_SCENARIOS[@]} - 1))" ]; then + destroy_completed_sandbox "${scenario#*:}" + fi +done print_summary diff --git a/test/e2e/test-overlayfs-autofix.sh b/test/e2e/test-overlayfs-autofix.sh index 95f81be57d..2a5df82f70 100755 --- a/test/e2e/test-overlayfs-autofix.sh +++ b/test/e2e/test-overlayfs-autofix.sh @@ -131,7 +131,7 @@ register_sandbox_for_teardown "$SANDBOX_NAME" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -if [ "$(uname -s)" = "Linux" ] && grep -q 'return platform === "linux";' "$REPO_ROOT/src/lib/onboard.ts"; then +if [ "$(uname -s)" = "Linux" ] && grep -q 'platform === "linux"' "$REPO_ROOT/src/lib/onboard/docker-driver-platform.ts"; then section "Applicability" skip "OpenShell Docker-driver onboarding is active on Linux; k3s overlayfs auto-fix is not in the runtime path" print_summary From ca9d434fc6c4a9cf1526d8ab388921e721130fa1 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 11 Jun 2026 20:55:29 +0700 Subject: [PATCH 2/2] fix(e2e): avoid deprecated countries probe --- test/e2e/test-common-egress-agent-e2e.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/e2e/test-common-egress-agent-e2e.sh b/test/e2e/test-common-egress-agent-e2e.sh index 3dfded1cfb..5bb702b2b8 100755 --- a/test/e2e/test-common-egress-agent-e2e.sh +++ b/test/e2e/test-common-egress-agent-e2e.sh @@ -6,9 +6,9 @@ # # Proves the safe common-egress defaults through real agent turns: # C1 OpenClaw balanced includes weather and the agent fetches Open-Meteo. -# C2 OpenClaw open includes public-reference and the agent fetches REST Countries. +# C2 OpenClaw open includes public-reference and the agent fetches Wikidata. # C3 Hermes open includes public-reference plus all Hermes Nous policy presets, -# and the Hermes agent fetches REST Countries through its API-server agent path. +# and the Hermes agent fetches Wikidata through its API-server agent path. # # Required env: # NVIDIA_API_KEY real NVIDIA Endpoints key for inference @@ -438,12 +438,12 @@ PROMPT section "Phase 2: OpenClaw open public reference" OPENCLAW_OPEN_SANDBOX="${NEMOCLAW_COMMON_EGRESS_OPENCLAW_OPEN_SANDBOX:-e2e-common-egress-openclaw-open}" run_onboard "$OPENCLAW_OPEN_SANDBOX" "openclaw" "open" - assert_policy_contains "$OPENCLAW_OPEN_SANDBOX" "C2 policy" "restcountries.com" "nominatim.openstreetmap.org" "query.wikidata.org" + assert_policy_contains "$OPENCLAW_OPEN_SANDBOX" "C2 policy" "www.wikidata.org" "nominatim.openstreetmap.org" "query.wikidata.org" REFERENCE_AGENT_PROMPT=$( cat <<'PROMPT' Use the web_fetch tool to fetch exactly this URL: -https://restcountries.com/v3.1/alpha/US?fields=name,cca3 -After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched response says the common name is United States and cca3 is USA. Do not fetch any other URL. +https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json +After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched response says entity Q30 has the English label United States. Do not fetch any other URL. PROMPT ) run_openclaw_agent_assertion "$OPENCLAW_OPEN_SANDBOX" "C2 agent reference" "$REFERENCE_AGENT_PROMPT" "REFERENCE_AGENT_OK" @@ -455,7 +455,7 @@ if [ "${NEMOCLAW_COMMON_EGRESS_SKIP_HERMES:-}" != "1" ]; then section "Phase 3: Hermes open public reference" HERMES_SANDBOX="${NEMOCLAW_COMMON_EGRESS_HERMES_SANDBOX:-e2e-common-egress-hermes-open}" run_onboard "$HERMES_SANDBOX" "hermes" "open" - assert_policy_contains "$HERMES_SANDBOX" "C3 common policy" "restcountries.com" "api.open-meteo.com" + assert_policy_contains "$HERMES_SANDBOX" "C3 common policy" "www.wikidata.org" "api.open-meteo.com" assert_policy_contains "$HERMES_SANDBOX" "C3 Hermes Nous policy" "/firecrawl" "/fal-queue" "/openai-audio" "/browser-use" "/modal" HERMES_REFERENCE_AGENT_PROMPT=$( cat <<'PROMPT' @@ -464,11 +464,10 @@ python3 - <<'PY' import json import urllib.request -url = "https://restcountries.com/v3.1/alpha/US?fields=name,cca3" +url = "https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json" with urllib.request.urlopen(url, timeout=20) as response: doc = json.load(response) -country = doc[0] if isinstance(doc, list) else doc -ok = country.get("name", {}).get("common") == "United States" and country.get("cca3") == "USA" +ok = doc.get("success") == 1 and doc.get("entities", {}).get("Q30", {}).get("labels", {}).get("en", {}).get("value") == "United States" print("HERMES_REFERENCE_AGENT_OK" if ok else "HERMES_REFERENCE_AGENT_BAD") PY After the command completes, reply exactly HERMES_REFERENCE_AGENT_OK if that exact token appeared. Do not fetch any other URL.