Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 34 additions & 11 deletions .github/workflows/nightly-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@ on:
issue-4462-scope-upgrade-approval-e2e,
issue-4462-gateway-pinned-approval-characterization-e2e,
messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e,
channels-add-remove-e2e, channels-stop-start-e2e, brave-search-e2e,
channels-add-remove-e2e, channels-stop-start-openclaw-e2e,
channels-stop-start-hermes-e2e, brave-search-e2e,
common-egress-agent-e2e, kimi-inference-compat-e2e,
bedrock-runtime-compatible-anthropic-e2e, token-rotation-e2e,
sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e,
Expand Down Expand Up @@ -750,24 +751,43 @@ jobs:
secrets: *nightly-e2e-default-secrets
# ── Channels stop/start lifecycle E2E (#3462) ───────────────────────
# Regression coverage for #3453 (stop must disable across rebuild), #3381
# (start must re-attach from cached credentials).
# Exercises OpenClaw and Hermes across telegram, discord, wechat, slack, and whatsapp.
channels-stop-start-e2e:
# (start must re-attach from cached credentials). Sharded by agent so the
# OpenClaw and Hermes coverage can run in parallel.
channels-stop-start-openclaw-e2e:
if: >-
github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
inputs.jobs == '' ||
contains(format(',{0},', inputs.jobs), ',channels-stop-start-e2e,'))
contains(format(',{0},', inputs.jobs), ',channels-stop-start-openclaw-e2e,'))
uses: ./.github/workflows/e2e-script.yaml
with:
ref: ${{ inputs.target_ref || github.ref }}
script: test/e2e/test-channels-stop-start.sh
timeout_minutes: 120
artifact_name: "install-log-channels-stop-start"
timeout_minutes: 75
artifact_name: "install-log-channels-stop-start-openclaw"
artifact_path: |
/tmp/nemoclaw-e2e-install.log
/tmp/nemoclaw-e2e-channels-*-install.log
/tmp/nc-channels-*.log
env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"openclaw","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}'
nvidia_api_key: true
github_token: true
secrets: *nightly-e2e-default-secrets
channels-stop-start-hermes-e2e:
if: >-
github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
inputs.jobs == '' ||
contains(format(',{0},', inputs.jobs), ',channels-stop-start-hermes-e2e,'))
uses: ./.github/workflows/e2e-script.yaml
with:
ref: ${{ inputs.target_ref || github.ref }}
script: test/e2e/test-channels-stop-start.sh
timeout_minutes: 75
artifact_name: "install-log-channels-stop-start-hermes"
artifact_path: |
/tmp/nemoclaw-e2e-install.log
/tmp/nemoclaw-e2e-channels-*-install.log
/tmp/nc-channels-*.log
env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}'
env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}'
nvidia_api_key: true
github_token: true
secrets: *nightly-e2e-default-secrets
Expand Down Expand Up @@ -2259,7 +2279,8 @@ jobs:
messaging-compatible-endpoint-e2e,
sessions-agents-cli-e2e,
channels-add-remove-e2e,
channels-stop-start-e2e,
channels-stop-start-openclaw-e2e,
channels-stop-start-hermes-e2e,
brave-search-e2e,
common-egress-agent-e2e,
kimi-inference-compat-e2e,
Expand Down Expand Up @@ -2376,7 +2397,8 @@ jobs:
messaging-compatible-endpoint-e2e,
sessions-agents-cli-e2e,
channels-add-remove-e2e,
channels-stop-start-e2e,
channels-stop-start-openclaw-e2e,
channels-stop-start-hermes-e2e,
brave-search-e2e,
common-egress-agent-e2e,
kimi-inference-compat-e2e,
Expand Down Expand Up @@ -2552,7 +2574,8 @@ jobs:
messaging-compatible-endpoint-e2e,
sessions-agents-cli-e2e,
channels-add-remove-e2e,
channels-stop-start-e2e,
channels-stop-start-openclaw-e2e,
channels-stop-start-hermes-e2e,
brave-search-e2e,
common-egress-agent-e2e,
kimi-inference-compat-e2e,
Expand Down
41 changes: 33 additions & 8 deletions test/e2e/test-channels-stop-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
# Channel stop/start lifecycle E2E test.
#
# Covers Test 1 from issue #3462 ("onboard telegram -> channels stop -> channels start").
# The regression surface is intentionally exercised for both supported agents
# (OpenClaw and Hermes) and every messaging channel (telegram, discord, wechat,
# slack, whatsapp).
# The regression surface is exercised for both supported agents (OpenClaw and
# Hermes) and every messaging channel (telegram, discord, wechat, slack,
# whatsapp). Set NEMOCLAW_CHANNELS_STOP_START_AGENT=openclaw or hermes to run a
# single-agent shard.
#
# Regression coverage:
# - #3453: `channels stop <ch>` + rebuild must actually remove the channel
Expand Down Expand Up @@ -96,10 +97,29 @@ fi
BASE_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-channels-stop-start}"
OPENCLAW_SANDBOX_NAME="${NEMOCLAW_CHANNELS_OPENCLAW_SANDBOX_NAME:-${BASE_SANDBOX_NAME}-openclaw}"
HERMES_SANDBOX_NAME="${NEMOCLAW_CHANNELS_HERMES_SANDBOX_NAME:-${BASE_SANDBOX_NAME}-hermes}"
REQUESTED_AGENT="${NEMOCLAW_CHANNELS_STOP_START_AGENT:-all}"
REGISTRY="$HOME/.nemoclaw/sandboxes.json"
OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}"
CHANNELS=(telegram discord wechat slack whatsapp)
TOKENLESS_CHANNELS=(whatsapp)
SELECTED_AGENT_SCENARIOS=()

case "$REQUESTED_AGENT" in
all)
SELECTED_AGENT_SCENARIOS=("openclaw:${OPENCLAW_SANDBOX_NAME}" "hermes:${HERMES_SANDBOX_NAME}")
;;
openclaw)
SELECTED_AGENT_SCENARIOS=("openclaw:${OPENCLAW_SANDBOX_NAME}")
;;
hermes)
SELECTED_AGENT_SCENARIOS=("hermes:${HERMES_SANDBOX_NAME}")
;;
*)
section "Phase 0: Prerequisites"
fail_msg "C0: NEMOCLAW_CHANNELS_STOP_START_AGENT must be all, openclaw, or hermes (got ${REQUESTED_AGENT})"
print_summary
;;
esac

ACTIVE_AGENT=""
ACTIVE_SANDBOX=""
Expand Down Expand Up @@ -132,8 +152,9 @@ openshell() {

# shellcheck source=test/e2e/lib/sandbox-teardown.sh
. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh"
register_sandbox_for_teardown "$OPENCLAW_SANDBOX_NAME"
register_sandbox_for_teardown "$HERMES_SANDBOX_NAME"
for scenario in "${SELECTED_AGENT_SCENARIOS[@]}"; do
register_sandbox_for_teardown "${scenario#*:}"
done

refresh_path() {
if [ -f "$HOME/.bashrc" ]; then
Expand Down Expand Up @@ -806,8 +827,12 @@ fi

refresh_path

run_agent_scenario "openclaw" "$OPENCLAW_SANDBOX_NAME"
destroy_completed_sandbox "$OPENCLAW_SANDBOX_NAME"
run_agent_scenario "hermes" "$HERMES_SANDBOX_NAME"
for index in "${!SELECTED_AGENT_SCENARIOS[@]}"; do
scenario="${SELECTED_AGENT_SCENARIOS[$index]}"
run_agent_scenario "${scenario%%:*}" "${scenario#*:}"
if [ "$index" -lt "$((${#SELECTED_AGENT_SCENARIOS[@]} - 1))" ]; then
destroy_completed_sandbox "${scenario#*:}"
fi
done

print_summary
17 changes: 8 additions & 9 deletions test/e2e/test-common-egress-agent-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
#
# Proves the safe common-egress defaults through real agent turns:
# C1 OpenClaw balanced includes weather and the agent fetches Open-Meteo.
# C2 OpenClaw open includes public-reference and the agent fetches REST Countries.
# C2 OpenClaw open includes public-reference and the agent fetches Wikidata.
# C3 Hermes open includes public-reference plus all Hermes Nous policy presets,
# and the Hermes agent fetches REST Countries through its API-server agent path.
# and the Hermes agent fetches Wikidata through its API-server agent path.
#
# Required env:
# NVIDIA_API_KEY real NVIDIA Endpoints key for inference
Expand Down Expand Up @@ -438,12 +438,12 @@ PROMPT
section "Phase 2: OpenClaw open public reference"
OPENCLAW_OPEN_SANDBOX="${NEMOCLAW_COMMON_EGRESS_OPENCLAW_OPEN_SANDBOX:-e2e-common-egress-openclaw-open}"
run_onboard "$OPENCLAW_OPEN_SANDBOX" "openclaw" "open"
assert_policy_contains "$OPENCLAW_OPEN_SANDBOX" "C2 policy" "restcountries.com" "nominatim.openstreetmap.org" "query.wikidata.org"
assert_policy_contains "$OPENCLAW_OPEN_SANDBOX" "C2 policy" "www.wikidata.org" "nominatim.openstreetmap.org" "query.wikidata.org"
REFERENCE_AGENT_PROMPT=$(
cat <<'PROMPT'
Use the web_fetch tool to fetch exactly this URL:
https://restcountries.com/v3.1/alpha/US?fields=name,cca3
After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched response says the common name is United States and cca3 is USA. Do not fetch any other URL.
https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json
After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched response says entity Q30 has the English label United States. Do not fetch any other URL.
PROMPT
)
run_openclaw_agent_assertion "$OPENCLAW_OPEN_SANDBOX" "C2 agent reference" "$REFERENCE_AGENT_PROMPT" "REFERENCE_AGENT_OK"
Expand All @@ -455,7 +455,7 @@ if [ "${NEMOCLAW_COMMON_EGRESS_SKIP_HERMES:-}" != "1" ]; then
section "Phase 3: Hermes open public reference"
HERMES_SANDBOX="${NEMOCLAW_COMMON_EGRESS_HERMES_SANDBOX:-e2e-common-egress-hermes-open}"
run_onboard "$HERMES_SANDBOX" "hermes" "open"
assert_policy_contains "$HERMES_SANDBOX" "C3 common policy" "restcountries.com" "api.open-meteo.com"
assert_policy_contains "$HERMES_SANDBOX" "C3 common policy" "www.wikidata.org" "api.open-meteo.com"
assert_policy_contains "$HERMES_SANDBOX" "C3 Hermes Nous policy" "/firecrawl" "/fal-queue" "/openai-audio" "/browser-use" "/modal"
HERMES_REFERENCE_AGENT_PROMPT=$(
cat <<'PROMPT'
Expand All @@ -464,11 +464,10 @@ python3 - <<'PY'
import json
import urllib.request

url = "https://restcountries.com/v3.1/alpha/US?fields=name,cca3"
url = "https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json"
with urllib.request.urlopen(url, timeout=20) as response:
doc = json.load(response)
country = doc[0] if isinstance(doc, list) else doc
ok = country.get("name", {}).get("common") == "United States" and country.get("cca3") == "USA"
ok = doc.get("success") == 1 and doc.get("entities", {}).get("Q30", {}).get("labels", {}).get("en", {}).get("value") == "United States"
print("HERMES_REFERENCE_AGENT_OK" if ok else "HERMES_REFERENCE_AGENT_BAD")
PY
After the command completes, reply exactly HERMES_REFERENCE_AGENT_OK if that exact token appeared. Do not fetch any other URL.
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/test-overlayfs-autofix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ register_sandbox_for_teardown "$SANDBOX_NAME"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

if [ "$(uname -s)" = "Linux" ] && grep -q 'return platform === "linux";' "$REPO_ROOT/src/lib/onboard.ts"; then
if [ "$(uname -s)" = "Linux" ] && grep -q 'platform === "linux"' "$REPO_ROOT/src/lib/onboard/docker-driver-platform.ts"; then
section "Applicability"
skip "OpenShell Docker-driver onboarding is active on Linux; k3s overlayfs auto-fix is not in the runtime path"
print_summary
Expand Down
Loading