Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 109 additions & 52 deletions .github/workflows/nightly-e2e.yaml

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions ci/env-var-doc-allowlist.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,13 @@
{
"name": "NEMOCLAW_TEST_NO_SLEEP",
"reason": "Test sentinel that bypasses real-time sleep() calls in onboard inference probes. Set to '1' only by Vitest tests; never user-set."
},
{
"name": "NEMOCLAW_E2E_FAILURE_INJECTION",
"reason": "Internal E2E-only sentinel that enables deterministic onboarding fault injection for resume/repair scripts. Never user-set in production."
},
{
"name": "NEMOCLAW_E2E_FORCE_FAIL_AT_STEP",
"reason": "Internal E2E-only selector naming the onboarding step where deterministic fault injection should exit. Used only with NEMOCLAW_E2E_FAILURE_INJECTION in test scripts."
}
]
6 changes: 3 additions & 3 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,8 @@ const { isLinuxDockerDriverGatewayEnabled }: typeof import("./onboard/docker-dri
const {
reconcileGatewayGpuReuseForGpuIntent,
}: typeof import("./onboard/gateway-gpu-passthrough") = require("./onboard/gateway-gpu-passthrough");
const {
syncPresetSelection,
}: typeof import("./onboard/policy-preset-sync") = require("./onboard/policy-preset-sync");
const { syncPresetSelection }: typeof import("./onboard/policy-preset-sync") = require("./onboard/policy-preset-sync");
const { maybeForceE2eStepFailure }: typeof import("./onboard/e2e-failure-injection") = require("./onboard/e2e-failure-injection");
const {
gatherWechatConfig,
hasWechatConfigDrift,
Expand Down Expand Up @@ -9037,6 +9036,7 @@ function startRecordedStep(
return session;
});
}
maybeForceE2eStepFailure(stepName);
}

const ONBOARD_STEP_INDEX: Record<string, { number: number; title: string }> = {
Expand Down
11 changes: 11 additions & 0 deletions src/lib/onboard/e2e-failure-injection.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

/** Testing-only hook for deterministic E2E resume/repair fault injection. */
export function maybeForceE2eStepFailure(stepName: string): void {
if (process.env.NEMOCLAW_E2E_FAILURE_INJECTION !== "1") return;
const forcedStep = (process.env.NEMOCLAW_E2E_FORCE_FAIL_AT_STEP || "").trim();
if (!forcedStep || forcedStep !== stepName) return;
console.error(` [e2e] Forced onboarding failure at step '${stepName}'.`);
process.exit(1);
}
36 changes: 36 additions & 0 deletions test/e2e/docs/parity-map.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,42 @@ scripts:
reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking
owner: e2e-maintainers
runner_requirement: Docker + sudo hosts edit + OpenShell
test-onboard-negative-paths.sh:
scenario: ubuntu-repo-cloud-openclaw-custom-policies
status: migrated
bucket: onboarding-negative-paths
assertions:
- legacy: NEMOCLAW_POLICY_MODE=restricted falls back to suggested presets
status: deferred
reason: pure policy fallback behavior belongs in focused TypeScript behavior tests
owner: e2e-maintainers
runner_requirement: local CLI build
- legacy: NEMOCLAW_POLICY_MODE=nonexistent falls back to suggested presets
status: deferred
reason: pure policy fallback behavior belongs in focused TypeScript behavior tests
owner: e2e-maintainers
runner_requirement: local CLI build
- legacy: Invalid NVIDIA API key exits cleanly with an explicit message and no stack trace
status: migrated
scenario: ubuntu-invalid-nvidia-key-negative
assertion_id: expected-state.failure.invalid-nvidia-api-key
- legacy: Non-NVIDIA provider key prefixes are accepted by provider-aware credential validation
status: deferred
reason: provider-aware prefix validation is pure validation logic and does not require live scenario coverage
owner: e2e-maintainers
runner_requirement: local CLI build
- legacy: Occupied onboard gateway port returns a user-friendly conflict error and no stack trace
status: migrated
scenario: ubuntu-gateway-port-conflict-negative
assertion_id: expected-state.failure.gateway-port-conflict
- legacy: Non-interactive onboard honors explicit NEMOCLAW_POLICY_PRESETS
status: migrated
scenario: ubuntu-repo-cloud-openclaw-custom-policies
assertion_id: onboarding-state.registry-provider-model-policies
- legacy: Non-interactive onboard honors NEMOCLAW_PROVIDER=cloud plus the requested model
status: migrated
scenario: ubuntu-repo-cloud-openclaw-custom-policies
assertion_id: onboarding-state.session-provider-model-policies
test-brave-search-e2e.sh:
scenario: ubuntu-repo-cloud-openclaw
status: migrated
Expand Down
56 changes: 56 additions & 0 deletions test/e2e/nemoclaw_scenarios/expected-states.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,32 @@ expected_states:
policy_engine: supported
shields: supported

cloud-openclaw-custom-policies-ready:
cli:
installed: true
gateway:
expected: present
health: healthy
sandbox:
expected: present
status: running
agent: openclaw
inference:
expected: available
provider: nvidia
route: inference-local
mode: gateway-routed
credentials:
expected: present
storage: gateway-managed
onboarding_state:
provider: nvidia-prod
model: nvidia/nemotron-3-super-120b-a12b
policy_presets: npm,pypi
security:
policy_engine: supported
shields: supported

cloud-hermes-ready:
cli:
installed: true
Expand Down Expand Up @@ -118,3 +144,33 @@ expected_states:
failure:
expected: true
stage: preflight

onboarding-failure-invalid-nvidia-key:
cli:
installed: true
gateway:
expected: absent
sandbox:
expected: absent
failure:
expected: true
stage: onboarding
reason: invalid-nvidia-api-key
exit_code: 1
message_contains: Invalid NVIDIA API key. Must start with nvapi-
no_stack_trace: true

onboarding-failure-gateway-port-conflict:
cli:
installed: true
gateway:
expected: absent
sandbox:
expected: absent
failure:
expected: true
stage: onboarding
reason: gateway-port-conflict
exit_code: 1
message_contains: Port 18080 is not available
no_stack_trace: true
11 changes: 11 additions & 0 deletions test/e2e/nemoclaw_scenarios/onboard/dispatch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,17 @@ e2e_onboard() {
cloud-openclaw)
e2e_onboard_cloud_openclaw
;;
cloud-openclaw-custom-policies)
E2E_ONBOARDING_MODEL="${E2E_ONBOARDING_MODEL:-nvidia/nemotron-3-super-120b-a12b}"
E2E_ONBOARDING_POLICY_PRESETS="${E2E_ONBOARDING_POLICY_PRESETS:-npm,pypi}"
e2e_context_set E2E_ONBOARDING_MODEL "${E2E_ONBOARDING_MODEL}"
e2e_context_set E2E_ONBOARDING_POLICY_PRESETS "${E2E_ONBOARDING_POLICY_PRESETS}"
e2e_context_set E2E_ONBOARDING_REGISTRY_PROVIDER "nvidia-prod"
NEMOCLAW_MODEL="${E2E_ONBOARDING_MODEL}" NEMOCLAW_POLICY_MODE=custom NEMOCLAW_POLICY_PRESETS="${E2E_ONBOARDING_POLICY_PRESETS}" e2e_onboard_cloud_openclaw
;;
cloud-openclaw-invalid-nvidia-key | cloud-openclaw-gateway-port-conflict)
e2e_onboard_cloud_openclaw
;;
cloud-hermes)
e2e_onboard_cloud_hermes
;;
Expand Down
49 changes: 49 additions & 0 deletions test/e2e/nemoclaw_scenarios/scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,27 @@ onboarding:
agent: openclaw
provider: nvidia
inference_route: inference-local
cloud-openclaw-custom-policies:
path: cloud
agent: openclaw
provider: nvidia
inference_route: inference-local
model: nvidia/nemotron-3-super-120b-a12b
policy_presets:
- npm
- pypi
cloud-openclaw-invalid-nvidia-key:
path: cloud
agent: openclaw
provider: nvidia
inference_route: inference-local
invalid_api_key: not-a-nvidia-key
cloud-openclaw-gateway-port-conflict:
path: cloud
agent: openclaw
provider: nvidia
inference_route: inference-local
gateway_port: 18080
cloud-hermes: &id002
path: cloud
agent: hermes
Expand Down Expand Up @@ -173,6 +194,34 @@ setup_scenarios:
onboarding: cloud-openclaw
expected_state: preflight-failure-no-sandbox
suites: []
ubuntu-repo-cloud-openclaw-custom-policies:
dimensions:
platform: ubuntu-local
install: repo-current
runtime: docker-running
onboarding: cloud-openclaw-custom-policies
expected_state: cloud-openclaw-custom-policies-ready
suites:
- smoke
- inference
- credentials
- onboarding-state
ubuntu-invalid-nvidia-key-negative:
dimensions:
platform: ubuntu-local
install: repo-current
runtime: docker-running
onboarding: cloud-openclaw-invalid-nvidia-key
expected_state: onboarding-failure-invalid-nvidia-key
suites: []
ubuntu-gateway-port-conflict-negative:
dimensions:
platform: ubuntu-local
install: repo-current
runtime: docker-running
onboarding: cloud-openclaw-gateway-port-conflict
expected_state: onboarding-failure-gateway-port-conflict
suites: []
base_scenarios:
ubuntu-repo-docker:
platform: ubuntu-local
Expand Down
34 changes: 34 additions & 0 deletions test/e2e/runtime/lib/negative.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Helpers for expected-failure E2E flows.

e2e_negative_output_has_stack_trace() {
local output="$1"
printf '%s\n' "${output}" | grep -Eq '(^|[[:space:]])(TypeError|ReferenceError|SyntaxError):|^[[:space:]]+at '
}

e2e_negative_assert_failure() {
local log_file="$1"
local actual_exit="$2"
local expected_exit="$3"
local message_contains="$4"
local no_stack_trace="${5:-0}"

if [[ "${actual_exit}" -ne "${expected_exit}" ]]; then
echo "expected failure exit ${expected_exit}, got ${actual_exit}" >&2
cat "${log_file}" >&2
return 1
fi
if [[ -n "${message_contains}" ]] && ! grep -Fq "${message_contains}" "${log_file}"; then
echo "expected failure output to contain: ${message_contains}" >&2
cat "${log_file}" >&2
return 1
fi
if [[ "${no_stack_trace}" == "1" ]] && e2e_negative_output_has_stack_trace "$(cat "${log_file}")"; then
echo "expected failure output not to contain a JavaScript stack trace" >&2
cat "${log_file}" >&2
return 1
fi
}
47 changes: 47 additions & 0 deletions test/e2e/runtime/lib/onboard-state.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Assertions for onboard registry/session provider, model, and policy state.

e2e_onboard_state_assert_registry() {
local registry_file="$1"
local sandbox_name="$2"
local expected_provider="$3"
local expected_model="$4"
local expected_presets_csv="$5"
node - "${registry_file}" "${sandbox_name}" "${expected_provider}" "${expected_model}" "${expected_presets_csv}" <<'NODE'
const fs = require("node:fs");
const [registryPath, sandboxName, expectedProvider, expectedModel, csv] = process.argv.slice(2);
const registry = JSON.parse(fs.readFileSync(registryPath, "utf8"));
const sandbox = registry.sandboxes && registry.sandboxes[sandboxName];
if (!sandbox) throw new Error(`missing sandbox registry entry: ${sandboxName}`);
if (sandbox.provider !== expectedProvider) throw new Error(`expected provider ${expectedProvider}, got ${sandbox.provider}`);
if (sandbox.model !== expectedModel) throw new Error(`expected model ${expectedModel}, got ${sandbox.model}`);
const policies = Array.isArray(sandbox.policies) ? sandbox.policies : [];
for (const preset of csv.split(",").filter(Boolean)) {
if (!policies.includes(preset)) throw new Error(`missing policy preset ${preset}; policies=${JSON.stringify(policies)}`);
}
NODE
}

e2e_onboard_state_assert_session() {
local session_file="$1"
local sandbox_name="$2"
local expected_provider="$3"
local expected_model="$4"
local expected_presets_csv="$5"
node - "${session_file}" "${sandbox_name}" "${expected_provider}" "${expected_model}" "${expected_presets_csv}" <<'NODE'
const fs = require("node:fs");
const [sessionPath, sandboxName, expectedProvider, expectedModel, csv] = process.argv.slice(2);
const session = JSON.parse(fs.readFileSync(sessionPath, "utf8"));
if (session.status !== "complete") throw new Error(`session status ${session.status}`);
if (session.sandboxName !== sandboxName) throw new Error(`session sandbox ${session.sandboxName}`);
if (session.provider !== expectedProvider) throw new Error(`session provider ${session.provider}`);
if (session.model !== expectedModel) throw new Error(`session model ${session.model}`);
const presets = Array.isArray(session.policyPresets) ? session.policyPresets : [];
for (const preset of csv.split(",").filter(Boolean)) {
if (!presets.includes(preset)) throw new Error(`missing session policy preset ${preset}; presets=${JSON.stringify(presets)}`);
}
NODE
}
54 changes: 54 additions & 0 deletions test/e2e/runtime/lib/port-holder.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Local TCP listener helper for deterministic gateway port-conflict tests.

E2E_PORT_HOLDER_PID="${E2E_PORT_HOLDER_PID:-}"

e2e_port_holder_start() {
local port="$1"
if [[ -n "${E2E_PORT_HOLDER_PID}" ]]; then
e2e_port_holder_stop
fi
E2E_PORT_HOLDER_PID=""
node - "${port}" <<'NODE' >/tmp/nemoclaw-e2e-port-holder.log 2>&1 &
const net = require("node:net");
const port = Number(process.argv[2]);
const server = net.createServer((socket) => socket.end());
server.on("error", (err) => {
console.error(err && err.message ? err.message : err);
process.exit(2);
});
server.listen(port, "127.0.0.1", () => {
console.log("ready");
});
setInterval(() => {}, 1000);
NODE
E2E_PORT_HOLDER_PID=$!
local _i
for _i in $(seq 1 40); do
if node -e 'const net=require("node:net"); const port=Number(process.argv[1]); const s=net.connect(port,"127.0.0.1"); s.once("connect",()=>{s.destroy(); process.exit(0);}); s.once("error",()=>process.exit(1)); setTimeout(()=>process.exit(1),250);' "${port}" >/dev/null 2>&1; then
return 0
fi
if ! kill -0 "${E2E_PORT_HOLDER_PID}" >/dev/null 2>&1; then
E2E_PORT_HOLDER_PID=""
return 1
fi
sleep 0.25
done
if [[ -n "${E2E_PORT_HOLDER_PID}" ]]; then
kill "${E2E_PORT_HOLDER_PID}" >/dev/null 2>&1 || true
wait "${E2E_PORT_HOLDER_PID}" >/dev/null 2>&1 || true
E2E_PORT_HOLDER_PID=""
fi
return 1
}
Comment thread
jyaunches marked this conversation as resolved.

e2e_port_holder_stop() {
if [[ -n "${E2E_PORT_HOLDER_PID}" ]]; then
kill "${E2E_PORT_HOLDER_PID}" >/dev/null 2>&1 || true
wait "${E2E_PORT_HOLDER_PID}" >/dev/null 2>&1 || true
E2E_PORT_HOLDER_PID=""
fi
}
Loading
Loading