From 0a18ea8de74fb6f44c8d82d0f8213eba5bfe6f21 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 08:36:19 -0400 Subject: [PATCH 1/3] test(e2e): migrate baseline onboarding coverage --- test/e2e/docs/parity-map.yaml | 521 +++++++++++++----- test/e2e/nemoclaw_scenarios/scenarios.yaml | 4 + .../e2e-lib-helpers.test.ts | 31 ++ .../00-cli-and-openshell.sh | 11 + .../baseline-onboarding/01-sandbox-state.sh | 11 + .../baseline-onboarding/02-route-and-smoke.sh | 9 + .../lib/baseline_onboarding.sh | 79 +++ test/e2e/validation_suites/suites.yaml | 11 + 8 files changed, 529 insertions(+), 148 deletions(-) create mode 100755 test/e2e/validation_suites/baseline-onboarding/00-cli-and-openshell.sh create mode 100755 test/e2e/validation_suites/baseline-onboarding/01-sandbox-state.sh create mode 100755 test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh create mode 100755 test/e2e/validation_suites/lib/baseline_onboarding.sh diff --git a/test/e2e/docs/parity-map.yaml b/test/e2e/docs/parity-map.yaml index 39c2cc8833..438dd62f1e 100644 --- a/test/e2e/docs/parity-map.yaml +++ b/test/e2e/docs/parity-map.yaml @@ -1,6 +1,3 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - scripts: brev-e2e.test.ts: scenario: '' @@ -11,28 +8,38 @@ scripts: test-onboard-inference-smoke.sh: scenario: '' status: deferred - bucket: inference-onboard-smoke + bucket: onboarding-baseline assertions: - - legacy: setupInference() accepted a configured route without proving the chat/completions path; onboard would later print Installation complete while the first real request returns HTTP 503 (#3253) + - legacy: setupInference() accepted a configured route without proving the chat/completions path; onboard would later + print Installation complete while the first real request returns HTTP 503 (#3253) status: deferred - reason: regression guard validates fix PR #3594 before migration to scenario framework + reason: regression guard validates fix PR owner: e2e-maintainers runner_requirement: local CLI build with mocked OpenShell runner + layer: validation + gap_domain: baseline-onboarding - legacy: setupInference() did not accept a runtime-broken inference route status: deferred - reason: regression guard validates fix PR #3594 before migration to scenario framework + reason: regression guard validates fix PR owner: e2e-maintainers runner_requirement: local CLI build with mocked OpenShell runner - - legacy: onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential env/upstream 503) + layer: validation + gap_domain: baseline-onboarding + - legacy: onboard did not surface actionable inference smoke diagnostics (expected provider/model/api_base/credential + env/upstream 503) status: deferred - reason: regression guard validates fix PR #3594 before migration to scenario framework + reason: regression guard validates fix PR owner: e2e-maintainers runner_requirement: local CLI build with mocked OpenShell runner + layer: validation + gap_domain: baseline-onboarding - legacy: onboard surfaced actionable inference smoke diagnostics for the broken route status: deferred - reason: regression guard validates fix PR #3594 before migration to scenario framework + reason: regression guard validates fix PR owner: e2e-maintainers runner_requirement: local CLI build with mocked OpenShell runner + layer: validation + gap_domain: baseline-onboarding test-bedrock-runtime-compatible-anthropic.sh: scenario: ubuntu-repo-cloud-openclaw status: deferred @@ -238,42 +245,42 @@ scripts: reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'Docker is running' + - legacy: Docker is running status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'Docker is not running' + - legacy: Docker is not running status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'python3 is available' + - legacy: python3 is available status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'python3 not found' + - legacy: python3 not found status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'NEMOCLAW_NON_INTERACTIVE=1' + - legacy: NEMOCLAW_NON_INTERACTIVE=1 status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'NEMOCLAW_NON_INTERACTIVE=1 is required' + - legacy: NEMOCLAW_NON_INTERACTIVE=1 is required status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'third-party software acceptance is set' + - legacy: third-party software acceptance is set status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker + sudo hosts edit + OpenShell - - legacy: 'NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required' + - legacy: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required status: deferred reason: live legacy behavior requires Docker, sudo hosts edit, and OpenShell; retained for bucket parity tracking owner: e2e-maintainers @@ -483,94 +490,144 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Docker is running status: mapped - id: legacy.cloud.inference.e2e.docker.is.running + id: validation.baseline_onboarding.docker_running + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: NVIDIA_API_KEY not set or invalid status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: NVIDIA_API_KEY is set status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Could not cd to repo root status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: install.sh failed (exit $install_exit) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: NemoClaw installed status: mapped - id: legacy.cloud.inference.e2e.nemoclaw.installed + id: validation.baseline_onboarding.nemoclaw_installed + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: nemoclaw not on PATH status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: openshell not on PATH status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: CLIs on PATH status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: python3 not on PATH status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Could not build chat payload status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: openshell sandbox ssh-config failed for '${SANDBOX_NAME}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Chat completion returned PONG (attempt ${attempt}/${MAX_ATTEMPTS}) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: 'Live chat: $last_fail' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Repo skill validation failed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Repo agent skills (SKILL.md) valid status: mapped id: legacy.cloud.inference.e2e.repo.agent.skills.skill.md.valid + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: 'Sandbox OpenClaw layout check failed (exit ${sb_rc}): ${sb_out:0:240}' status: mapped id: legacy.cloud.inference.e2e.sandbox.openclaw.layout.check.failed.exit.sb.rc.sb.out.0.240 + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: Sandbox /sandbox/.openclaw + openclaw.json OK status: mapped id: legacy.cloud.inference.e2e.sandbox.sandbox.openclaw.openclaw.json.ok + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: Sandbox /sandbox/.openclaw/skills present status: mapped id: legacy.cloud.inference.e2e.sandbox.sandbox.openclaw.skills.present + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: 'Unexpected sandbox check output: ${sb_out:0:240}' status: retired reason: legacy assertion is obsolete or negative cleanup behavior after scenario migration reviewer: e2e-maintainers approved_at: '2026-05-13' + layer: validation + gap_domain: baseline-onboarding test-cloud-onboard-e2e.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated @@ -581,139 +638,203 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Docker is running status: mapped - id: legacy.cloud.onboard.e2e.docker.is.running + id: validation.baseline_onboarding.docker_running + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: "Docker is not running \u2014 cannot continue" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: NVIDIA_API_KEY is set (starts with nvapi-) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: "NVIDIA_API_KEY not set or invalid \u2014 required for cloud onboard" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Network access to integrate.api.nvidia.com status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Cannot reach integrate.api.nvidia.com status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: NEMOCLAW_NON_INTERACTIVE=1 is required for non-interactive install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Non-interactive mode configured status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Host OS is Linux status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: "Interactive install (RUN_E2E_CLOUD_ONBOARD_INTERACTIVE_INSTALL=1) is not yet supported \u2014 use non-interactive\ \ mode" status: retired reason: legacy assertion is obsolete or negative cleanup behavior after scenario migration reviewer: e2e-maintainers approved_at: '2026-05-13' + layer: validation + gap_domain: baseline-onboarding - legacy: Public install completed (exit 0) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Public install failed (exit $install_exit) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Public install unexpectedly used the local source checkout status: retired reason: legacy assertion is obsolete or negative cleanup behavior after scenario migration reviewer: e2e-maintainers approved_at: '2026-05-13' + layer: validation + gap_domain: baseline-onboarding - legacy: Public install used the GitHub clone path status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Public install did not show the GitHub clone path status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Public install used requested ref ${PUBLIC_INSTALL_REF} status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Public install did not use requested ref ${PUBLIC_INSTALL_REF} status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: nemoclaw on PATH ($(command -v nemoclaw)) status: mapped id: legacy.cloud.onboard.e2e.nemoclaw.on.path.command.v.nemoclaw + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: nemoclaw not found on PATH after install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: openshell on PATH ($(openshell --version 2>&1 || echo unknown)) status: mapped - id: legacy.cloud.onboard.e2e.openshell.on.path.openshell.version.2.1.echo.unknown + id: validation.baseline_onboarding.openshell_on_path + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: openshell not found on PATH after install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: nemoclaw --help exits 0 status: mapped - id: legacy.cloud.onboard.e2e.nemoclaw.help.exits.0 + id: validation.baseline_onboarding.nemoclaw_help_exits_zero + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: nemoclaw --help failed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: '$(basename ' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: '$(basename ' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Cleanup or verification failed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Cleanup complete status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding test-channels-add-remove.sh: scenario: ubuntu-repo-cloud-openclaw status: not-started @@ -721,239 +842,240 @@ scripts: assertions: - legacy: 'C0: NVIDIA_API_KEY is required' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NVIDIA_API_KEY is set' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_NON_INTERACTIVE=1 is required' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_NON_INTERACTIVE=1 is set' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is set' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1a: Pre-cleanup complete' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1b: install.sh + onboard completed (exit 0)' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1b: install.sh failed (exit $install_exit)' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1c: openshell not on PATH after install' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1c: openshell installed' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1d: nemoclaw not on PATH after install' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1d: nemoclaw installed' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1e: Sandbox ''${SANDBOX_NAME}'' is Ready' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1e: Sandbox ''${SANDBOX_NAME}'' not Ready' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2a: Provider ''${SANDBOX_NAME}-telegram-bridge'' unexpectedly exists at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2a: No telegram-bridge provider at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2b: openclaw.json unexpectedly contains ''telegram'' at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2b: could not read openclaw.json inside sandbox at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2b: openclaw.json has no ''telegram'' channel block at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2c: ''telegram'' preset unexpectedly applied at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2c: ''telegram'' preset not applied at baseline' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3a: channels add telegram registered the bridge' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3a: channels add telegram did not register' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3b: rebuild (post-add) completed' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3b: rebuild (post-add) failed' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4a: ''telegram'' preset present in policy list after add+rebuild (#3437 fixed)' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: "C4a: REGRESSION \u2014 'telegram' preset missing from policy list after add+rebuild (#3437)" status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4b: openclaw.json contains ''telegram'' channel block after add+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4b: could not read openclaw.json inside sandbox post-add' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4b: openclaw.json missing ''telegram'' channel after add+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4c: telegram-bridge provider exists in gateway after add+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4c: telegram-bridge provider missing in gateway after add+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4d: egress to api.telegram.org reaches Telegram through L7 proxy' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4d: egress to api.telegram.org blocked by proxy (preset not in effect)' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5a: channels remove telegram unregistered the bridge' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5a: channels remove telegram did not unregister' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5b: rebuild (post-remove) completed' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5b: rebuild (post-remove) failed' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6a: openclaw.json still contains ''telegram'' after remove+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6a: could not read openclaw.json inside sandbox post-remove' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6a: openclaw.json excludes ''telegram'' after remove+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6b: telegram-bridge provider still exists in gateway after remove+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6b: telegram-bridge provider removed from gateway after remove+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: "C6c: REGRESSION \u2014 'telegram' preset still applied after remove+rebuild (#3671)" status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6c: ''telegram'' preset removed from policy list after remove+rebuild' status: deferred - reason: new regression test (issue #3462 Test 2); pending scenario-framework migration + reason: new regression test (issue owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY test-channels-stop-start.sh: scenario: ubuntu-repo-cloud-openclaw status: deferred bucket: providers-messaging - zero_assertion_review: dynamic PASS/FAIL assertions cover OpenClaw and Hermes across telegram, discord, wechat, and slack; pending scenario-framework migration + zero_assertion_review: dynamic PASS/FAIL assertions cover OpenClaw and Hermes across telegram, discord, wechat, and slack; + pending scenario-framework migration assertions: [] test-credential-migration.sh: scenario: ubuntu-repo-cloud-openclaw @@ -1434,7 +1556,7 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'TC-STATE-01: BackupCaptureFiles — 5/5 .md files captured in host backup' + - legacy: "TC-STATE-01: BackupCaptureFiles \u2014 5/5 .md files captured in host backup" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -1444,7 +1566,7 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'TC-STATE-01: BackupCaptureDir — memory directory captured in host backup' + - legacy: "TC-STATE-01: BackupCaptureDir \u2014 memory directory captured in host backup" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -1479,7 +1601,7 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'TC-STATE-01: FilesRestore — ${files_restored}/5 workspace files restored correctly' + - legacy: "TC-STATE-01: FilesRestore \u2014 ${files_restored}/5 workspace files restored correctly" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -1489,7 +1611,7 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'TC-STATE-01: MemoryDirRestore — memory directory contents restored correctly' + - legacy: "TC-STATE-01: MemoryDirRestore \u2014 memory directory contents restored correctly" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -1514,7 +1636,7 @@ scripts: status: migrated bucket: rebuild-runtime assertions: - - legacy: 'TC-DEPLOY-01a / TC-DEPLOY-01b / TC-DEPLOY-01c' + - legacy: TC-DEPLOY-01a / TC-DEPLOY-01b / TC-DEPLOY-01c status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -2331,180 +2453,279 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: Docker is running status: mapped - id: legacy.full.e2e.docker.is.running + id: validation.baseline_onboarding.docker_running + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: "Docker is not running \u2014 cannot continue" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: Docker daemon + layer: validation + gap_domain: baseline-onboarding - legacy: NVIDIA_API_KEY is set (starts with nvapi-) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: "NVIDIA_API_KEY not set or invalid \u2014 required for live inference" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Network access to integrate.api.nvidia.com status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: Cannot reach integrate.api.nvidia.com status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: NEMOCLAW_NON_INTERACTIVE=1 is required status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: 'Could not cd to repo root: $REPO' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: install.sh completed (exit 0) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: install.sh failed (exit $install_exit) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: nemoclaw installed at $(command -v nemoclaw) status: mapped id: legacy.full.e2e.nemoclaw.installed.at.command.v.nemoclaw + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: nemoclaw not found on PATH after install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: openshell installed ($(openshell --version 2>&1 || echo unknown)) status: mapped id: legacy.full.e2e.openshell.installed.openshell.version.2.1.echo.unknown + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: openshell not found on PATH after install status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: nemoclaw --help exits 0 status: mapped - id: legacy.full.e2e.nemoclaw.help.exits.0 + id: validation.baseline_onboarding.nemoclaw_help_exits_zero + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: nemoclaw --help failed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: nemoclaw list contains '${SANDBOX_NAME}' status: mapped id: legacy.full.e2e.nemoclaw.list.contains.sandbox.name + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: nemoclaw list does not contain '${SANDBOX_NAME}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: 'nemoclaw list failed: ${list_output:0:200}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: nemoclaw ${SANDBOX_NAME} status exits 0 status: mapped id: legacy.full.e2e.nemoclaw.sandbox.name.status.exits.0 + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: 'nemoclaw ${SANDBOX_NAME} status failed: ${status_output:0:200}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: Inference configured via onboard status: mapped id: legacy.full.e2e.inference.configured.via.onboard + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: "Inference not configured \u2014 onboard did not set up nvidia-prod provider" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: 'openshell inference get failed: ${inf_check:0:200}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: Policy applied to sandbox status: mapped id: legacy.full.e2e.policy.applied.to.sandbox + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: No network policy found on sandbox status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: Policy presets (npm/pypi) detected in sandbox policy status: mapped id: legacy.full.e2e.policy.presets.npm.pypi.detected.in.sandbox.policy + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: 'openshell policy get failed: ${policy_output:0:200}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: '[LIVE] Direct API: model responded with PONG' status: mapped - id: legacy.full.e2e.live.direct.api.model.responded.with.pong + id: validation.baseline_onboarding.sandbox_inference_local_chat + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: '[LIVE] Direct API: expected PONG, got: ${api_content:0:200}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: '[LIVE] Direct API: empty response from curl' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: '[ROUTING] inference.local: OpenShell routed curl to NVIDIA Endpoints and returned PONG' status: mapped - id: legacy.full.e2e.routing.inference.local.openshell.routed.curl.to.nvidia.endpoints.and.returned.pong + id: validation.baseline_onboarding.sandbox_inference_local_chat + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: '[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_content:0:200}' status: mapped - id: legacy.full.e2e.routing.inference.local.expected.pong.after.3.attempts.got.sandbox.content.0.200 + id: validation.baseline_onboarding.sandbox_inference_local_chat + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers + reusable: true - legacy: "[LIVE] openclaw agent: model answered 6\xD77=42 through openclaw \u2192 inference.local" status: mapped id: legacy.full.e2e.live.openclaw.agent.model.answered.6.7.42.through.openclaw.inference.local + layer: validation + gap_domain: baseline-onboarding + owner: e2e-maintainers - legacy: '[LIVE] openclaw agent: expected ''42'' in agent reply, got: ${agent_reply:0:200}' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: NVIDIA_API_KEY secret and network egress + layer: validation + gap_domain: baseline-onboarding - legacy: 'nemoclaw logs: produced output ($(echo ' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: 'nemoclaw logs: no output' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: Sandbox ${SANDBOX_NAME} still in registry after destroy status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs + layer: validation + gap_domain: baseline-onboarding - legacy: Sandbox ${SANDBOX_NAME} removed status: retired reason: legacy assertion is obsolete or negative cleanup behavior after scenario migration reviewer: e2e-maintainers approved_at: '2026-05-13' + layer: validation + gap_domain: baseline-onboarding test-gateway-drift-preflight.sh: scenario: ubuntu-repo-cloud-openclaw status: migrated @@ -3059,32 +3280,32 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: self-hosted GPU runner - - legacy: "Onboard GPU proof passed: nvidia-smi when available" + - legacy: 'Onboard GPU proof passed: nvidia-smi when available' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: self-hosted GPU runner - - legacy: "Onboard GPU proof missing: nvidia-smi when available" + - legacy: 'Onboard GPU proof missing: nvidia-smi when available' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: self-hosted GPU runner - - legacy: "Onboard GPU proof passed: /proc/self/task//comm write" + - legacy: 'Onboard GPU proof passed: /proc/self/task//comm write' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: self-hosted GPU runner - - legacy: "Onboard GPU proof missing: /proc comm write" + - legacy: 'Onboard GPU proof missing: /proc comm write' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: self-hosted GPU runner - - legacy: "Onboard GPU proof passed: cuInit(0)" + - legacy: 'Onboard GPU proof passed: cuInit(0)' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: self-hosted GPU runner - - legacy: "Onboard GPU proof missing: cuInit(0)" + - legacy: 'Onboard GPU proof missing: cuInit(0)' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -5842,8 +6063,8 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'M13-rest-b: Failed to apply fake Discord REST policy: $(tail -20 /tmp/nemoclaw-fake-discord-rest-policy.log 2>/dev/null - | tr ''\n'' '' '' | cut -c1-300)' + - legacy: 'M13-rest-b: Failed to apply fake Discord REST policy: $(tail -20 /tmp/nemoclaw-fake-discord-rest-policy.log + 2>/dev/null | tr ''\n'' '' '' | cut -c1-300)' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -6264,7 +6485,8 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: WeChat test credentials - - legacy: 'M-W1: Provider ''${SANDBOX_NAME}-wechat-bridge'' not found in gateway (non-interactive QR-skip path may be broken)' + - legacy: 'M-W1: Provider ''${SANDBOX_NAME}-wechat-bridge'' not found in gateway (non-interactive QR-skip path may be + broken)' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -6324,7 +6546,8 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers secret_requirement: WeChat test credentials - - legacy: 'M-W9: Real WeChat token spliced into accounts/${WECHAT_ACCOUNT}.json — seed-wechat-accounts.py placeholder regression' + - legacy: "M-W9: Real WeChat token spliced into accounts/${WECHAT_ACCOUNT}.json \u2014 seed-wechat-accounts.py placeholder\ + \ regression" status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7654,32 +7877,32 @@ scripts: status: deferred bucket: providers-messaging assertions: - - legacy: 'NVIDIA_API_KEY not set' + - legacy: NVIDIA_API_KEY not set status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'NVIDIA_API_KEY is set' + - legacy: NVIDIA_API_KEY is set status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Docker is not running' + - legacy: Docker is not running status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Docker is running' + - legacy: Docker is running status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Pre-cleanup complete' + - legacy: Pre-cleanup complete status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Slack network policy pre-merged into base policy' + - legacy: Slack network policy pre-merged into base policy status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7689,47 +7912,47 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Install completed' + - legacy: Install completed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'install.sh failed (exit $install_exit)' + - legacy: install.sh failed (exit $install_exit) status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Sandbox '$SANDBOX_NAME' is Ready" + - legacy: Sandbox '$SANDBOX_NAME' is Ready status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Sandbox '$SANDBOX_NAME' not Ready (list: ${sandbox_list:0:300})" + - legacy: 'Sandbox ''$SANDBOX_NAME'' not Ready (list: ${sandbox_list:0:300})' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Slack bot/app providers exist in OpenShell' + - legacy: Slack bot/app providers exist in OpenShell status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Slack bot/app providers missing in OpenShell' + - legacy: Slack bot/app providers missing in OpenShell status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Connect-shell OpenClaw env resolves to /sandbox/.openclaw' + - legacy: Connect-shell OpenClaw env resolves to /sandbox/.openclaw status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Connect-shell OpenClaw env does not resolve to the shared state root' + - legacy: Connect-shell OpenClaw env does not resolve to the shared state root status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'openclaw pairing list slack works in connect shell' + - legacy: openclaw pairing list slack works in connect shell status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7739,57 +7962,59 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Hermetic fake Slack API started on host port ${FAKE_SLACK_API_PORT}' + - legacy: Hermetic fake Slack API started on host port ${FAKE_SLACK_API_PORT} status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Failed to start hermetic fake Slack API' + - legacy: Failed to start hermetic fake Slack API status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Applied REST policy for fake Slack chat.postMessage' + - legacy: Applied REST policy for fake Slack chat.postMessage status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Failed to apply fake Slack REST policy: $(tail -20 /tmp/nemoclaw-fake-slack-pairing-rest-policy.log 2>/dev/null | tr '\\n' ' ' | cut -c1-300)" + - legacy: 'Failed to apply fake Slack REST policy: $(tail -20 /tmp/nemoclaw-fake-slack-pairing-rest-policy.log 2>/dev/null + | tr ''\n'' '' '' | cut -c1-300)' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Applied websocket policy for fake Slack Socket Mode' + - legacy: Applied websocket policy for fake Slack Socket Mode status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Failed to apply fake Slack websocket policy: $(tail -20 /tmp/nemoclaw-fake-slack-pairing-ws-policy.log 2>/dev/null | tr '\\n' ' ' | cut -c1-300)" + - legacy: 'Failed to apply fake Slack websocket policy: $(tail -20 /tmp/nemoclaw-fake-slack-pairing-ws-policy.log 2>/dev/null + | tr ''\n'' '' '' | cut -c1-300)' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'OpenShell-tracked Slack Socket Mode handler created a pairing request' + - legacy: OpenShell-tracked Slack Socket Mode handler created a pairing request status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'OpenShell-tracked Slack Socket Mode pairing request creation failed' + - legacy: OpenShell-tracked Slack Socket Mode pairing request creation failed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Pairing code extracted from fake Slack reply path' + - legacy: Pairing code extracted from fake Slack reply path status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Failed to extract pairing code' + - legacy: Failed to extract pairing code status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Fake Slack saw rewritten xapp websocket frame and xoxb chat.postMessage' + - legacy: Fake Slack saw rewritten xapp websocket frame and xoxb chat.postMessage status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7799,27 +8024,27 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Runtime-created Slack pending request is in the shared OpenClaw state root' + - legacy: Runtime-created Slack pending request is in the shared OpenClaw state root status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Slack pending request missing from /sandbox/.openclaw/credentials/slack-pairing.json' + - legacy: Slack pending request missing from /sandbox/.openclaw/credentials/slack-pairing.json status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Connect-shell openclaw pairing list sees runtime-created Slack request' + - legacy: Connect-shell openclaw pairing list sees runtime-created Slack request status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Connect-shell openclaw pairing list does not see the Slack request' + - legacy: Connect-shell openclaw pairing list does not see the Slack request status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Connect-shell openclaw pairing approve approved the Slack request' + - legacy: Connect-shell openclaw pairing approve approved the Slack request status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7834,27 +8059,27 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Approved Slack pairing code is still pending' + - legacy: Approved Slack pairing code is still pending status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Approved Slack pairing code was consumed' + - legacy: Approved Slack pairing code was consumed status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Slack allowFrom store contains the approved user' + - legacy: Slack allowFrom store contains the approved user status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Slack allowFrom store missing approved user' + - legacy: Slack allowFrom store missing approved user status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: 'Second approval fails closed after request consumption' + - legacy: Second approval fails closed after request consumption status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7864,17 +8089,17 @@ scripts: reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Cleanup: Sandbox '$SANDBOX_NAME' intentionally kept" + - legacy: 'Cleanup: Sandbox ''$SANDBOX_NAME'' intentionally kept' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Cleanup: Sandbox '$SANDBOX_NAME' still present after cleanup" + - legacy: 'Cleanup: Sandbox ''$SANDBOX_NAME'' still present after cleanup' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - - legacy: "Cleanup: Sandbox '$SANDBOX_NAME' removed" + - legacy: 'Cleanup: Sandbox ''$SANDBOX_NAME'' removed' status: deferred reason: live legacy behavior requires non-deterministic infrastructure; retained for bucket parity tracking owner: e2e-maintainers @@ -7886,22 +8111,22 @@ scripts: assertions: - legacy: Docker is running status: deferred - reason: regression guard validates #3513 before migration to scenario framework + reason: regression guard validates owner: e2e-maintainers runner_requirement: sandbox runner with Docker and NemoClaw/OpenShell CLIs - legacy: Docker is not running status: deferred - reason: prerequisite failure for #3513 regression guard + reason: prerequisite failure for owner: e2e-maintainers runner_requirement: sandbox runner with Docker and NemoClaw/OpenShell CLIs - legacy: NVIDIA_API_KEY is set status: deferred - reason: regression guard validates #3513 with live provider credentials before scenario migration + reason: regression guard validates owner: e2e-maintainers runner_requirement: sandbox runner with NVIDIA_API_KEY - legacy: NVIDIA_API_KEY is required and must start with nvapi- status: deferred - reason: prerequisite failure for #3513 regression guard + reason: prerequisite failure for owner: e2e-maintainers runner_requirement: sandbox runner with NVIDIA_API_KEY - legacy: 'nemoclaw is available: $(nemoclaw --version 2>/dev/null || echo unknown)' @@ -7911,42 +8136,42 @@ scripts: runner_requirement: local CLI build with installer - legacy: nemoclaw not found after install status: deferred - reason: prerequisite failure for #3513 regression guard + reason: prerequisite failure for owner: e2e-maintainers runner_requirement: local CLI build with installer - legacy: fresh sandbox onboard completed status: deferred - reason: regression guard validates #3513 on a fresh sandbox before scenario migration + reason: regression guard validates owner: e2e-maintainers runner_requirement: sandbox runner with Docker and OpenShell - legacy: fresh sandbox onboard failed (exit ${onboard_rc}); see ${ONBOARD_LOG} status: deferred - reason: setup failure evidence for #3513 regression guard + reason: setup failure evidence for owner: e2e-maintainers runner_requirement: sandbox runner with Docker and OpenShell - legacy: 'OpenClaw-style plugin runtime deps replacement hit #3513 EXDEV failure' status: deferred - reason: executable red assertion for #3513 regression guard + reason: executable red assertion for owner: e2e-maintainers runner_requirement: sandbox runner with cross-filesystem /dev/shm and /sandbox paths - legacy: runtime deps replacement exited ${agent_rc}; see ${AGENT_LOG} status: deferred - reason: runtime failure evidence for #3513 regression guard + reason: runtime failure evidence for owner: e2e-maintainers runner_requirement: sandbox runner with OpenShell exec - legacy: OpenClaw-style plugin runtime-deps replacement completed across filesystems status: deferred - reason: expected green assertion after #3513 fix before migration to scenario framework + reason: expected green assertion after owner: e2e-maintainers runner_requirement: sandbox runner with cross-filesystem /dev/shm and /sandbox paths - legacy: runtime deps replacement exited 0 but success marker was missing; see ${AGENT_LOG} status: deferred - reason: runtime success-marker failure evidence for #3513 regression guard + reason: runtime success-marker failure evidence for owner: e2e-maintainers runner_requirement: sandbox runner with OpenShell exec - legacy: OpenClaw plugin runtime-deps EXDEV guard passed status: deferred - reason: expected green summary after #3513 fix before migration to scenario framework + reason: expected green summary after owner: e2e-maintainers runner_requirement: sandbox runner with OpenShell exec test-openshell-gateway-upgrade.sh: @@ -11010,7 +11235,7 @@ scripts: runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: Model Router onboard failed (exit ${onboard_rc}); see ${ONBOARD_LOG} status: deferred - reason: live regression guard failure evidence for #3255 path; retained for bucket parity tracking + reason: live regression guard failure evidence for owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: model-router reports at least one healthy endpoint @@ -11018,24 +11243,24 @@ scripts: reason: live regression guard requires external Model Router health; retained for bucket parity tracking owner: e2e-maintainers runner_requirement: sandbox runner with NVIDIA_API_KEY - - legacy: "model-router has no healthy endpoints; expected #3255 main-equivalent failure" + - legacy: 'model-router has no healthy endpoints; expected #3255 main-equivalent failure' status: deferred - reason: live regression guard failure evidence for #3255; retained for bucket parity tracking + reason: live regression guard failure evidence for owner: e2e-maintainers runner_requirement: sandbox runner with NVIDIA_API_KEY - legacy: inference.local returned a routed Model Router completion status: deferred - reason: live regression guard assertion for #3255 routed inference; retained for bucket parity tracking + reason: live regression guard assertion for owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - - legacy: "Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure" + - legacy: 'Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure' status: deferred - reason: live regression guard failure evidence for #3255; retained for bucket parity tracking + reason: live regression guard failure evidence for owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: Model Router provider-routed inference guard passed status: deferred - reason: live regression guard success assertion for #3255; retained for bucket parity tracking + reason: live regression guard success assertion for owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY test-openshell-version-pin.sh: diff --git a/test/e2e/nemoclaw_scenarios/scenarios.yaml b/test/e2e/nemoclaw_scenarios/scenarios.yaml index dc898a746f..a37e2adcad 100644 --- a/test/e2e/nemoclaw_scenarios/scenarios.yaml +++ b/test/e2e/nemoclaw_scenarios/scenarios.yaml @@ -106,6 +106,7 @@ setup_scenarios: - smoke - inference - credentials + - baseline-onboarding ubuntu-repo-cloud-hermes: alias_for_plan: ubuntu-repo-docker__cloud-nvidia-hermes dimensions: @@ -118,6 +119,7 @@ setup_scenarios: - smoke - inference - hermes-specific + - baseline-onboarding gpu-repo-local-ollama-openclaw: alias_for_plan: gpu-repo-docker-cdi__local-ollama-openclaw dimensions: @@ -177,6 +179,7 @@ setup_scenarios: suites: - smoke - inference + - baseline-onboarding runner_requirements: - ubuntu-latest - brev-api-token @@ -206,6 +209,7 @@ setup_scenarios: - inference - credentials - onboarding-state + - baseline-onboarding ubuntu-invalid-nvidia-key-negative: dimensions: platform: ubuntu-local diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index d9072af70a..d49f125c25 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -435,3 +435,34 @@ describe("Phase 1.E install dispatcher splits", () => { expect(r.stdout + r.stderr).not.toMatch(/install-repo|install-curl|install-ollama/); }); }); + +describe("baseline onboarding validation helper", () => { + it("baseline_helper_should_source_under_strict_shell_options", () => { + const r = runBash(`set -euo pipefail; source "${VALIDATION_SUITES}/lib/baseline_onboarding.sh"`); + expect(r.status, r.stderr).toBe(0); + }); + + it("baseline_cli_assertions_should_use_mocked_binaries", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "baseline-cli-")); + try { + const bin = path.join(tmp, "bin"); + const ctx = path.join(tmp, "ctx"); + fs.mkdirSync(bin); fs.mkdirSync(ctx); + fs.writeFileSync(path.join(ctx, "context.env"), "E2E_SANDBOX_NAME=sb1\nE2E_PROVIDER=nvidia\nE2E_INFERENCE_ROUTE=inference-local\n"); + fs.writeFileSync(path.join(bin, "nemoclaw"), "#!/usr/bin/env bash\n[[ $1 == --help ]] && echo help && exit 0\n", { mode: 0o755 }); + fs.writeFileSync(path.join(bin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); + const r = runBash(` + set -euo pipefail + source "${VALIDATION_SUITES}/lib/baseline_onboarding.sh" + baseline_onboarding_load_context + baseline_assert_nemoclaw_on_path + baseline_assert_openshell_on_path + baseline_assert_nemoclaw_help_exits_zero + `, { E2E_CONTEXT_DIR: ctx, PATH: `${bin}:${process.env.PATH}` }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain("PASS: validation.baseline_onboarding.nemoclaw_on_path"); + expect(r.stdout).toContain("PASS: validation.baseline_onboarding.openshell_on_path"); + expect(r.stdout).toContain("PASS: validation.baseline_onboarding.nemoclaw_help_exits_zero"); + } finally { fs.rmSync(tmp, { recursive: true, force: true }); } + }); +}); diff --git a/test/e2e/validation_suites/baseline-onboarding/00-cli-and-openshell.sh b/test/e2e/validation_suites/baseline-onboarding/00-cli-and-openshell.sh new file mode 100755 index 0000000000..17a6376a03 --- /dev/null +++ b/test/e2e/validation_suites/baseline-onboarding/00-cli-and-openshell.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../lib/baseline_onboarding.sh +source "$SCRIPT_DIR/../lib/baseline_onboarding.sh" +baseline_onboarding_load_context +baseline_assert_nemoclaw_on_path +baseline_assert_openshell_on_path +baseline_assert_nemoclaw_help_exits_zero diff --git a/test/e2e/validation_suites/baseline-onboarding/01-sandbox-state.sh b/test/e2e/validation_suites/baseline-onboarding/01-sandbox-state.sh new file mode 100755 index 0000000000..f2c975bef3 --- /dev/null +++ b/test/e2e/validation_suites/baseline-onboarding/01-sandbox-state.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../lib/baseline_onboarding.sh +source "$SCRIPT_DIR/../lib/baseline_onboarding.sh" +baseline_onboarding_load_context +baseline_assert_sandbox_list_contains_context_sandbox +baseline_assert_sandbox_status_exits_zero +baseline_assert_logs_produce_output diff --git a/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh b/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh new file mode 100755 index 0000000000..f6f43ac7b7 --- /dev/null +++ b/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../lib/baseline_onboarding.sh +source "$SCRIPT_DIR/../lib/baseline_onboarding.sh" +baseline_onboarding_load_context +baseline_assert_inference_route_provider "$E2E_PROVIDER" diff --git a/test/e2e/validation_suites/lib/baseline_onboarding.sh b/test/e2e/validation_suites/lib/baseline_onboarding.sh new file mode 100755 index 0000000000..4806cc2301 --- /dev/null +++ b/test/e2e/validation_suites/lib/baseline_onboarding.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +baseline_onboarding_pass() { printf 'PASS: %s %s\n' "$1" "${2:-}"; } +baseline_onboarding_fail() { + printf 'FAIL: %s %s\n' "$1" "${2:-}" >&2 + return 1 +} + +baseline_onboarding_load_context() { + local context_file="${E2E_CONTEXT_DIR:?E2E_CONTEXT_DIR is required}/context.env" + # shellcheck disable=SC1090 + source "$context_file" + : "${E2E_SANDBOX_NAME:?E2E_SANDBOX_NAME is required}" + : "${E2E_PROVIDER:?E2E_PROVIDER is required}" + : "${E2E_INFERENCE_ROUTE:?E2E_INFERENCE_ROUTE is required}" +} + +baseline_assert_nemoclaw_on_path() { + if command -v nemoclaw >/dev/null 2>&1; then + baseline_onboarding_pass validation.baseline_onboarding.nemoclaw_on_path "nemoclaw found" + else + baseline_onboarding_fail validation.baseline_onboarding.nemoclaw_on_path "nemoclaw not on PATH" + fi +} + +baseline_assert_openshell_on_path() { + if command -v openshell >/dev/null 2>&1; then + baseline_onboarding_pass validation.baseline_onboarding.openshell_on_path "openshell found" + else + baseline_onboarding_fail validation.baseline_onboarding.openshell_on_path "openshell not on PATH" + fi +} + +baseline_assert_nemoclaw_help_exits_zero() { + local out + if out=$(nemoclaw --help 2>&1); then + baseline_onboarding_pass validation.baseline_onboarding.nemoclaw_help_exits_zero "nemoclaw --help exits 0" + else + baseline_onboarding_fail validation.baseline_onboarding.nemoclaw_help_exits_zero "nemoclaw --help failed: ${out:0:200}" + fi +} + +baseline_assert_sandbox_list_contains_context_sandbox() { + local out + if out=$(nemoclaw list 2>&1) && grep -Fq "$E2E_SANDBOX_NAME" <<<"$out"; then + baseline_onboarding_pass validation.baseline_onboarding.sandbox_listed "$E2E_SANDBOX_NAME listed" + else + baseline_onboarding_fail validation.baseline_onboarding.sandbox_listed "sandbox not listed: ${out:0:200}" + fi +} + +baseline_assert_sandbox_status_exits_zero() { + local out + if out=$(nemoclaw status "$E2E_SANDBOX_NAME" 2>&1); then + baseline_onboarding_pass validation.baseline_onboarding.sandbox_status "$E2E_SANDBOX_NAME status ok" + else + baseline_onboarding_fail validation.baseline_onboarding.sandbox_status "status failed: ${out:0:200}" + fi +} + +baseline_assert_logs_produce_output() { + local out + if out=$(nemoclaw logs "$E2E_SANDBOX_NAME" 2>&1) && [[ -n "$out" ]]; then + baseline_onboarding_pass validation.baseline_onboarding.logs_available "logs available" + else + baseline_onboarding_fail validation.baseline_onboarding.logs_available "logs unavailable" + fi +} + +baseline_assert_inference_route_provider() { + local expected="${1:-$E2E_PROVIDER}" + if [[ "${E2E_PROVIDER:-}" == "$expected" ]]; then + baseline_onboarding_pass validation.baseline_onboarding.inference_route_provider "provider=$expected route=${E2E_INFERENCE_ROUTE:-}" + else + baseline_onboarding_fail validation.baseline_onboarding.inference_route_provider "provider mismatch" + fi +} diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e/validation_suites/suites.yaml index 6ec38c4a3f..e176f7d098 100644 --- a/test/e2e/validation_suites/suites.yaml +++ b/test/e2e/validation_suites/suites.yaml @@ -30,6 +30,17 @@ suites: steps: &id008 - id: credentials-present script: security/credentials/00-credentials-present.sh + baseline-onboarding: + requires_state: + cli.installed: true + sandbox.status: running + steps: + - id: cli-and-openshell + script: baseline-onboarding/00-cli-and-openshell.sh + - id: sandbox-state + script: baseline-onboarding/01-sandbox-state.sh + - id: route-and-smoke + script: baseline-onboarding/02-route-and-smoke.sh onboarding-state: requires_state: sandbox.status: running From 4bbe94870278de5abd66f0eefd20c4c480addee7 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 08:45:20 -0400 Subject: [PATCH 2/3] fix(e2e): address baseline migration review feedback --- test/e2e/docs/parity-map.yaml | 104 +++++++++--------- test/e2e/nemoclaw_scenarios/scenarios.yaml | 3 + .../baseline-onboarding/02-route-and-smoke.sh | 2 +- .../lib/baseline_onboarding.sh | 2 +- test/e2e/validation_suites/suites.yaml | 3 + 5 files changed, 60 insertions(+), 54 deletions(-) diff --git a/test/e2e/docs/parity-map.yaml b/test/e2e/docs/parity-map.yaml index 438dd62f1e..1165f0263d 100644 --- a/test/e2e/docs/parity-map.yaml +++ b/test/e2e/docs/parity-map.yaml @@ -842,232 +842,232 @@ scripts: assertions: - legacy: 'C0: NVIDIA_API_KEY is required' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NVIDIA_API_KEY is set' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_NON_INTERACTIVE=1 is required' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_NON_INTERACTIVE=1 is set' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is set' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1a: Pre-cleanup complete' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1b: install.sh + onboard completed (exit 0)' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1b: install.sh failed (exit $install_exit)' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1c: openshell not on PATH after install' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1c: openshell installed' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1d: nemoclaw not on PATH after install' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1d: nemoclaw installed' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1e: Sandbox ''${SANDBOX_NAME}'' is Ready' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C1e: Sandbox ''${SANDBOX_NAME}'' not Ready' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2a: Provider ''${SANDBOX_NAME}-telegram-bridge'' unexpectedly exists at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2a: No telegram-bridge provider at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2b: openclaw.json unexpectedly contains ''telegram'' at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2b: could not read openclaw.json inside sandbox at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2b: openclaw.json has no ''telegram'' channel block at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2c: ''telegram'' preset unexpectedly applied at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C2c: ''telegram'' preset not applied at baseline' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3a: channels add telegram registered the bridge' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3a: channels add telegram did not register' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3b: rebuild (post-add) completed' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C3b: rebuild (post-add) failed' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4a: ''telegram'' preset present in policy list after add+rebuild (#3437 fixed)' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: "C4a: REGRESSION \u2014 'telegram' preset missing from policy list after add+rebuild (#3437)" status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4b: openclaw.json contains ''telegram'' channel block after add+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4b: could not read openclaw.json inside sandbox post-add' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4b: openclaw.json missing ''telegram'' channel after add+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4c: telegram-bridge provider exists in gateway after add+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4c: telegram-bridge provider missing in gateway after add+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4d: egress to api.telegram.org reaches Telegram through L7 proxy' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C4d: egress to api.telegram.org blocked by proxy (preset not in effect)' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5a: channels remove telegram unregistered the bridge' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5a: channels remove telegram did not unregister' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5b: rebuild (post-remove) completed' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C5b: rebuild (post-remove) failed' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6a: openclaw.json still contains ''telegram'' after remove+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6a: could not read openclaw.json inside sandbox post-remove' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6a: openclaw.json excludes ''telegram'' after remove+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6b: telegram-bridge provider still exists in gateway after remove+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6b: telegram-bridge provider removed from gateway after remove+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: "C6c: REGRESSION \u2014 'telegram' preset still applied after remove+rebuild (#3671)" status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'C6c: ''telegram'' preset removed from policy list after remove+rebuild' status: deferred - reason: new regression test (issue + reason: new regression test for issue #3809 baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY test-channels-stop-start.sh: @@ -8111,7 +8111,7 @@ scripts: assertions: - legacy: Docker is running status: deferred - reason: regression guard validates + reason: regression guard validates baseline onboarding scenario migration coverage owner: e2e-maintainers runner_requirement: sandbox runner with Docker and NemoClaw/OpenShell CLIs - legacy: Docker is not running @@ -8121,7 +8121,7 @@ scripts: runner_requirement: sandbox runner with Docker and NemoClaw/OpenShell CLIs - legacy: NVIDIA_API_KEY is set status: deferred - reason: regression guard validates + reason: regression guard validates baseline onboarding scenario migration coverage owner: e2e-maintainers runner_requirement: sandbox runner with NVIDIA_API_KEY - legacy: NVIDIA_API_KEY is required and must start with nvapi- @@ -8141,7 +8141,7 @@ scripts: runner_requirement: local CLI build with installer - legacy: fresh sandbox onboard completed status: deferred - reason: regression guard validates + reason: regression guard validates baseline onboarding scenario migration coverage owner: e2e-maintainers runner_requirement: sandbox runner with Docker and OpenShell - legacy: fresh sandbox onboard failed (exit ${onboard_rc}); see ${ONBOARD_LOG} @@ -11235,7 +11235,7 @@ scripts: runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: Model Router onboard failed (exit ${onboard_rc}); see ${ONBOARD_LOG} status: deferred - reason: live regression guard failure evidence for + reason: live regression guard failure evidence for baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: model-router reports at least one healthy endpoint @@ -11245,7 +11245,7 @@ scripts: runner_requirement: sandbox runner with NVIDIA_API_KEY - legacy: 'model-router has no healthy endpoints; expected #3255 main-equivalent failure' status: deferred - reason: live regression guard failure evidence for + reason: live regression guard failure evidence for baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NVIDIA_API_KEY - legacy: inference.local returned a routed Model Router completion @@ -11255,7 +11255,7 @@ scripts: runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: 'Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure' status: deferred - reason: live regression guard failure evidence for + reason: live regression guard failure evidence for baseline onboarding scenario migration owner: e2e-maintainers runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs and NVIDIA_API_KEY - legacy: Model Router provider-routed inference guard passed diff --git a/test/e2e/nemoclaw_scenarios/scenarios.yaml b/test/e2e/nemoclaw_scenarios/scenarios.yaml index a37e2adcad..1039056837 100644 --- a/test/e2e/nemoclaw_scenarios/scenarios.yaml +++ b/test/e2e/nemoclaw_scenarios/scenarios.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + platforms: ubuntu-local: os: ubuntu diff --git a/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh b/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh index f6f43ac7b7..64ae539487 100755 --- a/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh +++ b/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh @@ -6,4 +6,4 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../lib/baseline_onboarding.sh source "$SCRIPT_DIR/../lib/baseline_onboarding.sh" baseline_onboarding_load_context -baseline_assert_inference_route_provider "$E2E_PROVIDER" +baseline_assert_inference_route_provider "nvidia-prod" diff --git a/test/e2e/validation_suites/lib/baseline_onboarding.sh b/test/e2e/validation_suites/lib/baseline_onboarding.sh index 4806cc2301..231962a310 100755 --- a/test/e2e/validation_suites/lib/baseline_onboarding.sh +++ b/test/e2e/validation_suites/lib/baseline_onboarding.sh @@ -44,7 +44,7 @@ baseline_assert_nemoclaw_help_exits_zero() { baseline_assert_sandbox_list_contains_context_sandbox() { local out - if out=$(nemoclaw list 2>&1) && grep -Fq "$E2E_SANDBOX_NAME" <<<"$out"; then + if out=$(nemoclaw list 2>&1) && awk -v n="$E2E_SANDBOX_NAME" '$1 == n { found = 1 } END { exit !found }' <<<"$out"; then baseline_onboarding_pass validation.baseline_onboarding.sandbox_listed "$E2E_SANDBOX_NAME listed" else baseline_onboarding_fail validation.baseline_onboarding.sandbox_listed "sandbox not listed: ${out:0:200}" diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e/validation_suites/suites.yaml index e176f7d098..cb9901eff8 100644 --- a/test/e2e/validation_suites/suites.yaml +++ b/test/e2e/validation_suites/suites.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + suites: smoke: requires_state: &id001 From 0a4dc4b4f9f97d8e88357e68878c0980cd120789 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 08:54:53 -0400 Subject: [PATCH 3/3] fix(e2e): address baseline route review feedback --- test/e2e/docs/parity-map.yaml | 2 +- .../validation_suites/baseline-onboarding/02-route-and-smoke.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/docs/parity-map.yaml b/test/e2e/docs/parity-map.yaml index 1165f0263d..8142b3b19f 100644 --- a/test/e2e/docs/parity-map.yaml +++ b/test/e2e/docs/parity-map.yaml @@ -2652,7 +2652,7 @@ scripts: gap_domain: baseline-onboarding - legacy: '[LIVE] Direct API: model responded with PONG' status: mapped - id: validation.baseline_onboarding.sandbox_inference_local_chat + id: validation.baseline_onboarding.direct_api_inference_local_chat layer: validation gap_domain: baseline-onboarding owner: e2e-maintainers diff --git a/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh b/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh index 64ae539487..478b4dfa46 100755 --- a/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh +++ b/test/e2e/validation_suites/baseline-onboarding/02-route-and-smoke.sh @@ -6,4 +6,4 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../lib/baseline_onboarding.sh source "$SCRIPT_DIR/../lib/baseline_onboarding.sh" baseline_onboarding_load_context -baseline_assert_inference_route_provider "nvidia-prod" +baseline_assert_inference_route_provider