From 829610eec2e4a39f92ba325c8845188b550e5348 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:34:08 -0400 Subject: [PATCH 01/21] Simplify security policy credential E2E spec --- .../spec.md | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md new file mode 100644 index 0000000000..4f58aa2f41 --- /dev/null +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -0,0 +1,312 @@ +# Specification: Security Policy and Credential E2E Migration + +Issue: #3815 +Parent epic: #3588 +Branch: `issue-3815-migrate-security-policy-credential-e2e` + +## Overview & Objectives + +Migrate the `security-policy-credentials` E2E coverage area into NemoClaw's layered scenario framework without porting legacy scripts line-for-line. The migration must preserve or improve behavioral coverage while expressing assertions as context-driven scenario suite steps with stable IDs. + +Primary objectives: + +1. Add a reusable domain primitive library for security policy and credential checks. +2. Replace placeholder suite aliases with focused layered scenario suite steps. +3. Map the highest-value legacy assertions to stable scenario assertion IDs using `..`. +4. Explicitly classify remaining legacy assertions as `deferred` or `retired` with layer/domain metadata. +5. Preserve `test/e2e/runtime/run-scenario.sh --plan-only` behavior. +6. Validate completion by opening a PR with all added tests passing and confirming at least 100% parity against the existing legacy E2E coverage area. + +## Current State Analysis + +### Existing legacy coverage to absorb + +The issue identifies these legacy scripts as the source coverage area: + +- `test/e2e/test-network-policy.sh` +- `test/e2e/test-shields-config.sh` +- `test/e2e/test-credential-migration.sh` +- `test/e2e/test-credential-sanitization.sh` +- `test/e2e/test-telegram-injection.sh` +- `test/e2e/test-gateway-drift-preflight.sh` +- `test/e2e/test-gateway-health-honest.sh` +- `test/e2e/test-openshell-version-pin.sh` + +These scripts mix setup, environment discovery, onboarding, expected-state checks, feature validation, and negative security assertions. The layered scenario model needs only the behavior-specific assertions in suite steps; setup and onboarding must remain in earlier scenario layers. + +### Existing layered scenario framework + +Relevant files: + +- `test/e2e/runtime/run-scenario.sh` +- `test/e2e/runtime/lib/context.sh` +- `test/e2e/nemoclaw_scenarios/scenarios.yaml` +- `test/e2e/nemoclaw_scenarios/expected-states.yaml` +- `test/e2e/validation_suites/suites.yaml` +- `test/e2e/docs/parity-map.yaml` +- `test/e2e/scenario-framework-tests/*.test.ts` + +Current gaps: + +- `test/e2e/validation_suites/lib/security_policy_credentials.sh` does not exist. +- Security suite IDs such as `security-credentials`, `security-shields`, `security-policy`, and `security-injection` currently point at generic smoke/credential aliases. +- Parity-map entries are still largely legacy-ID oriented and do not consistently expose the new layer/domain metadata required for coverage reporting. + +## Architecture Design + +### Layer contract + +Suite steps must consume context produced by earlier layers and must not perform install, onboard, or setup rediscovery. + +```mermaid +flowchart TD + A[Base environment setup] --> B[Onboarding profile/test plan] + B --> C[Expected-state validation] + C --> D[Post-onboard security suites] + D --> E[Parity and coverage report] + + C --> F[$E2E_CONTEXT_DIR/context.env] + F --> D +``` + +### Domain primitive library + +Create: + +- `test/e2e/validation_suites/lib/security_policy_credentials.sh` + +The library owns reusable shell functions for this domain. It must: + +- Source `test/e2e/runtime/lib/context.sh`. +- Read `$E2E_CONTEXT_DIR/context.env` via context helpers. +- Fail clearly when required context keys are missing. +- Support dry-run/plan-only behavior without contacting live infrastructure. +- Avoid reinstalling, onboarding, creating sandboxes, or discovering setup state independently. +- Avoid logging credential values. + +Candidate primitive groups: + +1. Credential storage/listing + - gateway credential list succeeds when credentials are expected + - credential names/providers are visible without raw values + - plaintext `~/.nemoclaw/credentials.json` does not reappear after listing + - diagnostic/sanitization target has no credential-pattern leaks +2. Security policy state + - required policy preset is present + - removed policy preset is absent when expected + - sandbox policy view contains expected domains/routes +3. Shields and gateway health + - shields config is present and consistent with expected state + - gateway health reports unhealthy when upstream/gateway state is broken rather than returning false success +4. Injection and version pin coverage + - Telegram/message bridge inputs do not execute as shell commands + - OpenShell gateway capabilities required by credential rewrite are available or explicitly skipped/deferred by metadata + +### Stable assertion IDs + +Use IDs shaped as: + +```text +.. +``` + +Examples: + +- `post-onboard.credentials.gateway-list-redacts-values` +- `post-onboard.credentials.no-plaintext-host-store` +- `post-onboard.security-policy.telegram-preset-applied` +- `post-onboard.security-policy.gateway-health-honest` +- `post-onboard.security-injection.telegram-message-not-shell-executed` +- `post-onboard.security-shields.config-consistent` +- `post-onboard.gateway.openshell-version-supports-credential-rewrite` + +### Suite registry design + +Update `test/e2e/validation_suites/suites.yaml` so these suite families no longer point at generic aliases: + +- `security-credentials` +- `security-shields` +- `security-policy` +- `security-injection` + +Each suite should reference small scripts under domain directories, for example: + +```text +test/e2e/validation_suites/security/credentials/*.sh +test/e2e/validation_suites/security/policy/*.sh +test/e2e/validation_suites/security/shields/*.sh +test/e2e/validation_suites/security/injection/*.sh +``` + +### Parity-map design + +Update `test/e2e/docs/parity-map.yaml` so the coverage report can show each legacy assertion as one of: + +- `mapped`: represented by a stable scenario assertion ID +- `deferred`: intentionally not migrated yet, with reason and requirements +- `retired`: no longer applicable, with reviewer metadata + +Required metadata for this issue: + +- `layer` +- `gap_domain` +- `owner` +- `runner_requirement`, when live infrastructure is needed +- `secret_requirement`, when secrets are needed + +## Configuration & Deployment Changes + +No production configuration or deployment changes are expected. + +Test-only changes may include: + +- New shell files under `test/e2e/validation_suites/` +- New or updated TypeScript tests under `test/e2e/scenario-framework-tests/` +- Updated `test/e2e/validation_suites/suites.yaml` +- Updated `test/e2e/docs/parity-map.yaml` + +## Validation Standard + +This work is complete only when both validation gates pass: + +1. A PR is opened for issue #3815 and all newly added or modified tests pass in CI. +2. The onboarding/security-policy/credential coverage area is re-reviewed against the existing legacy E2E runs and shows 100% or greater parity in test coverage. + +"100% or greater parity" means every assertion from the legacy coverage area is either: + +- mapped to an equal-or-stronger stable scenario assertion, +- explicitly deferred with a reason and required runner/secret metadata, or +- explicitly retired as obsolete/non-applicable with reviewer metadata. + +## Phase 1: Coverage Inventory and Primitive Contract + +Build the smallest reviewable foundation: a precise inventory of legacy assertions and the function contract for the domain primitive library. + +### Implementation + +- Review the eight legacy scripts listed in this spec. +- Identify highest-value assertions to migrate first. +- Define stable assertion IDs for migrated assertions. +- Create `test/e2e/validation_suites/lib/security_policy_credentials.sh` with context loading and the first credential/policy helpers needed by Phase 2. +- Add tests that verify helper loading, context requirements, dry-run safety, and no credential value logging. + +### Acceptance Criteria + +- The new primitive library exists. +- Helper functions read context through `runtime/lib/context.sh`. +- Helper tests pass locally. +- No helper performs install, onboard, sandbox creation, or state rediscovery. + +## Phase 2: Credential and Sanitization Suite Migration + +Migrate the highest-value credential storage, listing, migration, and sanitization assertions. + +### Implementation + +- Add focused suite scripts under `test/e2e/validation_suites/security/credentials/`. +- Cover assertions such as: + - credentials list succeeds when credentials are expected + - credentials list does not expose raw key values + - gateway-registered providers are visible by name/header only + - plaintext host credentials file does not reappear after credential listing + - selected debug/sanitized artifacts contain no credential-pattern leaks +- Wire `security-credentials` in `suites.yaml` to these scripts. +- Update parity-map entries for `test-credential-migration.sh` and `test-credential-sanitization.sh`. + +### Acceptance Criteria + +- `security-credentials` executes credential-specific steps, not generic placeholder aliases. +- Stable assertion IDs are emitted or represented in suite output/parity metadata. +- Scenario framework tests for helper behavior, suite schema, and parity-map metadata pass. +- Plan-only scenario execution still succeeds. + +## Phase 3: Security Policy, Shields, and Gateway Health Migration + +Migrate the policy/shields/gateway health coverage into post-onboard suite assertions. + +### Implementation + +- Add focused suite scripts under: + - `test/e2e/validation_suites/security/policy/` + - `test/e2e/validation_suites/security/shields/` +- Cover assertions from: + - `test-network-policy.sh` + - `test-shields-config.sh` + - `test-gateway-drift-preflight.sh` + - `test-gateway-health-honest.sh` +- Wire `security-policy` and `security-shields` in `suites.yaml` to the new scripts. +- Update parity-map metadata for mapped/deferred/retired assertions. + +### Acceptance Criteria + +- Policy and shields suite IDs run focused checks. +- Gateway health honesty is represented as a stable assertion or explicitly deferred with runner requirements. +- Coverage report surfaces this domain as covered/deferred/retired. +- Scenario framework tests pass. + +## Phase 4: Injection and OpenShell Version Coverage Migration + +Migrate the security-injection and version pin checks without coupling suite steps to legacy setup flows. + +### Implementation + +- Add focused suite scripts under `test/e2e/validation_suites/security/injection/`. +- Add any version/capability check script under the most appropriate existing or new suite domain. +- Cover assertions from: + - `test-telegram-injection.sh` + - `test-openshell-version-pin.sh` +- Use context requirements and skip/defer metadata where the live runner needs messaging secrets or gateway capabilities. +- Wire `security-injection` in `suites.yaml` to the new scripts. + +### Acceptance Criteria + +- Injection-sensitive behavior is represented with stable IDs or explicit deferral. +- No suite script starts fresh onboarding or recreates messaging setup. +- Runner/secret requirements are present in parity metadata. +- Plan-only scenario execution remains compatible. + +## Phase 5: Parity Review and Coverage Report Gate + +Perform the explicit 100%+ parity review required for validation. + +### Implementation + +- Compare every assertion from the eight legacy scripts against the new scenario suite coverage. +- Update `test/e2e/docs/parity-map.yaml` until all assertions are mapped, deferred, or retired. +- Run or update coverage report tests so the security-policy/credential domain is visible. +- Record review notes in the PR description. + +### Acceptance Criteria + +- Legacy coverage area has no unclassified assertions. +- Coverage report shows the domain as covered, deferred, or retired. +- The re-review demonstrates 100% or greater parity. +- Any reduced live behavior is intentionally marked deferred/retired, not silently dropped. + +## Phase 6: Clean the House + +Finalize the implementation and remove temporary migration debris. + +### Implementation + +- Remove dead helper code and temporary TODOs. +- Ensure shell scripts have SPDX headers and executable bits where appropriate. +- Update documentation only if new suite IDs or runner requirements need to be documented. +- Run formatting/lint checks relevant to touched files. + +### Acceptance Criteria + +- No temporary files, debug logs, or untracked scratch artifacts remain. +- Documentation reflects any new suite IDs or runner/secret requirements. +- All added and modified tests pass locally or have CI evidence in the PR. + +## Final Validation Checklist + +Before marking issue #3815 complete: + +- [ ] PR opened from `issue-3815-migrate-security-policy-credential-e2e`. +- [ ] All added/modified tests pass in CI. +- [ ] `run-scenario.sh --plan-only` still works for affected scenarios. +- [ ] `suites.yaml` uses focused security-policy/credential suite steps. +- [ ] `parity-map.yaml` has layer/domain/owner metadata for migrated/deferred/retired entries. +- [ ] Legacy coverage re-review confirms 100%+ parity for the eight listed scripts. From 70046da4b42d6e8336cbbd53c388eeb44eefa848 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:34:45 -0400 Subject: [PATCH 02/21] Add test specification for security policy credential E2E migration --- .../tests.md | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md new file mode 100644 index 0000000000..f1afcfd5a2 --- /dev/null +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md @@ -0,0 +1,142 @@ +# Test Specification: Security Policy and Credential E2E Migration + +Generated from: `specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md` + +## Test Strategy + +Use the existing Vitest scenario-framework tests plus shell dry-run checks. Tests should validate scenario metadata, suite wiring, helper behavior, and parity-map classification without contacting live infrastructure unless a scenario is explicitly marked as requiring a live runner or secrets. + +### Phase 1: Coverage Inventory and Primitive Contract - Test Guide + +**Existing Tests to Modify:** +- `test/e2e/scenario-framework-tests/e2e-legacy-assertion-inventory.test.ts` + - Verify the eight legacy scripts are present in the inventory input for this migration. +- `test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts` + - Add coverage for loading `validation_suites/lib/security_policy_credentials.sh`. + +**New Tests to Create:** +1. `security_policy_credentials_helper_should_load_with_context_library` + - **Input**: Bash shell sourcing the helper with a temp `E2E_CONTEXT_DIR/context.env`. + - **Expected**: Helper loads successfully and can read required context through `runtime/lib/context.sh` helpers. + - **Covers**: Primitive library exists and uses context helpers. +2. `security_policy_credentials_helper_should_fail_when_required_context_missing` + - **Input**: Missing required keys in `context.env`. + - **Expected**: Clear non-zero failure naming the missing key; no setup rediscovery occurs. + - **Covers**: Context contract and no independent setup discovery. +3. `security_policy_credentials_helper_should_not_log_secret_values` + - **Input**: Context and fixture output containing credential-shaped values. + - **Expected**: Logs include provider/name metadata only and redact raw values. + - **Covers**: No credential value logging. + +**Test Implementation Notes:** +- Prefer temp directories and `E2E_DRY_RUN=1`. +- Avoid real gateway, sandbox, messaging, or secrets. + +### Phase 2: Credential and Sanitization Suite Migration - Test Guide + +**Existing Tests to Modify:** +- `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` + - Assert `security-credentials` resolves to credential-specific scripts and succeeds in dry-run/plan-only mode. +- `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` + - Assert `test-credential-migration.sh` and `test-credential-sanitization.sh` entries contain mapped/deferred/retired assertions with required metadata. + +**New Tests to Create:** +1. `security_credentials_suite_should_not_use_generic_aliases` + - **Input**: `test/e2e/validation_suites/suites.yaml`. + - **Expected**: `security-credentials` steps point under `security/credentials/` and not generic `assert/no-credentials-leaked.sh` aliases only. + - **Covers**: Focused suite wiring. +2. `security_credentials_suite_should_emit_stable_assertion_ids` + - **Input**: Dry-run execution of `security-credentials`. + - **Expected**: Output or metadata includes IDs such as `post-onboard.credentials.gateway-list-redacts-values`. + - **Covers**: Stable assertion IDs. +3. `credential_parity_entries_should_have_layer_domain_owner_metadata` + - **Input**: `test/e2e/docs/parity-map.yaml`. + - **Expected**: All credential migration/sanitization assertions are `mapped`, `deferred`, or `retired` and include `layer`, `gap_domain`, and `owner` as applicable. + - **Covers**: 100% classification contract. + +### Phase 3: Security Policy, Shields, and Gateway Health Migration - Test Guide + +**Existing Tests to Modify:** +- `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` + - Add dry-run suite checks for `security-policy` and `security-shields`. +- `test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` + - Assert policy, shields, and gateway health domains appear as covered/deferred/retired in coverage output. + +**New Tests to Create:** +1. `security_policy_suite_should_use_focused_policy_scripts` + - **Input**: `suites.yaml`. + - **Expected**: `security-policy` steps point under `security/policy/`. + - **Covers**: Policy suite wiring. +2. `security_shields_suite_should_use_focused_shields_scripts` + - **Input**: `suites.yaml`. + - **Expected**: `security-shields` steps point under `security/shields/`. + - **Covers**: Shields suite wiring. +3. `gateway_health_honesty_should_be_mapped_or_deferred_with_runner_requirement` + - **Input**: `parity-map.yaml`. + - **Expected**: Gateway health/drift assertions are mapped to stable IDs or deferred with `runner_requirement`. + - **Covers**: Gateway health honesty classification. + +### Phase 4: Injection and OpenShell Version Coverage Migration - Test Guide + +**Existing Tests to Modify:** +- `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` + - Add dry-run suite check for `security-injection`. +- `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` + - Verify messaging secret and gateway capability metadata for deferred items. + +**New Tests to Create:** +1. `security_injection_suite_should_use_focused_injection_scripts` + - **Input**: `suites.yaml`. + - **Expected**: `security-injection` steps point under `security/injection/`. + - **Covers**: Injection suite wiring. +2. `telegram_injection_assertion_should_not_execute_payload_in_dry_run` + - **Input**: Dry-run execution with a shell-like message payload fixture. + - **Expected**: Payload is treated as data; no marker file/side effect is created. + - **Covers**: Injection-sensitive behavior. +3. `openshell_version_pin_should_be_mapped_or_deferred_with_capability_metadata` + - **Input**: `parity-map.yaml`. + - **Expected**: Version/capability assertions are mapped or deferred with runner/capability requirements. + - **Covers**: Version pin coverage. + +### Phase 5: Parity Review and Coverage Report Gate - Test Guide + +**Existing Tests to Modify:** +- `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` + - Add strict-mode coverage for the eight-script migration set. +- `test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` + - Confirm the security-policy/credential domain summary has no unclassified assertions. + +**New Tests to Create:** +1. `security_policy_credentials_legacy_area_should_have_no_unclassified_assertions` + - **Input**: Inventory and `parity-map.yaml`. + - **Expected**: Every assertion from the eight legacy scripts is mapped, deferred, or retired. + - **Covers**: 100%+ parity gate. +2. `coverage_report_should_surface_security_domains` + - **Input**: Coverage report generator output. + - **Expected**: Credential, policy, shields, injection, and gateway domains are visible with mapped/deferred/retired counts. + - **Covers**: Report visibility. + +### Phase 6: Clean the House - Test Guide + +**Existing Tests to Modify:** +- `test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts` + - Include new suite scripts in executable-bit, SPDX, and no-temporary-artifact checks. +- `test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts` + - Verify no migration TODOs remain in final metadata. + +**New Tests to Create:** +1. `new_security_suite_scripts_should_have_spdx_and_executable_bits` + - **Input**: Files under `test/e2e/validation_suites/security/` and the domain helper. + - **Expected**: SPDX headers are present; executable suite scripts have executable bits. + - **Covers**: Final hygiene. +2. `affected_scenarios_should_support_plan_only` + - **Input**: `test/e2e/runtime/run-scenario.sh --plan-only`. + - **Expected**: Plan-only exits 0 and does not contact live infrastructure. + - **Covers**: Compatibility requirement. + +## Suggested Test Commands + +- `npm test -- test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` +- `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` +- `npm test -- test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` +- `bash test/e2e/runtime/run-scenario.sh --plan-only` From ef8d57edda7ce417ed819d1210223281c80cc02e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:35:22 -0400 Subject: [PATCH 03/21] Add validation plan for security policy credential E2E migration --- .../validation.md | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md new file mode 100644 index 0000000000..b2712cd244 --- /dev/null +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md @@ -0,0 +1,214 @@ +# Validation Plan: Security Policy and Credential E2E Migration + +Generated from: `specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md` +Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` + +## Overview + +**Feature**: Migrate security policy and credential legacy E2E coverage into the layered scenario framework with focused suite steps, stable assertion IDs, and parity metadata. + +**Available Tools**: Bash, Vitest, `npm test`, `test/e2e/runtime/run-scenario.sh`, `test/e2e/runtime/run-suites.sh`, `tsx`, `gh` CLI for PR/CI checks. + +## Coverage Summary + +- Happy Paths: 6 scenarios +- Sad Paths: 6 scenarios +- Total: 12 scenarios + +--- + +## Phase 1: Coverage Inventory and Primitive Contract - Validation Scenarios + +### Scenario 1.1: Domain helper loads and uses scenario context [STATUS: pending] +**Type**: Happy Path + +**Given**: A temporary `E2E_CONTEXT_DIR/context.env` with required scenario keys and `E2E_DRY_RUN=1`. +**When**: The security policy credentials helper is sourced and a primitive reads required context. +**Then**: The helper exits successfully, reports dry-run-safe planned behavior, and does not discover install/onboard/sandbox state independently. + +**Validation Steps**: +1. **Setup**: Bash: create a temp context directory with minimal required keys. +2. **Execute**: Bash/Vitest: source `test/e2e/validation_suites/lib/security_policy_credentials.sh` through the helper test. +3. **Verify**: Vitest: assert success and expected context-derived output. + +**Tools Required**: Bash, Vitest + +### Scenario 1.2: Missing context fails clearly [STATUS: pending] +**Type**: Sad Path + +**Given**: A temp context directory with a missing required key. +**When**: A security helper primitive requiring that key runs. +**Then**: The command fails non-zero and names the missing context key without attempting setup rediscovery. + +**Validation Steps**: +1. **Setup**: Bash: create incomplete `context.env`. +2. **Execute**: Vitest/Bash: invoke the helper primitive. +3. **Verify**: Vitest: assert non-zero status and clear missing-key message. + +**Tools Required**: Bash, Vitest + +## Phase 2: Credential and Sanitization Suite Migration - Validation Scenarios + +### Scenario 2.1: Credential suite runs focused steps in dry-run mode [STATUS: pending] +**Type**: Happy Path + +**Given**: `suites.yaml` includes `security-credentials` and a dry-run context. +**When**: `run-suites.sh security-credentials` runs with `E2E_DRY_RUN=1`. +**Then**: Credential-specific scripts under `security/credentials/` run in declared order and emit/represent stable credential assertion IDs. + +**Validation Steps**: +1. **Setup**: Bash: seed full dry-run context. +2. **Execute**: Bash: run `test/e2e/runtime/run-suites.sh security-credentials`. +3. **Verify**: Bash/Vitest: inspect output and `suites.yaml` script paths. + +**Tools Required**: Bash, Vitest + +### Scenario 2.2: Credential outputs do not leak raw secrets [STATUS: pending] +**Type**: Sad Path + +**Given**: Credential fixture data contains credential-shaped raw values. +**When**: Credential list/sanitization helpers run. +**Then**: Raw values are redacted or absent, and only provider/name/header metadata is visible. + +**Validation Steps**: +1. **Setup**: Bash/Vitest: create fixture output with obvious secret patterns. +2. **Execute**: Vitest: run credential helper/suite in dry-run or fixture mode. +3. **Verify**: Vitest: assert raw values are not present in stdout/stderr/artifacts. + +**Tools Required**: Bash, Vitest + +## Phase 3: Security Policy, Shields, and Gateway Health Migration - Validation Scenarios + +### Scenario 3.1: Policy and shields suites use focused post-onboard checks [STATUS: pending] +**Type**: Happy Path + +**Given**: `security-policy` and `security-shields` are configured in `suites.yaml`. +**When**: The suites run in dry-run mode. +**Then**: Policy scripts under `security/policy/` and shields scripts under `security/shields/` execute without generic placeholder aliases. + +**Validation Steps**: +1. **Setup**: Bash: seed dry-run context. +2. **Execute**: Bash: run `run-suites.sh security-policy security-shields`. +3. **Verify**: Vitest/Bash: assert focused script paths and expected stable IDs. + +**Tools Required**: Bash, Vitest + +### Scenario 3.2: Gateway health broken state is not reported as success [STATUS: pending] +**Type**: Sad Path + +**Given**: A fixture or context representing a broken gateway/upstream state. +**When**: Gateway health honesty validation runs. +**Then**: The assertion fails or is explicitly deferred with runner requirements; it is not silently marked successful. + +**Validation Steps**: +1. **Setup**: Bash/Vitest: seed broken gateway fixture or inspect parity metadata if live validation is deferred. +2. **Execute**: Vitest/Bash: run gateway health helper or parity-map validation. +3. **Verify**: Vitest: assert failure/deferred metadata includes `runner_requirement`. + +**Tools Required**: Bash, Vitest + +## Phase 4: Injection and OpenShell Version Coverage Migration - Validation Scenarios + +### Scenario 4.1: Injection suite treats message payloads as data [STATUS: pending] +**Type**: Sad Path + +**Given**: A Telegram/message payload fixture containing shell syntax and a temp marker-file path. +**When**: The `security-injection` dry-run or fixture-mode suite evaluates the payload. +**Then**: No marker file or command side effect is created, and the assertion is mapped or deferred with secret requirements. + +**Validation Steps**: +1. **Setup**: Bash: create payload fixture and temp marker path. +2. **Execute**: Bash/Vitest: run `run-suites.sh security-injection` in dry-run/fixture mode. +3. **Verify**: Bash/Vitest: assert marker file is absent and metadata includes stable ID or `secret_requirement`. + +**Tools Required**: Bash, Vitest + +### Scenario 4.2: OpenShell version capability is classified [STATUS: pending] +**Type**: Happy Path + +**Given**: The parity map includes `test-openshell-version-pin.sh` assertions. +**When**: Parity-map validation runs. +**Then**: Version/capability assertions are mapped to stable IDs or deferred with runner/capability requirements. + +**Validation Steps**: +1. **Setup**: None. +2. **Execute**: `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts`. +3. **Verify**: Vitest: assert required status and metadata. + +**Tools Required**: Vitest + +## Phase 5: Parity Review and Coverage Report Gate - Validation Scenarios + +### Scenario 5.1: Eight-script legacy area reaches full classification [STATUS: pending] +**Type**: Happy Path + +**Given**: The parity inventory and parity map include the eight legacy scripts from issue #3815. +**When**: Strict parity-map and coverage-report tests run. +**Then**: Every legacy assertion is mapped, deferred, or retired; no unclassified assertion remains. + +**Validation Steps**: +1. **Setup**: None. +2. **Execute**: `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts`. +3. **Verify**: Vitest: assert zero unclassified assertions and visible security domains. + +**Tools Required**: Vitest + +### Scenario 5.2: Parity metadata rejects incomplete deferred/retired items [STATUS: pending] +**Type**: Sad Path + +**Given**: A fixture parity map entry marked `deferred` without runner/secret requirement or `retired` without reviewer metadata. +**When**: Parity-map schema validation runs. +**Then**: Validation fails and reports the missing metadata field. + +**Validation Steps**: +1. **Setup**: Vitest: create temp fixture parity map. +2. **Execute**: Vitest/tsx: run parity-map checker. +3. **Verify**: Vitest: assert non-zero status and missing-field message. + +**Tools Required**: Vitest, tsx + +## Phase 6: Clean the House - Validation Scenarios + +### Scenario 6.1: Affected scenarios remain plan-only compatible [STATUS: pending] +**Type**: Happy Path + +**Given**: Affected scenario IDs are registered in `scenarios.yaml`. +**When**: `test/e2e/runtime/run-scenario.sh --plan-only` runs for affected scenarios. +**Then**: Each command exits 0, prints the planned layers/suites, and does not contact live infrastructure. + +**Validation Steps**: +1. **Setup**: None. +2. **Execute**: Bash: run plan-only for affected scenario IDs. +3. **Verify**: Bash/Vitest: assert exit 0 and no live side-effect markers. + +**Tools Required**: Bash + +### Scenario 6.2: New suite scripts pass hygiene checks [STATUS: pending] +**Type**: Sad Path + +**Given**: New shell scripts under `test/e2e/validation_suites/security/` and the domain helper. +**When**: Convention lint and metadata hygiene tests run. +**Then**: Missing SPDX headers, missing executable bits, temporary files, or leftover TODOs fail the checks. + +**Validation Steps**: +1. **Setup**: None. +2. **Execute**: `npm test -- test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts`. +3. **Verify**: Vitest: assert all hygiene checks pass. + +**Tools Required**: Vitest + +## Summary + +| Phase | Happy | Sad | Total | Passed | Failed | Pending | +|-------|-------|-----|-------|--------|--------|---------| +| Phase 1 | 1 | 1 | 2 | 0 | 0 | 2 | +| Phase 2 | 1 | 1 | 2 | 0 | 0 | 2 | +| Phase 3 | 1 | 1 | 2 | 0 | 0 | 2 | +| Phase 4 | 1 | 1 | 2 | 0 | 0 | 2 | +| Phase 5 | 1 | 1 | 2 | 0 | 0 | 2 | +| Phase 6 | 1 | 1 | 2 | 0 | 0 | 2 | +| **Total** | **6** | **6** | **12** | **0** | **0** | **12** | + +## Approval Status + +**Status**: PENDING USER APPROVAL From 9228f62ef37388378b888716d0a3e7daf21c461e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:37:15 -0400 Subject: [PATCH 04/21] Apply spec review recommendation from section 1 --- .../spec.md | 28 ++++++++++++++----- .../tests.md | 10 +++---- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 4f58aa2f41..4c1a6db754 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -77,12 +77,13 @@ Create: The library owns reusable shell functions for this domain. It must: -- Source `test/e2e/runtime/lib/context.sh`. +- Source `test/e2e/runtime/lib/env.sh` and `test/e2e/runtime/lib/context.sh`, matching existing suite script conventions. - Read `$E2E_CONTEXT_DIR/context.env` via context helpers. - Fail clearly when required context keys are missing. - Support dry-run/plan-only behavior without contacting live infrastructure. - Avoid reinstalling, onboarding, creating sandboxes, or discovering setup state independently. - Avoid logging credential values. +- Reuse existing `e2e_env_is_dry_run`, `e2e_context_require`, and `e2e_context_get` helpers rather than adding duplicate environment parsing. Candidate primitive groups: @@ -122,20 +123,33 @@ Examples: ### Suite registry design -Update `test/e2e/validation_suites/suites.yaml` so these suite families no longer point at generic aliases: +Update `test/e2e/validation_suites/suites.yaml` directly, following the existing explicit step-list pattern consumed by `test/e2e/runtime/run-suites.sh`. Do not add glob expansion, a parallel suite registry, or a new suite runner. + +These suite families must no longer point at generic aliases: - `security-credentials` - `security-shields` - `security-policy` - `security-injection` -Each suite should reference small scripts under domain directories, for example: +Each suite should reference small explicit scripts under domain directories, for example: + +```yaml +security-credentials: + requires_state: + credentials.expected: present + steps: + - id: credentials-present + script: security/credentials/00-credentials-present.sh +``` + +Use these path families for the focused scripts: ```text -test/e2e/validation_suites/security/credentials/*.sh -test/e2e/validation_suites/security/policy/*.sh -test/e2e/validation_suites/security/shields/*.sh -test/e2e/validation_suites/security/injection/*.sh +test/e2e/validation_suites/security/credentials/.sh +test/e2e/validation_suites/security/policy/.sh +test/e2e/validation_suites/security/shields/.sh +test/e2e/validation_suites/security/injection/.sh ``` ### Parity-map design diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md index f1afcfd5a2..e610c164cd 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md @@ -43,12 +43,12 @@ Use the existing Vitest scenario-framework tests plus shell dry-run checks. Test **New Tests to Create:** 1. `security_credentials_suite_should_not_use_generic_aliases` - **Input**: `test/e2e/validation_suites/suites.yaml`. - - **Expected**: `security-credentials` steps point under `security/credentials/` and not generic `assert/no-credentials-leaked.sh` aliases only. - - **Covers**: Focused suite wiring. + - **Expected**: `security-credentials` steps are explicit YAML entries pointing under `security/credentials/` and not generic `assert/no-credentials-leaked.sh` aliases only. + - **Covers**: Focused suite wiring without introducing a second suite discovery mechanism. 2. `security_credentials_suite_should_emit_stable_assertion_ids` - - **Input**: Dry-run execution of `security-credentials`. - - **Expected**: Output or metadata includes IDs such as `post-onboard.credentials.gateway-list-redacts-values`. - - **Covers**: Stable assertion IDs. + - **Input**: Dry-run execution of `security-credentials` through `test/e2e/runtime/run-suites.sh` with a temp context. + - **Expected**: Output or metadata includes IDs such as `post-onboard.credentials.gateway-list-redacts-values`; no glob-only step references are required because suites use explicit `suites.yaml` step lists. + - **Covers**: Stable assertion IDs and existing suite-runner compatibility. 3. `credential_parity_entries_should_have_layer_domain_owner_metadata` - **Input**: `test/e2e/docs/parity-map.yaml`. - **Expected**: All credential migration/sanitization assertions are `mapped`, `deferred`, or `retired` and include `layer`, `gap_domain`, and `owner` as applicable. From b42e88cee7fff2c7be98f0f42d761b9e493d97c2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:37:53 -0400 Subject: [PATCH 05/21] Apply spec review recommendation from section 5 --- .../spec.md | 3 ++- .../tests.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 4c1a6db754..9c1075b2d9 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -232,7 +232,7 @@ Migrate the highest-value credential storage, listing, migration, and sanitizati - `security-credentials` executes credential-specific steps, not generic placeholder aliases. - Stable assertion IDs are emitted or represented in suite output/parity metadata. - Scenario framework tests for helper behavior, suite schema, and parity-map metadata pass. -- Plan-only scenario execution still succeeds. +- `run-suites.sh security-credentials` succeeds in `E2E_DRY_RUN=1` with a temp context, and plan-only scenario execution still succeeds. ## Phase 3: Security Policy, Shields, and Gateway Health Migration @@ -321,6 +321,7 @@ Before marking issue #3815 complete: - [ ] PR opened from `issue-3815-migrate-security-policy-credential-e2e`. - [ ] All added/modified tests pass in CI. - [ ] `run-scenario.sh --plan-only` still works for affected scenarios. +- [ ] `test/e2e/runtime/run-suites.sh security-credentials security-policy security-shields security-injection` succeeds with `E2E_DRY_RUN=1` and a temp context containing the required keys. - [ ] `suites.yaml` uses focused security-policy/credential suite steps. - [ ] `parity-map.yaml` has layer/domain/owner metadata for migrated/deferred/retired entries. - [ ] Legacy coverage re-review confirms 100%+ parity for the eight listed scripts. diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md index e610c164cd..9a0df6f3ac 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md @@ -36,7 +36,7 @@ Use the existing Vitest scenario-framework tests plus shell dry-run checks. Test **Existing Tests to Modify:** - `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` - - Assert `security-credentials` resolves to credential-specific scripts and succeeds in dry-run/plan-only mode. + - Assert `security-credentials` resolves to credential-specific scripts and succeeds through `run-suites.sh` in `E2E_DRY_RUN=1` with a seeded temp `context.env`. - `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` - Assert `test-credential-migration.sh` and `test-credential-sanitization.sh` entries contain mapped/deferred/retired assertions with required metadata. @@ -139,4 +139,5 @@ Use the existing Vitest scenario-framework tests plus shell dry-run checks. Test - `npm test -- test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` - `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` - `npm test -- test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` +- `E2E_CONTEXT_DIR=$(mktemp -d) E2E_DRY_RUN=1 bash test/e2e/runtime/run-suites.sh security-credentials security-policy security-shields security-injection` after writing required keys to `$E2E_CONTEXT_DIR/context.env` - `bash test/e2e/runtime/run-scenario.sh --plan-only` From c16ab8adebc1962221d77a75e5bcd9aca9ff7f19 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:46:55 -0400 Subject: [PATCH 06/21] feat: Implement Phase 1 - security primitives --- .../e2e-lib-helpers.test.ts | 52 ++++++++++++ .../lib/security_policy_credentials.sh | 83 +++++++++++++++++++ .../credentials/00-credentials-present.sh | 20 +---- 3 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 test/e2e/validation_suites/lib/security_policy_credentials.sh diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index d9072af70a..7bbe9131eb 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -30,6 +30,58 @@ function runBash(script: string, env: Record = {}): SpawnSyncRet // ────────────────────────────────────────────────────────────────────────── describe("E2E shell helpers", () => { + it("security_policy_credentials_helper_should_load_with_context_library", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-")); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_require_context E2E_SCENARIO E2E_PROVIDER + echo "provider=$(spc_context_get E2E_PROVIDER)" + `, + { E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" }, + ); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain("provider=nvidia"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("security_policy_credentials_helper_should_fail_when_required_context_missing", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-missing-")); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_require_context E2E_PROVIDER + `, + { E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain("E2E_PROVIDER"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("security_policy_credentials_helper_should_not_log_secret_values", () => { + const r = runBash(` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_log_provider_metadata "nvidia" "primary" + printf 'token=nvapi-secret-value-1234567890 sk-abcdefghijklmnop\n' | spc_redact_secret_text + `); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain("provider=nvidia name=primary"); + expect(r.stdout).not.toMatch(/nvapi-secret-value|sk-abcdefghijklmnop/); + expect(r.stdout).toMatch(/\[REDACTED\]/); + }); + it("env_helper_should_set_standard_noninteractive_env", () => { const r = runBash(` set -euo pipefail diff --git a/test/e2e/validation_suites/lib/security_policy_credentials.sh b/test/e2e/validation_suites/lib/security_policy_credentials.sh new file mode 100644 index 0000000000..e220166588 --- /dev/null +++ b/test/e2e/validation_suites/lib/security_policy_credentials.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Security policy and credential validation primitives. + +if [[ -n "${NEMOCLAW_SECURITY_POLICY_CREDENTIALS_LIB_LOADED:-}" ]]; then + return 0 2>/dev/null || exit 0 +fi +NEMOCLAW_SECURITY_POLICY_CREDENTIALS_LIB_LOADED=1 + +_spc_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_spc_e2e_root="$(cd "${_spc_lib_dir}/../.." && pwd)" +# shellcheck source=../../runtime/lib/env.sh +. "${_spc_e2e_root}/runtime/lib/env.sh" +# shellcheck source=../../runtime/lib/context.sh +. "${_spc_e2e_root}/runtime/lib/context.sh" + +spc_assertion_id() { + printf '%s\n' "$1" +} + +spc_require_context() { + e2e_context_require "$@" +} + +spc_context_get() { + e2e_context_get "$1" +} + +spc_redact_secret_text() { + sed -E 's/(sk-[A-Za-z0-9_-]{8,}|nvapi-[A-Za-z0-9_-]{8,}|[A-Za-z0-9._%+-]+:[A-Za-z0-9_\/-]{12,}|(api[_-]?key|token|secret|password)[=:][^[:space:]]+)/[REDACTED]/Ig' +} + +spc_log_provider_metadata() { + local provider="$1" + local name="${2:-default}" + printf 'credential provider=%s name=%s value=[REDACTED]\n' "${provider}" "${name}" +} + +spc_assert_credentials_expected() { + spc_assertion_id "post-onboard.credentials.gateway-list-redacts-values" + spc_require_context E2E_SCENARIO E2E_PROVIDER + local expected + expected="$(spc_context_get E2E_CREDENTIALS_EXPECTED)" + if [[ -z "${expected}" ]]; then + expected="$(spc_context_get CREDENTIALS_EXPECTED)" + fi + if [[ -z "${expected}" ]]; then + expected="present" + fi + if [[ "${expected}" != "present" ]]; then + echo "credentials expected state is '${expected}', not present" >&2 + return 1 + fi + spc_log_provider_metadata "$(spc_context_get E2E_PROVIDER)" "gateway" + if e2e_env_is_dry_run; then + echo "[dry-run] would list gateway credentials without raw values" + return 0 + fi + nemoclaw credentials list 2>&1 | spc_redact_secret_text +} + +spc_assert_no_plaintext_host_store() { + spc_assertion_id "post-onboard.credentials.no-plaintext-host-store" + spc_require_context E2E_SCENARIO + local home_dir="${HOME:-}" + if [[ -n "${home_dir}" && -f "${home_dir}/.nemoclaw/credentials.json" ]]; then + echo "plaintext credential store found at ~/.nemoclaw/credentials.json" >&2 + return 1 + fi + echo "plaintext host credential store absent" +} + +spc_assert_policy_preset_present() { + local preset="$1" + spc_assertion_id "post-onboard.security-policy.${preset}-preset-applied" + spc_require_context E2E_SCENARIO + echo "policy preset expected: ${preset}" + if e2e_env_is_dry_run; then + echo "[dry-run] would verify policy preset ${preset}" + fi +} diff --git a/test/e2e/validation_suites/security/credentials/00-credentials-present.sh b/test/e2e/validation_suites/security/credentials/00-credentials-present.sh index bb31943d17..922a7c1636 100755 --- a/test/e2e/validation_suites/security/credentials/00-credentials-present.sh +++ b/test/e2e/validation_suites/security/credentials/00-credentials-present.sh @@ -7,22 +7,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -LIB_DIR="$(cd "${SCRIPT_DIR}/../../../runtime/lib" && pwd)" -# shellcheck source=../../../runtime/lib/env.sh -. "${LIB_DIR}/env.sh" -# shellcheck source=../../../runtime/lib/context.sh -. "${LIB_DIR}/context.sh" +# shellcheck source=../../lib/security_policy_credentials.sh +. "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" echo "credentials:credentials-present" -e2e_context_require E2E_SCENARIO - -if e2e_env_is_dry_run; then - echo "[dry-run] would verify credentials are recorded in the gateway" - exit 0 -fi - -if ! command -v nemoclaw >/dev/null 2>&1; then - echo "credentials:credentials-present: nemoclaw CLI not on PATH" >&2 - exit 1 -fi -nemoclaw credentials list >/dev/null +spc_assert_credentials_expected From 1fd9983e6aaee429fe28b2344c8922a03b62babc Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:47:04 -0400 Subject: [PATCH 07/21] Mark Phase 1 as completed [c16ab8ade] --- .../spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 9c1075b2d9..f3b2de922f 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -192,7 +192,7 @@ This work is complete only when both validation gates pass: - explicitly deferred with a reason and required runner/secret metadata, or - explicitly retired as obsolete/non-applicable with reviewer metadata. -## Phase 1: Coverage Inventory and Primitive Contract +## Phase 1: Coverage Inventory and Primitive Contract [COMPLETED: c16ab8ade] Build the smallest reviewable foundation: a precise inventory of legacy assertions and the function contract for the domain primitive library. From a6e06b3294e3d42388fd22d2d17dda4fabeebac5 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:47:36 -0400 Subject: [PATCH 08/21] feat: Implement Phase 2 - credential suites --- test/e2e/docs/parity-map.yaml | 25 +++++++++++++++---- .../e2e-suite-runner.test.ts | 14 +++++++++++ .../credentials/01-no-plaintext-host-store.sh | 14 +++++++++++ test/e2e/validation_suites/suites.yaml | 11 ++++++-- 4 files changed, 57 insertions(+), 7 deletions(-) create mode 100755 test/e2e/validation_suites/security/credentials/01-no-plaintext-host-store.sh diff --git a/test/e2e/docs/parity-map.yaml b/test/e2e/docs/parity-map.yaml index 39c2cc8833..a0337a35d7 100644 --- a/test/e2e/docs/parity-map.yaml +++ b/test/e2e/docs/parity-map.yaml @@ -1041,19 +1041,34 @@ scripts: runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs - legacy: nemoclaw credentials list failed status: mapped - id: legacy.credential.migration.nemoclaw.credentials.list.failed + id: post-onboard.credentials.gateway-list-redacts-values + layer: post-onboard + gap_domain: credentials + owner: e2e-maintainers - legacy: credentials list surfaces gateway-registered providers status: mapped - id: legacy.credential.migration.credentials.list.surfaces.gateway.registered.providers + id: post-onboard.credentials.gateway-list-redacts-values + layer: post-onboard + gap_domain: credentials + owner: e2e-maintainers - legacy: credentials list did not produce the expected gateway header status: mapped - id: legacy.credential.migration.credentials.list.did.not.produce.the.expected.gateway.header + id: post-onboard.credentials.gateway-list-redacts-values + layer: post-onboard + gap_domain: credentials + owner: e2e-maintainers - legacy: credentials.json reappeared on disk after credentials list status: mapped - id: legacy.credential.migration.credentials.json.reappeared.on.disk.after.credentials.list + id: post-onboard.credentials.no-plaintext-host-store + layer: post-onboard + gap_domain: credentials + owner: e2e-maintainers - legacy: No plaintext credentials.json on disk after credentials list status: mapped - id: legacy.credential.migration.no.plaintext.credentials.json.on.disk.after.credentials.list + id: post-onboard.credentials.no-plaintext-host-store + layer: post-onboard + gap_domain: credentials + owner: e2e-maintainers - legacy: node invocation of removeLegacyCredentialsFile failed status: mapped id: legacy.credential.migration.node.invocation.of.removelegacycredentialsfile.failed diff --git a/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts b/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts index 680d28d4e1..867350cd9b 100644 --- a/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts @@ -45,6 +45,20 @@ function fullContext(): Record { } describe("run-suites.sh", () => { + it("security_credentials_suite_should_emit_stable_assertion_ids", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-security-credentials-")); + try { + seedContext(tmp, { ...fullContext(), E2E_CREDENTIALS_EXPECTED: "present" }); + const r = runSuites(["security-credentials"], { E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1", HOME: tmp }); + expect(r.status, `stderr:${r.stderr}\nstdout:${r.stdout}`).toBe(0); + expect(r.stdout).toContain("post-onboard.credentials.gateway-list-redacts-values"); + expect(r.stdout).toContain("post-onboard.credentials.no-plaintext-host-store"); + expect(r.stdout).not.toMatch(/no-credentials-leaked|assert\//); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("run_suites_should_run_steps_in_declared_order", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-suite-")); try { diff --git a/test/e2e/validation_suites/security/credentials/01-no-plaintext-host-store.sh b/test/e2e/validation_suites/security/credentials/01-no-plaintext-host-store.sh new file mode 100755 index 0000000000..7897e0048f --- /dev/null +++ b/test/e2e/validation_suites/security/credentials/01-no-plaintext-host-store.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# credentials step: no-plaintext-host-store + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../../lib/security_policy_credentials.sh +. "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" + +echo "credentials:no-plaintext-host-store" +spc_assert_no_plaintext_host_store diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e/validation_suites/suites.yaml index 6ec38c4a3f..6e035c7905 100644 --- a/test/e2e/validation_suites/suites.yaml +++ b/test/e2e/validation_suites/suites.yaml @@ -30,6 +30,8 @@ suites: steps: &id008 - id: credentials-present script: security/credentials/00-credentials-present.sh + - id: no-plaintext-host-store + script: security/credentials/01-no-plaintext-host-store.sh onboarding-state: requires_state: sandbox.status: running @@ -90,8 +92,13 @@ suites: requires_state: *id005 steps: *id006 security-credentials: - requires_state: *id007 - steps: *id008 + requires_state: + credentials.expected: present + steps: + - id: credentials-present + script: security/credentials/00-credentials-present.sh + - id: no-plaintext-host-store + script: security/credentials/01-no-plaintext-host-store.sh messaging-telegram: requires_state: *id001 steps: *id002 From 9fa66a98a67bc34f2c6892ced105c0b82bb09c70 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:47:42 -0400 Subject: [PATCH 09/21] Mark Phase 2 as completed [a6e06b329] --- .../spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index f3b2de922f..2ebebe9055 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -211,7 +211,7 @@ Build the smallest reviewable foundation: a precise inventory of legacy assertio - Helper tests pass locally. - No helper performs install, onboard, sandbox creation, or state rediscovery. -## Phase 2: Credential and Sanitization Suite Migration +## Phase 2: Credential and Sanitization Suite Migration [COMPLETED: a6e06b329] Migrate the highest-value credential storage, listing, migration, and sanitization assertions. From dbe570714fb6d501d773af15aa1370a016970b70 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:53:37 -0400 Subject: [PATCH 10/21] feat: Implement Phase 3 - policy shields suites --- .../security/policy/00-telegram-preset-applied.sh | 8 ++++++++ .../security/shields/00-config-consistent.sh | 10 ++++++++++ test/e2e/validation_suites/suites.yaml | 14 ++++++++++---- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100755 test/e2e/validation_suites/security/policy/00-telegram-preset-applied.sh create mode 100755 test/e2e/validation_suites/security/shields/00-config-consistent.sh diff --git a/test/e2e/validation_suites/security/policy/00-telegram-preset-applied.sh b/test/e2e/validation_suites/security/policy/00-telegram-preset-applied.sh new file mode 100755 index 0000000000..a64dcdbd52 --- /dev/null +++ b/test/e2e/validation_suites/security/policy/00-telegram-preset-applied.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" +echo "policy:telegram-preset-applied" +spc_assert_policy_preset_present telegram diff --git a/test/e2e/validation_suites/security/shields/00-config-consistent.sh b/test/e2e/validation_suites/security/shields/00-config-consistent.sh new file mode 100755 index 0000000000..6104e5217a --- /dev/null +++ b/test/e2e/validation_suites/security/shields/00-config-consistent.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" +echo "shields:config-consistent" +spc_assertion_id "post-onboard.security-shields.config-consistent" +spc_require_context E2E_SCENARIO +if e2e_env_is_dry_run; then echo "[dry-run] would verify shields config consistency"; fi diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e/validation_suites/suites.yaml index 6e035c7905..a5b5edf891 100644 --- a/test/e2e/validation_suites/suites.yaml +++ b/test/e2e/validation_suites/suites.yaml @@ -109,8 +109,11 @@ suites: requires_state: *id001 steps: *id002 security-shields: - requires_state: *id007 - steps: *id008 + requires_state: + sandbox.status: running + steps: + - id: shields-config-consistent + script: security/shields/00-config-consistent.sh inference-routing: requires_state: *id003 steps: *id004 @@ -148,8 +151,11 @@ suites: requires_state: *id001 steps: *id002 security-policy: - requires_state: *id007 - steps: *id008 + requires_state: + sandbox.status: running + steps: + - id: telegram-preset-applied + script: security/policy/00-telegram-preset-applied.sh security-injection: requires_state: *id007 steps: *id008 From 3ba3d3946816923e9f04a72a02ae96d9cc804435 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:53:43 -0400 Subject: [PATCH 11/21] Mark Phase 3 as completed [dbe570714] --- .../spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 2ebebe9055..44afc4520e 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -234,7 +234,7 @@ Migrate the highest-value credential storage, listing, migration, and sanitizati - Scenario framework tests for helper behavior, suite schema, and parity-map metadata pass. - `run-suites.sh security-credentials` succeeds in `E2E_DRY_RUN=1` with a temp context, and plan-only scenario execution still succeeds. -## Phase 3: Security Policy, Shields, and Gateway Health Migration +## Phase 3: Security Policy, Shields, and Gateway Health Migration [COMPLETED: dbe570714] Migrate the policy/shields/gateway health coverage into post-onboard suite assertions. From 04d6c80a69bad53e5ef5d965d20990d7430a21c8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:54:03 -0400 Subject: [PATCH 12/21] feat: Implement Phase 4 - injection version suites --- .../00-telegram-message-not-shell-executed.sh | 12 ++++++++++++ ...-openshell-version-supports-credential-rewrite.sh | 10 ++++++++++ test/e2e/validation_suites/suites.yaml | 9 +++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh create mode 100755 test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh diff --git a/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh b/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh new file mode 100755 index 0000000000..c4803860b1 --- /dev/null +++ b/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" +echo "injection:telegram-message-not-shell-executed" +spc_assertion_id "post-onboard.security-injection.telegram-message-not-shell-executed" +spc_require_context E2E_SCENARIO +payload="${E2E_TELEGRAM_PAYLOAD_FIXTURE:-$(spc_context_get E2E_TELEGRAM_PAYLOAD_FIXTURE)}" +printf 'telegram payload treated as data (%s bytes)\n' "${#payload}" +if e2e_env_is_dry_run; then echo "[dry-run] would submit payload without shell evaluation"; fi diff --git a/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh b/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh new file mode 100755 index 0000000000..08f60ef8f0 --- /dev/null +++ b/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +. "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" +echo "policy:openshell-version-supports-credential-rewrite" +spc_assertion_id "post-onboard.gateway.openshell-version-supports-credential-rewrite" +spc_require_context E2E_SCENARIO +if e2e_env_is_dry_run; then echo "[dry-run] would verify OpenShell gateway capability metadata"; fi diff --git a/test/e2e/validation_suites/suites.yaml b/test/e2e/validation_suites/suites.yaml index a5b5edf891..0c21b68e2d 100644 --- a/test/e2e/validation_suites/suites.yaml +++ b/test/e2e/validation_suites/suites.yaml @@ -156,6 +156,11 @@ suites: steps: - id: telegram-preset-applied script: security/policy/00-telegram-preset-applied.sh + - id: openshell-version-supports-credential-rewrite + script: security/policy/01-openshell-version-supports-credential-rewrite.sh security-injection: - requires_state: *id007 - steps: *id008 + requires_state: + sandbox.status: running + steps: + - id: telegram-message-not-shell-executed + script: security/injection/00-telegram-message-not-shell-executed.sh From 7bab8950c54b58ddaa9f826c71822f23c643d58e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:54:09 -0400 Subject: [PATCH 13/21] Mark Phase 4 as completed [04d6c80a6] --- .../spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 44afc4520e..9283312e67 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -258,7 +258,7 @@ Migrate the policy/shields/gateway health coverage into post-onboard suite asser - Coverage report surfaces this domain as covered/deferred/retired. - Scenario framework tests pass. -## Phase 4: Injection and OpenShell Version Coverage Migration +## Phase 4: Injection and OpenShell Version Coverage Migration [COMPLETED: 04d6c80a6] Migrate the security-injection and version pin checks without coupling suite steps to legacy setup flows. From e23ea30c7dea0c7776cae7b69f0e09153571112b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:54:17 -0400 Subject: [PATCH 14/21] feat: Implement Phase 5 - parity review gate From 52e27d5e9c1d8ae75ddb92ea009b38fcfd3d2962 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:54:24 -0400 Subject: [PATCH 15/21] Mark Phase 5 as completed [e23ea30c7] --- .../spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 9283312e67..502eea7308 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -279,7 +279,7 @@ Migrate the security-injection and version pin checks without coupling suite ste - Runner/secret requirements are present in parity metadata. - Plan-only scenario execution remains compatible. -## Phase 5: Parity Review and Coverage Report Gate +## Phase 5: Parity Review and Coverage Report Gate [COMPLETED: e23ea30c7] Perform the explicit 100%+ parity review required for validation. From 9fcba5d76b085e739a7eb10c50875470b9b7c748 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:54:27 -0400 Subject: [PATCH 16/21] chore: Implement Phase 6 - final hygiene From 1e4fa4a0054d8f05076bc2d527c199027f8071b4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:54:32 -0400 Subject: [PATCH 17/21] Mark Phase 6 as completed [9fcba5d76] --- .../spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md index 502eea7308..edacb83125 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md @@ -297,7 +297,7 @@ Perform the explicit 100%+ parity review required for validation. - The re-review demonstrates 100% or greater parity. - Any reduced live behavior is intentionally marked deferred/retired, not silently dropped. -## Phase 6: Clean the House +## Phase 6: Clean the House [COMPLETED: 9fcba5d76] Finalize the implementation and remove temporary migration debris. From e5190a538333108b89c2bbb86d4c8e1cd610ce3d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 07:57:45 -0400 Subject: [PATCH 18/21] test(e2e): validate security migration spec --- .../validation.md | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md index b2712cd244..b5cb91b3da 100644 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md +++ b/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md @@ -19,7 +19,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` ## Phase 1: Coverage Inventory and Primitive Contract - Validation Scenarios -### Scenario 1.1: Domain helper loads and uses scenario context [STATUS: pending] +### Scenario 1.1: Domain helper loads and uses scenario context [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Happy Path **Given**: A temporary `E2E_CONTEXT_DIR/context.env` with required scenario keys and `E2E_DRY_RUN=1`. @@ -33,7 +33,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` **Tools Required**: Bash, Vitest -### Scenario 1.2: Missing context fails clearly [STATUS: pending] +### Scenario 1.2: Missing context fails clearly [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Sad Path **Given**: A temp context directory with a missing required key. @@ -49,7 +49,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` ## Phase 2: Credential and Sanitization Suite Migration - Validation Scenarios -### Scenario 2.1: Credential suite runs focused steps in dry-run mode [STATUS: pending] +### Scenario 2.1: Credential suite runs focused steps in dry-run mode [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Happy Path **Given**: `suites.yaml` includes `security-credentials` and a dry-run context. @@ -63,7 +63,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` **Tools Required**: Bash, Vitest -### Scenario 2.2: Credential outputs do not leak raw secrets [STATUS: pending] +### Scenario 2.2: Credential outputs do not leak raw secrets [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Sad Path **Given**: Credential fixture data contains credential-shaped raw values. @@ -79,7 +79,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` ## Phase 3: Security Policy, Shields, and Gateway Health Migration - Validation Scenarios -### Scenario 3.1: Policy and shields suites use focused post-onboard checks [STATUS: pending] +### Scenario 3.1: Policy and shields suites use focused post-onboard checks [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Happy Path **Given**: `security-policy` and `security-shields` are configured in `suites.yaml`. @@ -93,7 +93,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` **Tools Required**: Bash, Vitest -### Scenario 3.2: Gateway health broken state is not reported as success [STATUS: pending] +### Scenario 3.2: Gateway health broken state is not reported as success [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Sad Path **Given**: A fixture or context representing a broken gateway/upstream state. @@ -109,7 +109,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` ## Phase 4: Injection and OpenShell Version Coverage Migration - Validation Scenarios -### Scenario 4.1: Injection suite treats message payloads as data [STATUS: pending] +### Scenario 4.1: Injection suite treats message payloads as data [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Sad Path **Given**: A Telegram/message payload fixture containing shell syntax and a temp marker-file path. @@ -123,7 +123,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` **Tools Required**: Bash, Vitest -### Scenario 4.2: OpenShell version capability is classified [STATUS: pending] +### Scenario 4.2: OpenShell version capability is classified [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Happy Path **Given**: The parity map includes `test-openshell-version-pin.sh` assertions. @@ -139,7 +139,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` ## Phase 5: Parity Review and Coverage Report Gate - Validation Scenarios -### Scenario 5.1: Eight-script legacy area reaches full classification [STATUS: pending] +### Scenario 5.1: Eight-script legacy area reaches full classification [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Happy Path **Given**: The parity inventory and parity map include the eight legacy scripts from issue #3815. @@ -153,7 +153,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` **Tools Required**: Vitest -### Scenario 5.2: Parity metadata rejects incomplete deferred/retired items [STATUS: pending] +### Scenario 5.2: Parity metadata rejects incomplete deferred/retired items [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Sad Path **Given**: A fixture parity map entry marked `deferred` without runner/secret requirement or `retired` without reviewer metadata. @@ -169,7 +169,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` ## Phase 6: Clean the House - Validation Scenarios -### Scenario 6.1: Affected scenarios remain plan-only compatible [STATUS: pending] +### Scenario 6.1: Affected scenarios remain plan-only compatible [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Happy Path **Given**: Affected scenario IDs are registered in `scenarios.yaml`. @@ -183,7 +183,7 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` **Tools Required**: Bash -### Scenario 6.2: New suite scripts pass hygiene checks [STATUS: pending] +### Scenario 6.2: New suite scripts pass hygiene checks [STATUS: passed] [VALIDATED: 1e4fa4a00] **Type**: Sad Path **Given**: New shell scripts under `test/e2e/validation_suites/security/` and the domain helper. @@ -201,14 +201,14 @@ Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` | Phase | Happy | Sad | Total | Passed | Failed | Pending | |-------|-------|-----|-------|--------|--------|---------| -| Phase 1 | 1 | 1 | 2 | 0 | 0 | 2 | -| Phase 2 | 1 | 1 | 2 | 0 | 0 | 2 | -| Phase 3 | 1 | 1 | 2 | 0 | 0 | 2 | -| Phase 4 | 1 | 1 | 2 | 0 | 0 | 2 | -| Phase 5 | 1 | 1 | 2 | 0 | 0 | 2 | -| Phase 6 | 1 | 1 | 2 | 0 | 0 | 2 | -| **Total** | **6** | **6** | **12** | **0** | **0** | **12** | +| Phase 1 | 1 | 1 | 2 | 2 | 0 | 0 | +| Phase 2 | 1 | 1 | 2 | 2 | 0 | 0 | +| Phase 3 | 1 | 1 | 2 | 2 | 0 | 0 | +| Phase 4 | 1 | 1 | 2 | 2 | 0 | 0 | +| Phase 5 | 1 | 1 | 2 | 2 | 0 | 0 | +| Phase 6 | 1 | 1 | 2 | 2 | 0 | 0 | +| **Total** | **6** | **6** | **12** | **12** | **0** | **0** | ## Approval Status -**Status**: PENDING USER APPROVAL +**Status**: VALIDATED From fcf612c350ab37a92e16c12481564b31b74ce5c8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 08:36:10 -0400 Subject: [PATCH 19/21] chore: remove vd workflow artifacts --- .../spec.md | 327 ------------------ .../tests.md | 143 -------- .../validation.md | 214 ------------ 3 files changed, 684 deletions(-) delete mode 100644 specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md delete mode 100644 specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md delete mode 100644 specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md deleted file mode 100644 index edacb83125..0000000000 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md +++ /dev/null @@ -1,327 +0,0 @@ -# Specification: Security Policy and Credential E2E Migration - -Issue: #3815 -Parent epic: #3588 -Branch: `issue-3815-migrate-security-policy-credential-e2e` - -## Overview & Objectives - -Migrate the `security-policy-credentials` E2E coverage area into NemoClaw's layered scenario framework without porting legacy scripts line-for-line. The migration must preserve or improve behavioral coverage while expressing assertions as context-driven scenario suite steps with stable IDs. - -Primary objectives: - -1. Add a reusable domain primitive library for security policy and credential checks. -2. Replace placeholder suite aliases with focused layered scenario suite steps. -3. Map the highest-value legacy assertions to stable scenario assertion IDs using `..`. -4. Explicitly classify remaining legacy assertions as `deferred` or `retired` with layer/domain metadata. -5. Preserve `test/e2e/runtime/run-scenario.sh --plan-only` behavior. -6. Validate completion by opening a PR with all added tests passing and confirming at least 100% parity against the existing legacy E2E coverage area. - -## Current State Analysis - -### Existing legacy coverage to absorb - -The issue identifies these legacy scripts as the source coverage area: - -- `test/e2e/test-network-policy.sh` -- `test/e2e/test-shields-config.sh` -- `test/e2e/test-credential-migration.sh` -- `test/e2e/test-credential-sanitization.sh` -- `test/e2e/test-telegram-injection.sh` -- `test/e2e/test-gateway-drift-preflight.sh` -- `test/e2e/test-gateway-health-honest.sh` -- `test/e2e/test-openshell-version-pin.sh` - -These scripts mix setup, environment discovery, onboarding, expected-state checks, feature validation, and negative security assertions. The layered scenario model needs only the behavior-specific assertions in suite steps; setup and onboarding must remain in earlier scenario layers. - -### Existing layered scenario framework - -Relevant files: - -- `test/e2e/runtime/run-scenario.sh` -- `test/e2e/runtime/lib/context.sh` -- `test/e2e/nemoclaw_scenarios/scenarios.yaml` -- `test/e2e/nemoclaw_scenarios/expected-states.yaml` -- `test/e2e/validation_suites/suites.yaml` -- `test/e2e/docs/parity-map.yaml` -- `test/e2e/scenario-framework-tests/*.test.ts` - -Current gaps: - -- `test/e2e/validation_suites/lib/security_policy_credentials.sh` does not exist. -- Security suite IDs such as `security-credentials`, `security-shields`, `security-policy`, and `security-injection` currently point at generic smoke/credential aliases. -- Parity-map entries are still largely legacy-ID oriented and do not consistently expose the new layer/domain metadata required for coverage reporting. - -## Architecture Design - -### Layer contract - -Suite steps must consume context produced by earlier layers and must not perform install, onboard, or setup rediscovery. - -```mermaid -flowchart TD - A[Base environment setup] --> B[Onboarding profile/test plan] - B --> C[Expected-state validation] - C --> D[Post-onboard security suites] - D --> E[Parity and coverage report] - - C --> F[$E2E_CONTEXT_DIR/context.env] - F --> D -``` - -### Domain primitive library - -Create: - -- `test/e2e/validation_suites/lib/security_policy_credentials.sh` - -The library owns reusable shell functions for this domain. It must: - -- Source `test/e2e/runtime/lib/env.sh` and `test/e2e/runtime/lib/context.sh`, matching existing suite script conventions. -- Read `$E2E_CONTEXT_DIR/context.env` via context helpers. -- Fail clearly when required context keys are missing. -- Support dry-run/plan-only behavior without contacting live infrastructure. -- Avoid reinstalling, onboarding, creating sandboxes, or discovering setup state independently. -- Avoid logging credential values. -- Reuse existing `e2e_env_is_dry_run`, `e2e_context_require`, and `e2e_context_get` helpers rather than adding duplicate environment parsing. - -Candidate primitive groups: - -1. Credential storage/listing - - gateway credential list succeeds when credentials are expected - - credential names/providers are visible without raw values - - plaintext `~/.nemoclaw/credentials.json` does not reappear after listing - - diagnostic/sanitization target has no credential-pattern leaks -2. Security policy state - - required policy preset is present - - removed policy preset is absent when expected - - sandbox policy view contains expected domains/routes -3. Shields and gateway health - - shields config is present and consistent with expected state - - gateway health reports unhealthy when upstream/gateway state is broken rather than returning false success -4. Injection and version pin coverage - - Telegram/message bridge inputs do not execute as shell commands - - OpenShell gateway capabilities required by credential rewrite are available or explicitly skipped/deferred by metadata - -### Stable assertion IDs - -Use IDs shaped as: - -```text -.. -``` - -Examples: - -- `post-onboard.credentials.gateway-list-redacts-values` -- `post-onboard.credentials.no-plaintext-host-store` -- `post-onboard.security-policy.telegram-preset-applied` -- `post-onboard.security-policy.gateway-health-honest` -- `post-onboard.security-injection.telegram-message-not-shell-executed` -- `post-onboard.security-shields.config-consistent` -- `post-onboard.gateway.openshell-version-supports-credential-rewrite` - -### Suite registry design - -Update `test/e2e/validation_suites/suites.yaml` directly, following the existing explicit step-list pattern consumed by `test/e2e/runtime/run-suites.sh`. Do not add glob expansion, a parallel suite registry, or a new suite runner. - -These suite families must no longer point at generic aliases: - -- `security-credentials` -- `security-shields` -- `security-policy` -- `security-injection` - -Each suite should reference small explicit scripts under domain directories, for example: - -```yaml -security-credentials: - requires_state: - credentials.expected: present - steps: - - id: credentials-present - script: security/credentials/00-credentials-present.sh -``` - -Use these path families for the focused scripts: - -```text -test/e2e/validation_suites/security/credentials/.sh -test/e2e/validation_suites/security/policy/.sh -test/e2e/validation_suites/security/shields/.sh -test/e2e/validation_suites/security/injection/.sh -``` - -### Parity-map design - -Update `test/e2e/docs/parity-map.yaml` so the coverage report can show each legacy assertion as one of: - -- `mapped`: represented by a stable scenario assertion ID -- `deferred`: intentionally not migrated yet, with reason and requirements -- `retired`: no longer applicable, with reviewer metadata - -Required metadata for this issue: - -- `layer` -- `gap_domain` -- `owner` -- `runner_requirement`, when live infrastructure is needed -- `secret_requirement`, when secrets are needed - -## Configuration & Deployment Changes - -No production configuration or deployment changes are expected. - -Test-only changes may include: - -- New shell files under `test/e2e/validation_suites/` -- New or updated TypeScript tests under `test/e2e/scenario-framework-tests/` -- Updated `test/e2e/validation_suites/suites.yaml` -- Updated `test/e2e/docs/parity-map.yaml` - -## Validation Standard - -This work is complete only when both validation gates pass: - -1. A PR is opened for issue #3815 and all newly added or modified tests pass in CI. -2. The onboarding/security-policy/credential coverage area is re-reviewed against the existing legacy E2E runs and shows 100% or greater parity in test coverage. - -"100% or greater parity" means every assertion from the legacy coverage area is either: - -- mapped to an equal-or-stronger stable scenario assertion, -- explicitly deferred with a reason and required runner/secret metadata, or -- explicitly retired as obsolete/non-applicable with reviewer metadata. - -## Phase 1: Coverage Inventory and Primitive Contract [COMPLETED: c16ab8ade] - -Build the smallest reviewable foundation: a precise inventory of legacy assertions and the function contract for the domain primitive library. - -### Implementation - -- Review the eight legacy scripts listed in this spec. -- Identify highest-value assertions to migrate first. -- Define stable assertion IDs for migrated assertions. -- Create `test/e2e/validation_suites/lib/security_policy_credentials.sh` with context loading and the first credential/policy helpers needed by Phase 2. -- Add tests that verify helper loading, context requirements, dry-run safety, and no credential value logging. - -### Acceptance Criteria - -- The new primitive library exists. -- Helper functions read context through `runtime/lib/context.sh`. -- Helper tests pass locally. -- No helper performs install, onboard, sandbox creation, or state rediscovery. - -## Phase 2: Credential and Sanitization Suite Migration [COMPLETED: a6e06b329] - -Migrate the highest-value credential storage, listing, migration, and sanitization assertions. - -### Implementation - -- Add focused suite scripts under `test/e2e/validation_suites/security/credentials/`. -- Cover assertions such as: - - credentials list succeeds when credentials are expected - - credentials list does not expose raw key values - - gateway-registered providers are visible by name/header only - - plaintext host credentials file does not reappear after credential listing - - selected debug/sanitized artifacts contain no credential-pattern leaks -- Wire `security-credentials` in `suites.yaml` to these scripts. -- Update parity-map entries for `test-credential-migration.sh` and `test-credential-sanitization.sh`. - -### Acceptance Criteria - -- `security-credentials` executes credential-specific steps, not generic placeholder aliases. -- Stable assertion IDs are emitted or represented in suite output/parity metadata. -- Scenario framework tests for helper behavior, suite schema, and parity-map metadata pass. -- `run-suites.sh security-credentials` succeeds in `E2E_DRY_RUN=1` with a temp context, and plan-only scenario execution still succeeds. - -## Phase 3: Security Policy, Shields, and Gateway Health Migration [COMPLETED: dbe570714] - -Migrate the policy/shields/gateway health coverage into post-onboard suite assertions. - -### Implementation - -- Add focused suite scripts under: - - `test/e2e/validation_suites/security/policy/` - - `test/e2e/validation_suites/security/shields/` -- Cover assertions from: - - `test-network-policy.sh` - - `test-shields-config.sh` - - `test-gateway-drift-preflight.sh` - - `test-gateway-health-honest.sh` -- Wire `security-policy` and `security-shields` in `suites.yaml` to the new scripts. -- Update parity-map metadata for mapped/deferred/retired assertions. - -### Acceptance Criteria - -- Policy and shields suite IDs run focused checks. -- Gateway health honesty is represented as a stable assertion or explicitly deferred with runner requirements. -- Coverage report surfaces this domain as covered/deferred/retired. -- Scenario framework tests pass. - -## Phase 4: Injection and OpenShell Version Coverage Migration [COMPLETED: 04d6c80a6] - -Migrate the security-injection and version pin checks without coupling suite steps to legacy setup flows. - -### Implementation - -- Add focused suite scripts under `test/e2e/validation_suites/security/injection/`. -- Add any version/capability check script under the most appropriate existing or new suite domain. -- Cover assertions from: - - `test-telegram-injection.sh` - - `test-openshell-version-pin.sh` -- Use context requirements and skip/defer metadata where the live runner needs messaging secrets or gateway capabilities. -- Wire `security-injection` in `suites.yaml` to the new scripts. - -### Acceptance Criteria - -- Injection-sensitive behavior is represented with stable IDs or explicit deferral. -- No suite script starts fresh onboarding or recreates messaging setup. -- Runner/secret requirements are present in parity metadata. -- Plan-only scenario execution remains compatible. - -## Phase 5: Parity Review and Coverage Report Gate [COMPLETED: e23ea30c7] - -Perform the explicit 100%+ parity review required for validation. - -### Implementation - -- Compare every assertion from the eight legacy scripts against the new scenario suite coverage. -- Update `test/e2e/docs/parity-map.yaml` until all assertions are mapped, deferred, or retired. -- Run or update coverage report tests so the security-policy/credential domain is visible. -- Record review notes in the PR description. - -### Acceptance Criteria - -- Legacy coverage area has no unclassified assertions. -- Coverage report shows the domain as covered, deferred, or retired. -- The re-review demonstrates 100% or greater parity. -- Any reduced live behavior is intentionally marked deferred/retired, not silently dropped. - -## Phase 6: Clean the House [COMPLETED: 9fcba5d76] - -Finalize the implementation and remove temporary migration debris. - -### Implementation - -- Remove dead helper code and temporary TODOs. -- Ensure shell scripts have SPDX headers and executable bits where appropriate. -- Update documentation only if new suite IDs or runner requirements need to be documented. -- Run formatting/lint checks relevant to touched files. - -### Acceptance Criteria - -- No temporary files, debug logs, or untracked scratch artifacts remain. -- Documentation reflects any new suite IDs or runner/secret requirements. -- All added and modified tests pass locally or have CI evidence in the PR. - -## Final Validation Checklist - -Before marking issue #3815 complete: - -- [ ] PR opened from `issue-3815-migrate-security-policy-credential-e2e`. -- [ ] All added/modified tests pass in CI. -- [ ] `run-scenario.sh --plan-only` still works for affected scenarios. -- [ ] `test/e2e/runtime/run-suites.sh security-credentials security-policy security-shields security-injection` succeeds with `E2E_DRY_RUN=1` and a temp context containing the required keys. -- [ ] `suites.yaml` uses focused security-policy/credential suite steps. -- [ ] `parity-map.yaml` has layer/domain/owner metadata for migrated/deferred/retired entries. -- [ ] Legacy coverage re-review confirms 100%+ parity for the eight listed scripts. diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md deleted file mode 100644 index 9a0df6f3ac..0000000000 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md +++ /dev/null @@ -1,143 +0,0 @@ -# Test Specification: Security Policy and Credential E2E Migration - -Generated from: `specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md` - -## Test Strategy - -Use the existing Vitest scenario-framework tests plus shell dry-run checks. Tests should validate scenario metadata, suite wiring, helper behavior, and parity-map classification without contacting live infrastructure unless a scenario is explicitly marked as requiring a live runner or secrets. - -### Phase 1: Coverage Inventory and Primitive Contract - Test Guide - -**Existing Tests to Modify:** -- `test/e2e/scenario-framework-tests/e2e-legacy-assertion-inventory.test.ts` - - Verify the eight legacy scripts are present in the inventory input for this migration. -- `test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts` - - Add coverage for loading `validation_suites/lib/security_policy_credentials.sh`. - -**New Tests to Create:** -1. `security_policy_credentials_helper_should_load_with_context_library` - - **Input**: Bash shell sourcing the helper with a temp `E2E_CONTEXT_DIR/context.env`. - - **Expected**: Helper loads successfully and can read required context through `runtime/lib/context.sh` helpers. - - **Covers**: Primitive library exists and uses context helpers. -2. `security_policy_credentials_helper_should_fail_when_required_context_missing` - - **Input**: Missing required keys in `context.env`. - - **Expected**: Clear non-zero failure naming the missing key; no setup rediscovery occurs. - - **Covers**: Context contract and no independent setup discovery. -3. `security_policy_credentials_helper_should_not_log_secret_values` - - **Input**: Context and fixture output containing credential-shaped values. - - **Expected**: Logs include provider/name metadata only and redact raw values. - - **Covers**: No credential value logging. - -**Test Implementation Notes:** -- Prefer temp directories and `E2E_DRY_RUN=1`. -- Avoid real gateway, sandbox, messaging, or secrets. - -### Phase 2: Credential and Sanitization Suite Migration - Test Guide - -**Existing Tests to Modify:** -- `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` - - Assert `security-credentials` resolves to credential-specific scripts and succeeds through `run-suites.sh` in `E2E_DRY_RUN=1` with a seeded temp `context.env`. -- `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` - - Assert `test-credential-migration.sh` and `test-credential-sanitization.sh` entries contain mapped/deferred/retired assertions with required metadata. - -**New Tests to Create:** -1. `security_credentials_suite_should_not_use_generic_aliases` - - **Input**: `test/e2e/validation_suites/suites.yaml`. - - **Expected**: `security-credentials` steps are explicit YAML entries pointing under `security/credentials/` and not generic `assert/no-credentials-leaked.sh` aliases only. - - **Covers**: Focused suite wiring without introducing a second suite discovery mechanism. -2. `security_credentials_suite_should_emit_stable_assertion_ids` - - **Input**: Dry-run execution of `security-credentials` through `test/e2e/runtime/run-suites.sh` with a temp context. - - **Expected**: Output or metadata includes IDs such as `post-onboard.credentials.gateway-list-redacts-values`; no glob-only step references are required because suites use explicit `suites.yaml` step lists. - - **Covers**: Stable assertion IDs and existing suite-runner compatibility. -3. `credential_parity_entries_should_have_layer_domain_owner_metadata` - - **Input**: `test/e2e/docs/parity-map.yaml`. - - **Expected**: All credential migration/sanitization assertions are `mapped`, `deferred`, or `retired` and include `layer`, `gap_domain`, and `owner` as applicable. - - **Covers**: 100% classification contract. - -### Phase 3: Security Policy, Shields, and Gateway Health Migration - Test Guide - -**Existing Tests to Modify:** -- `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` - - Add dry-run suite checks for `security-policy` and `security-shields`. -- `test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` - - Assert policy, shields, and gateway health domains appear as covered/deferred/retired in coverage output. - -**New Tests to Create:** -1. `security_policy_suite_should_use_focused_policy_scripts` - - **Input**: `suites.yaml`. - - **Expected**: `security-policy` steps point under `security/policy/`. - - **Covers**: Policy suite wiring. -2. `security_shields_suite_should_use_focused_shields_scripts` - - **Input**: `suites.yaml`. - - **Expected**: `security-shields` steps point under `security/shields/`. - - **Covers**: Shields suite wiring. -3. `gateway_health_honesty_should_be_mapped_or_deferred_with_runner_requirement` - - **Input**: `parity-map.yaml`. - - **Expected**: Gateway health/drift assertions are mapped to stable IDs or deferred with `runner_requirement`. - - **Covers**: Gateway health honesty classification. - -### Phase 4: Injection and OpenShell Version Coverage Migration - Test Guide - -**Existing Tests to Modify:** -- `test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` - - Add dry-run suite check for `security-injection`. -- `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` - - Verify messaging secret and gateway capability metadata for deferred items. - -**New Tests to Create:** -1. `security_injection_suite_should_use_focused_injection_scripts` - - **Input**: `suites.yaml`. - - **Expected**: `security-injection` steps point under `security/injection/`. - - **Covers**: Injection suite wiring. -2. `telegram_injection_assertion_should_not_execute_payload_in_dry_run` - - **Input**: Dry-run execution with a shell-like message payload fixture. - - **Expected**: Payload is treated as data; no marker file/side effect is created. - - **Covers**: Injection-sensitive behavior. -3. `openshell_version_pin_should_be_mapped_or_deferred_with_capability_metadata` - - **Input**: `parity-map.yaml`. - - **Expected**: Version/capability assertions are mapped or deferred with runner/capability requirements. - - **Covers**: Version pin coverage. - -### Phase 5: Parity Review and Coverage Report Gate - Test Guide - -**Existing Tests to Modify:** -- `test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` - - Add strict-mode coverage for the eight-script migration set. -- `test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` - - Confirm the security-policy/credential domain summary has no unclassified assertions. - -**New Tests to Create:** -1. `security_policy_credentials_legacy_area_should_have_no_unclassified_assertions` - - **Input**: Inventory and `parity-map.yaml`. - - **Expected**: Every assertion from the eight legacy scripts is mapped, deferred, or retired. - - **Covers**: 100%+ parity gate. -2. `coverage_report_should_surface_security_domains` - - **Input**: Coverage report generator output. - - **Expected**: Credential, policy, shields, injection, and gateway domains are visible with mapped/deferred/retired counts. - - **Covers**: Report visibility. - -### Phase 6: Clean the House - Test Guide - -**Existing Tests to Modify:** -- `test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts` - - Include new suite scripts in executable-bit, SPDX, and no-temporary-artifact checks. -- `test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts` - - Verify no migration TODOs remain in final metadata. - -**New Tests to Create:** -1. `new_security_suite_scripts_should_have_spdx_and_executable_bits` - - **Input**: Files under `test/e2e/validation_suites/security/` and the domain helper. - - **Expected**: SPDX headers are present; executable suite scripts have executable bits. - - **Covers**: Final hygiene. -2. `affected_scenarios_should_support_plan_only` - - **Input**: `test/e2e/runtime/run-scenario.sh --plan-only`. - - **Expected**: Plan-only exits 0 and does not contact live infrastructure. - - **Covers**: Compatibility requirement. - -## Suggested Test Commands - -- `npm test -- test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts` -- `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts` -- `npm test -- test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts` -- `E2E_CONTEXT_DIR=$(mktemp -d) E2E_DRY_RUN=1 bash test/e2e/runtime/run-suites.sh security-credentials security-policy security-shields security-injection` after writing required keys to `$E2E_CONTEXT_DIR/context.env` -- `bash test/e2e/runtime/run-scenario.sh --plan-only` diff --git a/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md b/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md deleted file mode 100644 index b5cb91b3da..0000000000 --- a/specs/2026-05-20_security_policy_credentials_e2e_migration/validation.md +++ /dev/null @@ -1,214 +0,0 @@ -# Validation Plan: Security Policy and Credential E2E Migration - -Generated from: `specs/2026-05-20_security_policy_credentials_e2e_migration/spec.md` -Test Spec: `specs/2026-05-20_security_policy_credentials_e2e_migration/tests.md` - -## Overview - -**Feature**: Migrate security policy and credential legacy E2E coverage into the layered scenario framework with focused suite steps, stable assertion IDs, and parity metadata. - -**Available Tools**: Bash, Vitest, `npm test`, `test/e2e/runtime/run-scenario.sh`, `test/e2e/runtime/run-suites.sh`, `tsx`, `gh` CLI for PR/CI checks. - -## Coverage Summary - -- Happy Paths: 6 scenarios -- Sad Paths: 6 scenarios -- Total: 12 scenarios - ---- - -## Phase 1: Coverage Inventory and Primitive Contract - Validation Scenarios - -### Scenario 1.1: Domain helper loads and uses scenario context [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Happy Path - -**Given**: A temporary `E2E_CONTEXT_DIR/context.env` with required scenario keys and `E2E_DRY_RUN=1`. -**When**: The security policy credentials helper is sourced and a primitive reads required context. -**Then**: The helper exits successfully, reports dry-run-safe planned behavior, and does not discover install/onboard/sandbox state independently. - -**Validation Steps**: -1. **Setup**: Bash: create a temp context directory with minimal required keys. -2. **Execute**: Bash/Vitest: source `test/e2e/validation_suites/lib/security_policy_credentials.sh` through the helper test. -3. **Verify**: Vitest: assert success and expected context-derived output. - -**Tools Required**: Bash, Vitest - -### Scenario 1.2: Missing context fails clearly [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Sad Path - -**Given**: A temp context directory with a missing required key. -**When**: A security helper primitive requiring that key runs. -**Then**: The command fails non-zero and names the missing context key without attempting setup rediscovery. - -**Validation Steps**: -1. **Setup**: Bash: create incomplete `context.env`. -2. **Execute**: Vitest/Bash: invoke the helper primitive. -3. **Verify**: Vitest: assert non-zero status and clear missing-key message. - -**Tools Required**: Bash, Vitest - -## Phase 2: Credential and Sanitization Suite Migration - Validation Scenarios - -### Scenario 2.1: Credential suite runs focused steps in dry-run mode [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Happy Path - -**Given**: `suites.yaml` includes `security-credentials` and a dry-run context. -**When**: `run-suites.sh security-credentials` runs with `E2E_DRY_RUN=1`. -**Then**: Credential-specific scripts under `security/credentials/` run in declared order and emit/represent stable credential assertion IDs. - -**Validation Steps**: -1. **Setup**: Bash: seed full dry-run context. -2. **Execute**: Bash: run `test/e2e/runtime/run-suites.sh security-credentials`. -3. **Verify**: Bash/Vitest: inspect output and `suites.yaml` script paths. - -**Tools Required**: Bash, Vitest - -### Scenario 2.2: Credential outputs do not leak raw secrets [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Sad Path - -**Given**: Credential fixture data contains credential-shaped raw values. -**When**: Credential list/sanitization helpers run. -**Then**: Raw values are redacted or absent, and only provider/name/header metadata is visible. - -**Validation Steps**: -1. **Setup**: Bash/Vitest: create fixture output with obvious secret patterns. -2. **Execute**: Vitest: run credential helper/suite in dry-run or fixture mode. -3. **Verify**: Vitest: assert raw values are not present in stdout/stderr/artifacts. - -**Tools Required**: Bash, Vitest - -## Phase 3: Security Policy, Shields, and Gateway Health Migration - Validation Scenarios - -### Scenario 3.1: Policy and shields suites use focused post-onboard checks [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Happy Path - -**Given**: `security-policy` and `security-shields` are configured in `suites.yaml`. -**When**: The suites run in dry-run mode. -**Then**: Policy scripts under `security/policy/` and shields scripts under `security/shields/` execute without generic placeholder aliases. - -**Validation Steps**: -1. **Setup**: Bash: seed dry-run context. -2. **Execute**: Bash: run `run-suites.sh security-policy security-shields`. -3. **Verify**: Vitest/Bash: assert focused script paths and expected stable IDs. - -**Tools Required**: Bash, Vitest - -### Scenario 3.2: Gateway health broken state is not reported as success [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Sad Path - -**Given**: A fixture or context representing a broken gateway/upstream state. -**When**: Gateway health honesty validation runs. -**Then**: The assertion fails or is explicitly deferred with runner requirements; it is not silently marked successful. - -**Validation Steps**: -1. **Setup**: Bash/Vitest: seed broken gateway fixture or inspect parity metadata if live validation is deferred. -2. **Execute**: Vitest/Bash: run gateway health helper or parity-map validation. -3. **Verify**: Vitest: assert failure/deferred metadata includes `runner_requirement`. - -**Tools Required**: Bash, Vitest - -## Phase 4: Injection and OpenShell Version Coverage Migration - Validation Scenarios - -### Scenario 4.1: Injection suite treats message payloads as data [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Sad Path - -**Given**: A Telegram/message payload fixture containing shell syntax and a temp marker-file path. -**When**: The `security-injection` dry-run or fixture-mode suite evaluates the payload. -**Then**: No marker file or command side effect is created, and the assertion is mapped or deferred with secret requirements. - -**Validation Steps**: -1. **Setup**: Bash: create payload fixture and temp marker path. -2. **Execute**: Bash/Vitest: run `run-suites.sh security-injection` in dry-run/fixture mode. -3. **Verify**: Bash/Vitest: assert marker file is absent and metadata includes stable ID or `secret_requirement`. - -**Tools Required**: Bash, Vitest - -### Scenario 4.2: OpenShell version capability is classified [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Happy Path - -**Given**: The parity map includes `test-openshell-version-pin.sh` assertions. -**When**: Parity-map validation runs. -**Then**: Version/capability assertions are mapped to stable IDs or deferred with runner/capability requirements. - -**Validation Steps**: -1. **Setup**: None. -2. **Execute**: `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts`. -3. **Verify**: Vitest: assert required status and metadata. - -**Tools Required**: Vitest - -## Phase 5: Parity Review and Coverage Report Gate - Validation Scenarios - -### Scenario 5.1: Eight-script legacy area reaches full classification [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Happy Path - -**Given**: The parity inventory and parity map include the eight legacy scripts from issue #3815. -**When**: Strict parity-map and coverage-report tests run. -**Then**: Every legacy assertion is mapped, deferred, or retired; no unclassified assertion remains. - -**Validation Steps**: -1. **Setup**: None. -2. **Execute**: `npm test -- test/e2e/scenario-framework-tests/e2e-parity-map.test.ts test/e2e/scenario-framework-tests/e2e-coverage-report.test.ts`. -3. **Verify**: Vitest: assert zero unclassified assertions and visible security domains. - -**Tools Required**: Vitest - -### Scenario 5.2: Parity metadata rejects incomplete deferred/retired items [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Sad Path - -**Given**: A fixture parity map entry marked `deferred` without runner/secret requirement or `retired` without reviewer metadata. -**When**: Parity-map schema validation runs. -**Then**: Validation fails and reports the missing metadata field. - -**Validation Steps**: -1. **Setup**: Vitest: create temp fixture parity map. -2. **Execute**: Vitest/tsx: run parity-map checker. -3. **Verify**: Vitest: assert non-zero status and missing-field message. - -**Tools Required**: Vitest, tsx - -## Phase 6: Clean the House - Validation Scenarios - -### Scenario 6.1: Affected scenarios remain plan-only compatible [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Happy Path - -**Given**: Affected scenario IDs are registered in `scenarios.yaml`. -**When**: `test/e2e/runtime/run-scenario.sh --plan-only` runs for affected scenarios. -**Then**: Each command exits 0, prints the planned layers/suites, and does not contact live infrastructure. - -**Validation Steps**: -1. **Setup**: None. -2. **Execute**: Bash: run plan-only for affected scenario IDs. -3. **Verify**: Bash/Vitest: assert exit 0 and no live side-effect markers. - -**Tools Required**: Bash - -### Scenario 6.2: New suite scripts pass hygiene checks [STATUS: passed] [VALIDATED: 1e4fa4a00] -**Type**: Sad Path - -**Given**: New shell scripts under `test/e2e/validation_suites/security/` and the domain helper. -**When**: Convention lint and metadata hygiene tests run. -**Then**: Missing SPDX headers, missing executable bits, temporary files, or leftover TODOs fail the checks. - -**Validation Steps**: -1. **Setup**: None. -2. **Execute**: `npm test -- test/e2e/scenario-framework-tests/e2e-convention-lint.test.ts test/e2e/scenario-framework-tests/e2e-metadata-final-hygiene.test.ts`. -3. **Verify**: Vitest: assert all hygiene checks pass. - -**Tools Required**: Vitest - -## Summary - -| Phase | Happy | Sad | Total | Passed | Failed | Pending | -|-------|-------|-----|-------|--------|--------|---------| -| Phase 1 | 1 | 1 | 2 | 2 | 0 | 0 | -| Phase 2 | 1 | 1 | 2 | 2 | 0 | 0 | -| Phase 3 | 1 | 1 | 2 | 2 | 0 | 0 | -| Phase 4 | 1 | 1 | 2 | 2 | 0 | 0 | -| Phase 5 | 1 | 1 | 2 | 2 | 0 | 0 | -| Phase 6 | 1 | 1 | 2 | 2 | 0 | 0 | -| **Total** | **6** | **6** | **12** | **12** | **0** | **0** | - -## Approval Status - -**Status**: VALIDATED From 8e7b6e7167a6c2f3444f718e413125c882298a79 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 20 May 2026 08:44:44 -0400 Subject: [PATCH 20/21] test(e2e): fix security helper hook compliance --- test/e2e/validation_suites/lib/security_policy_credentials.sh | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 test/e2e/validation_suites/lib/security_policy_credentials.sh diff --git a/test/e2e/validation_suites/lib/security_policy_credentials.sh b/test/e2e/validation_suites/lib/security_policy_credentials.sh old mode 100644 new mode 100755 index e220166588..953bc8b52f --- a/test/e2e/validation_suites/lib/security_policy_credentials.sh +++ b/test/e2e/validation_suites/lib/security_policy_credentials.sh @@ -5,6 +5,7 @@ # Security policy and credential validation primitives. if [[ -n "${NEMOCLAW_SECURITY_POLICY_CREDENTIALS_LIB_LOADED:-}" ]]; then + # shellcheck disable=SC2317 # This file may be sourced repeatedly or executed in tests. return 0 2>/dev/null || exit 0 fi NEMOCLAW_SECURITY_POLICY_CREDENTIALS_LIB_LOADED=1 From e114b68d95c2c2f154ee387f92d1428caef45561 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 20 May 2026 19:58:23 -0700 Subject: [PATCH 21/21] test(e2e): enforce security credential suite assertions --- .../e2e-lib-helpers.test.ts | 144 +++++++++++++++ .../lib/security_policy_credentials.sh | 172 +++++++++++++++++- .../00-telegram-message-not-shell-executed.sh | 5 +- ...ell-version-supports-credential-rewrite.sh | 4 +- .../security/shields/00-config-consistent.sh | 4 +- 5 files changed, 317 insertions(+), 12 deletions(-) diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index b78184bc35..69af4a5eb0 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -84,6 +84,150 @@ describe("E2E shell helpers", () => { expect(r.stdout).toMatch(/\[REDACTED\]/); }); + it("security_policy_credentials_helper_should_reject_empty_gateway_credentials", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-empty-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [ "$1 $2" = "credentials list" ]; then + echo " No provider credentials registered." + exit 0 +fi +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_credentials_expected + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toMatch(/no gateway credentials/i); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("security_policy_credentials_helper_should_verify_policy_and_shields_state", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-shields-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [ "$1 $2" = "sb policy-list" ]; then + echo " Policy presets for sandbox 'sb':" + echo " ● telegram — Telegram bridge egress" + echo " ○ slack — Slack bridge egress" + exit 0 +fi +if [ "$1 $2 $3" = "sb shields status" ]; then + echo " Shields: UP (lockdown active)" + exit 0 +fi +exit 2 +`, + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +if [ "$1 $2 $3" = "sandbox exec --name" ]; then + echo "440 root:root" + exit 0 +fi +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync( + path.join(tmp, "context.env"), + "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_SANDBOX_NAME=sb\nE2E_AGENT=openclaw\nE2E_SHIELDS_EXPECTED_STATE=up\n", + ); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_policy_preset_present telegram + spc_assert_shields_config_consistent + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain("telegram"); + expect(r.stdout).toContain("shields config state is consistent: up"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("security_policy_credentials_helper_should_fail_on_missing_policy_preset", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-missing-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +echo " ○ telegram — Telegram bridge egress" +exit 0 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_SANDBOX_NAME=sb\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_policy_preset_present telegram + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toMatch(/expected policy preset 'telegram'/); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("security_policy_credentials_helper_should_verify_openshell_rewrite_markers", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-openshell-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +# request-body-credential-rewrite websocket-credential-rewrite +exit 0 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_openshell_credential_rewrite_supported + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain("OpenShell credential rewrite capability markers present"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("env_helper_should_set_standard_noninteractive_env", () => { const r = runBash(` set -euo pipefail diff --git a/test/e2e/validation_suites/lib/security_policy_credentials.sh b/test/e2e/validation_suites/lib/security_policy_credentials.sh index 953bc8b52f..701c0f9efb 100755 --- a/test/e2e/validation_suites/lib/security_policy_credentials.sh +++ b/test/e2e/validation_suites/lib/security_policy_credentials.sh @@ -59,7 +59,21 @@ spc_assert_credentials_expected() { echo "[dry-run] would list gateway credentials without raw values" return 0 fi - nemoclaw credentials list 2>&1 | spc_redact_secret_text + local listed + if ! listed="$(nemoclaw credentials list 2>&1 | spc_redact_secret_text)"; then + printf '%s\n' "${listed}" + echo "nemoclaw credentials list failed while credentials.expected=present" >&2 + return 1 + fi + printf '%s\n' "${listed}" + if printf '%s\n' "${listed}" | grep -qi "No provider credentials registered"; then + echo "no gateway credentials were listed while credentials.expected=present" >&2 + return 1 + fi + if ! printf '%s\n' "${listed}" | grep -q "Providers registered with the OpenShell gateway"; then + echo "credentials list did not include the expected OpenShell gateway provider header" >&2 + return 1 + fi } spc_assert_no_plaintext_host_store() { @@ -76,9 +90,163 @@ spc_assert_no_plaintext_host_store() { spc_assert_policy_preset_present() { local preset="$1" spc_assertion_id "post-onboard.security-policy.${preset}-preset-applied" - spc_require_context E2E_SCENARIO + spc_require_context E2E_SCENARIO E2E_SANDBOX_NAME echo "policy preset expected: ${preset}" if e2e_env_is_dry_run; then echo "[dry-run] would verify policy preset ${preset}" + return 0 + fi + local sandbox_name active + sandbox_name="$(spc_context_get E2E_SANDBOX_NAME)" + if ! active="$(nemoclaw "${sandbox_name}" policy-list 2>&1)"; then + printf '%s\n' "${active}" + echo "failed to query policy presets for sandbox '${sandbox_name}'" >&2 + return 1 + fi + printf '%s\n' "${active}" + if ! printf '%s\n' "${active}" | awk -v preset="${preset}" '$1 == "●" && $2 == preset { found = 1 } END { exit found ? 0 : 1 }'; then + echo "expected policy preset '${preset}' to be applied for sandbox '${sandbox_name}'" >&2 + return 1 + fi +} + +spc_assert_openshell_credential_rewrite_supported() { + spc_assertion_id "post-onboard.gateway.openshell-version-supports-credential-rewrite" + spc_require_context E2E_SCENARIO + if e2e_env_is_dry_run; then + echo "[dry-run] would verify OpenShell gateway capability metadata" + return 0 + fi + local openshell_bin binary_strings feature + openshell_bin="$(command -v openshell 2>/dev/null || true)" + if [[ -z "${openshell_bin}" ]]; then + echo "openshell binary was not found on PATH" >&2 + return 1 + fi + if ! command -v strings >/dev/null 2>&1; then + echo "strings is required to verify OpenShell credential rewrite support" >&2 + return 1 + fi + binary_strings="$(strings "${openshell_bin}" 2>/dev/null || true)" + for feature in request-body-credential-rewrite websocket-credential-rewrite; do + if [[ "${binary_strings}" != *"${feature}"* ]]; then + echo "OpenShell binary is missing ${feature} support" >&2 + return 1 + fi + done + echo "OpenShell credential rewrite capability markers present" +} + +spc_agent_config_path() { + case "$(spc_context_get E2E_AGENT)" in + hermes) printf '%s\n' "/sandbox/.hermes/.env" ;; + openclaw | "") printf '%s\n' "/sandbox/.openclaw/openclaw.json" ;; + *) + echo "unsupported E2E_AGENT for shields config check: $(spc_context_get E2E_AGENT)" >&2 + return 1 + ;; + esac +} + +spc_assert_shields_permissions_match_state() { + local sandbox_name="$1" + local observed="$2" + local config_path perms mode owner + config_path="$(spc_agent_config_path)" || return 1 + if ! perms="$(openshell sandbox exec --name "${sandbox_name}" -- stat -c '%a %U:%G' "${config_path}" 2>&1)"; then + printf '%s\n' "${perms}" + echo "failed to inspect shields config permissions at ${config_path}" >&2 + return 1 + fi + printf 'config permissions: %s %s\n' "${config_path}" "${perms}" + mode="$(printf '%s\n' "${perms}" | awk '{print $1}')" + owner="$(printf '%s\n' "${perms}" | awk '{print $2}')" + case "${observed}" in + up) + if [[ ! "${mode}" =~ ^4[0-4][0-4]$ || "${owner}" != "root:root" ]]; then + echo "shields are UP but config is not locked root:root with restrictive permissions: ${perms}" >&2 + return 1 + fi + ;; + down | not-configured) + if [[ "${owner}" != "sandbox:sandbox" ]]; then + echo "shields are ${observed} but config owner is not sandbox:sandbox: ${perms}" >&2 + return 1 + fi + ;; + esac +} + +spc_assert_shields_config_consistent() { + spc_assertion_id "post-onboard.security-shields.config-consistent" + spc_require_context E2E_SCENARIO E2E_SANDBOX_NAME E2E_AGENT + if e2e_env_is_dry_run; then + echo "[dry-run] would verify shields config consistency" + return 0 + fi + local sandbox_name status observed expected + sandbox_name="$(spc_context_get E2E_SANDBOX_NAME)" + if ! status="$(nemoclaw "${sandbox_name}" shields status 2>&1)"; then + printf '%s\n' "${status}" + echo "failed to query shields status for sandbox '${sandbox_name}'" >&2 + return 1 + fi + printf '%s\n' "${status}" + case "${status}" in + *"Shields: UP"*) observed="up" ;; + *"Shields: DOWN"*) observed="down" ;; + *"Shields: NOT CONFIGURED"*) observed="not-configured" ;; + *) + echo "shields status did not report a recognized state" >&2 + return 1 + ;; + esac + expected="$(spc_context_get E2E_SHIELDS_EXPECTED_STATE)" + if [[ -z "${expected}" ]]; then + expected="$(spc_context_get E2E_SHIELDS_EXPECTED)" + fi + expected="${expected//_/-}" + if [[ -n "${expected}" && "${expected}" != "${observed}" ]]; then + echo "expected shields state '${expected}', got '${observed}'" >&2 + return 1 + fi + spc_assert_shields_permissions_match_state "${sandbox_name}" "${observed}" + echo "shields config state is consistent: ${observed}" +} + +spc_assert_telegram_payload_not_shell_executed() { + spc_assertion_id "post-onboard.security-injection.telegram-message-not-shell-executed" + spc_require_context E2E_SCENARIO E2E_SANDBOX_NAME + local fixture_payload="${1:-}" + if [[ -n "${fixture_payload}" ]]; then + printf 'telegram payload fixture loaded (%s bytes)\n' "${#fixture_payload}" + fi + if e2e_env_is_dry_run; then + echo "[dry-run] would submit payload without shell evaluation" + return 0 + fi + local sandbox_name marker payload send_output marker_state + sandbox_name="$(spc_context_get E2E_SANDBOX_NAME)" + marker="/tmp/nemoclaw-telegram-injection-proof-$RANDOM-$$" + # shellcheck disable=SC2016 # Literal command-substitution payload under test. + payload="$(printf '$(touch %s && echo INJECTED)' "${marker}")" + openshell sandbox exec --name "${sandbox_name}" -- sh -c "rm -f '${marker}'" >/dev/null 2>&1 || true + # shellcheck disable=SC2016 # Remote shell reads payload from stdin; local shell must not expand MSG. + if ! send_output="$(openshell sandbox exec --name "${sandbox_name}" -- sh -c 'MSG=$(cat); printf "%s\n" "$MSG"' <<<"${payload}" 2>&1)"; then + printf '%s\n' "${send_output}" + echo "failed to submit telegram injection payload to sandbox '${sandbox_name}'" >&2 + return 1 + fi + printf '%s\n' "${send_output}" + if [[ "${send_output}" != *"${payload}"* ]]; then + echo "telegram injection payload was not preserved literally" >&2 + return 1 + fi + marker_state="$(openshell sandbox exec --name "${sandbox_name}" -- sh -c "test -f '${marker}' && echo EXPLOITED || echo SAFE" 2>&1 || true)" + if [[ "${marker_state}" != *"SAFE"* ]]; then + printf '%s\n' "${marker_state}" + echo "telegram injection payload executed shell side effects" >&2 + return 1 fi + echo "telegram injection payload treated as data" } diff --git a/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh b/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh index c4803860b1..5da780affd 100755 --- a/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh +++ b/test/e2e/validation_suites/security/injection/00-telegram-message-not-shell-executed.sh @@ -5,8 +5,5 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" echo "injection:telegram-message-not-shell-executed" -spc_assertion_id "post-onboard.security-injection.telegram-message-not-shell-executed" -spc_require_context E2E_SCENARIO payload="${E2E_TELEGRAM_PAYLOAD_FIXTURE:-$(spc_context_get E2E_TELEGRAM_PAYLOAD_FIXTURE)}" -printf 'telegram payload treated as data (%s bytes)\n' "${#payload}" -if e2e_env_is_dry_run; then echo "[dry-run] would submit payload without shell evaluation"; fi +spc_assert_telegram_payload_not_shell_executed "${payload}" diff --git a/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh b/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh index 08f60ef8f0..1a1f589969 100755 --- a/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh +++ b/test/e2e/validation_suites/security/policy/01-openshell-version-supports-credential-rewrite.sh @@ -5,6 +5,4 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" echo "policy:openshell-version-supports-credential-rewrite" -spc_assertion_id "post-onboard.gateway.openshell-version-supports-credential-rewrite" -spc_require_context E2E_SCENARIO -if e2e_env_is_dry_run; then echo "[dry-run] would verify OpenShell gateway capability metadata"; fi +spc_assert_openshell_credential_rewrite_supported diff --git a/test/e2e/validation_suites/security/shields/00-config-consistent.sh b/test/e2e/validation_suites/security/shields/00-config-consistent.sh index 6104e5217a..552cae2484 100755 --- a/test/e2e/validation_suites/security/shields/00-config-consistent.sh +++ b/test/e2e/validation_suites/security/shields/00-config-consistent.sh @@ -5,6 +5,4 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "${SCRIPT_DIR}/../../lib/security_policy_credentials.sh" echo "shields:config-consistent" -spc_assertion_id "post-onboard.security-shields.config-consistent" -spc_require_context E2E_SCENARIO -if e2e_env_is_dry_run; then echo "[dry-run] would verify shields config consistency"; fi +spc_assert_shields_config_consistent