Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
829610e
Simplify security policy credential E2E spec
jyaunches May 20, 2026
70046da
Add test specification for security policy credential E2E migration
jyaunches May 20, 2026
ef8d57e
Add validation plan for security policy credential E2E migration
jyaunches May 20, 2026
9228f62
Apply spec review recommendation from section 1
jyaunches May 20, 2026
b42e88c
Apply spec review recommendation from section 5
jyaunches May 20, 2026
c16ab8a
feat: Implement Phase 1 - security primitives
jyaunches May 20, 2026
1fd9983
Mark Phase 1 as completed [c16ab8ade]
jyaunches May 20, 2026
a6e06b3
feat: Implement Phase 2 - credential suites
jyaunches May 20, 2026
9fa66a9
Mark Phase 2 as completed [a6e06b329]
jyaunches May 20, 2026
dbe5707
feat: Implement Phase 3 - policy shields suites
jyaunches May 20, 2026
3ba3d39
Mark Phase 3 as completed [dbe570714]
jyaunches May 20, 2026
04d6c80
feat: Implement Phase 4 - injection version suites
jyaunches May 20, 2026
7bab895
Mark Phase 4 as completed [04d6c80a6]
jyaunches May 20, 2026
e23ea30
feat: Implement Phase 5 - parity review gate
jyaunches May 20, 2026
52e27d5
Mark Phase 5 as completed [e23ea30c7]
jyaunches May 20, 2026
9fcba5d
chore: Implement Phase 6 - final hygiene
jyaunches May 20, 2026
1e4fa4a
Mark Phase 6 as completed [9fcba5d76]
jyaunches May 20, 2026
e5190a5
test(e2e): validate security migration spec
jyaunches May 20, 2026
fcf612c
chore: remove vd workflow artifacts
jyaunches May 20, 2026
8e7b6e7
test(e2e): fix security helper hook compliance
jyaunches May 20, 2026
c5fc179
merge: resolve main conflicts in e2e credential suites
cv May 21, 2026
e114b68
test(e2e): enforce security credential suite assertions
cv May 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions test/e2e/docs/parity-map.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1169,19 +1169,34 @@ scripts:
runner_requirement: sandbox runner with NemoClaw/OpenShell CLIs
- legacy: nemoclaw credentials list failed
status: mapped
id: legacy.credential.migration.nemoclaw.credentials.list.failed
id: post-onboard.credentials.gateway-list-redacts-values
layer: post-onboard
gap_domain: credentials
owner: e2e-maintainers
- legacy: credentials list surfaces gateway-registered providers
status: mapped
id: legacy.credential.migration.credentials.list.surfaces.gateway.registered.providers
id: post-onboard.credentials.gateway-list-redacts-values
layer: post-onboard
gap_domain: credentials
owner: e2e-maintainers
- legacy: credentials list did not produce the expected gateway header
status: mapped
id: legacy.credential.migration.credentials.list.did.not.produce.the.expected.gateway.header
id: post-onboard.credentials.gateway-list-redacts-values
layer: post-onboard
gap_domain: credentials
owner: e2e-maintainers
- legacy: credentials.json reappeared on disk after credentials list
status: mapped
id: legacy.credential.migration.credentials.json.reappeared.on.disk.after.credentials.list
id: post-onboard.credentials.no-plaintext-host-store
layer: post-onboard
gap_domain: credentials
owner: e2e-maintainers
- legacy: No plaintext credentials.json on disk after credentials list
status: mapped
id: legacy.credential.migration.no.plaintext.credentials.json.on.disk.after.credentials.list
id: post-onboard.credentials.no-plaintext-host-store
layer: post-onboard
gap_domain: credentials
owner: e2e-maintainers
- legacy: node invocation of removeLegacyCredentialsFile failed
status: mapped
id: legacy.credential.migration.node.invocation.of.removelegacycredentialsfile.failed
Expand Down
196 changes: 196 additions & 0 deletions test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,202 @@ function runBash(script: string, env: Record<string, string> = {}): SpawnSyncRet
// ──────────────────────────────────────────────────────────────────────────

describe("E2E shell helpers", () => {
it("security_policy_credentials_helper_should_load_with_context_library", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-"));
try {
fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n");
const r = runBash(
`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_require_context E2E_SCENARIO E2E_PROVIDER
echo "provider=$(spc_context_get E2E_PROVIDER)"
`,
{ E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" },
);
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain("provider=nvidia");
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("security_policy_credentials_helper_should_fail_when_required_context_missing", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-context-missing-"));
try {
fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n");
const r = runBash(
`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_require_context E2E_PROVIDER
`,
{ E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1" },
);
expect(r.status).not.toBe(0);
expect(r.stderr).toContain("E2E_PROVIDER");
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("security_policy_credentials_helper_should_not_log_secret_values", () => {
const r = runBash(`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_log_provider_metadata "nvidia" "primary"
printf 'token=nvapi-secret-value-1234567890 sk-abcdefghijklmnop\n' | spc_redact_secret_text
`);
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain("provider=nvidia name=primary");
expect(r.stdout).not.toMatch(/nvapi-secret-value|sk-abcdefghijklmnop/);
expect(r.stdout).toMatch(/\[REDACTED\]/);
});

it("security_policy_credentials_helper_should_reject_empty_gateway_credentials", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-empty-"));
const fakeBin = path.join(tmp, "bin");
fs.mkdirSync(fakeBin);
fs.writeFileSync(
path.join(fakeBin, "nemoclaw"),
`#!/usr/bin/env bash
if [ "$1 $2" = "credentials list" ]; then
echo " No provider credentials registered."
exit 0
fi
exit 2
`,
{ mode: 0o755 },
);
try {
fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n");
const r = runBash(
`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_assert_credentials_expected
`,
{ E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` },
);
expect(r.status).not.toBe(0);
expect(r.stderr).toMatch(/no gateway credentials/i);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("security_policy_credentials_helper_should_verify_policy_and_shields_state", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-shields-"));
const fakeBin = path.join(tmp, "bin");
fs.mkdirSync(fakeBin);
fs.writeFileSync(
path.join(fakeBin, "nemoclaw"),
`#!/usr/bin/env bash
if [ "$1 $2" = "sb policy-list" ]; then
echo " Policy presets for sandbox 'sb':"
echo " ● telegram — Telegram bridge egress"
echo " ○ slack — Slack bridge egress"
exit 0
fi
if [ "$1 $2 $3" = "sb shields status" ]; then
echo " Shields: UP (lockdown active)"
exit 0
fi
exit 2
`,
{ mode: 0o755 },
);
fs.writeFileSync(
path.join(fakeBin, "openshell"),
`#!/usr/bin/env bash
if [ "$1 $2 $3" = "sandbox exec --name" ]; then
echo "440 root:root"
exit 0
fi
exit 2
`,
{ mode: 0o755 },
);
try {
fs.writeFileSync(
path.join(tmp, "context.env"),
"E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_SANDBOX_NAME=sb\nE2E_AGENT=openclaw\nE2E_SHIELDS_EXPECTED_STATE=up\n",
);
const r = runBash(
`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_assert_policy_preset_present telegram
spc_assert_shields_config_consistent
`,
{ E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` },
);
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain("telegram");
expect(r.stdout).toContain("shields config state is consistent: up");
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("security_policy_credentials_helper_should_fail_on_missing_policy_preset", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-missing-"));
const fakeBin = path.join(tmp, "bin");
fs.mkdirSync(fakeBin);
fs.writeFileSync(
path.join(fakeBin, "nemoclaw"),
`#!/usr/bin/env bash
echo " ○ telegram — Telegram bridge egress"
exit 0
`,
{ mode: 0o755 },
);
try {
fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_SANDBOX_NAME=sb\n");
const r = runBash(
`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_assert_policy_preset_present telegram
`,
{ E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` },
);
expect(r.status).not.toBe(0);
expect(r.stderr).toMatch(/expected policy preset 'telegram'/);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("security_policy_credentials_helper_should_verify_openshell_rewrite_markers", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-openshell-"));
const fakeBin = path.join(tmp, "bin");
fs.mkdirSync(fakeBin);
fs.writeFileSync(
path.join(fakeBin, "openshell"),
`#!/usr/bin/env bash
# request-body-credential-rewrite websocket-credential-rewrite
exit 0
`,
{ mode: 0o755 },
);
try {
fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n");
const r = runBash(
`
set -euo pipefail
. "${VALIDATION_SUITES}/lib/security_policy_credentials.sh"
spc_assert_openshell_credential_rewrite_supported
`,
{ E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` },
);
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain("OpenShell credential rewrite capability markers present");
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("env_helper_should_set_standard_noninteractive_env", () => {
const r = runBash(`
set -euo pipefail
Expand Down
14 changes: 14 additions & 0 deletions test/e2e/scenario-framework-tests/e2e-suite-runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,20 @@ describe("Issue #3810 messaging suite wiring", () => {
});

describe("run-suites.sh", () => {
it("security_credentials_suite_should_emit_stable_assertion_ids", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-security-credentials-"));
try {
seedContext(tmp, { ...fullContext(), E2E_CREDENTIALS_EXPECTED: "present" });
const r = runSuites(["security-credentials"], { E2E_CONTEXT_DIR: tmp, E2E_DRY_RUN: "1", HOME: tmp });
expect(r.status, `stderr:${r.stderr}\nstdout:${r.stdout}`).toBe(0);
expect(r.stdout).toContain("post-onboard.credentials.gateway-list-redacts-values");
expect(r.stdout).toContain("post-onboard.credentials.no-plaintext-host-store");
expect(r.stdout).not.toMatch(/no-credentials-leaked|assert\//);
} finally {
fs.rmSync(tmp, { recursive: true, force: true });
}
});

it("run_suites_should_run_steps_in_declared_order", () => {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-suite-"));
try {
Expand Down
Loading
Loading