From 032c4c942c81dd41976cf3e3a57399637e14c39c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 21 May 2026 14:34:20 -0700 Subject: [PATCH 1/2] test(e2e): strengthen credential leak validation --- .../e2e-lib-helpers.test.ts | 40 +++++++++++++++- .../lib/security_policy_credentials.sh | 48 +++++++++++++++++-- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index d5318cccef..80b2f18849 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -116,6 +116,40 @@ exit 2 } }); + it("security_policy_credentials_helper_should_reject_raw_credential_leaks", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-leak-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [ "$1 $2" = "credentials list" ]; then + echo " Providers registered with the OpenShell gateway:" + echo " nvidia token=nvapi-secret-value-1234567890" + exit 0 +fi +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_credentials_expected + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toMatch(/secret-looking raw output/i); + expect(r.stdout).not.toContain("nvapi-secret-value"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("security_policy_credentials_helper_should_verify_policy_and_shields_state", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-shields-")); const fakeBin = path.join(tmp, "bin"); @@ -207,6 +241,10 @@ exit 0 path.join(fakeBin, "openshell"), `#!/usr/bin/env bash # request-body-credential-rewrite websocket-credential-rewrite +if [ "$1" = "--version" ]; then + echo "openshell 0.0.39" + exit 0 +fi exit 0 `, { mode: 0o755 }, @@ -222,7 +260,7 @@ exit 0 { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, ); expect(r.status, r.stderr).toBe(0); - expect(r.stdout).toContain("OpenShell credential rewrite capability markers present"); + expect(r.stdout).toContain("OpenShell 0.0.39 credential rewrite capability markers present"); } finally { fs.rmSync(tmp, { recursive: true, force: true }); } diff --git a/test/e2e/validation_suites/lib/security_policy_credentials.sh b/test/e2e/validation_suites/lib/security_policy_credentials.sh index 701c0f9efb..7baf1ed5c7 100755 --- a/test/e2e/validation_suites/lib/security_policy_credentials.sh +++ b/test/e2e/validation_suites/lib/security_policy_credentials.sh @@ -59,13 +59,25 @@ spc_assert_credentials_expected() { echo "[dry-run] would list gateway credentials without raw values" return 0 fi - local listed - if ! listed="$(nemoclaw credentials list 2>&1 | spc_redact_secret_text)"; then + local raw_file listed_raw listed + raw_file="$(mktemp "${TMPDIR:-/tmp}/nemoclaw-credentials-list.XXXXXX")" + chmod 600 "${raw_file}" + if ! nemoclaw credentials list >"${raw_file}" 2>&1; then + listed_raw="$(cat "${raw_file}")" + listed="$(printf '%s\n' "${listed_raw}" | spc_redact_secret_text)" + rm -f "${raw_file}" printf '%s\n' "${listed}" echo "nemoclaw credentials list failed while credentials.expected=present" >&2 return 1 fi + listed_raw="$(cat "${raw_file}")" + listed="$(printf '%s\n' "${listed_raw}" | spc_redact_secret_text)" + rm -f "${raw_file}" printf '%s\n' "${listed}" + if [[ "${listed_raw}" != "${listed}" ]]; then + echo "credentials list emitted secret-looking raw output before redaction" >&2 + return 1 + fi if printf '%s\n' "${listed}" | grep -qi "No provider credentials registered"; then echo "no gateway credentials were listed while credentials.expected=present" >&2 return 1 @@ -110,6 +122,23 @@ spc_assert_policy_preset_present() { fi } +spc_semver_ge() { + local have="$1" want="$2" h_major h_minor h_patch w_major w_minor w_patch + IFS=. read -r h_major h_minor h_patch <<<"${have}" + IFS=. read -r w_major w_minor w_patch <<<"${want}" + h_major=$((10#${h_major:-0})) + h_minor=$((10#${h_minor:-0})) + h_patch=$((10#${h_patch:-0})) + w_major=$((10#${w_major:-0})) + w_minor=$((10#${w_minor:-0})) + w_patch=$((10#${w_patch:-0})) + ((h_major > w_major)) && return 0 + ((h_major < w_major)) && return 1 + ((h_minor > w_minor)) && return 0 + ((h_minor < w_minor)) && return 1 + ((h_patch >= w_patch)) +} + spc_assert_openshell_credential_rewrite_supported() { spc_assertion_id "post-onboard.gateway.openshell-version-supports-credential-rewrite" spc_require_context E2E_SCENARIO @@ -117,12 +146,23 @@ spc_assert_openshell_credential_rewrite_supported() { echo "[dry-run] would verify OpenShell gateway capability metadata" return 0 fi - local openshell_bin binary_strings feature + local openshell_bin version_output version minimum_version binary_strings feature + minimum_version="0.0.39" openshell_bin="$(command -v openshell 2>/dev/null || true)" if [[ -z "${openshell_bin}" ]]; then echo "openshell binary was not found on PATH" >&2 return 1 fi + version_output="$(${openshell_bin} --version 2>&1 || true)" + version="$(printf '%s\n' "${version_output}" | grep -oE '[0-9]+(\.[0-9]+){1,2}' | head -n1 || true)" + if [[ -z "${version}" ]]; then + echo "could not determine OpenShell version from: ${version_output}" >&2 + return 1 + fi + if ! spc_semver_ge "${version}" "${minimum_version}"; then + echo "OpenShell ${version} is below credential rewrite minimum ${minimum_version}" >&2 + return 1 + fi if ! command -v strings >/dev/null 2>&1; then echo "strings is required to verify OpenShell credential rewrite support" >&2 return 1 @@ -134,7 +174,7 @@ spc_assert_openshell_credential_rewrite_supported() { return 1 fi done - echo "OpenShell credential rewrite capability markers present" + echo "OpenShell ${version} credential rewrite capability markers present" } spc_agent_config_path() { From e27e4d0057295d46f9c182d67817f97f2433a04e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Thu, 21 May 2026 15:13:24 -0700 Subject: [PATCH 2/2] test(e2e): check failed credential list leaks --- .../e2e-lib-helpers.test.ts | 67 +++++++++++++++++++ .../lib/security_policy_credentials.sh | 17 ++--- 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts index 268d8e705d..9d3a89a683 100644 --- a/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts +++ b/test/e2e/scenario-framework-tests/e2e-lib-helpers.test.ts @@ -224,6 +224,40 @@ exit 2 } }); + it("security_policy_credentials_helper_should_reject_raw_credential_leaks_from_failed_list", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-credentials-failed-leak-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "nemoclaw"), + `#!/usr/bin/env bash +if [ "$1 $2" = "credentials list" ]; then + echo "gateway error token=nvapi-secret-value-1234567890" >&2 + exit 1 +fi +exit 2 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\nE2E_PROVIDER=nvidia\nE2E_CREDENTIALS_EXPECTED=present\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_credentials_expected + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toMatch(/secret-looking raw output/i); + expect(r.stderr).not.toMatch(/credentials list failed/); + expect(r.stdout).not.toContain("nvapi-secret-value"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("security_policy_credentials_helper_should_verify_policy_and_shields_state", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-policy-shields-")); const fakeBin = path.join(tmp, "bin"); @@ -340,6 +374,39 @@ exit 0 } }); + it("security_policy_credentials_helper_should_reject_below_minimum_openshell_version", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "spc-openshell-old-")); + const fakeBin = path.join(tmp, "bin"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + path.join(fakeBin, "openshell"), + `#!/usr/bin/env bash +# request-body-credential-rewrite websocket-credential-rewrite +if [ "$1" = "--version" ]; then + echo "openshell 0.0.38" + exit 0 +fi +exit 0 +`, + { mode: 0o755 }, + ); + try { + fs.writeFileSync(path.join(tmp, "context.env"), "E2E_SCENARIO=test\n"); + const r = runBash( + ` + set -euo pipefail + . "${VALIDATION_SUITES}/lib/security_policy_credentials.sh" + spc_assert_openshell_credential_rewrite_supported + `, + { E2E_CONTEXT_DIR: tmp, PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + ); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain("below credential rewrite minimum"); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("env_helper_should_set_standard_noninteractive_env", () => { const r = runBash(` set -euo pipefail diff --git a/test/e2e/validation_suites/lib/security_policy_credentials.sh b/test/e2e/validation_suites/lib/security_policy_credentials.sh index 7baf1ed5c7..3e1872d62a 100755 --- a/test/e2e/validation_suites/lib/security_policy_credentials.sh +++ b/test/e2e/validation_suites/lib/security_policy_credentials.sh @@ -59,16 +59,13 @@ spc_assert_credentials_expected() { echo "[dry-run] would list gateway credentials without raw values" return 0 fi - local raw_file listed_raw listed + local raw_file listed_raw listed list_rc raw_file="$(mktemp "${TMPDIR:-/tmp}/nemoclaw-credentials-list.XXXXXX")" chmod 600 "${raw_file}" - if ! nemoclaw credentials list >"${raw_file}" 2>&1; then - listed_raw="$(cat "${raw_file}")" - listed="$(printf '%s\n' "${listed_raw}" | spc_redact_secret_text)" - rm -f "${raw_file}" - printf '%s\n' "${listed}" - echo "nemoclaw credentials list failed while credentials.expected=present" >&2 - return 1 + if nemoclaw credentials list >"${raw_file}" 2>&1; then + list_rc=0 + else + list_rc=$? fi listed_raw="$(cat "${raw_file}")" listed="$(printf '%s\n' "${listed_raw}" | spc_redact_secret_text)" @@ -78,6 +75,10 @@ spc_assert_credentials_expected() { echo "credentials list emitted secret-looking raw output before redaction" >&2 return 1 fi + if ((list_rc != 0)); then + echo "nemoclaw credentials list failed while credentials.expected=present" >&2 + return 1 + fi if printf '%s\n' "${listed}" | grep -qi "No provider credentials registered"; then echo "no gateway credentials were listed while credentials.expected=present" >&2 return 1