diff --git a/actions/deploy-preview/run.sh b/actions/deploy-preview/run.sh index 33ac009..79f3615 100755 --- a/actions/deploy-preview/run.sh +++ b/actions/deploy-preview/run.sh @@ -173,7 +173,8 @@ compute_scorecard() { # contain secrets, so the check correctly passes in that case. local no_raw_secrets="pass" local raw_secrets_file - raw_secrets_file=$(grep -rl 'kind: Secret' out/manifests/ 2>/dev/null | head -1 || true) + raw_secrets_file=$(grep -rlE '^kind:[[:space:]]*Secret[[:space:]]*$' out/manifests/ 2>/dev/null \ + | head -1 || true) if [[ -n "$raw_secrets_file" ]]; then no_raw_secrets="fail:raw-secret-in:${raw_secrets_file}" fi @@ -502,8 +503,9 @@ main() { '.context_pinned = "fail" | .no_latest_images = "fail"') fi - # Emit render failure diagnostics (renders failing does not itself fail any - # scorecard check — the individual checks handle missing manifests correctly). + # Emit render failure diagnostics. Render failures never fail a scorecard + # check (the checks handle missing manifests correctly); they fail the action + # itself at the end via E_RENDER_FAILED, naming the failed fragments. if [[ "${#render_failures[@]}" -gt 0 ]]; then for key in "${!render_failures[@]}"; do warn "render failure [${key}]: ${render_failures[$key]}" @@ -526,17 +528,36 @@ main() { warn "Deploy Preview: escape hatch in use — one or more SC-11 checks are not_applicable" fi - # (7) Emit SC-4 gate summary + # (7) Emit SC-4 gate summary. Render failures fail the gate with the failed + # fragments named in the reason — never misattributed to a scorecard check. local overall overall=$(printf '%s' "$scorecard" \ | jq -r '[.[] | select(type == "string" and startswith("fail"))] | length == 0 | if . then "pass" else "fail" end' \ 2>/dev/null || echo "fail") - emit_gate_summary "deploy-validate" "Deploy Validate" "$overall" "scorecard-evaluated" "none" + local gate_reason="scorecard-evaluated" + local failed_fragments="" + if [[ "${#render_failures[@]}" -gt 0 ]]; then + failed_fragments=$(printf '%s\n' "${!render_failures[@]}" | sort | paste -sd, -) + overall="fail" + gate_reason="render-failed:${failed_fragments}" + fi + emit_gate_summary "deploy-validate" "Deploy Validate" "$overall" "$gate_reason" "none" # Exit nonzero when rendering is impossible (emit-contract failed) — the PR - # check should fail, not silently pass with a broken scorecard. + # check should fail, not silently pass with a broken scorecard. Failed + # fragment renders are named alongside so neither cause is hidden. if [[ "$emit_exit" -ne 0 ]]; then - fail "E_EMIT_CONTRACT_FAILED: artifact emit-contract returned ${emit_exit}; rendering impossible" + local emit_extra="" + if [[ -n "$failed_fragments" ]]; then + emit_extra="; fragment renders also failed: ${failed_fragments}" + fi + fail "E_EMIT_CONTRACT_FAILED: artifact emit-contract returned ${emit_exit}; rendering impossible${emit_extra}" + fi + + # Exit nonzero when any fragment render failed, naming the concrete + # fragments; per-fragment diagnostics were already emitted as warnings. + if [[ -n "$failed_fragments" ]]; then + fail "E_RENDER_FAILED: fragment renders failed: ${failed_fragments}" fi if [[ "$overall" != "pass" ]]; then diff --git a/tests/test_scorecard.py b/tests/test_scorecard.py index 9e8fb96..6edfde0 100644 --- a/tests/test_scorecard.py +++ b/tests/test_scorecard.py @@ -258,6 +258,27 @@ def test_no_raw_secrets_pass_when_manifests_have_no_secrets(self) -> None: f"Expected pass when manifests contain no Secret, got: {scorecard['no_raw_secrets']}", ) + def test_no_raw_secrets_pass_for_secretstore_kinds(self) -> None: + # 'kind: SecretStore' / 'kind: ClusterSecretStore' are not raw Secrets + # and must not trip the anchored kind match. + setup = ( + "mkdir -p out/manifests/preview/production && " + "printf 'apiVersion: external-secrets.io/v1\\nkind: SecretStore\\n' " + "> out/manifests/preview/production/store.yaml && " + "printf 'apiVersion: external-secrets.io/v1\\nkind: ClusterSecretStore\\n' " + "> out/manifests/preview/production/cluster-store.yaml" + ) + result = _run_compute_scorecard( + MINIMAL_DEPLOYMENT_YML, MINIMAL_CONTRACT_YAML, extra_setup=setup + ) + self.assertEqual(result.returncode, 0, result.stderr) + scorecard = json.loads(result.stdout) + self.assertEqual( + scorecard["no_raw_secrets"], + "pass", + f"SecretStore kinds must not count as raw Secrets, got: {scorecard['no_raw_secrets']}", + ) + # --------------------------------------------------------------------------- # T-SC4: no_raw_secrets fails with file name when a raw Secret is found @@ -535,6 +556,28 @@ def test_detail_suffixed_fail_counts_as_fail(self) -> None: "A detail-suffixed fail value (fail:raw-secret-in:...) must fail the gate", ) + def test_render_failures_fail_action_with_named_fragments(self) -> None: + """ + Render failures must fail the action via E_RENDER_FAILED, naming the + failed fragments — never by overriding a scorecard check. + """ + run_sh_text = DEPLOY_PREVIEW_RUN.read_text(encoding="utf-8") + self.assertIn( + "E_RENDER_FAILED", + run_sh_text, + "run.sh must fail with E_RENDER_FAILED when any fragment render fails", + ) + self.assertIn( + "render-failed:", + run_sh_text, + "gate-summary reason must name the failed fragments (render-failed:)", + ) + self.assertNotIn( + "'.no_raw_secrets = \"fail\"'", + run_sh_text, + "render failures must not be misattributed to no_raw_secrets", + ) + def test_gate_filter_in_tests_matches_run_sh(self) -> None: """ The jq filter used by these tests must appear verbatim in run.sh so the @@ -548,5 +591,129 @@ def test_gate_filter_in_tests_matches_run_sh(self) -> None: ) +# --------------------------------------------------------------------------- +# T-SC8: executable render-failure path — main() must exit via E_RENDER_FAILED +# --------------------------------------------------------------------------- + + +class RenderFailureEndToEndTest(unittest.TestCase): + """ + Run main() with stubbed CLI tooling (oras, npm, deploy-config-schema) so + every fragment render fails while emit-contract succeeds. The action must + exit nonzero via E_RENDER_FAILED naming the failed env/fragment pairs, and + the gate summary must carry a render-failed: reason — proving the + failure path is reachable and ordered after the sticky-comment/summary + steps, not just present in the source text. + """ + + STUB_ORAS = """#!/usr/bin/env bash +# Stub oras: create the expected pulled-context layout under --output . +out="" +prev="" +for a in "$@"; do + if [[ "$prev" == "--output" ]]; then out="$a"; fi + prev="$a" +done +mkdir -p "${out}/context/public" +printf 'cluster: stub\n' > "${out}/context/public/cluster-context-public.yml" +exit 0 +""" + + STUB_NPM = """#!/usr/bin/env bash +exit 0 +""" + + STUB_SCHEMA_CLI = """#!/usr/bin/env bash +# Stub deploy-config-schema: every fragment render fails; emit-contract +# succeeds and writes a minimal contract to --out. +if [[ "$1" == "render" ]]; then + printf '{"error":"stub render failure"}\n' >&2 + exit 1 +fi +if [[ "$1" == "artifact" ]]; then + out="" + prev="" + for a in "$@"; do + if [[ "$prev" == "--out" ]]; then out="$a"; fi + prev="$a" + done + printf 'spec:\n contextRef: ghcr.io/stub/context@sha256:abc\n' > "$out" + exit 0 +fi +exit 0 +""" + + def test_render_failure_exits_via_e_render_failed(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + tmppath = Path(tmpdir) + stub_bin = tmppath / "stub-bin" + stub_bin.mkdir() + for name, content in ( + ("oras", self.STUB_ORAS), + ("npm", self.STUB_NPM), + ("deploy-config-schema", self.STUB_SCHEMA_CLI), + ): + stub = stub_bin / name + stub.write_text(content, encoding="utf-8") + stub.chmod(0o755) + + workdir = tmppath / "work" + (workdir / "deploy").mkdir(parents=True) + (workdir / "deploy" / "deployment.yml").write_text( + MINIMAL_DEPLOYMENT_YML, encoding="utf-8" + ) + + env = { + "PATH": f"{stub_bin}:{os.environ.get('PATH', '/usr/bin:/bin')}", + "HOME": os.environ.get("HOME", "/root"), + "RUNNER_TEMP": str(tmppath / "runner-temp"), + "DEPLOY_DIR": "deploy", + "SCHEMA_VERSION": "0.0.0", + "IMAGE_LOCK_PATH": "deploy/images.lock.json", + "CONTEXT_REF": "ghcr.io/stub/context@sha256:abc", + "ENVIRONMENTS": "production", + "COMMENT": "false", + } + result = subprocess.run( + ["bash", str(DEPLOY_PREVIEW_RUN)], + env=env, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=str(workdir), + ) + + self.assertNotEqual( + result.returncode, + 0, + f"main() must exit nonzero when fragment renders fail: {result.stdout}", + ) + self.assertIn( + "E_RENDER_FAILED", + result.stderr, + f"stderr must carry E_RENDER_FAILED: {result.stderr}", + ) + self.assertIn( + "production/kubernetes-workload-fragment", + result.stderr, + f"E_RENDER_FAILED must name the failed env/fragment pairs: {result.stderr}", + ) + + gate_summary = json.loads( + (workdir / "gate-summary.json").read_text(encoding="utf-8") + ) + self.assertEqual(gate_summary["status"], "fail") + self.assertTrue( + gate_summary["reason"].startswith("render-failed:"), + f"gate reason must be render-failed:, got: {gate_summary['reason']!r}", + ) + self.assertIn( + "production/kubernetes-workload-fragment", + gate_summary["reason"], + f"gate reason must name the failed fragments: {gate_summary['reason']!r}", + ) + + if __name__ == "__main__": unittest.main()