From c5758493989478da934b57780799762496a7a5f9 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:45:43 -0600 Subject: [PATCH 1/4] Switch stackrox prow jobs to apollo-ci stable tag Replace all versioned stackrox-ui-test tags (0.4.9, 0.5.2, 0.5.5) with the stackrox-ui-test-stable floating tag. This avoids needing to update these configs on every rox-ci-image release. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../config/stackrox/stackrox/stackrox-stackrox-master.yaml | 2 +- .../stackrox/stackrox/stackrox-stackrox-master__ocp-4-12.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-14-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-15-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-16-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-17-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-18-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-19-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-20-lp-interop.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-4-21-lp-interop.yaml | 2 +- .../stackrox/stackrox/stackrox-stackrox-master__ocp-4-21.yaml | 2 +- ...krox-stackrox-master__ocp-4.21-lp-interop-cr-acs-latest.yaml | 2 +- ...tackrox-stackrox-master__ocp-4.22-lp-interop-acs-latest.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-dev-preview.yaml | 2 +- .../stackrox/stackrox-stackrox-master__ocp-next-candidate.yaml | 2 +- ...stackrox-stackrox-master__ocp-stable-scanner-v4-install.yaml | 2 +- .../config/stackrox/stackrox/stackrox-stackrox-nightlies.yaml | 2 +- .../stackrox/stackrox-stackrox-nightlies__ocp-4-12.yaml | 2 +- .../stackrox/stackrox-stackrox-nightlies__ocp-4-19.yaml | 2 +- .../stackrox/stackrox-stackrox-nightlies__ocp-4-21.yaml | 2 +- .../stackrox/stackrox-stackrox-nightlies__perf-scale.yaml | 2 +- .../stackrox/stackrox/stackrox-stackrox-release-4.10.yaml | 2 +- .../stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml | 2 +- .../stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml | 2 +- .../stackrox-stackrox-release-4.10__ocp-dev-preview.yaml | 2 +- .../stackrox-stackrox-release-4.10__ocp-next-candidate.yaml | 2 +- .../config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml | 2 +- .../stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml | 2 +- .../stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml | 2 +- .../stackrox-stackrox-release-4.8__ocp-dev-preview.yaml | 2 +- .../stackrox-stackrox-release-4.8__ocp-next-candidate.yaml | 2 +- .../config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml | 2 +- .../stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml | 2 +- .../stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml | 2 +- .../stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml | 2 +- .../stackrox-stackrox-release-4.9__ocp-dev-preview.yaml | 2 +- .../stackrox-stackrox-release-4.9__ocp-next-candidate.yaml | 2 +- .../config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml | 2 +- .../stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml | 2 +- .../stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml | 2 +- .../stackrox-stackrox-release-x.y__ocp-dev-preview.yaml | 2 +- .../stackrox-stackrox-release-x.y__ocp-next-candidate.yaml | 2 +- 42 files changed, 42 insertions(+), 42 deletions(-) diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master.yaml index 6af44de57262a..6180e1bdfd985 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master.yaml @@ -31,7 +31,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: release: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-12.yaml index a180eb8a3399b..188eb5a9baf4d 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-14-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-14-lp-interop.yaml index b8d335d376838..54967761c8450 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-14-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-14-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-15-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-15-lp-interop.yaml index 3c1e022d3c07a..8fdaf700075e6 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-15-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-15-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-16-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-16-lp-interop.yaml index 6f38e336a430b..bf2d1e3a00c1f 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-16-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-16-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-17-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-17-lp-interop.yaml index 563ae1b177dbd..8ab6efb1a5a5f 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-17-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-17-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: prerelease: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-18-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-18-lp-interop.yaml index 952f6417a154c..604d7b9586d75 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-18-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-18-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: prerelease: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-19-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-19-lp-interop.yaml index bedf94b73b0dc..2f20147923206 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-19-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-19-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-20-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-20-lp-interop.yaml index 5311370663a98..e6dbfe1d5eef6 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-20-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-20-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21-lp-interop.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21-lp-interop.yaml index 960730568fd52..927d86581ea8a 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21-lp-interop.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21-lp-interop.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21.yaml index 0f7f58afd26b7..3cdaadcb0c7f0 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4-21.yaml @@ -11,7 +11,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.21-lp-interop-cr-acs-latest.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.21-lp-interop-cr-acs-latest.yaml index 47fc6939f59a1..f0c7da3385e10 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.21-lp-interop-cr-acs-latest.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.21-lp-interop-cr-acs-latest.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.22-lp-interop-acs-latest.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.22-lp-interop-acs-latest.yaml index 248e99b406856..11fe47bcf11a1 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.22-lp-interop-acs-latest.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-4.22-lp-interop-acs-latest.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-dev-preview.yaml index 4cd6b14777557..780fdb55e794b 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-next-candidate.yaml index 221a7d8461687..ec270ae1f26aa 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-stable-scanner-v4-install.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-stable-scanner-v4-install.yaml index b16955bdab06f..44106b0e1a615 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-stable-scanner-v4-install.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-master__ocp-stable-scanner-v4-install.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies.yaml index 682eafb131d78..683045563fab6 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-12.yaml index e9283a7257753..fca7406a56ea2 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-19.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-19.yaml index 5a90a98fd57a9..93ef01500a539 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-19.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-19.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-21.yaml index 91e208dbe380f..ef4480f8767c7 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__perf-scale.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__perf-scale.yaml index c6b04512ae5bb..45b749672362f 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__perf-scale.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-nightlies__perf-scale.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.7 + tag: stackrox-ui-test-stable releases: latest: release: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml index 71cb5e6da32ec..0dcb430fc8209 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml index 8a2ed9008a0cd..9c6e8baf6efa2 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml index 0d690a1994e72..66a02ed886940 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml index e18f46d1c976b..c7d7180172b8f 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml index 349e3ef8e8e98..994682a1b140b 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml index 83f325e8de893..ff416aeae8d04 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml index c9da46b1ef152..7fa3b7c0255bf 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml index 2d5be1fc7f888..ccbaf8673466e 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml index 065bea421c505..7664e493b1fcd 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml index bbb4554fd89d1..55f0072a8b0f3 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml index eaa75677caa59..c9c4d2c211457 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml index 7083b2ef79513..d3e6f783434c9 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml index 10785faa049bc..f9333188d9693 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.4.9 + tag: stackrox-ui-test-stable releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml index 9c4d2fae8ad70..09af32ced99f0 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml index a1ed21943f949..4dc2054a9e674 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml index 355c89f173515..8ff59ef71c47c 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml index 0c592deaa2ed4..c339f47f2f6bb 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml index bbf29cd76a47f..beaa428535e71 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml index 10b79d745dc81..20e68e3857073 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml index 46a9eec0903c7..18adb91ef5232 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml index 02ebf8e29fb3a..6e6f66d0d3bd9 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-0.5.2 + tag: stackrox-ui-test-stable resources: '*': requests: From e970a3ee095c6bd131fbce68e2359a9688ea59e8 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 9 Apr 2026 22:10:01 -0600 Subject: [PATCH 2/4] Revert release branch configs to current versions Only update master and nightly jobs to stable tag. Release branches keep their current image versions to avoid potential compatibility issues. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../stackrox-stackrox-release-4.10.yaml | 2 +- ...ckrox-stackrox-release-4.10__ocp-4-12.yaml | 2 +- ...ckrox-stackrox-release-4.10__ocp-4-21.yaml | 2 +- ...tackrox-release-4.10__ocp-dev-preview.yaml | 2 +- ...krox-release-4.10__ocp-next-candidate.yaml | 2 +- .../stackrox-stackrox-release-4.8.yaml | 2 +- ...ackrox-stackrox-release-4.8__ocp-4-12.yaml | 2 +- ...ackrox-stackrox-release-4.8__ocp-4-20.yaml | 2 +- ...stackrox-release-4.8__ocp-dev-preview.yaml | 2 +- ...ckrox-release-4.8__ocp-next-candidate.yaml | 2 +- .../stackrox-stackrox-release-4.9.yaml | 2 +- ...ackrox-stackrox-release-4.9__ocp-4-12.yaml | 2 +- ...x-release-4.9__ocp-4-21-lp-interop-cr.yaml | 2 +- ...ackrox-stackrox-release-4.9__ocp-4-21.yaml | 2 +- ...stackrox-release-4.9__ocp-dev-preview.yaml | 2 +- ...ckrox-release-4.9__ocp-next-candidate.yaml | 2 +- .../stackrox-stackrox-release-x.y.yaml | 2 +- ...ackrox-stackrox-release-x.y__ocp-4-12.yaml | 2 +- ...ackrox-stackrox-release-x.y__ocp-4-21.yaml | 2 +- ...stackrox-release-x.y__ocp-dev-preview.yaml | 2 +- ...ckrox-release-x.y__ocp-next-candidate.yaml | 2 +- nuke-ci-cluster.sh | 537 ++++++++++++++ nuke-ci-queue.sh | 694 ++++++++++++++++++ quick-helm-install.sh | 121 +++ sa-disable-chain.sh | 396 ++++++++++ test.sh | 32 + test.yq.sh | 41 ++ tst.sh | 35 + update.sh | 69 ++ 29 files changed, 1946 insertions(+), 21 deletions(-) create mode 100755 nuke-ci-cluster.sh create mode 100755 nuke-ci-queue.sh create mode 100644 quick-helm-install.sh create mode 100755 sa-disable-chain.sh create mode 100755 test.sh create mode 100755 test.yq.sh create mode 100755 tst.sh create mode 100755 update.sh diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml index 0dcb430fc8209..71cb5e6da32ec 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml index 9c6e8baf6efa2..8a2ed9008a0cd 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml index 66a02ed886940..0d690a1994e72 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml index c7d7180172b8f..e18f46d1c976b 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml index 994682a1b140b..349e3ef8e8e98 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.10__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml index ff416aeae8d04..83f325e8de893 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml index 7fa3b7c0255bf..c9da46b1ef152 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml index ccbaf8673466e..2d5be1fc7f888 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-4-20.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml index 7664e493b1fcd..065bea421c505 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml index 55f0072a8b0f3..bbb4554fd89d1 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.8__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml index c9c4d2c211457..eaa75677caa59 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml index d3e6f783434c9..7083b2ef79513 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml index f9333188d9693..10785faa049bc 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21-lp-interop-cr.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.4.9 releases: latest: candidate: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml index 09af32ced99f0..9c4d2fae8ad70 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml index 4dc2054a9e674..a1ed21943f949 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml index 8ff59ef71c47c..355c89f173515 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-4.9__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml index c339f47f2f6bb..0c592deaa2ed4 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y.yaml @@ -27,7 +27,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml index beaa428535e71..bbf29cd76a47f 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-12.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml index 20e68e3857073..10b79d745dc81 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-4-21.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml index 18adb91ef5232..46a9eec0903c7 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-dev-preview.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml index 6e6f66d0d3bd9..02ebf8e29fb3a 100644 --- a/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml +++ b/ci-operator/config/stackrox/stackrox/stackrox-stackrox-release-x.y__ocp-next-candidate.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: stackrox-ui-test-stable + tag: stackrox-ui-test-0.5.2 resources: '*': requests: diff --git a/nuke-ci-cluster.sh b/nuke-ci-cluster.sh new file mode 100755 index 0000000000000..fd181a3e6d3cf --- /dev/null +++ b/nuke-ci-cluster.sh @@ -0,0 +1,537 @@ +#!/usr/bin/env bash +# +# Nuke a StackRox CI OpenShift cluster by Prow job ID. +# +# Usage: +# ./nuke-ci-cluster.sh +# ./nuke-ci-cluster.sh --dry-run +# ./nuke-ci-cluster.sh --force +# ./nuke-ci-cluster.sh --force --wait +# ./nuke-ci-cluster.sh --force --after-create +# +# The script derives the cluster name prefix from the last 8 digits of the +# job ID (rox-ci-XXXXXXXX), discovers the full infra-id from running VMs, +# and deletes all associated GCP resources. +# +# Modes: +# +# Default: Immediately nuke the cluster (sabotage auth, kill VMs, cleanup). +# This fights the installer retry loop (up to 10 retries / 90 min). +# Best when the cluster is already mid-install or you want to act now. +# +# --after-create: Wait for the cluster install to succeed, then nuke it +# right before tests run. This is FASTER overall (~50 min vs ~90 min) +# because a successful install takes ~45 min, and then tests fail in +# minutes when the cluster disappears. The CI post step handles cleanup. +# +# Strategy (default mode): +# 1. Delete the workload identity pool (prevents installer retries from +# authenticating to GCP, making them fail faster) +# 2. Delete service accounts +# 3. Kill all VMs +# 4. Clean up remaining infrastructure in dependency order +# 5. Optionally loop (--wait) killing new VMs until the job finishes +# +# Strategy (--after-create mode): +# 1. Wait for ocp-4-create step to finish successfully +# 2. Immediately kill all VMs (just instances is enough) +# 3. Tests fail quickly, CI post step runs ocp-4-destroy to clean up +# +# Requires: gcloud CLI authenticated to the acs-san-stackroxci project. + +set -euo pipefail + +GCP_PROJECT="${GCP_PROJECT:-acs-san-stackroxci}" +DRY_RUN=false +FORCE=false +WAIT=false +AFTER_CREATE=false +# GCS prefix for job artifacts - can be overridden for non-default job names +GCS_JOB_PATH="${GCS_JOB_PATH:-}" + +usage() { + echo "Usage: $0 [--dry-run] [--force] [--wait] [--after-create]" + echo "" + echo " prow-job-id The numeric Prow job build ID" + echo " --dry-run Show what would be deleted without deleting" + echo " --force Skip confirmation prompt" + echo " --wait After nuking, loop killing new VMs until job finishes" + echo " --after-create Wait for cluster install to succeed, then nuke (faster)" + echo "" + echo "Environment variables:" + echo " GCP_PROJECT GCP project (default: acs-san-stackroxci)" + echo " GCS_JOB_PATH Override GCS path for job artifacts" + exit 1 +} + +[[ $# -lt 1 ]] && usage + +JOB_ID="$1" +shift +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true ;; + --force) FORCE=true ;; + --wait) WAIT=true ;; + --after-create) AFTER_CREATE=true ;; + *) usage ;; + esac + shift +done + +# Derive cluster name prefix from last 8 digits of job ID +CLUSTER_PREFIX="rox-ci-${JOB_ID: -8}" +echo "Job ID: $JOB_ID" +echo "Cluster prefix: $CLUSTER_PREFIX" +echo "GCP Project: $GCP_PROJECT" +echo "" + +# Helper: delete a global-or-regional resource +delete_global_or_regional() { + local resource_type="$1" name="$2" region="$3" + region=$(basename "$region") + if [[ -z "$region" || "$region" == "$name" ]]; then + echo " deleting global $resource_type $name" + gcloud compute "$resource_type" delete "$name" --global --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + else + echo " deleting $resource_type $name (region: $region)" + gcloud compute "$resource_type" delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + fi +} + +# Phase 1: Sabotage authentication (fastest way to prevent retries) +sabotage_auth() { + echo "=== Phase 1: Sabotaging cluster authentication ===" + + echo "Deleting workload identity pool..." + if gcloud iam workload-identity-pools describe "$CLUSTER_PREFIX" \ + --location=global --project="$GCP_PROJECT" &>/dev/null; then + gcloud iam workload-identity-pools delete "$CLUSTER_PREFIX" \ + --location=global --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + echo " Deleted." + else + echo " Not found (may already be deleted)." + fi + + echo "Deleting service accounts..." + local sa_list + sa_list=$(gcloud iam service-accounts list --project="$GCP_PROJECT" \ + --filter="email~${INFRA_ID}" --format="value(email)" 2>/dev/null || true) + if [[ -n "$sa_list" ]]; then + echo "$sa_list" | while read -r email; do + echo " deleting $email" + gcloud iam service-accounts delete "$email" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + else + echo " None found." + fi + echo "" +} + +# Phase 2: Kill all VMs +kill_instances() { + echo "=== Phase 2: Killing instances ===" + local instances + instances=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,zone)" 2>/dev/null || true) + if [[ -z "$instances" ]]; then + echo " No instances found." + return + fi + echo "$instances" | while IFS=, read -r name zone; do + zone=$(basename "$zone") + echo " killing $name ($zone)" + gcloud compute instances delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & + done + wait + echo "" +} + +# Phase 3: Clean up infrastructure in correct dependency order +cleanup_infra() { + echo "=== Phase 3: Cleaning up infrastructure ===" + + # 1. Forwarding rules (must go before target proxies and backend services) + echo "[1/13] Forwarding rules..." + gcloud compute forwarding-rules list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ + | while IFS=, read -r name region; do + [[ -z "$name" ]] && continue + delete_global_or_regional "forwarding-rules" "$name" "$region" + done + + # 2. Target TCP proxies (sits between forwarding rules and backend services) + echo "[2/13] Target TCP proxies..." + gcloud compute target-tcp-proxies list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ + | while read -r name; do + [[ -z "$name" ]] && continue + echo " deleting target-tcp-proxy $name" + gcloud compute target-tcp-proxies delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 3. Backend services (before instance groups and health checks) + echo "[3/13] Backend services..." + gcloud compute backend-services list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ + | while IFS=, read -r name region; do + [[ -z "$name" ]] && continue + delete_global_or_regional "backend-services" "$name" "$region" + done + + # 4. Instance groups + echo "[4/13] Instance groups..." + gcloud compute instance-groups list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,zone)" 2>/dev/null \ + | while IFS=, read -r name zone; do + [[ -z "$name" ]] && continue + zone=$(basename "$zone") + echo " deleting instance group $name ($zone)" + gcloud compute instance-groups unmanaged delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 5. Target pools + echo "[5/13] Target pools..." + gcloud compute target-pools list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ + | while IFS=, read -r name region; do + [[ -z "$name" ]] && continue + region=$(basename "$region") + echo " deleting target pool $name ($region)" + gcloud compute target-pools delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 6. Health checks + echo "[6/13] Health checks..." + gcloud compute health-checks list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ + | while read -r name; do + [[ -z "$name" ]] && continue + echo " deleting health check $name" + gcloud compute health-checks delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 7. Firewall rules + echo "[7/13] Firewall rules..." + gcloud compute firewall-rules list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ + | while read -r name; do + [[ -z "$name" ]] && continue + echo " deleting firewall rule $name" + gcloud compute firewall-rules delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 8. Addresses + echo "[8/13] Addresses..." + gcloud compute addresses list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ + | while IFS=, read -r name region; do + [[ -z "$name" ]] && continue + delete_global_or_regional "addresses" "$name" "$region" + done + + # 9. Routers + echo "[9/13] Routers..." + gcloud compute routers list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ + | while IFS=, read -r name region; do + [[ -z "$name" ]] && continue + region=$(basename "$region") + echo " deleting router $name ($region)" + gcloud compute routers delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 10. Subnets + echo "[10/13] Subnets..." + gcloud compute networks subnets list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ + | while IFS=, read -r name region; do + [[ -z "$name" ]] && continue + region=$(basename "$region") + echo " deleting subnet $name ($region)" + gcloud compute networks subnets delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 11. Networks + echo "[11/13] Networks..." + gcloud compute networks list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ + | while read -r name; do + [[ -z "$name" ]] && continue + echo " deleting network $name" + gcloud compute networks delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + # 12. Remaining disks + echo "[12/13] Remaining disks..." + gcloud compute disks list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="csv[no-heading](name,zone)" 2>/dev/null \ + | while IFS=, read -r name zone; do + [[ -z "$name" ]] && continue + zone=$(basename "$zone") + echo " deleting disk $name ($zone)" + gcloud compute disks delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & + done + wait + + # 13. DNS zones + echo "[13/13] DNS zones..." + gcloud dns managed-zones list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ + | while read -r zone_name; do + [[ -z "$zone_name" ]] && continue + echo " clearing DNS records in $zone_name" + gcloud dns record-sets list --zone="$zone_name" --project="$GCP_PROJECT" \ + --format="csv[no-heading](name,type)" 2>/dev/null \ + | grep -v ',NS$' | grep -v ',SOA$' \ + | while IFS=, read -r rname rtype; do + gcloud dns record-sets delete "$rname" --zone="$zone_name" \ + --type="$rtype" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + echo " deleting DNS zone $zone_name" + gcloud dns managed-zones delete "$zone_name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true + done + + echo "" +} + +# Phase 4: Wait loop - keep killing new VMs until job finishes +wait_for_job() { + echo "=== Phase 4: Monitoring for new instances until job finishes ===" + local max_iterations=120 # 60 minutes at 30s intervals + for (( i=1; i<=max_iterations; i++ )); do + # Kill any new instances + local instances + instances=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~${CLUSTER_PREFIX}" --format="csv[no-heading](name,zone)" 2>/dev/null || true) + if [[ -n "$instances" ]]; then + echo "$(date): Found new instances, killing..." + echo "$instances" | while IFS=, read -r name zone; do + zone=$(basename "$zone") + echo " killing $name ($zone)" + gcloud compute instances delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & + done + wait + fi + + # Check if the job has finished + resolve_gcs_path + if gsutil -q stat "${GCS_JOB_PATH}/finished.json" 2>/dev/null; then + local result + result=$(gsutil cat "${GCS_JOB_PATH}/finished.json" 2>/dev/null || echo "unknown") + echo "$(date): Job finished! $result" + return 0 + fi + + sleep 30 + done + echo "$(date): Timed out waiting for job to finish after $((max_iterations * 30 / 60)) minutes." + return 1 +} + +# Resolve GCS job path for checking job artifacts +resolve_gcs_path() { + if [[ -n "$GCS_JOB_PATH" ]]; then + return + fi + # Try to find the job path from GCS by searching common patterns + for path_pattern in \ + "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests/${JOB_ID}" \ + "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-22-merge-qa-e2e-tests/${JOB_ID}"; do + if gsutil -q stat "${path_pattern}/prowjob.json" 2>/dev/null; then + GCS_JOB_PATH="$path_pattern" + return + fi + done + # Fallback: try to find via prowjob.json + GCS_JOB_PATH="gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests/${JOB_ID}" +} + +# Phase 0: --after-create mode - wait for install, then nuke +after_create_mode() { + resolve_gcs_path + echo "=== After-create mode: waiting for cluster install to succeed ===" + echo "Monitoring: ${GCS_JOB_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" + echo "" + + # Wait for the create step to finish + while true; do + local create_finished + create_finished=$(gsutil cat "${GCS_JOB_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null || true) + if [[ -n "$create_finished" ]]; then + echo "" + echo "$(date): Create step finished!" + echo "$create_finished" + break + fi + + # Also check if the overall job already finished (e.g. timed out) + local job_finished + job_finished=$(gsutil cat "${GCS_JOB_PATH}/finished.json" 2>/dev/null || true) + if [[ -n "$job_finished" ]]; then + echo "" + echo "$(date): Job already finished: $job_finished" + echo "Nothing to nuke." + exit 0 + fi + + echo -n "." + sleep 30 + done + + echo "" + echo "Cluster install complete. Nuking cluster to fail tests..." + echo "" + + # Discover infra-id from instances or logs + INFRA_ID=$(gcloud compute instances list \ + --project="$GCP_PROJECT" \ + --filter="name~^${CLUSTER_PREFIX}" \ + --format="value(name)" 2>/dev/null \ + | head -1 \ + | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) + + if [[ -z "$INFRA_ID" ]]; then + # Try from logs + INFRA_ID=$(gsutil cat "${GCS_JOB_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/build-log.txt" 2>/dev/null \ + | gunzip 2>/dev/null \ + | grep -oP 'CLUSTER_NAME=\K\S+' \ + | head -1 || true) + fi + + if [[ -z "$INFRA_ID" ]]; then + echo "ERROR: Could not determine cluster infra-id." + exit 1 + fi + + echo "Infra-id: $INFRA_ID" + + # Just kill the instances - that's enough to fail the tests. + # The CI post step (ocp-4-destroy) will handle full cleanup. + kill_instances + + echo "=========================================" + echo " Cluster instances killed." + echo " Tests should fail shortly." + echo " CI post step will handle cleanup." + echo "=========================================" + + if $WAIT; then + wait_for_job + fi + exit 0 +} + +# === Main === + +# Handle --after-create mode early +if $AFTER_CREATE; then + after_create_mode +fi + +# Discover the full infra-id from running instances +echo "Discovering infra-id from running instances..." +INFRA_ID=$(gcloud compute instances list \ + --project="$GCP_PROJECT" \ + --filter="name~^${CLUSTER_PREFIX}" \ + --format="value(name)" 2>/dev/null \ + | head -1 \ + | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) + +if [[ -z "$INFRA_ID" ]]; then + echo "No instances found matching prefix '${CLUSTER_PREFIX}'." + echo "" + echo "Checking GCS for cluster name from job artifacts..." + INFRA_ID=$(gsutil cat \ + "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests/${JOB_ID}/artifacts/merge-qa-e2e-tests/ocp-4-create/build-log.txt" 2>/dev/null \ + | gunzip 2>/dev/null \ + | grep -oP 'CLUSTER_NAME=\K\S+' \ + | head -1 || true) + if [[ -n "$INFRA_ID" ]]; then + echo "Found cluster name from logs: $INFRA_ID" + echo "No running instances, but will proceed with cleanup of other resources." + echo "" + else + echo "No cluster found. The cluster may not have been provisioned yet." + if $WAIT; then + echo "Waiting for cluster to appear..." + while true; do + INFRA_ID=$(gcloud compute instances list \ + --project="$GCP_PROJECT" \ + --filter="name~^${CLUSTER_PREFIX}" \ + --format="value(name)" 2>/dev/null \ + | head -1 \ + | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) + if [[ -n "$INFRA_ID" ]]; then + echo "Found infra-id: $INFRA_ID" + break + fi + echo "$(date): waiting..." + sleep 30 + done + else + exit 1 + fi + fi +fi + +echo "Infra-id: $INFRA_ID" +echo "" + +if $DRY_RUN; then + echo "[DRY RUN] Surveying resources..." + echo "" + for desc_filter_fmt in \ + "Instances|instances list|name~${INFRA_ID}|csv[no-heading](name,zone)" \ + "Instance Groups|instance-groups list|name~${INFRA_ID}|csv[no-heading](name,zone)" \ + "Forwarding Rules|forwarding-rules list|name~${INFRA_ID}|csv[no-heading](name,region)" \ + "Target TCP Proxies|target-tcp-proxies list|name~${INFRA_ID}|value(name)" \ + "Backend Services|backend-services list|name~${INFRA_ID}|csv[no-heading](name,region)" \ + "Health Checks|health-checks list|name~${INFRA_ID}|value(name)" \ + "Firewall Rules|firewall-rules list|name~${INFRA_ID}|value(name)" \ + "Routers|routers list|name~${INFRA_ID}|csv[no-heading](name,region)" \ + "Subnets|networks subnets list|name~${INFRA_ID}|csv[no-heading](name,region)" \ + "Networks|networks list|name~${INFRA_ID}|value(name)" \ + "Addresses|addresses list|name~${INFRA_ID}|csv[no-heading](name,region)" \ + "Disks|disks list|name~${INFRA_ID}|csv[no-heading](name,zone)"; do + IFS='|' read -r desc cmd filter fmt <<< "$desc_filter_fmt" + echo "--- $desc ---" + # shellcheck disable=SC2086 + gcloud compute $cmd --project="$GCP_PROJECT" --filter="$filter" --format="$fmt" 2>/dev/null | grep . || echo "(none)" + echo "" + done + echo "--- Workload Identity Pool ---" + gcloud iam workload-identity-pools describe "$CLUSTER_PREFIX" \ + --location=global --project="$GCP_PROJECT" --format="value(name)" 2>/dev/null || echo "(none)" + echo "" + echo "--- DNS Zones ---" + gcloud dns managed-zones list --project="$GCP_PROJECT" \ + --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null | grep . || echo "(none)" + echo "" + echo "--- Service Accounts ---" + gcloud iam service-accounts list --project="$GCP_PROJECT" \ + --filter="email~${INFRA_ID}" --format="value(email)" 2>/dev/null | grep . || echo "(none)" + echo "" + echo "[DRY RUN] Would delete all resources listed above." + exit 0 +fi + +if ! $FORCE; then + read -rp "Nuke cluster ${INFRA_ID} (job ${JOB_ID})? [y/N] " confirm + [[ "$confirm" =~ ^[yY]$ ]] || { echo "Aborted."; exit 1; } +fi + +echo "" + +# Execute in order: sabotage auth first (fastest impact), then kill VMs, then cleanup +sabotage_auth +kill_instances +cleanup_infra + +echo "=========================================" +echo " Cluster $INFRA_ID nuked." +echo "=========================================" + +if $WAIT; then + wait_for_job +fi diff --git a/nuke-ci-queue.sh b/nuke-ci-queue.sh new file mode 100755 index 0000000000000..cc567d913ca1a --- /dev/null +++ b/nuke-ci-queue.sh @@ -0,0 +1,694 @@ +#!/usr/bin/env bash +# +# Automatically chain-kill queued StackRox CI jobs using --after-create strategy. +# +# Usage: +# ./nuke-ci-queue.sh # Kill all queued jobs +# ./nuke-ci-queue.sh --keep-after TIME # Keep jobs triggered after TIME (UTC) +# ./nuke-ci-queue.sh --max-kills N # Stop after killing N jobs +# ./nuke-ci-queue.sh --dry-run # Show what would be killed +# +# Strategy: +# 1. Query prowjobs.js for pending/triggered ocp-4-21-merge-qa-e2e jobs +# 2. Wait for the current pending job's create step to succeed +# 3. Kill the cluster instances to fail the tests +# 4. Wait for the job to finish +# 5. Repeat for the next job in the queue +# +# Requires: gcloud CLI, gsutil, curl, python3 + +set -euo pipefail + +GCP_PROJECT="${GCP_PROJECT:-acs-san-stackroxci}" +JOB_NAME="${JOB_NAME:-branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests}" +GCS_BASE="${GCS_BASE:-gs://test-platform-results/logs/${JOB_NAME}}" +PROVISIONER_SA="${PROVISIONER_SA:-openshift-ipi-provisioner@acs-san-stackroxci.iam.gserviceaccount.com}" +KEEP_AFTER="${KEEP_AFTER:-}" +MAX_KILLS="${MAX_KILLS:-0}" +DRY_RUN=false +DISABLE_SA=false +KILL_COUNT=0 + +usage() { + echo "Usage: $0 [--keep-after TIME] [--max-kills N] [--dry-run] [--disable-sa]" + echo "" + echo " --keep-after TIME Keep jobs triggered after this UTC time (e.g. 2026-03-10T14:35:00Z)" + echo " --max-kills N Stop after killing N jobs (0 = unlimited)" + echo " --dry-run Show queue status without killing" + echo " --disable-sa Use SA-disable strategy (fastest, ~2 min per job)" + echo " Only used when no other jobs are in create/destroy phase" + echo "" + echo "Environment variables:" + echo " GCP_PROJECT GCP project (default: acs-san-stackroxci)" + echo " JOB_NAME Prow job name to target" + echo " PROVISIONER_SA GCP SA to disable (default: openshift-ipi-provisioner@...)" + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --keep-after) KEEP_AFTER="$2"; shift ;; + --max-kills) MAX_KILLS="$2"; shift ;; + --dry-run) DRY_RUN=true ;; + --disable-sa) DISABLE_SA=true ;; + --help|-h) usage ;; + *) echo "Unknown arg: $1"; usage ;; + esac + shift +done + +# Get the current queue state from prowjobs.js +get_queue() { + curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ + | sed 's/^var allBuilds = //;s/;$//' \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +job_name = '${JOB_NAME}' +keep_after = '${KEEP_AFTER}' +jobs = [j for j in data.get('items', []) + if j.get('spec',{}).get('job','') == job_name + and j['status'].get('state','') in ('pending', 'triggered')] +jobs.sort(key=lambda j: j['metadata'].get('creationTimestamp','')) +for j in jobs: + created = j['metadata'].get('creationTimestamp','') + if keep_after and created > keep_after: + continue + state = j['status'].get('state','') + build_id = j['status'].get('build_id','') + print(f'{state}|{created}|{build_id}') +" 2>/dev/null +} + +# Get the pending job (the one currently running or about to run) +get_pending_job_id() { + curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ + | sed 's/^var allBuilds = //;s/;$//' \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +job_name = '${JOB_NAME}' +keep_after = '${KEEP_AFTER}' +jobs = [j for j in data.get('items', []) + if j.get('spec',{}).get('job','') == job_name + and j['status'].get('state','') == 'pending' + and j['status'].get('build_id','')] +jobs.sort(key=lambda j: j['metadata'].get('creationTimestamp','')) +for j in jobs: + created = j['metadata'].get('creationTimestamp','') + if keep_after and created > keep_after: + continue + print(j['status']['build_id']) + break +" 2>/dev/null +} + +# Count remaining jobs to kill +count_remaining() { + get_queue | wc -l | tr -d ' ' +} + +# Wait for instances to appear for a cluster prefix +wait_for_instances() { + local prefix="$1" + local job_id="$2" + local gcs_path="${GCS_BASE}/${job_id}" + echo " Waiting for instances to appear..." + for i in $(seq 1 120); do + # Check if job or create step already finished (no point waiting) + if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then + echo " Job already finished while waiting for instances." + return 1 + fi + if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then + echo " Create step finished while waiting. Checking for instances one more time..." + local inst + inst=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~^${prefix}" --format="value(name)" 2>/dev/null | head -1) + if [[ -n "$inst" ]]; then + echo " Found instance: $inst" + return 0 + fi + echo " No instances found after create finished." + return 1 + fi + local inst + inst=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~^${prefix}" --format="value(name)" 2>/dev/null | head -1) + if [[ -n "$inst" ]]; then + echo " Found instance: $inst" + return 0 + fi + sleep 15 + done + echo " Timed out waiting for instances" + return 1 +} + +# Extract kubeconfig from the bootstrap ignition stored in GCS +extract_kubeconfig() { + local prefix="$1" + local infra_id="$2" + local outfile="$3" + local ign_bucket="${infra_id}-bootstrap-ignition" + echo " Extracting kubeconfig from gs://${ign_bucket}/bootstrap.ign..." + gsutil cat "gs://${ign_bucket}/bootstrap.ign" 2>/dev/null | python3 -c " +import sys, json, base64 +data = json.load(sys.stdin) +for f in data.get('storage', {}).get('files', []): + if f.get('path') == '/opt/openshift/auth/kubeconfig': + source = f['contents']['source'] + _, encoded = source.split(',', 1) + decoded = base64.b64decode(encoded).decode() + # Replace api-int with api for external access + print(decoded.replace('api-int.', 'api.')) + break +" > "$outfile" 2>/dev/null + if [[ -s "$outfile" ]]; then + echo " Kubeconfig saved to $outfile" + return 0 + fi + echo " Failed to extract kubeconfig" + return 1 +} + +# Wait for the cluster API to be reachable +wait_for_api() { + local cluster_name="$1" + local api_host="api.${cluster_name}.ocp.ci.rox.systems" + echo " Waiting for API at ${api_host}:6443..." + for i in $(seq 1 60); do + if timeout 5 curl -sk "https://${api_host}:6443/version" &>/dev/null; then + echo " API is up!" + return 0 + fi + sleep 10 + done + echo " Timed out waiting for API" + return 1 +} + +# Create the bootstrap-complete configmap to trick the installer +signal_bootstrap_complete() { + local kubeconfig="$1" + echo " Creating bootstrap-complete configmap..." + if KUBECONFIG="$kubeconfig" kubectl get configmap bootstrap -n kube-system &>/dev/null; then + KUBECONFIG="$kubeconfig" kubectl patch configmap bootstrap -n kube-system \ + -p '{"data":{"status":"complete"}}' 2>&1 | sed 's/^/ /' + else + KUBECONFIG="$kubeconfig" kubectl create configmap bootstrap -n kube-system \ + --from-literal=status=complete 2>&1 | sed 's/^/ /' + fi +} + +# Fake the ClusterVersion to signal install-complete. +# Strategy: delete admission webhooks that block status changes, +# scale down CVO to prevent it from overwriting, then patch. +signal_install_complete() { + local kubeconfig="$1" + echo " Signaling install-complete..." + + # Step 1: Delete admission webhooks that may block status patches + echo " Deleting admission webhooks..." + KUBECONFIG="$kubeconfig" kubectl delete validatingwebhookconfigurations --all 2>&1 | sed 's/^/ /' + KUBECONFIG="$kubeconfig" kubectl delete mutatingwebhookconfigurations --all 2>&1 | sed 's/^/ /' + + # Step 2: Scale down CVO to prevent it from overwriting our status + echo " Scaling down CVO..." + KUBECONFIG="$kubeconfig" kubectl scale deployment cluster-version-operator \ + -n openshift-cluster-version --replicas=0 2>&1 | sed 's/^/ /' + KUBECONFIG="$kubeconfig" kubectl delete pod -n openshift-cluster-version \ + --all --force --grace-period=0 2>&1 | sed 's/^/ /' + sleep 5 + + # Step 3: Wait for ClusterVersion to exist + echo " Waiting for ClusterVersion 'version' to exist..." + for i in $(seq 1 30); do + if KUBECONFIG="$kubeconfig" kubectl get clusterversion version &>/dev/null; then + echo " ClusterVersion exists." + break + fi + sleep 10 + done + + # Step 4: Build and apply patch preserving required fields + echo " Patching ClusterVersion status..." + local current_status + current_status=$(KUBECONFIG="$kubeconfig" kubectl get clusterversion version -o json 2>/dev/null) + if [[ -n "$current_status" ]]; then + echo "$current_status" | python3 -c " +import sys, json +cv = json.load(sys.stdin) +status = cv.get('status', {}) +patch = { + 'status': { + 'desired': status.get('desired', {'version': '4.21.0', 'image': 'unknown'}), + 'observedGeneration': status.get('observedGeneration', 1), + 'versionHash': status.get('versionHash', 'fake'), + 'availableUpdates': status.get('availableUpdates') or [], + 'conditions': [ + {'type': 'Available', 'status': 'True', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': 'Done'}, + {'type': 'Failing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''}, + {'type': 'Progressing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''} + ] + } +} +print(json.dumps(patch)) +" > /tmp/cv-patch.json + KUBECONFIG="$kubeconfig" kubectl patch clusterversion version --type=merge --subresource=status \ + -p "$(cat /tmp/cv-patch.json)" 2>&1 | sed 's/^/ /' + rm -f /tmp/cv-patch.json + else + echo " Could not get current ClusterVersion." + fi + + # Step 5: Verify the patch stuck and keep patching if CVO respawns + echo " Verifying ClusterVersion conditions..." + local patch_success=false + for attempt in $(seq 1 10); do + local conditions + conditions=$(KUBECONFIG="$kubeconfig" kubectl get clusterversion version \ + -o jsonpath='{.status.conditions[?(@.type=="Available")].status},{.status.conditions[?(@.type=="Failing")].status},{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null) + echo " Attempt $attempt: Available,Failing,Progressing = $conditions" + if [[ "$conditions" == "True,False,False" ]]; then + echo " [OK] ClusterVersion conditions are correct!" + patch_success=true + break + fi + # Re-kill CVO if it respawned + KUBECONFIG="$kubeconfig" kubectl delete pod -n openshift-cluster-version --all --force --grace-period=0 &>/dev/null + # Re-apply patch + if [[ -n "$current_status" ]]; then + echo "$current_status" | python3 -c " +import sys, json +cv = json.load(sys.stdin) +status = cv.get('status', {}) +patch = { + 'status': { + 'desired': status.get('desired', {'version': '4.21.0', 'image': 'unknown'}), + 'observedGeneration': status.get('observedGeneration', 1), + 'versionHash': status.get('versionHash', 'fake'), + 'availableUpdates': status.get('availableUpdates') or [], + 'conditions': [ + {'type': 'Available', 'status': 'True', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': 'Done'}, + {'type': 'Failing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''}, + {'type': 'Progressing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''} + ] + } +} +print(json.dumps(patch)) +" | KUBECONFIG="$kubeconfig" kubectl patch clusterversion version --type=merge --subresource=status -p "$(cat -)" &>/dev/null + fi + sleep 5 + done + if ! $patch_success; then + echo " [WARN] Could not get ClusterVersion conditions to stick after 10 attempts." + echo " [WARN] Falling back to wait-for-create approach." + fi +} + +# Check if any other jobs have clusters being created/destroyed right now. +# Checks for bootstrap nodes (active provisioning) and also for clusters +# that are still installing (have bootstrap but no workers yet). +# The provisioner SA (OCP_4_GCP_SA) is used by all ocp-4 jobs during +# create and destroy phases. +other_jobs_creating() { + local our_prefix="$1" + + # Check for bootstrap nodes from other jobs (active provisioning) + local other_bootstraps + other_bootstraps=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~bootstrap AND NOT name~${our_prefix}" \ + --format="value(name)" 2>/dev/null || true) + if [[ -n "$other_bootstraps" ]]; then + echo "$other_bootstraps" + return 0 # other jobs ARE creating + fi + + # Check for very recently created instances (< 5 min old) from other jobs + # These might be in early provisioning before bootstrap appears + local recent + recent=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~^rox-ci- AND NOT name~${our_prefix} AND creationTimestamp>-PT5M" \ + --format="value(name)" 2>/dev/null | head -1 || true) + if [[ -n "$recent" ]]; then + echo "(recently created) $recent" + return 0 + fi + + return 1 # safe to proceed +} + +# Disable the provisioner SA to make the installer fail instantly +disable_provisioner_sa() { + echo " [SA] Disabling provisioner SA: $PROVISIONER_SA" + gcloud iam service-accounts disable "$PROVISIONER_SA" \ + --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' +} + +# Re-enable the provisioner SA +enable_provisioner_sa() { + echo " [SA] Re-enabling provisioner SA: $PROVISIONER_SA" + gcloud iam service-accounts enable "$PROVISIONER_SA" \ + --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' +} + +# SA-disable abort: disable the provisioner SA so the installer fails instantly. +# Only used when no other jobs are in create/destroy phase. +sa_disable_abort() { + local job_id="$1" + local prefix="$2" + local gcs_path="${GCS_BASE}/${job_id}" + + echo " [SA] Checking if SA-disable is safe..." + + # Check if other jobs are creating clusters + local others + if others=$(other_jobs_creating "$prefix"); then + echo " [SA] Other jobs have clusters being created:" + echo "$others" | sed 's/^/ /' + echo " [SA] Not safe to disable SA. Falling back to fast_abort." + return 1 + fi + echo " [SA] No other jobs creating. Safe to proceed." + + # Disable the SA + disable_provisioner_sa + + # Set trap to always re-enable SA on exit/error + trap 'enable_provisioner_sa' EXIT + + # Wait for the create step to finish (should be very fast) + echo " [SA] Waiting for create step to fail (should be ~2-5 min)..." + local sa_start + sa_start=$(date +%s) + for i in $(seq 1 60); do + # Safety check: re-enable if other jobs start creating + if others=$(other_jobs_creating "$prefix"); then + echo " [SA] WARNING: Other job started creating! Re-enabling SA immediately." + echo "$others" | sed 's/^/ /' + enable_provisioner_sa + trap - EXIT + echo " [SA] SA re-enabled. Falling back to fast_abort." + return 1 + fi + + # Check if create step finished + if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then + local elapsed=$(( $(date +%s) - sa_start )) + local create_result + create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) + echo " [SA] Create step finished in ${elapsed}s! $create_result" + break + fi + + # Check if job already finished + if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then + echo " [SA] Job already finished." + enable_provisioner_sa + trap - EXIT + return 0 + fi + + sleep 10 + done + + # Re-enable SA immediately + enable_provisioner_sa + trap - EXIT + + echo " [SA] SA re-enabled. Waiting for job to fail..." + return 0 +} + +# Fast abort: extract kubeconfig from ignition, signal bootstrap complete, +# then kill instances. Falls back to wait-for-create if anything fails. +fast_abort() { + local job_id="$1" + local prefix="$2" + local gcs_path="${GCS_BASE}/${job_id}" + local cluster_name="$prefix" # e.g. rox-ci-49586688 + + echo " [FAST] Attempting ignition-kubeconfig abort..." + + # Wait for instances to appear + if ! wait_for_instances "$prefix" "$job_id"; then + echo " [FAST] No instances found, falling back to wait-for-create." + return 1 + fi + + # Discover infra-id from instances + local infra_id + infra_id=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~^${prefix}" --format="value(name)" 2>/dev/null \ + | head -1 | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) + if [[ -z "$infra_id" ]]; then + echo " [FAST] Could not determine infra-id, falling back." + return 1 + fi + echo " [FAST] Infra-id: $infra_id" + + # Extract kubeconfig from ignition + local kubeconfig="/tmp/${infra_id}-kubeconfig.yaml" + if ! extract_kubeconfig "$prefix" "$infra_id" "$kubeconfig"; then + echo " [FAST] Kubeconfig extraction failed, falling back." + return 1 + fi + + local gcs_path="${GCS_BASE}/${job_id}" + + # Check if create step already finished — if so, skip bootstrap signal and just kill + if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then + local create_result + create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) + echo " [FAST] Create step already finished: $create_result" + echo " [FAST] Killing cluster instances to fail tests..." + kill_cluster_instances "$prefix" || true + rm -f "$kubeconfig" + echo " [FAST] Instances killed. Waiting for job to fail..." + return 0 + fi + + # Wait for API to come up, but also check if create step finishes while waiting + echo " [FAST] Waiting for API..." + local api_host="api.${cluster_name}.ocp.ci.rox.systems" + local api_up=false + for i in $(seq 1 60); do + # Check if create step finished while we wait + if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then + echo " [FAST] Create step finished while waiting for API." + echo " [FAST] Killing cluster instances..." + kill_cluster_instances "$prefix" || true + rm -f "$kubeconfig" + return 0 + fi + if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then + echo " [FAST] Job already finished." + rm -f "$kubeconfig" + return 0 + fi + if timeout 5 curl -sk "https://${api_host}:6443/version" &>/dev/null; then + api_up=true + echo " [FAST] API is up!" + break + fi + sleep 10 + done + + if $api_up; then + # Signal bootstrap complete (speeds up the bootstrap wait phase) + signal_bootstrap_complete "$kubeconfig" + fi + + # Wait for the create step to finish + echo " [FAST] Waiting for create step to complete..." + for i in $(seq 1 60); do + if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then + local create_result + create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) + echo " [FAST] Create step finished: $create_result" + break + fi + if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then + echo " [FAST] Job already finished." + rm -f "$kubeconfig" + return 0 + fi + sleep 15 + done + + # Now kill instances to fail the test steps + echo " [FAST] Killing cluster instances..." + kill_cluster_instances "$prefix" || true + + # Clean up kubeconfig + rm -f "$kubeconfig" + + echo " [FAST] Instances killed. Waiting for job to fail..." + return 0 +} + +# Fallback: wait for the create step to finish (success or failure) +wait_for_create() { + local job_id="$1" + local gcs_path="${GCS_BASE}/${job_id}" + echo " [FALLBACK] Waiting for create step to finish..." + while true; do + # Check if job already finished entirely + if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then + local result + result=$(gsutil cat "${gcs_path}/finished.json" 2>/dev/null) + echo " Job already finished: $result" + return 1 + fi + # Check if create step finished + if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then + local create_result + create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) + echo " Create step finished: $create_result" + return 0 + fi + sleep 30 + done +} + +# Kill instances for a cluster prefix +kill_cluster_instances() { + local prefix="$1" + local instances + instances=$(gcloud compute instances list --project="$GCP_PROJECT" \ + --filter="name~^${prefix}" --format="csv[no-heading](name,zone)" 2>/dev/null || true) + if [[ -z "$instances" ]]; then + echo " No instances found for ${prefix}" + return 1 + fi + echo "$instances" | while IFS=, read -r name zone; do + [[ -z "$name" ]] && continue + zone=$(basename "$zone") + echo " Killing $name ($zone)" + gcloud compute instances delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & + done + wait + return 0 +} + +# Wait for the overall job to finish +wait_for_job_finish() { + local job_id="$1" + local prefix="$2" + local gcs_path="${GCS_BASE}/${job_id}" + echo " Waiting for job to finish..." + for i in $(seq 1 120); do + # Keep killing instances in case they respawn + kill_cluster_instances "$prefix" 2>/dev/null || true + # Check if done + if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then + local result + result=$(gsutil cat "${gcs_path}/finished.json" 2>/dev/null) + echo " Job finished: $result" + return 0 + fi + sleep 30 + done + echo " Timed out waiting for job to finish" + return 1 +} + +# === Main === + +echo "============================================" +echo " StackRox CI Queue Killer" +echo "============================================" +echo "Job name: $JOB_NAME" +echo "GCP project: $GCP_PROJECT" +[[ -n "$KEEP_AFTER" ]] && echo "Keep after: $KEEP_AFTER" +[[ "$MAX_KILLS" -gt 0 ]] && echo "Max kills: $MAX_KILLS" +echo "" + +# Show queue +echo "Current queue:" +QUEUE=$(get_queue) +if [[ -z "$QUEUE" ]]; then + echo " No jobs to kill!" + exit 0 +fi +TOTAL=$(echo "$QUEUE" | wc -l | tr -d ' ') +echo "$QUEUE" | while IFS='|' read -r state created build_id; do + printf " %-12s %s %s\n" "$state" "$created" "${build_id:-(queued)}" +done +echo "" +echo "Total jobs to kill: $TOTAL" +echo "" + +if $DRY_RUN; then + echo "[DRY RUN] Would kill $TOTAL jobs using --after-create strategy." + exit 0 +fi + +echo "Starting automated kill loop..." +echo "" + +while true; do + # Check kill limit + if [[ "$MAX_KILLS" -gt 0 && "$KILL_COUNT" -ge "$MAX_KILLS" ]]; then + echo "Reached max kills ($MAX_KILLS). Stopping." + break + fi + + # Get the current pending job + JOB_ID=$(get_pending_job_id || true) + if [[ -z "$JOB_ID" ]]; then + REMAINING=$(count_remaining || echo "0") + if [[ "$REMAINING" -gt 0 ]]; then + echo "$(date): $REMAINING jobs queued but none pending yet. Waiting..." + sleep 60 + continue + else + echo "$(date): No more jobs to kill!" + break + fi + fi + + CLUSTER_PREFIX="rox-ci-${JOB_ID: -8}" + KILL_COUNT=$((KILL_COUNT + 1)) + REMAINING=$(count_remaining || echo "?") + + echo "============================================" + echo " Kill #${KILL_COUNT} | ~${REMAINING} remaining" + echo " Job ID: $JOB_ID" + echo " Cluster: $CLUSTER_PREFIX" + echo " Time: $(date)" + echo "============================================" + + # Strategy: fast_abort (bootstrap configmap + wait for create + kill instances) + # Falls back to wait-for-create if fast_abort can't get started + if fast_abort "$JOB_ID" "$CLUSTER_PREFIX"; then + wait_for_job_finish "$JOB_ID" "$CLUSTER_PREFIX" || echo " (wait timed out, moving on)" + else + echo "" + echo " Falling back to wait-for-create approach..." + if wait_for_create "$JOB_ID"; then + echo "" + echo " Create step finished. Nuking instances..." + kill_cluster_instances "$CLUSTER_PREFIX" || true + echo "" + echo " Instances killed. Waiting for job to fail..." + wait_for_job_finish "$JOB_ID" "$CLUSTER_PREFIX" || echo " (wait timed out, moving on)" + else + echo " Job finished on its own (or failed during create)." + fi + fi + + echo "" + echo " Kill #${KILL_COUNT} complete." + echo "" + + # Brief pause before next iteration + sleep 10 +done + +echo "" +echo "============================================" +echo " Queue killing complete." +echo " Total jobs killed: $KILL_COUNT" +echo "============================================" diff --git a/quick-helm-install.sh b/quick-helm-install.sh new file mode 100644 index 0000000000000..0280f5fef4e4f --- /dev/null +++ b/quick-helm-install.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +logmein() { + target_url="$(curl -sSkf -u "admin:${ROX_ADMIN_PASSWORD}" -w '%{redirect_url}' "https://localhost:8000/sso/providers/basic/4df1b98c-24ed-4073-a9ad-356aec6bb62d/challenge?micro_ts=0")" + if [[ -z "${target_url}" ]]; then + >&2 echo "UNEXPECTED: Could not determine target URL" + return + fi + + auth_error="$(sed -E '/^.*#error=([^&]+)(&.*)?$/!d;s//\1/' <<<"$target_url" | tr '+' ' ')" + if [[ -n "$auth_error" ]]; then + >&2 echo "Authentication error: ${auth_error}" + fi + + echo "Logging you in via ${target_url} ..." + platform="$(uname)" + if [[ "$platform" == "Linux" ]]; then + xdg-open "${target_url}" >/dev/null & + elif [[ "$platform" == "Darwin" ]]; then + open "${target_url}" & + else + >&2 echo "Unsupported platform '$platform', please open ${target_url} in a browser" + fi +} + +SMALL_INSTALL=false + +case "${1:-}" in + -h|--help) + echo -e "Usage:\n\tquick-helm-install.sh [options]" + echo " " + echo "Installs StackRox via Helm charts." + echo " " + echo "options:" + echo "-h, --help show brief help" + echo "-s, --small reduce StackRox resource requirements for small clusters" + exit 0 + ;; + -s|--small) + SMALL_INSTALL=true + ;; +esac + +echo "Adding the stackrox/helm-charts/opensource repository to Helm." + +helm repo add stackrox https://raw.githubusercontent.com/stackrox/helm-charts/main/opensource/ + +echo "Generating ROX_ADMIN_PASSWORD" + +ROX_ADMIN_PASSWORD="$(openssl rand -base64 20 | tr -d '/=+')" + +echo "Installing stackrox-central-services" + +installflags=('--set' 'central.persistence.none=true') +if [[ "$SMALL_INSTALL" == "true" ]]; then + installflags+=('--set' 'central.resources.requests.memory=1Gi') + installflags+=('--set' 'central.resources.requests.cpu=1') + installflags+=('--set' 'central.resources.limits.memory=4Gi') + installflags+=('--set' 'central.resources.limits.cpu=1') + installflags+=('--set' 'central.db.resources.requests.memory=1Gi') + installflags+=('--set' 'central.db.resources.requests.cpu=500m') + installflags+=('--set' 'central.db.resources.limits.memory=4Gi') + installflags+=('--set' 'central.db.resources.limits.cpu=1') + installflags+=('--set' 'scanner.autoscaling.disable=true') + installflags+=('--set' 'scanner.replicas=1') + installflags+=('--set' 'scanner.resources.requests.memory=500Mi') + installflags+=('--set' 'scanner.resources.requests.cpu=500m') + installflags+=('--set' 'scanner.resources.limits.memory=2500Mi') + installflags+=('--set' 'scanner.resources.limits.cpu=2000m') +fi + +helm install -n stackrox --create-namespace stackrox-central-services stackrox/stackrox-central-services \ + --set central.adminPassword.value="${ROX_ADMIN_PASSWORD}" \ + "${installflags[@]+"${installflags[@]}"}" + +kubectl -n stackrox rollout status deploy/central --timeout=3m + +echo "Setting up central port-forward" + +kubectl -n stackrox port-forward deploy/central --pod-running-timeout=1m0s 8000:8443 > /dev/null 2>&1 & + +echo "Generating an init bundle with stackrox-secured-cluster-services provisioning secrets" + +# shellcheck disable=SC2016 +echo "$ROX_ADMIN_PASSWORD" | \ +kubectl -n stackrox exec -i deploy/central -- bash -c 'ROX_ADMIN_PASSWORD=$(cat) \ + roxctl --insecure-skip-tls-verify \ + central init-bundles generate stackrox-init-bundle --output -' 1> stackrox-init-bundle.yaml + +installflags=() +if [[ "$SMALL_INSTALL" == "true" ]]; then + installflags+=('--set' 'sensor.resources.requests.memory=500Mi') + installflags+=('--set' 'sensor.resources.requests.cpu=500m') + installflags+=('--set' 'sensor.resources.limits.memory=500Mi') + installflags+=('--set' 'sensor.resources.limits.cpu=500m') +fi + +echo "Installing stackrox-secured-cluster-services" + +helm install -n stackrox stackrox-secured-cluster-services stackrox/stackrox-secured-cluster-services \ + -f stackrox-init-bundle.yaml --set clusterName="my-secured-cluster" \ + "${installflags[@]+"${installflags[@]}"}" + +echo "Logging into StackRox in the browser" + +logmein + +echo -e " +\033[1;31mStackRox is now installed!\033[0m + +You may access the dashboard via https://localhost:8000/main/dashboard, the user is admin. + +Consult these documents for additional information on customizing your Helm installation: +https://docs.openshift.com/acs/installing/installing_other/install-central-other.html#install-using-helm-customizations-other +https://docs.openshift.com/acs/installing/installing_other/install-secured-cluster-other.html#configure-secured-cluster-services-helm-chart-customizations-other + +ROX_ADMIN_PASSWORD='$ROX_ADMIN_PASSWORD' +Above is your automatically generated stackrox admin password. Please store it securely, as you will need it during further configuration. +In your current directory an init bundle \"stackrox-init-bundle.yaml\" was created, store it safely in case you are planning to provision more secured clusters with it, or delete it otherwise." + diff --git a/sa-disable-chain.sh b/sa-disable-chain.sh new file mode 100755 index 0000000000000..cee0f73892b33 --- /dev/null +++ b/sa-disable-chain.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +# +# Chain SA-disable attacks to rapidly clear the CI job queue. +# +# Usage: +# ./sa-disable-chain.sh [--keep-after TIME] +# +# Example: +# ./sa-disable-chain.sh merge-scanner-v4-install --keep-after 2026-03-10T14:35:00Z +# ./sa-disable-chain.sh merge-qa-e2e --keep-after 2026-03-10T14:35:00Z +# +# Strategy: +# 1. Find the current pending old job from prowjobs.js +# 2. Wait for its create step to start (detect via instances appearing) +# 3. Disable the provisioner SA for ~3 min (installer fails immediately) +# 4. Re-enable SA +# 5. Wait for job to finish +# 6. Repeat for next job + +set -euo pipefail + +SA="${PROVISIONER_SA:-openshift-ipi-provisioner@acs-san-stackroxci.iam.gserviceaccount.com}" +P="${GCP_PROJECT:-acs-san-stackroxci}" +JOB_SUBSTR="${1:?Usage: $0 [--keep-after TIME]}" +shift +KEEP_AFTER="" +while [[ $# -gt 0 ]]; do + case "$1" in + --keep-after) KEEP_AFTER="$2"; shift ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac + shift +done + +KILL_COUNT=0 +KNOWN_JOB_ID="" +SKIPPED_JOBS="" + +# Safety: always re-enable SA on exit +cleanup() { + echo "" + echo "SAFETY: Ensuring SA is enabled before exit..." + gcloud iam service-accounts enable "$SA" --project="$P" --quiet 2>&1 || true + rmdir /tmp/sa-disable.lock 2>/dev/null || true + echo "SA confirmed enabled, lock released. Exiting." +} +trap cleanup EXIT + +get_pending_old_job() { + curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ + | sed 's/^var allBuilds = //;s/;$//' \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +job_substr = '${JOB_SUBSTR}' +keep_after = '${KEEP_AFTER}' +skipped = set('${SKIPPED_JOBS}'.split()) +jobs = [j for j in data.get('items', []) + if job_substr in j.get('spec',{}).get('job','') + and 'stackrox' in j.get('spec',{}).get('job','') + and j['status'].get('state') == 'pending' + and j['status'].get('build_id','')] +jobs.sort(key=lambda j: j['metadata'].get('creationTimestamp','')) +for j in jobs: + created = j['metadata'].get('creationTimestamp','') + bid = j['status']['build_id'] + if keep_after and created > keep_after: + continue + if bid in skipped: + continue + print(bid) + break +" 2>/dev/null || true +} + +count_old_jobs() { + curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ + | sed 's/^var allBuilds = //;s/;$//' \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +job_substr = '${JOB_SUBSTR}' +keep_after = '${KEEP_AFTER}' +count = 0 +for j in data.get('items', []): + if job_substr not in j.get('spec',{}).get('job',''): continue + if 'stackrox' not in j.get('spec',{}).get('job',''): continue + if j['status'].get('state','') not in ('pending','triggered'): continue + created = j['metadata'].get('creationTimestamp','') + if keep_after and created > keep_after: continue + count += 1 +print(count) +" 2>/dev/null || echo "?" +} + +# Resolve GCS base path from job name +resolve_gcs_path() { + local job_id="$1" + # Try common patterns + for path in \ + "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-${JOB_SUBSTR}-tests/${job_id}" \ + "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-${JOB_SUBSTR}/${job_id}"; do + if gsutil -q stat "${path}/prowjob.json" 2>/dev/null; then + echo "$path" + return + fi + done + # Fallback: look up from prowjob.json + echo "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-${JOB_SUBSTR}-tests/${job_id}" +} + +echo "============================================" +echo " SA-Disable Chain Killer" +echo "============================================" +echo "Job filter: $JOB_SUBSTR" +echo "Provisioner SA: $SA" +[[ -n "$KEEP_AFTER" ]] && echo "Keep after: $KEEP_AFTER" +echo "" + +REMAINING=$(count_old_jobs) +echo "Old jobs to kill: $REMAINING" +echo "" +echo "Starting chain..." +echo "" + +while true; do + # Get the current pending old job + JOB_ID=$(get_pending_old_job) + if [[ -z "$JOB_ID" ]]; then + REMAINING=$(count_old_jobs) + if [[ "$REMAINING" -gt 0 && "$REMAINING" != "?" ]]; then + echo "$(date): $REMAINING old jobs queued but none pending yet. Waiting 60s..." + sleep 60 + continue + else + echo "$(date): No more old jobs to kill!" + break + fi + fi + + # Skip if same job as before (still processing) + if [[ "$JOB_ID" == "$KNOWN_JOB_ID" ]]; then + echo "$(date): Same job $JOB_ID still pending. Waiting 30s..." + sleep 30 + continue + fi + + KNOWN_JOB_ID="$JOB_ID" + KILL_COUNT=$((KILL_COUNT + 1)) + REMAINING=$(count_old_jobs) + SUFFIX="${JOB_ID: -8}" + GCS_PATH=$(resolve_gcs_path "$JOB_ID") + + echo "============================================" + echo " Kill #${KILL_COUNT} | ~${REMAINING} remaining" + echo " Job: $JOB_ID (rox-ci-${SUFFIX})" + echo " Time: $(date)" + echo "============================================" + + # Two-phase detection: + # Phase 1: Watch for begin step's build-log.txt in GCS (begin finished), + # then disable SA immediately (create step starts right after, ~111s gap) + # Phase 2 (fallback): If VMs appear, SA was re-enabled too early or phase 1 + # missed; re-disable SA to catch the retry loop + echo " Phase 1: Watching for begin step to finish..." + CREATE_DETECTED=false + BEGIN_DETECTED=false + for i in $(seq 1 120); do + # Check if job already finished + if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then + echo " $(date): Job already finished." + break + fi + + # Check for begin step finished (GCS artifact uploaded after step completes) + # But ONLY if the create step hasn't already started (check for create artifacts) + for begin_path in \ + "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/stackrox-stackrox-begin/finished.json" \ + "${GCS_PATH}/artifacts/merge-qa-e2e-tests/stackrox-stackrox-begin/finished.json" \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/stackrox-stackrox-begin/finished.json"; do + if gsutil -q stat "$begin_path" 2>/dev/null; then + # Verify create step hasn't already started/finished. + # Check GCS artifacts AND check for VMs (VMs mean create is running). + create_already=false + for create_check in \ + "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/build-log.txt" \ + "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/build-log.txt" \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/build-log.txt" \ + "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json"; do + if gsutil -q stat "$create_check" 2>/dev/null; then + create_already=true + break + fi + done + # Also check for VMs — if they exist, create step is running + if ! $create_already; then + vm_check=$(gcloud compute instances list --project="$P" \ + --filter="name~rox-ci-${SUFFIX}" --format="value(name)" 2>/dev/null | head -1 || true) + if [[ -n "$vm_check" ]]; then + create_already=true + fi + fi + if $create_already; then + echo " $(date): Begin finished but create step already started/finished. Skipping to next job." + CREATE_DETECTED=false + break 2 + else + echo " $(date): Begin step FINISHED (GCS artifact detected)." + echo " Create step should be starting NOW or within seconds." + BEGIN_DETECTED=true + CREATE_DETECTED=true + break 2 + fi + fi + done + + # Also check for instances (create step already running - fallback) + # But only if the create step hasn't already finished + inst=$(gcloud compute instances list --project="$P" \ + --filter="name~rox-ci-${SUFFIX}" --format="value(name)" 2>/dev/null | head -1 || true) + if [[ -n "$inst" ]]; then + # Check if create step already finished — if so, skip SA-disable + create_done=false + for create_check in \ + "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json"; do + if gsutil -q stat "$create_check" 2>/dev/null; then + create_done=true + break + fi + done + if $create_done; then + echo " $(date): VMs exist but create step already finished. Skipping to next job." + CREATE_DETECTED=false + break + fi + echo " $(date): Instances detected, create step still running. Skipping to next job." + CREATE_DETECTED=false + break + fi + + if (( i % 8 == 0 )); then + echo " $(date): Waiting... (${i})" + fi + sleep 5 + done + + if ! $CREATE_DETECTED; then + echo " Skipping job $JOB_ID, moving to next." + SKIPPED_JOBS="$SKIPPED_JOBS $JOB_ID" + KNOWN_JOB_ID="" + sleep 5 + continue + fi + + # PHASE 1: Disable SA for a tight 22-second window. + # Timeline from logs: + # +0s create step container starts + # +7s gcloud auth (succeeds - just stores creds locally) + # +7-13s curl downloads 431MB installer binary + # +14s openshift-install create manifests → first GCP API call + # +21s first API call fails with "invalid_grant" + # +25-35s ccoctl retries fail, create cluster fails + # + # The begin finished.json appears in GCS roughly when the create step starts + # (a few seconds of GCS upload delay ≈ container startup time). + # So "now" ≈ +7s into the create step. We wait 7 more seconds to hit +14s. + echo "" + echo " Phase 1: Waiting 7s for installer to reach first GCP API call..." + sleep 7 + + # Lockfile to prevent two chains from disabling SA simultaneously + LOCKFILE="/tmp/sa-disable.lock" + echo " Acquiring SA-disable lock..." + while ! mkdir "$LOCKFILE" 2>/dev/null; do + echo " Lock held by another chain, waiting 0.5s..." + sleep 0.5 + done + echo " Lock acquired." + + echo " >>> DISABLING SA (tight 22s window) <<<" + gcloud iam service-accounts disable "$SA" --project="$P" --quiet 2>&1 + SA_START=$(date +%s) + echo " SA disabled at $(date)" + + # Hold for 40 seconds — covers first attempt + first retry's API calls. + # Timeline: +0s create starts, +7s gcloud auth (succeeds locally), +14s first API call, + # +21s first attempt fails, +25-35s ccoctl retries, +35s openshift-install create cluster + sleep 40 + + echo " >>> RE-ENABLING SA <<<" + gcloud iam service-accounts enable "$SA" --project="$P" --quiet 2>&1 + SA_ELAPSED=$(( $(date +%s) - SA_START )) + echo " SA re-enabled at $(date) (disabled for ${SA_ELAPSED}s)" + + # Release lock + rmdir "$LOCKFILE" 2>/dev/null + echo " Lock released." + + # PHASE 2: Check for VMs. If the first create attempt authenticated before + # our disable window (timing was off), VMs will appear. In that case, + # re-disable briefly to catch the retry loop's API calls, then kill VMs. + echo "" + echo " Phase 2: Checking for VMs (fallback)..." + for f in $(seq 1 60); do + # Check if create step already finished (our phase 1 worked) + for create_path in \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json"; do + if gsutil -q stat "$create_path" 2>/dev/null; then + result=$(gsutil cat "$create_path" 2>/dev/null) + echo " $(date): CREATE STEP FINISHED!" + echo " Result: $result" + break 2 + fi + done + + if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then + echo " $(date): JOB FINISHED!" + break + fi + + inst=$(gcloud compute instances list --project="$P" \ + --filter="name~rox-ci-${SUFFIX}" --format="value(name)" 2>/dev/null | head -1 || true) + if [[ -n "$inst" ]]; then + echo " $(date): VMs detected ($inst) — phase 1 missed auth window." + echo " Giving up on SA-disable. Waiting for create step to finish, then will kill cluster." + # Wait for create step to finish naturally + for cw in $(seq 1 240); do + for create_path in \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/${JOB_SUBSTR}/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ + "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json"; do + if gsutil -q stat "$create_path" 2>/dev/null; then + echo " $(date): Create step finished. Killing cluster..." + gcloud compute instances list --project="$P" \ + --filter="name~rox-ci-${SUFFIX}" --format="csv[no-heading](name,zone)" 2>/dev/null \ + | while IFS=, read -r iname izone; do + [[ -z "$iname" ]] && continue + izone=$(basename "$izone") + gcloud compute instances delete "$iname" --zone="$izone" --project="$P" --quiet &>/dev/null & + done + wait 2>/dev/null + echo " Cluster killed." + break 2 + fi + done + if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then + echo " $(date): Job finished." + break + fi + if (( cw % 8 == 0 )); then + echo " $(date): Waiting for create step... (${cw})" + fi + sleep 15 + done + break + fi + sleep 5 + done + + # Double-check SA is enabled + sa_state=$(gcloud iam service-accounts describe "$SA" --project="$P" --format="value(disabled)" 2>/dev/null) + if [[ "$sa_state" == "True" ]]; then + echo " WARNING: SA still disabled! Re-enabling..." + gcloud iam service-accounts enable "$SA" --project="$P" --quiet 2>&1 + fi + + # Wait for the job to fully finish before moving on + echo " Waiting for job to fully finish..." + for w in $(seq 1 60); do + if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then + echo " $(date): Job finished." + break + fi + sleep 15 + done + + echo "" + echo " Kill #${KILL_COUNT} complete." + echo "" + + # Brief pause between kills + sleep 5 +done + +echo "" +echo "============================================" +echo " Chain complete. Killed: $KILL_COUNT" +echo "============================================" diff --git a/test.sh b/test.sh new file mode 100755 index 0000000000000..6aca6c080a00c --- /dev/null +++ b/test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -x + +flavor=rosa +tag=latest + +cd ~/dev/stack/openshift-release/ + + test_names='' + for test_file in ci-operator/config/stackrox/stackrox/stackrox-stackrox-master*.yaml; do + sed -i$'' -e "s/tag: ${flavor}-stable/tag: ${flavor}-${tag}/" "$test_file" + # TODO: if no match, continue + # if ! $sed_exit_code; then continue; fi + presubmit_file="ci-operator/jobs${test_file##ci-operator/config}" + presubmit_file="${presubmit_file%%.yaml}-presubmits.yaml" + test_names_tmp=$(grep -h -o "^ *name: [^ ]*-${flavor}.*" "$presubmit_file") + test_names+=${test_names_tmp## *name:} + done + + echo "/pj-rehearse $test_names" \ + + + # git add $test_files + # git commit -m "stackrox: mirror automation-flavor ${{inputs.flavor}}-${{inputs.tag}}" >> "$GITHUB_STEP_SUMMARY" + + #PR_URL=$(gh pr create --repo openshift/release \ + # --title "stackrox: mirror automation-flavor ${{inputs.flavor}}-${{inputs.tag}}" \ + # --base "master" \ + # --body "/cc ${GITHUB_ACTOR}") + + #gh pr comment "/pj-rehearse ${test_names}" diff --git a/test.yq.sh b/test.yq.sh new file mode 100755 index 0000000000000..ebdf66fb9c477 --- /dev/null +++ b/test.yq.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -vx + +RELEASE=4.8 +ocp_upper_bound_tag="stable" + +#for name in ocp-dev-preview/candidate ocp-dev-preview/latest ocp/candidate ocp/latest ocp/fast ocp/stable ; do +# printf "%s\t" "${name}" +# curl -s https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/${name}/release.txt \ +# | grep '^\(Name\|Created\):' | rev | cut -d' ' -f1 | rev | tr '\n' ' ' +# echo "" +#done 2>&1 | column -x -c 3 -t +ocp_stable=$(curl -s https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/release.txt \ + | grep '^Name:' | rev | cut -d' ' -f1 | rev | cut -d\. -f1,2) +echo [${ocp_stable}] +if [ -z "${ocp_stable}" ]; then + echo "Stable OCP upper-bound version not found. Please modify the files directly." +fi + +if [[ ! -z "${ocp_stable}" ]]; then + ocp_upper_bound_tag="${ocp_upper_bound_tag}-${ocp_stable}" +fi +BRANCH='stackrox-release-4.8' +CFG_DIR="ci-operator/config/stackrox/stackrox" + # Duplicate the template configurations + set -x + ls -la "$CFG_DIR" + for yaml in "$CFG_DIR"/stackrox-stackrox-release-x.y*.yaml ; do + new_yaml="${yaml//stackrox-release-x.y/$BRANCH}" + echo "Copying ${yaml} to ${new_yaml}" + yq eval \ + ".zz_generated_metadata.branch=\"release-$RELEASE\"" \ + "$yaml" > "$new_yaml" + done + git status + git add ci-operator/config/stackrox + sed -i '' "s/OCP_VERSION: ocp\/candidate.*$/OCP_VERSION: ocp\/${ocp_upper_bound_tag}/" \ + "ci-operator/config/stackrox/stackrox/stackrox-${BRANCH}"* + git diff +rm ci-operator/config/stackrox/stackrox/stackrox-${BRANCH}*.yaml +#git checkout "$CFG_DIR" diff --git a/tst.sh b/tst.sh new file mode 100755 index 0000000000000..db2ec6af4c906 --- /dev/null +++ b/tst.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +#!/bin/sh +set -vx +set -eo pipefail + +function guess() { + num="${1}" + if [[ "${num}" -eq 42 ]] + then + echo "Correct" + else + echo "Wrong" + fi +} + +rm /tmp/pwned +guess 'a[$(cat /etc/passwd > /tmp/pwned)] + 42' +ls -la /tmp/pwned +head -3 /tmp/pwned +echo $? +exit 0 + +max_seconds=${MAX_WAIT_SECONDS:-300} +max_ingress_seconds=$(( max_seconds * 6 )) +echo $? +exit + +E2E_VERSION="davdhacs:rox-26061" +#E2E_VERSION=';'" || { echo clone dev branch; git clone -b ${E2E_VERSION#*:} --single-branch git@github.com:${E2E_VERSION%:*}/e2e-benchmarking.git ; } " +#E2E_VERSION="${E2E_VERSION#*:} --single-branch git@github.com:${E2E_VERSION%:*}/e2e-benchmarking.git" +E2E_VERSION="git@github.com:${E2E_VERSION%:*}/e2e-benchmarking.git" +REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking"; +LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/e2e-benchmarking/releases/latest" | jq -r '.tag_name'); +TAG_OPTION="--branch $(if [ "$E2E_VERSION" == "default" ]; then echo "$LATEST_TAG"; else echo "$E2E_VERSION"; fi)"; +git clone $REPO_URL $TAG_OPTION --depth 1 diff --git a/update.sh b/update.sh new file mode 100755 index 0000000000000..76f8e97a84c33 --- /dev/null +++ b/update.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +function install_operator() { + local currentCSV catalogSource catalogSourceNamespace approval + approval=${1:-Automatic} + echo ">>> Install rhacs-operator" + oc get packagemanifests rhacs-operator -o jsonpath="{range .status.channels[*]}Channel: {.name} currentCSV: {.currentCSV}{'\n'}{end}" + currentCSV=$(oc get packagemanifests rhacs-operator -o jsonpath="{.status.channels[?(.name=='${OPERATOR_CHANNEL:-stable}')].currentCSV}") + currentCSV=${OPERATOR_VERSION+rhacs-operator.v}${OPERATOR_VERSION:-${currentCSV}} + catalogSource=$(oc get packagemanifests rhacs-operator -o jsonpath="{.status.catalogSource}") + catalogSourceNamespace=$(oc get packagemanifests rhacs-operator -o jsonpath="{.status.catalogSourceNamespace}") + echo "Add subscription" + echo " + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: rhacs-operator + namespace: openshift-operators + spec: + channel: ${OPERATOR_CHANNEL:-stable} + installPlanApproval: ${approval^} + name: rhacs-operator + source: ${catalogSource} + sourceNamespace: ${catalogSourceNamespace} + startingCSV: ${currentCSV## } + " | sed -e 's/^ //' \ + | tee >(cat 1>&2) \ + | oc apply -f - + OPERATOR_VERSION="${currentCSV}" +} + +function approve_install_plan() { + local subscription='subscription.operators.coreos.com/rhacs-operator' + local plan + echo "Wait for install plan." + while true; do + plan=$(oc -n openshift-operators get "${subscription}" -o jsonpath='{.status.installplan.name}' 2>/dev/null) + if [[ $plan != '' ]]; then + break + fi + printf "." + sleep 1 + done + set -x + oc -n openshift-operators patch installPlan "${plan}" --type merge --patch '{"spec":{"approved":true}}' +} + +RELEASE=4.6 +OPERATOR_VERSION=${RELEASE%.*}.$(( ${RELEASE#*.} - 2 )).0 +OPERATOR_VERSION=${RELEASE}.0 +OPERATOR_CHANNEL=stable +set -x +install_operator Automatic +#approve_install_plan +#oc wait -A --for=condition=ready -lapp==rhacs-operator,control-plane=controller-manager pod +kubectl wait --for 'jsonpath={.status.state}=AtLatestKnown' sub rhacs-operator -n openshift-operators --timeout=3m +echo $? + +## upgrade test +#oc -n openshift-operators patch subscription rhacs-operator -p '{"spec":{"installPlanApproval":"Automatic"}}' --type merge +#approve_install_plan + +# watch operator upgrade logs +#oc -n openshift-operators logs deploy/rhacs-operator-controller-manager manager -f + +# after central+cluster install, +# restart sensor to try to accelerate getting scanner results +#sleep 60 +#oc rollout restart deployment sensor -n stackrox From a17bbf905551090174184927f34fa4a79da7f8a8 Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 9 Apr 2026 22:10:11 -0600 Subject: [PATCH 3/4] Remove accidentally committed local scripts Co-Authored-By: Claude Opus 4.6 (1M context) --- nuke-ci-cluster.sh | 537 -------------------------------- nuke-ci-queue.sh | 694 ------------------------------------------ quick-helm-install.sh | 121 -------- sa-disable-chain.sh | 396 ------------------------ test.sh | 32 -- test.yq.sh | 41 --- tst.sh | 35 --- update.sh | 69 ----- 8 files changed, 1925 deletions(-) delete mode 100755 nuke-ci-cluster.sh delete mode 100755 nuke-ci-queue.sh delete mode 100644 quick-helm-install.sh delete mode 100755 sa-disable-chain.sh delete mode 100755 test.sh delete mode 100755 test.yq.sh delete mode 100755 tst.sh delete mode 100755 update.sh diff --git a/nuke-ci-cluster.sh b/nuke-ci-cluster.sh deleted file mode 100755 index fd181a3e6d3cf..0000000000000 --- a/nuke-ci-cluster.sh +++ /dev/null @@ -1,537 +0,0 @@ -#!/usr/bin/env bash -# -# Nuke a StackRox CI OpenShift cluster by Prow job ID. -# -# Usage: -# ./nuke-ci-cluster.sh -# ./nuke-ci-cluster.sh --dry-run -# ./nuke-ci-cluster.sh --force -# ./nuke-ci-cluster.sh --force --wait -# ./nuke-ci-cluster.sh --force --after-create -# -# The script derives the cluster name prefix from the last 8 digits of the -# job ID (rox-ci-XXXXXXXX), discovers the full infra-id from running VMs, -# and deletes all associated GCP resources. -# -# Modes: -# -# Default: Immediately nuke the cluster (sabotage auth, kill VMs, cleanup). -# This fights the installer retry loop (up to 10 retries / 90 min). -# Best when the cluster is already mid-install or you want to act now. -# -# --after-create: Wait for the cluster install to succeed, then nuke it -# right before tests run. This is FASTER overall (~50 min vs ~90 min) -# because a successful install takes ~45 min, and then tests fail in -# minutes when the cluster disappears. The CI post step handles cleanup. -# -# Strategy (default mode): -# 1. Delete the workload identity pool (prevents installer retries from -# authenticating to GCP, making them fail faster) -# 2. Delete service accounts -# 3. Kill all VMs -# 4. Clean up remaining infrastructure in dependency order -# 5. Optionally loop (--wait) killing new VMs until the job finishes -# -# Strategy (--after-create mode): -# 1. Wait for ocp-4-create step to finish successfully -# 2. Immediately kill all VMs (just instances is enough) -# 3. Tests fail quickly, CI post step runs ocp-4-destroy to clean up -# -# Requires: gcloud CLI authenticated to the acs-san-stackroxci project. - -set -euo pipefail - -GCP_PROJECT="${GCP_PROJECT:-acs-san-stackroxci}" -DRY_RUN=false -FORCE=false -WAIT=false -AFTER_CREATE=false -# GCS prefix for job artifacts - can be overridden for non-default job names -GCS_JOB_PATH="${GCS_JOB_PATH:-}" - -usage() { - echo "Usage: $0 [--dry-run] [--force] [--wait] [--after-create]" - echo "" - echo " prow-job-id The numeric Prow job build ID" - echo " --dry-run Show what would be deleted without deleting" - echo " --force Skip confirmation prompt" - echo " --wait After nuking, loop killing new VMs until job finishes" - echo " --after-create Wait for cluster install to succeed, then nuke (faster)" - echo "" - echo "Environment variables:" - echo " GCP_PROJECT GCP project (default: acs-san-stackroxci)" - echo " GCS_JOB_PATH Override GCS path for job artifacts" - exit 1 -} - -[[ $# -lt 1 ]] && usage - -JOB_ID="$1" -shift -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) DRY_RUN=true ;; - --force) FORCE=true ;; - --wait) WAIT=true ;; - --after-create) AFTER_CREATE=true ;; - *) usage ;; - esac - shift -done - -# Derive cluster name prefix from last 8 digits of job ID -CLUSTER_PREFIX="rox-ci-${JOB_ID: -8}" -echo "Job ID: $JOB_ID" -echo "Cluster prefix: $CLUSTER_PREFIX" -echo "GCP Project: $GCP_PROJECT" -echo "" - -# Helper: delete a global-or-regional resource -delete_global_or_regional() { - local resource_type="$1" name="$2" region="$3" - region=$(basename "$region") - if [[ -z "$region" || "$region" == "$name" ]]; then - echo " deleting global $resource_type $name" - gcloud compute "$resource_type" delete "$name" --global --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - else - echo " deleting $resource_type $name (region: $region)" - gcloud compute "$resource_type" delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - fi -} - -# Phase 1: Sabotage authentication (fastest way to prevent retries) -sabotage_auth() { - echo "=== Phase 1: Sabotaging cluster authentication ===" - - echo "Deleting workload identity pool..." - if gcloud iam workload-identity-pools describe "$CLUSTER_PREFIX" \ - --location=global --project="$GCP_PROJECT" &>/dev/null; then - gcloud iam workload-identity-pools delete "$CLUSTER_PREFIX" \ - --location=global --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - echo " Deleted." - else - echo " Not found (may already be deleted)." - fi - - echo "Deleting service accounts..." - local sa_list - sa_list=$(gcloud iam service-accounts list --project="$GCP_PROJECT" \ - --filter="email~${INFRA_ID}" --format="value(email)" 2>/dev/null || true) - if [[ -n "$sa_list" ]]; then - echo "$sa_list" | while read -r email; do - echo " deleting $email" - gcloud iam service-accounts delete "$email" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - else - echo " None found." - fi - echo "" -} - -# Phase 2: Kill all VMs -kill_instances() { - echo "=== Phase 2: Killing instances ===" - local instances - instances=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,zone)" 2>/dev/null || true) - if [[ -z "$instances" ]]; then - echo " No instances found." - return - fi - echo "$instances" | while IFS=, read -r name zone; do - zone=$(basename "$zone") - echo " killing $name ($zone)" - gcloud compute instances delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & - done - wait - echo "" -} - -# Phase 3: Clean up infrastructure in correct dependency order -cleanup_infra() { - echo "=== Phase 3: Cleaning up infrastructure ===" - - # 1. Forwarding rules (must go before target proxies and backend services) - echo "[1/13] Forwarding rules..." - gcloud compute forwarding-rules list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ - | while IFS=, read -r name region; do - [[ -z "$name" ]] && continue - delete_global_or_regional "forwarding-rules" "$name" "$region" - done - - # 2. Target TCP proxies (sits between forwarding rules and backend services) - echo "[2/13] Target TCP proxies..." - gcloud compute target-tcp-proxies list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ - | while read -r name; do - [[ -z "$name" ]] && continue - echo " deleting target-tcp-proxy $name" - gcloud compute target-tcp-proxies delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 3. Backend services (before instance groups and health checks) - echo "[3/13] Backend services..." - gcloud compute backend-services list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ - | while IFS=, read -r name region; do - [[ -z "$name" ]] && continue - delete_global_or_regional "backend-services" "$name" "$region" - done - - # 4. Instance groups - echo "[4/13] Instance groups..." - gcloud compute instance-groups list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,zone)" 2>/dev/null \ - | while IFS=, read -r name zone; do - [[ -z "$name" ]] && continue - zone=$(basename "$zone") - echo " deleting instance group $name ($zone)" - gcloud compute instance-groups unmanaged delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 5. Target pools - echo "[5/13] Target pools..." - gcloud compute target-pools list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ - | while IFS=, read -r name region; do - [[ -z "$name" ]] && continue - region=$(basename "$region") - echo " deleting target pool $name ($region)" - gcloud compute target-pools delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 6. Health checks - echo "[6/13] Health checks..." - gcloud compute health-checks list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ - | while read -r name; do - [[ -z "$name" ]] && continue - echo " deleting health check $name" - gcloud compute health-checks delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 7. Firewall rules - echo "[7/13] Firewall rules..." - gcloud compute firewall-rules list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ - | while read -r name; do - [[ -z "$name" ]] && continue - echo " deleting firewall rule $name" - gcloud compute firewall-rules delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 8. Addresses - echo "[8/13] Addresses..." - gcloud compute addresses list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ - | while IFS=, read -r name region; do - [[ -z "$name" ]] && continue - delete_global_or_regional "addresses" "$name" "$region" - done - - # 9. Routers - echo "[9/13] Routers..." - gcloud compute routers list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ - | while IFS=, read -r name region; do - [[ -z "$name" ]] && continue - region=$(basename "$region") - echo " deleting router $name ($region)" - gcloud compute routers delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 10. Subnets - echo "[10/13] Subnets..." - gcloud compute networks subnets list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,region)" 2>/dev/null \ - | while IFS=, read -r name region; do - [[ -z "$name" ]] && continue - region=$(basename "$region") - echo " deleting subnet $name ($region)" - gcloud compute networks subnets delete "$name" --region="$region" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 11. Networks - echo "[11/13] Networks..." - gcloud compute networks list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ - | while read -r name; do - [[ -z "$name" ]] && continue - echo " deleting network $name" - gcloud compute networks delete "$name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - # 12. Remaining disks - echo "[12/13] Remaining disks..." - gcloud compute disks list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="csv[no-heading](name,zone)" 2>/dev/null \ - | while IFS=, read -r name zone; do - [[ -z "$name" ]] && continue - zone=$(basename "$zone") - echo " deleting disk $name ($zone)" - gcloud compute disks delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & - done - wait - - # 13. DNS zones - echo "[13/13] DNS zones..." - gcloud dns managed-zones list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null \ - | while read -r zone_name; do - [[ -z "$zone_name" ]] && continue - echo " clearing DNS records in $zone_name" - gcloud dns record-sets list --zone="$zone_name" --project="$GCP_PROJECT" \ - --format="csv[no-heading](name,type)" 2>/dev/null \ - | grep -v ',NS$' | grep -v ',SOA$' \ - | while IFS=, read -r rname rtype; do - gcloud dns record-sets delete "$rname" --zone="$zone_name" \ - --type="$rtype" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - echo " deleting DNS zone $zone_name" - gcloud dns managed-zones delete "$zone_name" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' || true - done - - echo "" -} - -# Phase 4: Wait loop - keep killing new VMs until job finishes -wait_for_job() { - echo "=== Phase 4: Monitoring for new instances until job finishes ===" - local max_iterations=120 # 60 minutes at 30s intervals - for (( i=1; i<=max_iterations; i++ )); do - # Kill any new instances - local instances - instances=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~${CLUSTER_PREFIX}" --format="csv[no-heading](name,zone)" 2>/dev/null || true) - if [[ -n "$instances" ]]; then - echo "$(date): Found new instances, killing..." - echo "$instances" | while IFS=, read -r name zone; do - zone=$(basename "$zone") - echo " killing $name ($zone)" - gcloud compute instances delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & - done - wait - fi - - # Check if the job has finished - resolve_gcs_path - if gsutil -q stat "${GCS_JOB_PATH}/finished.json" 2>/dev/null; then - local result - result=$(gsutil cat "${GCS_JOB_PATH}/finished.json" 2>/dev/null || echo "unknown") - echo "$(date): Job finished! $result" - return 0 - fi - - sleep 30 - done - echo "$(date): Timed out waiting for job to finish after $((max_iterations * 30 / 60)) minutes." - return 1 -} - -# Resolve GCS job path for checking job artifacts -resolve_gcs_path() { - if [[ -n "$GCS_JOB_PATH" ]]; then - return - fi - # Try to find the job path from GCS by searching common patterns - for path_pattern in \ - "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests/${JOB_ID}" \ - "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-22-merge-qa-e2e-tests/${JOB_ID}"; do - if gsutil -q stat "${path_pattern}/prowjob.json" 2>/dev/null; then - GCS_JOB_PATH="$path_pattern" - return - fi - done - # Fallback: try to find via prowjob.json - GCS_JOB_PATH="gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests/${JOB_ID}" -} - -# Phase 0: --after-create mode - wait for install, then nuke -after_create_mode() { - resolve_gcs_path - echo "=== After-create mode: waiting for cluster install to succeed ===" - echo "Monitoring: ${GCS_JOB_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" - echo "" - - # Wait for the create step to finish - while true; do - local create_finished - create_finished=$(gsutil cat "${GCS_JOB_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null || true) - if [[ -n "$create_finished" ]]; then - echo "" - echo "$(date): Create step finished!" - echo "$create_finished" - break - fi - - # Also check if the overall job already finished (e.g. timed out) - local job_finished - job_finished=$(gsutil cat "${GCS_JOB_PATH}/finished.json" 2>/dev/null || true) - if [[ -n "$job_finished" ]]; then - echo "" - echo "$(date): Job already finished: $job_finished" - echo "Nothing to nuke." - exit 0 - fi - - echo -n "." - sleep 30 - done - - echo "" - echo "Cluster install complete. Nuking cluster to fail tests..." - echo "" - - # Discover infra-id from instances or logs - INFRA_ID=$(gcloud compute instances list \ - --project="$GCP_PROJECT" \ - --filter="name~^${CLUSTER_PREFIX}" \ - --format="value(name)" 2>/dev/null \ - | head -1 \ - | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) - - if [[ -z "$INFRA_ID" ]]; then - # Try from logs - INFRA_ID=$(gsutil cat "${GCS_JOB_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/build-log.txt" 2>/dev/null \ - | gunzip 2>/dev/null \ - | grep -oP 'CLUSTER_NAME=\K\S+' \ - | head -1 || true) - fi - - if [[ -z "$INFRA_ID" ]]; then - echo "ERROR: Could not determine cluster infra-id." - exit 1 - fi - - echo "Infra-id: $INFRA_ID" - - # Just kill the instances - that's enough to fail the tests. - # The CI post step (ocp-4-destroy) will handle full cleanup. - kill_instances - - echo "=========================================" - echo " Cluster instances killed." - echo " Tests should fail shortly." - echo " CI post step will handle cleanup." - echo "=========================================" - - if $WAIT; then - wait_for_job - fi - exit 0 -} - -# === Main === - -# Handle --after-create mode early -if $AFTER_CREATE; then - after_create_mode -fi - -# Discover the full infra-id from running instances -echo "Discovering infra-id from running instances..." -INFRA_ID=$(gcloud compute instances list \ - --project="$GCP_PROJECT" \ - --filter="name~^${CLUSTER_PREFIX}" \ - --format="value(name)" 2>/dev/null \ - | head -1 \ - | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) - -if [[ -z "$INFRA_ID" ]]; then - echo "No instances found matching prefix '${CLUSTER_PREFIX}'." - echo "" - echo "Checking GCS for cluster name from job artifacts..." - INFRA_ID=$(gsutil cat \ - "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests/${JOB_ID}/artifacts/merge-qa-e2e-tests/ocp-4-create/build-log.txt" 2>/dev/null \ - | gunzip 2>/dev/null \ - | grep -oP 'CLUSTER_NAME=\K\S+' \ - | head -1 || true) - if [[ -n "$INFRA_ID" ]]; then - echo "Found cluster name from logs: $INFRA_ID" - echo "No running instances, but will proceed with cleanup of other resources." - echo "" - else - echo "No cluster found. The cluster may not have been provisioned yet." - if $WAIT; then - echo "Waiting for cluster to appear..." - while true; do - INFRA_ID=$(gcloud compute instances list \ - --project="$GCP_PROJECT" \ - --filter="name~^${CLUSTER_PREFIX}" \ - --format="value(name)" 2>/dev/null \ - | head -1 \ - | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) - if [[ -n "$INFRA_ID" ]]; then - echo "Found infra-id: $INFRA_ID" - break - fi - echo "$(date): waiting..." - sleep 30 - done - else - exit 1 - fi - fi -fi - -echo "Infra-id: $INFRA_ID" -echo "" - -if $DRY_RUN; then - echo "[DRY RUN] Surveying resources..." - echo "" - for desc_filter_fmt in \ - "Instances|instances list|name~${INFRA_ID}|csv[no-heading](name,zone)" \ - "Instance Groups|instance-groups list|name~${INFRA_ID}|csv[no-heading](name,zone)" \ - "Forwarding Rules|forwarding-rules list|name~${INFRA_ID}|csv[no-heading](name,region)" \ - "Target TCP Proxies|target-tcp-proxies list|name~${INFRA_ID}|value(name)" \ - "Backend Services|backend-services list|name~${INFRA_ID}|csv[no-heading](name,region)" \ - "Health Checks|health-checks list|name~${INFRA_ID}|value(name)" \ - "Firewall Rules|firewall-rules list|name~${INFRA_ID}|value(name)" \ - "Routers|routers list|name~${INFRA_ID}|csv[no-heading](name,region)" \ - "Subnets|networks subnets list|name~${INFRA_ID}|csv[no-heading](name,region)" \ - "Networks|networks list|name~${INFRA_ID}|value(name)" \ - "Addresses|addresses list|name~${INFRA_ID}|csv[no-heading](name,region)" \ - "Disks|disks list|name~${INFRA_ID}|csv[no-heading](name,zone)"; do - IFS='|' read -r desc cmd filter fmt <<< "$desc_filter_fmt" - echo "--- $desc ---" - # shellcheck disable=SC2086 - gcloud compute $cmd --project="$GCP_PROJECT" --filter="$filter" --format="$fmt" 2>/dev/null | grep . || echo "(none)" - echo "" - done - echo "--- Workload Identity Pool ---" - gcloud iam workload-identity-pools describe "$CLUSTER_PREFIX" \ - --location=global --project="$GCP_PROJECT" --format="value(name)" 2>/dev/null || echo "(none)" - echo "" - echo "--- DNS Zones ---" - gcloud dns managed-zones list --project="$GCP_PROJECT" \ - --filter="name~${INFRA_ID}" --format="value(name)" 2>/dev/null | grep . || echo "(none)" - echo "" - echo "--- Service Accounts ---" - gcloud iam service-accounts list --project="$GCP_PROJECT" \ - --filter="email~${INFRA_ID}" --format="value(email)" 2>/dev/null | grep . || echo "(none)" - echo "" - echo "[DRY RUN] Would delete all resources listed above." - exit 0 -fi - -if ! $FORCE; then - read -rp "Nuke cluster ${INFRA_ID} (job ${JOB_ID})? [y/N] " confirm - [[ "$confirm" =~ ^[yY]$ ]] || { echo "Aborted."; exit 1; } -fi - -echo "" - -# Execute in order: sabotage auth first (fastest impact), then kill VMs, then cleanup -sabotage_auth -kill_instances -cleanup_infra - -echo "=========================================" -echo " Cluster $INFRA_ID nuked." -echo "=========================================" - -if $WAIT; then - wait_for_job -fi diff --git a/nuke-ci-queue.sh b/nuke-ci-queue.sh deleted file mode 100755 index cc567d913ca1a..0000000000000 --- a/nuke-ci-queue.sh +++ /dev/null @@ -1,694 +0,0 @@ -#!/usr/bin/env bash -# -# Automatically chain-kill queued StackRox CI jobs using --after-create strategy. -# -# Usage: -# ./nuke-ci-queue.sh # Kill all queued jobs -# ./nuke-ci-queue.sh --keep-after TIME # Keep jobs triggered after TIME (UTC) -# ./nuke-ci-queue.sh --max-kills N # Stop after killing N jobs -# ./nuke-ci-queue.sh --dry-run # Show what would be killed -# -# Strategy: -# 1. Query prowjobs.js for pending/triggered ocp-4-21-merge-qa-e2e jobs -# 2. Wait for the current pending job's create step to succeed -# 3. Kill the cluster instances to fail the tests -# 4. Wait for the job to finish -# 5. Repeat for the next job in the queue -# -# Requires: gcloud CLI, gsutil, curl, python3 - -set -euo pipefail - -GCP_PROJECT="${GCP_PROJECT:-acs-san-stackroxci}" -JOB_NAME="${JOB_NAME:-branch-ci-stackrox-stackrox-master-ocp-4-21-merge-qa-e2e-tests}" -GCS_BASE="${GCS_BASE:-gs://test-platform-results/logs/${JOB_NAME}}" -PROVISIONER_SA="${PROVISIONER_SA:-openshift-ipi-provisioner@acs-san-stackroxci.iam.gserviceaccount.com}" -KEEP_AFTER="${KEEP_AFTER:-}" -MAX_KILLS="${MAX_KILLS:-0}" -DRY_RUN=false -DISABLE_SA=false -KILL_COUNT=0 - -usage() { - echo "Usage: $0 [--keep-after TIME] [--max-kills N] [--dry-run] [--disable-sa]" - echo "" - echo " --keep-after TIME Keep jobs triggered after this UTC time (e.g. 2026-03-10T14:35:00Z)" - echo " --max-kills N Stop after killing N jobs (0 = unlimited)" - echo " --dry-run Show queue status without killing" - echo " --disable-sa Use SA-disable strategy (fastest, ~2 min per job)" - echo " Only used when no other jobs are in create/destroy phase" - echo "" - echo "Environment variables:" - echo " GCP_PROJECT GCP project (default: acs-san-stackroxci)" - echo " JOB_NAME Prow job name to target" - echo " PROVISIONER_SA GCP SA to disable (default: openshift-ipi-provisioner@...)" - exit 1 -} - -while [[ $# -gt 0 ]]; do - case "$1" in - --keep-after) KEEP_AFTER="$2"; shift ;; - --max-kills) MAX_KILLS="$2"; shift ;; - --dry-run) DRY_RUN=true ;; - --disable-sa) DISABLE_SA=true ;; - --help|-h) usage ;; - *) echo "Unknown arg: $1"; usage ;; - esac - shift -done - -# Get the current queue state from prowjobs.js -get_queue() { - curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ - | sed 's/^var allBuilds = //;s/;$//' \ - | python3 -c " -import sys, json -data = json.load(sys.stdin) -job_name = '${JOB_NAME}' -keep_after = '${KEEP_AFTER}' -jobs = [j for j in data.get('items', []) - if j.get('spec',{}).get('job','') == job_name - and j['status'].get('state','') in ('pending', 'triggered')] -jobs.sort(key=lambda j: j['metadata'].get('creationTimestamp','')) -for j in jobs: - created = j['metadata'].get('creationTimestamp','') - if keep_after and created > keep_after: - continue - state = j['status'].get('state','') - build_id = j['status'].get('build_id','') - print(f'{state}|{created}|{build_id}') -" 2>/dev/null -} - -# Get the pending job (the one currently running or about to run) -get_pending_job_id() { - curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ - | sed 's/^var allBuilds = //;s/;$//' \ - | python3 -c " -import sys, json -data = json.load(sys.stdin) -job_name = '${JOB_NAME}' -keep_after = '${KEEP_AFTER}' -jobs = [j for j in data.get('items', []) - if j.get('spec',{}).get('job','') == job_name - and j['status'].get('state','') == 'pending' - and j['status'].get('build_id','')] -jobs.sort(key=lambda j: j['metadata'].get('creationTimestamp','')) -for j in jobs: - created = j['metadata'].get('creationTimestamp','') - if keep_after and created > keep_after: - continue - print(j['status']['build_id']) - break -" 2>/dev/null -} - -# Count remaining jobs to kill -count_remaining() { - get_queue | wc -l | tr -d ' ' -} - -# Wait for instances to appear for a cluster prefix -wait_for_instances() { - local prefix="$1" - local job_id="$2" - local gcs_path="${GCS_BASE}/${job_id}" - echo " Waiting for instances to appear..." - for i in $(seq 1 120); do - # Check if job or create step already finished (no point waiting) - if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then - echo " Job already finished while waiting for instances." - return 1 - fi - if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then - echo " Create step finished while waiting. Checking for instances one more time..." - local inst - inst=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~^${prefix}" --format="value(name)" 2>/dev/null | head -1) - if [[ -n "$inst" ]]; then - echo " Found instance: $inst" - return 0 - fi - echo " No instances found after create finished." - return 1 - fi - local inst - inst=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~^${prefix}" --format="value(name)" 2>/dev/null | head -1) - if [[ -n "$inst" ]]; then - echo " Found instance: $inst" - return 0 - fi - sleep 15 - done - echo " Timed out waiting for instances" - return 1 -} - -# Extract kubeconfig from the bootstrap ignition stored in GCS -extract_kubeconfig() { - local prefix="$1" - local infra_id="$2" - local outfile="$3" - local ign_bucket="${infra_id}-bootstrap-ignition" - echo " Extracting kubeconfig from gs://${ign_bucket}/bootstrap.ign..." - gsutil cat "gs://${ign_bucket}/bootstrap.ign" 2>/dev/null | python3 -c " -import sys, json, base64 -data = json.load(sys.stdin) -for f in data.get('storage', {}).get('files', []): - if f.get('path') == '/opt/openshift/auth/kubeconfig': - source = f['contents']['source'] - _, encoded = source.split(',', 1) - decoded = base64.b64decode(encoded).decode() - # Replace api-int with api for external access - print(decoded.replace('api-int.', 'api.')) - break -" > "$outfile" 2>/dev/null - if [[ -s "$outfile" ]]; then - echo " Kubeconfig saved to $outfile" - return 0 - fi - echo " Failed to extract kubeconfig" - return 1 -} - -# Wait for the cluster API to be reachable -wait_for_api() { - local cluster_name="$1" - local api_host="api.${cluster_name}.ocp.ci.rox.systems" - echo " Waiting for API at ${api_host}:6443..." - for i in $(seq 1 60); do - if timeout 5 curl -sk "https://${api_host}:6443/version" &>/dev/null; then - echo " API is up!" - return 0 - fi - sleep 10 - done - echo " Timed out waiting for API" - return 1 -} - -# Create the bootstrap-complete configmap to trick the installer -signal_bootstrap_complete() { - local kubeconfig="$1" - echo " Creating bootstrap-complete configmap..." - if KUBECONFIG="$kubeconfig" kubectl get configmap bootstrap -n kube-system &>/dev/null; then - KUBECONFIG="$kubeconfig" kubectl patch configmap bootstrap -n kube-system \ - -p '{"data":{"status":"complete"}}' 2>&1 | sed 's/^/ /' - else - KUBECONFIG="$kubeconfig" kubectl create configmap bootstrap -n kube-system \ - --from-literal=status=complete 2>&1 | sed 's/^/ /' - fi -} - -# Fake the ClusterVersion to signal install-complete. -# Strategy: delete admission webhooks that block status changes, -# scale down CVO to prevent it from overwriting, then patch. -signal_install_complete() { - local kubeconfig="$1" - echo " Signaling install-complete..." - - # Step 1: Delete admission webhooks that may block status patches - echo " Deleting admission webhooks..." - KUBECONFIG="$kubeconfig" kubectl delete validatingwebhookconfigurations --all 2>&1 | sed 's/^/ /' - KUBECONFIG="$kubeconfig" kubectl delete mutatingwebhookconfigurations --all 2>&1 | sed 's/^/ /' - - # Step 2: Scale down CVO to prevent it from overwriting our status - echo " Scaling down CVO..." - KUBECONFIG="$kubeconfig" kubectl scale deployment cluster-version-operator \ - -n openshift-cluster-version --replicas=0 2>&1 | sed 's/^/ /' - KUBECONFIG="$kubeconfig" kubectl delete pod -n openshift-cluster-version \ - --all --force --grace-period=0 2>&1 | sed 's/^/ /' - sleep 5 - - # Step 3: Wait for ClusterVersion to exist - echo " Waiting for ClusterVersion 'version' to exist..." - for i in $(seq 1 30); do - if KUBECONFIG="$kubeconfig" kubectl get clusterversion version &>/dev/null; then - echo " ClusterVersion exists." - break - fi - sleep 10 - done - - # Step 4: Build and apply patch preserving required fields - echo " Patching ClusterVersion status..." - local current_status - current_status=$(KUBECONFIG="$kubeconfig" kubectl get clusterversion version -o json 2>/dev/null) - if [[ -n "$current_status" ]]; then - echo "$current_status" | python3 -c " -import sys, json -cv = json.load(sys.stdin) -status = cv.get('status', {}) -patch = { - 'status': { - 'desired': status.get('desired', {'version': '4.21.0', 'image': 'unknown'}), - 'observedGeneration': status.get('observedGeneration', 1), - 'versionHash': status.get('versionHash', 'fake'), - 'availableUpdates': status.get('availableUpdates') or [], - 'conditions': [ - {'type': 'Available', 'status': 'True', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': 'Done'}, - {'type': 'Failing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''}, - {'type': 'Progressing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''} - ] - } -} -print(json.dumps(patch)) -" > /tmp/cv-patch.json - KUBECONFIG="$kubeconfig" kubectl patch clusterversion version --type=merge --subresource=status \ - -p "$(cat /tmp/cv-patch.json)" 2>&1 | sed 's/^/ /' - rm -f /tmp/cv-patch.json - else - echo " Could not get current ClusterVersion." - fi - - # Step 5: Verify the patch stuck and keep patching if CVO respawns - echo " Verifying ClusterVersion conditions..." - local patch_success=false - for attempt in $(seq 1 10); do - local conditions - conditions=$(KUBECONFIG="$kubeconfig" kubectl get clusterversion version \ - -o jsonpath='{.status.conditions[?(@.type=="Available")].status},{.status.conditions[?(@.type=="Failing")].status},{.status.conditions[?(@.type=="Progressing")].status}' 2>/dev/null) - echo " Attempt $attempt: Available,Failing,Progressing = $conditions" - if [[ "$conditions" == "True,False,False" ]]; then - echo " [OK] ClusterVersion conditions are correct!" - patch_success=true - break - fi - # Re-kill CVO if it respawned - KUBECONFIG="$kubeconfig" kubectl delete pod -n openshift-cluster-version --all --force --grace-period=0 &>/dev/null - # Re-apply patch - if [[ -n "$current_status" ]]; then - echo "$current_status" | python3 -c " -import sys, json -cv = json.load(sys.stdin) -status = cv.get('status', {}) -patch = { - 'status': { - 'desired': status.get('desired', {'version': '4.21.0', 'image': 'unknown'}), - 'observedGeneration': status.get('observedGeneration', 1), - 'versionHash': status.get('versionHash', 'fake'), - 'availableUpdates': status.get('availableUpdates') or [], - 'conditions': [ - {'type': 'Available', 'status': 'True', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': 'Done'}, - {'type': 'Failing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''}, - {'type': 'Progressing', 'status': 'False', 'lastTransitionTime': '2026-01-01T00:00:00Z', 'reason': 'Done', 'message': ''} - ] - } -} -print(json.dumps(patch)) -" | KUBECONFIG="$kubeconfig" kubectl patch clusterversion version --type=merge --subresource=status -p "$(cat -)" &>/dev/null - fi - sleep 5 - done - if ! $patch_success; then - echo " [WARN] Could not get ClusterVersion conditions to stick after 10 attempts." - echo " [WARN] Falling back to wait-for-create approach." - fi -} - -# Check if any other jobs have clusters being created/destroyed right now. -# Checks for bootstrap nodes (active provisioning) and also for clusters -# that are still installing (have bootstrap but no workers yet). -# The provisioner SA (OCP_4_GCP_SA) is used by all ocp-4 jobs during -# create and destroy phases. -other_jobs_creating() { - local our_prefix="$1" - - # Check for bootstrap nodes from other jobs (active provisioning) - local other_bootstraps - other_bootstraps=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~bootstrap AND NOT name~${our_prefix}" \ - --format="value(name)" 2>/dev/null || true) - if [[ -n "$other_bootstraps" ]]; then - echo "$other_bootstraps" - return 0 # other jobs ARE creating - fi - - # Check for very recently created instances (< 5 min old) from other jobs - # These might be in early provisioning before bootstrap appears - local recent - recent=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~^rox-ci- AND NOT name~${our_prefix} AND creationTimestamp>-PT5M" \ - --format="value(name)" 2>/dev/null | head -1 || true) - if [[ -n "$recent" ]]; then - echo "(recently created) $recent" - return 0 - fi - - return 1 # safe to proceed -} - -# Disable the provisioner SA to make the installer fail instantly -disable_provisioner_sa() { - echo " [SA] Disabling provisioner SA: $PROVISIONER_SA" - gcloud iam service-accounts disable "$PROVISIONER_SA" \ - --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' -} - -# Re-enable the provisioner SA -enable_provisioner_sa() { - echo " [SA] Re-enabling provisioner SA: $PROVISIONER_SA" - gcloud iam service-accounts enable "$PROVISIONER_SA" \ - --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' -} - -# SA-disable abort: disable the provisioner SA so the installer fails instantly. -# Only used when no other jobs are in create/destroy phase. -sa_disable_abort() { - local job_id="$1" - local prefix="$2" - local gcs_path="${GCS_BASE}/${job_id}" - - echo " [SA] Checking if SA-disable is safe..." - - # Check if other jobs are creating clusters - local others - if others=$(other_jobs_creating "$prefix"); then - echo " [SA] Other jobs have clusters being created:" - echo "$others" | sed 's/^/ /' - echo " [SA] Not safe to disable SA. Falling back to fast_abort." - return 1 - fi - echo " [SA] No other jobs creating. Safe to proceed." - - # Disable the SA - disable_provisioner_sa - - # Set trap to always re-enable SA on exit/error - trap 'enable_provisioner_sa' EXIT - - # Wait for the create step to finish (should be very fast) - echo " [SA] Waiting for create step to fail (should be ~2-5 min)..." - local sa_start - sa_start=$(date +%s) - for i in $(seq 1 60); do - # Safety check: re-enable if other jobs start creating - if others=$(other_jobs_creating "$prefix"); then - echo " [SA] WARNING: Other job started creating! Re-enabling SA immediately." - echo "$others" | sed 's/^/ /' - enable_provisioner_sa - trap - EXIT - echo " [SA] SA re-enabled. Falling back to fast_abort." - return 1 - fi - - # Check if create step finished - if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then - local elapsed=$(( $(date +%s) - sa_start )) - local create_result - create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) - echo " [SA] Create step finished in ${elapsed}s! $create_result" - break - fi - - # Check if job already finished - if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then - echo " [SA] Job already finished." - enable_provisioner_sa - trap - EXIT - return 0 - fi - - sleep 10 - done - - # Re-enable SA immediately - enable_provisioner_sa - trap - EXIT - - echo " [SA] SA re-enabled. Waiting for job to fail..." - return 0 -} - -# Fast abort: extract kubeconfig from ignition, signal bootstrap complete, -# then kill instances. Falls back to wait-for-create if anything fails. -fast_abort() { - local job_id="$1" - local prefix="$2" - local gcs_path="${GCS_BASE}/${job_id}" - local cluster_name="$prefix" # e.g. rox-ci-49586688 - - echo " [FAST] Attempting ignition-kubeconfig abort..." - - # Wait for instances to appear - if ! wait_for_instances "$prefix" "$job_id"; then - echo " [FAST] No instances found, falling back to wait-for-create." - return 1 - fi - - # Discover infra-id from instances - local infra_id - infra_id=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~^${prefix}" --format="value(name)" 2>/dev/null \ - | head -1 | grep -oP "^rox-ci-\d+-[a-z0-9]+" || true) - if [[ -z "$infra_id" ]]; then - echo " [FAST] Could not determine infra-id, falling back." - return 1 - fi - echo " [FAST] Infra-id: $infra_id" - - # Extract kubeconfig from ignition - local kubeconfig="/tmp/${infra_id}-kubeconfig.yaml" - if ! extract_kubeconfig "$prefix" "$infra_id" "$kubeconfig"; then - echo " [FAST] Kubeconfig extraction failed, falling back." - return 1 - fi - - local gcs_path="${GCS_BASE}/${job_id}" - - # Check if create step already finished — if so, skip bootstrap signal and just kill - if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then - local create_result - create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) - echo " [FAST] Create step already finished: $create_result" - echo " [FAST] Killing cluster instances to fail tests..." - kill_cluster_instances "$prefix" || true - rm -f "$kubeconfig" - echo " [FAST] Instances killed. Waiting for job to fail..." - return 0 - fi - - # Wait for API to come up, but also check if create step finishes while waiting - echo " [FAST] Waiting for API..." - local api_host="api.${cluster_name}.ocp.ci.rox.systems" - local api_up=false - for i in $(seq 1 60); do - # Check if create step finished while we wait - if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then - echo " [FAST] Create step finished while waiting for API." - echo " [FAST] Killing cluster instances..." - kill_cluster_instances "$prefix" || true - rm -f "$kubeconfig" - return 0 - fi - if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then - echo " [FAST] Job already finished." - rm -f "$kubeconfig" - return 0 - fi - if timeout 5 curl -sk "https://${api_host}:6443/version" &>/dev/null; then - api_up=true - echo " [FAST] API is up!" - break - fi - sleep 10 - done - - if $api_up; then - # Signal bootstrap complete (speeds up the bootstrap wait phase) - signal_bootstrap_complete "$kubeconfig" - fi - - # Wait for the create step to finish - echo " [FAST] Waiting for create step to complete..." - for i in $(seq 1 60); do - if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then - local create_result - create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) - echo " [FAST] Create step finished: $create_result" - break - fi - if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then - echo " [FAST] Job already finished." - rm -f "$kubeconfig" - return 0 - fi - sleep 15 - done - - # Now kill instances to fail the test steps - echo " [FAST] Killing cluster instances..." - kill_cluster_instances "$prefix" || true - - # Clean up kubeconfig - rm -f "$kubeconfig" - - echo " [FAST] Instances killed. Waiting for job to fail..." - return 0 -} - -# Fallback: wait for the create step to finish (success or failure) -wait_for_create() { - local job_id="$1" - local gcs_path="${GCS_BASE}/${job_id}" - echo " [FALLBACK] Waiting for create step to finish..." - while true; do - # Check if job already finished entirely - if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then - local result - result=$(gsutil cat "${gcs_path}/finished.json" 2>/dev/null) - echo " Job already finished: $result" - return 1 - fi - # Check if create step finished - if gsutil -q stat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null; then - local create_result - create_result=$(gsutil cat "${gcs_path}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" 2>/dev/null) - echo " Create step finished: $create_result" - return 0 - fi - sleep 30 - done -} - -# Kill instances for a cluster prefix -kill_cluster_instances() { - local prefix="$1" - local instances - instances=$(gcloud compute instances list --project="$GCP_PROJECT" \ - --filter="name~^${prefix}" --format="csv[no-heading](name,zone)" 2>/dev/null || true) - if [[ -z "$instances" ]]; then - echo " No instances found for ${prefix}" - return 1 - fi - echo "$instances" | while IFS=, read -r name zone; do - [[ -z "$name" ]] && continue - zone=$(basename "$zone") - echo " Killing $name ($zone)" - gcloud compute instances delete "$name" --zone="$zone" --project="$GCP_PROJECT" --quiet 2>&1 | sed 's/^/ /' & - done - wait - return 0 -} - -# Wait for the overall job to finish -wait_for_job_finish() { - local job_id="$1" - local prefix="$2" - local gcs_path="${GCS_BASE}/${job_id}" - echo " Waiting for job to finish..." - for i in $(seq 1 120); do - # Keep killing instances in case they respawn - kill_cluster_instances "$prefix" 2>/dev/null || true - # Check if done - if gsutil -q stat "${gcs_path}/finished.json" 2>/dev/null; then - local result - result=$(gsutil cat "${gcs_path}/finished.json" 2>/dev/null) - echo " Job finished: $result" - return 0 - fi - sleep 30 - done - echo " Timed out waiting for job to finish" - return 1 -} - -# === Main === - -echo "============================================" -echo " StackRox CI Queue Killer" -echo "============================================" -echo "Job name: $JOB_NAME" -echo "GCP project: $GCP_PROJECT" -[[ -n "$KEEP_AFTER" ]] && echo "Keep after: $KEEP_AFTER" -[[ "$MAX_KILLS" -gt 0 ]] && echo "Max kills: $MAX_KILLS" -echo "" - -# Show queue -echo "Current queue:" -QUEUE=$(get_queue) -if [[ -z "$QUEUE" ]]; then - echo " No jobs to kill!" - exit 0 -fi -TOTAL=$(echo "$QUEUE" | wc -l | tr -d ' ') -echo "$QUEUE" | while IFS='|' read -r state created build_id; do - printf " %-12s %s %s\n" "$state" "$created" "${build_id:-(queued)}" -done -echo "" -echo "Total jobs to kill: $TOTAL" -echo "" - -if $DRY_RUN; then - echo "[DRY RUN] Would kill $TOTAL jobs using --after-create strategy." - exit 0 -fi - -echo "Starting automated kill loop..." -echo "" - -while true; do - # Check kill limit - if [[ "$MAX_KILLS" -gt 0 && "$KILL_COUNT" -ge "$MAX_KILLS" ]]; then - echo "Reached max kills ($MAX_KILLS). Stopping." - break - fi - - # Get the current pending job - JOB_ID=$(get_pending_job_id || true) - if [[ -z "$JOB_ID" ]]; then - REMAINING=$(count_remaining || echo "0") - if [[ "$REMAINING" -gt 0 ]]; then - echo "$(date): $REMAINING jobs queued but none pending yet. Waiting..." - sleep 60 - continue - else - echo "$(date): No more jobs to kill!" - break - fi - fi - - CLUSTER_PREFIX="rox-ci-${JOB_ID: -8}" - KILL_COUNT=$((KILL_COUNT + 1)) - REMAINING=$(count_remaining || echo "?") - - echo "============================================" - echo " Kill #${KILL_COUNT} | ~${REMAINING} remaining" - echo " Job ID: $JOB_ID" - echo " Cluster: $CLUSTER_PREFIX" - echo " Time: $(date)" - echo "============================================" - - # Strategy: fast_abort (bootstrap configmap + wait for create + kill instances) - # Falls back to wait-for-create if fast_abort can't get started - if fast_abort "$JOB_ID" "$CLUSTER_PREFIX"; then - wait_for_job_finish "$JOB_ID" "$CLUSTER_PREFIX" || echo " (wait timed out, moving on)" - else - echo "" - echo " Falling back to wait-for-create approach..." - if wait_for_create "$JOB_ID"; then - echo "" - echo " Create step finished. Nuking instances..." - kill_cluster_instances "$CLUSTER_PREFIX" || true - echo "" - echo " Instances killed. Waiting for job to fail..." - wait_for_job_finish "$JOB_ID" "$CLUSTER_PREFIX" || echo " (wait timed out, moving on)" - else - echo " Job finished on its own (or failed during create)." - fi - fi - - echo "" - echo " Kill #${KILL_COUNT} complete." - echo "" - - # Brief pause before next iteration - sleep 10 -done - -echo "" -echo "============================================" -echo " Queue killing complete." -echo " Total jobs killed: $KILL_COUNT" -echo "============================================" diff --git a/quick-helm-install.sh b/quick-helm-install.sh deleted file mode 100644 index 0280f5fef4e4f..0000000000000 --- a/quick-helm-install.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -logmein() { - target_url="$(curl -sSkf -u "admin:${ROX_ADMIN_PASSWORD}" -w '%{redirect_url}' "https://localhost:8000/sso/providers/basic/4df1b98c-24ed-4073-a9ad-356aec6bb62d/challenge?micro_ts=0")" - if [[ -z "${target_url}" ]]; then - >&2 echo "UNEXPECTED: Could not determine target URL" - return - fi - - auth_error="$(sed -E '/^.*#error=([^&]+)(&.*)?$/!d;s//\1/' <<<"$target_url" | tr '+' ' ')" - if [[ -n "$auth_error" ]]; then - >&2 echo "Authentication error: ${auth_error}" - fi - - echo "Logging you in via ${target_url} ..." - platform="$(uname)" - if [[ "$platform" == "Linux" ]]; then - xdg-open "${target_url}" >/dev/null & - elif [[ "$platform" == "Darwin" ]]; then - open "${target_url}" & - else - >&2 echo "Unsupported platform '$platform', please open ${target_url} in a browser" - fi -} - -SMALL_INSTALL=false - -case "${1:-}" in - -h|--help) - echo -e "Usage:\n\tquick-helm-install.sh [options]" - echo " " - echo "Installs StackRox via Helm charts." - echo " " - echo "options:" - echo "-h, --help show brief help" - echo "-s, --small reduce StackRox resource requirements for small clusters" - exit 0 - ;; - -s|--small) - SMALL_INSTALL=true - ;; -esac - -echo "Adding the stackrox/helm-charts/opensource repository to Helm." - -helm repo add stackrox https://raw.githubusercontent.com/stackrox/helm-charts/main/opensource/ - -echo "Generating ROX_ADMIN_PASSWORD" - -ROX_ADMIN_PASSWORD="$(openssl rand -base64 20 | tr -d '/=+')" - -echo "Installing stackrox-central-services" - -installflags=('--set' 'central.persistence.none=true') -if [[ "$SMALL_INSTALL" == "true" ]]; then - installflags+=('--set' 'central.resources.requests.memory=1Gi') - installflags+=('--set' 'central.resources.requests.cpu=1') - installflags+=('--set' 'central.resources.limits.memory=4Gi') - installflags+=('--set' 'central.resources.limits.cpu=1') - installflags+=('--set' 'central.db.resources.requests.memory=1Gi') - installflags+=('--set' 'central.db.resources.requests.cpu=500m') - installflags+=('--set' 'central.db.resources.limits.memory=4Gi') - installflags+=('--set' 'central.db.resources.limits.cpu=1') - installflags+=('--set' 'scanner.autoscaling.disable=true') - installflags+=('--set' 'scanner.replicas=1') - installflags+=('--set' 'scanner.resources.requests.memory=500Mi') - installflags+=('--set' 'scanner.resources.requests.cpu=500m') - installflags+=('--set' 'scanner.resources.limits.memory=2500Mi') - installflags+=('--set' 'scanner.resources.limits.cpu=2000m') -fi - -helm install -n stackrox --create-namespace stackrox-central-services stackrox/stackrox-central-services \ - --set central.adminPassword.value="${ROX_ADMIN_PASSWORD}" \ - "${installflags[@]+"${installflags[@]}"}" - -kubectl -n stackrox rollout status deploy/central --timeout=3m - -echo "Setting up central port-forward" - -kubectl -n stackrox port-forward deploy/central --pod-running-timeout=1m0s 8000:8443 > /dev/null 2>&1 & - -echo "Generating an init bundle with stackrox-secured-cluster-services provisioning secrets" - -# shellcheck disable=SC2016 -echo "$ROX_ADMIN_PASSWORD" | \ -kubectl -n stackrox exec -i deploy/central -- bash -c 'ROX_ADMIN_PASSWORD=$(cat) \ - roxctl --insecure-skip-tls-verify \ - central init-bundles generate stackrox-init-bundle --output -' 1> stackrox-init-bundle.yaml - -installflags=() -if [[ "$SMALL_INSTALL" == "true" ]]; then - installflags+=('--set' 'sensor.resources.requests.memory=500Mi') - installflags+=('--set' 'sensor.resources.requests.cpu=500m') - installflags+=('--set' 'sensor.resources.limits.memory=500Mi') - installflags+=('--set' 'sensor.resources.limits.cpu=500m') -fi - -echo "Installing stackrox-secured-cluster-services" - -helm install -n stackrox stackrox-secured-cluster-services stackrox/stackrox-secured-cluster-services \ - -f stackrox-init-bundle.yaml --set clusterName="my-secured-cluster" \ - "${installflags[@]+"${installflags[@]}"}" - -echo "Logging into StackRox in the browser" - -logmein - -echo -e " -\033[1;31mStackRox is now installed!\033[0m - -You may access the dashboard via https://localhost:8000/main/dashboard, the user is admin. - -Consult these documents for additional information on customizing your Helm installation: -https://docs.openshift.com/acs/installing/installing_other/install-central-other.html#install-using-helm-customizations-other -https://docs.openshift.com/acs/installing/installing_other/install-secured-cluster-other.html#configure-secured-cluster-services-helm-chart-customizations-other - -ROX_ADMIN_PASSWORD='$ROX_ADMIN_PASSWORD' -Above is your automatically generated stackrox admin password. Please store it securely, as you will need it during further configuration. -In your current directory an init bundle \"stackrox-init-bundle.yaml\" was created, store it safely in case you are planning to provision more secured clusters with it, or delete it otherwise." - diff --git a/sa-disable-chain.sh b/sa-disable-chain.sh deleted file mode 100755 index cee0f73892b33..0000000000000 --- a/sa-disable-chain.sh +++ /dev/null @@ -1,396 +0,0 @@ -#!/usr/bin/env bash -# -# Chain SA-disable attacks to rapidly clear the CI job queue. -# -# Usage: -# ./sa-disable-chain.sh [--keep-after TIME] -# -# Example: -# ./sa-disable-chain.sh merge-scanner-v4-install --keep-after 2026-03-10T14:35:00Z -# ./sa-disable-chain.sh merge-qa-e2e --keep-after 2026-03-10T14:35:00Z -# -# Strategy: -# 1. Find the current pending old job from prowjobs.js -# 2. Wait for its create step to start (detect via instances appearing) -# 3. Disable the provisioner SA for ~3 min (installer fails immediately) -# 4. Re-enable SA -# 5. Wait for job to finish -# 6. Repeat for next job - -set -euo pipefail - -SA="${PROVISIONER_SA:-openshift-ipi-provisioner@acs-san-stackroxci.iam.gserviceaccount.com}" -P="${GCP_PROJECT:-acs-san-stackroxci}" -JOB_SUBSTR="${1:?Usage: $0 [--keep-after TIME]}" -shift -KEEP_AFTER="" -while [[ $# -gt 0 ]]; do - case "$1" in - --keep-after) KEEP_AFTER="$2"; shift ;; - *) echo "Unknown arg: $1"; exit 1 ;; - esac - shift -done - -KILL_COUNT=0 -KNOWN_JOB_ID="" -SKIPPED_JOBS="" - -# Safety: always re-enable SA on exit -cleanup() { - echo "" - echo "SAFETY: Ensuring SA is enabled before exit..." - gcloud iam service-accounts enable "$SA" --project="$P" --quiet 2>&1 || true - rmdir /tmp/sa-disable.lock 2>/dev/null || true - echo "SA confirmed enabled, lock released. Exiting." -} -trap cleanup EXIT - -get_pending_old_job() { - curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ - | sed 's/^var allBuilds = //;s/;$//' \ - | python3 -c " -import sys, json -data = json.load(sys.stdin) -job_substr = '${JOB_SUBSTR}' -keep_after = '${KEEP_AFTER}' -skipped = set('${SKIPPED_JOBS}'.split()) -jobs = [j for j in data.get('items', []) - if job_substr in j.get('spec',{}).get('job','') - and 'stackrox' in j.get('spec',{}).get('job','') - and j['status'].get('state') == 'pending' - and j['status'].get('build_id','')] -jobs.sort(key=lambda j: j['metadata'].get('creationTimestamp','')) -for j in jobs: - created = j['metadata'].get('creationTimestamp','') - bid = j['status']['build_id'] - if keep_after and created > keep_after: - continue - if bid in skipped: - continue - print(bid) - break -" 2>/dev/null || true -} - -count_old_jobs() { - curl -s "https://prow.ci.openshift.org/prowjobs.js?var=allBuilds&omit=annotations,labels,decoration_config,pod_spec" 2>/dev/null \ - | sed 's/^var allBuilds = //;s/;$//' \ - | python3 -c " -import sys, json -data = json.load(sys.stdin) -job_substr = '${JOB_SUBSTR}' -keep_after = '${KEEP_AFTER}' -count = 0 -for j in data.get('items', []): - if job_substr not in j.get('spec',{}).get('job',''): continue - if 'stackrox' not in j.get('spec',{}).get('job',''): continue - if j['status'].get('state','') not in ('pending','triggered'): continue - created = j['metadata'].get('creationTimestamp','') - if keep_after and created > keep_after: continue - count += 1 -print(count) -" 2>/dev/null || echo "?" -} - -# Resolve GCS base path from job name -resolve_gcs_path() { - local job_id="$1" - # Try common patterns - for path in \ - "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-${JOB_SUBSTR}-tests/${job_id}" \ - "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-${JOB_SUBSTR}/${job_id}"; do - if gsutil -q stat "${path}/prowjob.json" 2>/dev/null; then - echo "$path" - return - fi - done - # Fallback: look up from prowjob.json - echo "gs://test-platform-results/logs/branch-ci-stackrox-stackrox-master-ocp-4-21-${JOB_SUBSTR}-tests/${job_id}" -} - -echo "============================================" -echo " SA-Disable Chain Killer" -echo "============================================" -echo "Job filter: $JOB_SUBSTR" -echo "Provisioner SA: $SA" -[[ -n "$KEEP_AFTER" ]] && echo "Keep after: $KEEP_AFTER" -echo "" - -REMAINING=$(count_old_jobs) -echo "Old jobs to kill: $REMAINING" -echo "" -echo "Starting chain..." -echo "" - -while true; do - # Get the current pending old job - JOB_ID=$(get_pending_old_job) - if [[ -z "$JOB_ID" ]]; then - REMAINING=$(count_old_jobs) - if [[ "$REMAINING" -gt 0 && "$REMAINING" != "?" ]]; then - echo "$(date): $REMAINING old jobs queued but none pending yet. Waiting 60s..." - sleep 60 - continue - else - echo "$(date): No more old jobs to kill!" - break - fi - fi - - # Skip if same job as before (still processing) - if [[ "$JOB_ID" == "$KNOWN_JOB_ID" ]]; then - echo "$(date): Same job $JOB_ID still pending. Waiting 30s..." - sleep 30 - continue - fi - - KNOWN_JOB_ID="$JOB_ID" - KILL_COUNT=$((KILL_COUNT + 1)) - REMAINING=$(count_old_jobs) - SUFFIX="${JOB_ID: -8}" - GCS_PATH=$(resolve_gcs_path "$JOB_ID") - - echo "============================================" - echo " Kill #${KILL_COUNT} | ~${REMAINING} remaining" - echo " Job: $JOB_ID (rox-ci-${SUFFIX})" - echo " Time: $(date)" - echo "============================================" - - # Two-phase detection: - # Phase 1: Watch for begin step's build-log.txt in GCS (begin finished), - # then disable SA immediately (create step starts right after, ~111s gap) - # Phase 2 (fallback): If VMs appear, SA was re-enabled too early or phase 1 - # missed; re-disable SA to catch the retry loop - echo " Phase 1: Watching for begin step to finish..." - CREATE_DETECTED=false - BEGIN_DETECTED=false - for i in $(seq 1 120); do - # Check if job already finished - if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then - echo " $(date): Job already finished." - break - fi - - # Check for begin step finished (GCS artifact uploaded after step completes) - # But ONLY if the create step hasn't already started (check for create artifacts) - for begin_path in \ - "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/stackrox-stackrox-begin/finished.json" \ - "${GCS_PATH}/artifacts/merge-qa-e2e-tests/stackrox-stackrox-begin/finished.json" \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/stackrox-stackrox-begin/finished.json"; do - if gsutil -q stat "$begin_path" 2>/dev/null; then - # Verify create step hasn't already started/finished. - # Check GCS artifacts AND check for VMs (VMs mean create is running). - create_already=false - for create_check in \ - "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/build-log.txt" \ - "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/build-log.txt" \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/build-log.txt" \ - "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json"; do - if gsutil -q stat "$create_check" 2>/dev/null; then - create_already=true - break - fi - done - # Also check for VMs — if they exist, create step is running - if ! $create_already; then - vm_check=$(gcloud compute instances list --project="$P" \ - --filter="name~rox-ci-${SUFFIX}" --format="value(name)" 2>/dev/null | head -1 || true) - if [[ -n "$vm_check" ]]; then - create_already=true - fi - fi - if $create_already; then - echo " $(date): Begin finished but create step already started/finished. Skipping to next job." - CREATE_DETECTED=false - break 2 - else - echo " $(date): Begin step FINISHED (GCS artifact detected)." - echo " Create step should be starting NOW or within seconds." - BEGIN_DETECTED=true - CREATE_DETECTED=true - break 2 - fi - fi - done - - # Also check for instances (create step already running - fallback) - # But only if the create step hasn't already finished - inst=$(gcloud compute instances list --project="$P" \ - --filter="name~rox-ci-${SUFFIX}" --format="value(name)" 2>/dev/null | head -1 || true) - if [[ -n "$inst" ]]; then - # Check if create step already finished — if so, skip SA-disable - create_done=false - for create_check in \ - "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json"; do - if gsutil -q stat "$create_check" 2>/dev/null; then - create_done=true - break - fi - done - if $create_done; then - echo " $(date): VMs exist but create step already finished. Skipping to next job." - CREATE_DETECTED=false - break - fi - echo " $(date): Instances detected, create step still running. Skipping to next job." - CREATE_DETECTED=false - break - fi - - if (( i % 8 == 0 )); then - echo " $(date): Waiting... (${i})" - fi - sleep 5 - done - - if ! $CREATE_DETECTED; then - echo " Skipping job $JOB_ID, moving to next." - SKIPPED_JOBS="$SKIPPED_JOBS $JOB_ID" - KNOWN_JOB_ID="" - sleep 5 - continue - fi - - # PHASE 1: Disable SA for a tight 22-second window. - # Timeline from logs: - # +0s create step container starts - # +7s gcloud auth (succeeds - just stores creds locally) - # +7-13s curl downloads 431MB installer binary - # +14s openshift-install create manifests → first GCP API call - # +21s first API call fails with "invalid_grant" - # +25-35s ccoctl retries fail, create cluster fails - # - # The begin finished.json appears in GCS roughly when the create step starts - # (a few seconds of GCS upload delay ≈ container startup time). - # So "now" ≈ +7s into the create step. We wait 7 more seconds to hit +14s. - echo "" - echo " Phase 1: Waiting 7s for installer to reach first GCP API call..." - sleep 7 - - # Lockfile to prevent two chains from disabling SA simultaneously - LOCKFILE="/tmp/sa-disable.lock" - echo " Acquiring SA-disable lock..." - while ! mkdir "$LOCKFILE" 2>/dev/null; do - echo " Lock held by another chain, waiting 0.5s..." - sleep 0.5 - done - echo " Lock acquired." - - echo " >>> DISABLING SA (tight 22s window) <<<" - gcloud iam service-accounts disable "$SA" --project="$P" --quiet 2>&1 - SA_START=$(date +%s) - echo " SA disabled at $(date)" - - # Hold for 40 seconds — covers first attempt + first retry's API calls. - # Timeline: +0s create starts, +7s gcloud auth (succeeds locally), +14s first API call, - # +21s first attempt fails, +25-35s ccoctl retries, +35s openshift-install create cluster - sleep 40 - - echo " >>> RE-ENABLING SA <<<" - gcloud iam service-accounts enable "$SA" --project="$P" --quiet 2>&1 - SA_ELAPSED=$(( $(date +%s) - SA_START )) - echo " SA re-enabled at $(date) (disabled for ${SA_ELAPSED}s)" - - # Release lock - rmdir "$LOCKFILE" 2>/dev/null - echo " Lock released." - - # PHASE 2: Check for VMs. If the first create attempt authenticated before - # our disable window (timing was off), VMs will appear. In that case, - # re-disable briefly to catch the retry loop's API calls, then kill VMs. - echo "" - echo " Phase 2: Checking for VMs (fallback)..." - for f in $(seq 1 60); do - # Check if create step already finished (our phase 1 worked) - for create_path in \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json"; do - if gsutil -q stat "$create_path" 2>/dev/null; then - result=$(gsutil cat "$create_path" 2>/dev/null) - echo " $(date): CREATE STEP FINISHED!" - echo " Result: $result" - break 2 - fi - done - - if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then - echo " $(date): JOB FINISHED!" - break - fi - - inst=$(gcloud compute instances list --project="$P" \ - --filter="name~rox-ci-${SUFFIX}" --format="value(name)" 2>/dev/null | head -1 || true) - if [[ -n "$inst" ]]; then - echo " $(date): VMs detected ($inst) — phase 1 missed auth window." - echo " Giving up on SA-disable. Waiting for create step to finish, then will kill cluster." - # Wait for create step to finish naturally - for cw in $(seq 1 240); do - for create_path in \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/${JOB_SUBSTR}/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/merge-scanner-v4-install-tests/ocp-4-create/finished.json" \ - "${GCS_PATH}/artifacts/merge-qa-e2e-tests/ocp-4-create/finished.json"; do - if gsutil -q stat "$create_path" 2>/dev/null; then - echo " $(date): Create step finished. Killing cluster..." - gcloud compute instances list --project="$P" \ - --filter="name~rox-ci-${SUFFIX}" --format="csv[no-heading](name,zone)" 2>/dev/null \ - | while IFS=, read -r iname izone; do - [[ -z "$iname" ]] && continue - izone=$(basename "$izone") - gcloud compute instances delete "$iname" --zone="$izone" --project="$P" --quiet &>/dev/null & - done - wait 2>/dev/null - echo " Cluster killed." - break 2 - fi - done - if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then - echo " $(date): Job finished." - break - fi - if (( cw % 8 == 0 )); then - echo " $(date): Waiting for create step... (${cw})" - fi - sleep 15 - done - break - fi - sleep 5 - done - - # Double-check SA is enabled - sa_state=$(gcloud iam service-accounts describe "$SA" --project="$P" --format="value(disabled)" 2>/dev/null) - if [[ "$sa_state" == "True" ]]; then - echo " WARNING: SA still disabled! Re-enabling..." - gcloud iam service-accounts enable "$SA" --project="$P" --quiet 2>&1 - fi - - # Wait for the job to fully finish before moving on - echo " Waiting for job to fully finish..." - for w in $(seq 1 60); do - if gsutil -q stat "${GCS_PATH}/finished.json" 2>/dev/null; then - echo " $(date): Job finished." - break - fi - sleep 15 - done - - echo "" - echo " Kill #${KILL_COUNT} complete." - echo "" - - # Brief pause between kills - sleep 5 -done - -echo "" -echo "============================================" -echo " Chain complete. Killed: $KILL_COUNT" -echo "============================================" diff --git a/test.sh b/test.sh deleted file mode 100755 index 6aca6c080a00c..0000000000000 --- a/test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -x - -flavor=rosa -tag=latest - -cd ~/dev/stack/openshift-release/ - - test_names='' - for test_file in ci-operator/config/stackrox/stackrox/stackrox-stackrox-master*.yaml; do - sed -i$'' -e "s/tag: ${flavor}-stable/tag: ${flavor}-${tag}/" "$test_file" - # TODO: if no match, continue - # if ! $sed_exit_code; then continue; fi - presubmit_file="ci-operator/jobs${test_file##ci-operator/config}" - presubmit_file="${presubmit_file%%.yaml}-presubmits.yaml" - test_names_tmp=$(grep -h -o "^ *name: [^ ]*-${flavor}.*" "$presubmit_file") - test_names+=${test_names_tmp## *name:} - done - - echo "/pj-rehearse $test_names" \ - - - # git add $test_files - # git commit -m "stackrox: mirror automation-flavor ${{inputs.flavor}}-${{inputs.tag}}" >> "$GITHUB_STEP_SUMMARY" - - #PR_URL=$(gh pr create --repo openshift/release \ - # --title "stackrox: mirror automation-flavor ${{inputs.flavor}}-${{inputs.tag}}" \ - # --base "master" \ - # --body "/cc ${GITHUB_ACTOR}") - - #gh pr comment "/pj-rehearse ${test_names}" diff --git a/test.yq.sh b/test.yq.sh deleted file mode 100755 index ebdf66fb9c477..0000000000000 --- a/test.yq.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -vx - -RELEASE=4.8 -ocp_upper_bound_tag="stable" - -#for name in ocp-dev-preview/candidate ocp-dev-preview/latest ocp/candidate ocp/latest ocp/fast ocp/stable ; do -# printf "%s\t" "${name}" -# curl -s https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/${name}/release.txt \ -# | grep '^\(Name\|Created\):' | rev | cut -d' ' -f1 | rev | tr '\n' ' ' -# echo "" -#done 2>&1 | column -x -c 3 -t -ocp_stable=$(curl -s https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/release.txt \ - | grep '^Name:' | rev | cut -d' ' -f1 | rev | cut -d\. -f1,2) -echo [${ocp_stable}] -if [ -z "${ocp_stable}" ]; then - echo "Stable OCP upper-bound version not found. Please modify the files directly." -fi - -if [[ ! -z "${ocp_stable}" ]]; then - ocp_upper_bound_tag="${ocp_upper_bound_tag}-${ocp_stable}" -fi -BRANCH='stackrox-release-4.8' -CFG_DIR="ci-operator/config/stackrox/stackrox" - # Duplicate the template configurations - set -x - ls -la "$CFG_DIR" - for yaml in "$CFG_DIR"/stackrox-stackrox-release-x.y*.yaml ; do - new_yaml="${yaml//stackrox-release-x.y/$BRANCH}" - echo "Copying ${yaml} to ${new_yaml}" - yq eval \ - ".zz_generated_metadata.branch=\"release-$RELEASE\"" \ - "$yaml" > "$new_yaml" - done - git status - git add ci-operator/config/stackrox - sed -i '' "s/OCP_VERSION: ocp\/candidate.*$/OCP_VERSION: ocp\/${ocp_upper_bound_tag}/" \ - "ci-operator/config/stackrox/stackrox/stackrox-${BRANCH}"* - git diff -rm ci-operator/config/stackrox/stackrox/stackrox-${BRANCH}*.yaml -#git checkout "$CFG_DIR" diff --git a/tst.sh b/tst.sh deleted file mode 100755 index db2ec6af4c906..0000000000000 --- a/tst.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -#!/bin/sh -set -vx -set -eo pipefail - -function guess() { - num="${1}" - if [[ "${num}" -eq 42 ]] - then - echo "Correct" - else - echo "Wrong" - fi -} - -rm /tmp/pwned -guess 'a[$(cat /etc/passwd > /tmp/pwned)] + 42' -ls -la /tmp/pwned -head -3 /tmp/pwned -echo $? -exit 0 - -max_seconds=${MAX_WAIT_SECONDS:-300} -max_ingress_seconds=$(( max_seconds * 6 )) -echo $? -exit - -E2E_VERSION="davdhacs:rox-26061" -#E2E_VERSION=';'" || { echo clone dev branch; git clone -b ${E2E_VERSION#*:} --single-branch git@github.com:${E2E_VERSION%:*}/e2e-benchmarking.git ; } " -#E2E_VERSION="${E2E_VERSION#*:} --single-branch git@github.com:${E2E_VERSION%:*}/e2e-benchmarking.git" -E2E_VERSION="git@github.com:${E2E_VERSION%:*}/e2e-benchmarking.git" -REPO_URL="https://github.com/cloud-bulldozer/e2e-benchmarking"; -LATEST_TAG=$(curl -s "https://api.github.com/repos/cloud-bulldozer/e2e-benchmarking/releases/latest" | jq -r '.tag_name'); -TAG_OPTION="--branch $(if [ "$E2E_VERSION" == "default" ]; then echo "$LATEST_TAG"; else echo "$E2E_VERSION"; fi)"; -git clone $REPO_URL $TAG_OPTION --depth 1 diff --git a/update.sh b/update.sh deleted file mode 100755 index 76f8e97a84c33..0000000000000 --- a/update.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -function install_operator() { - local currentCSV catalogSource catalogSourceNamespace approval - approval=${1:-Automatic} - echo ">>> Install rhacs-operator" - oc get packagemanifests rhacs-operator -o jsonpath="{range .status.channels[*]}Channel: {.name} currentCSV: {.currentCSV}{'\n'}{end}" - currentCSV=$(oc get packagemanifests rhacs-operator -o jsonpath="{.status.channels[?(.name=='${OPERATOR_CHANNEL:-stable}')].currentCSV}") - currentCSV=${OPERATOR_VERSION+rhacs-operator.v}${OPERATOR_VERSION:-${currentCSV}} - catalogSource=$(oc get packagemanifests rhacs-operator -o jsonpath="{.status.catalogSource}") - catalogSourceNamespace=$(oc get packagemanifests rhacs-operator -o jsonpath="{.status.catalogSourceNamespace}") - echo "Add subscription" - echo " - apiVersion: operators.coreos.com/v1alpha1 - kind: Subscription - metadata: - name: rhacs-operator - namespace: openshift-operators - spec: - channel: ${OPERATOR_CHANNEL:-stable} - installPlanApproval: ${approval^} - name: rhacs-operator - source: ${catalogSource} - sourceNamespace: ${catalogSourceNamespace} - startingCSV: ${currentCSV## } - " | sed -e 's/^ //' \ - | tee >(cat 1>&2) \ - | oc apply -f - - OPERATOR_VERSION="${currentCSV}" -} - -function approve_install_plan() { - local subscription='subscription.operators.coreos.com/rhacs-operator' - local plan - echo "Wait for install plan." - while true; do - plan=$(oc -n openshift-operators get "${subscription}" -o jsonpath='{.status.installplan.name}' 2>/dev/null) - if [[ $plan != '' ]]; then - break - fi - printf "." - sleep 1 - done - set -x - oc -n openshift-operators patch installPlan "${plan}" --type merge --patch '{"spec":{"approved":true}}' -} - -RELEASE=4.6 -OPERATOR_VERSION=${RELEASE%.*}.$(( ${RELEASE#*.} - 2 )).0 -OPERATOR_VERSION=${RELEASE}.0 -OPERATOR_CHANNEL=stable -set -x -install_operator Automatic -#approve_install_plan -#oc wait -A --for=condition=ready -lapp==rhacs-operator,control-plane=controller-manager pod -kubectl wait --for 'jsonpath={.status.state}=AtLatestKnown' sub rhacs-operator -n openshift-operators --timeout=3m -echo $? - -## upgrade test -#oc -n openshift-operators patch subscription rhacs-operator -p '{"spec":{"installPlanApproval":"Automatic"}}' --type merge -#approve_install_plan - -# watch operator upgrade logs -#oc -n openshift-operators logs deploy/rhacs-operator-controller-manager manager -f - -# after central+cluster install, -# restart sensor to try to accelerate getting scanner results -#sleep 60 -#oc rollout restart deployment sensor -n stackrox From 74555f078faa494b37f99c7efe2258a394af764d Mon Sep 17 00:00:00 2001 From: davdhacs <105243888+davdhacs@users.noreply.github.com> Date: Thu, 9 Apr 2026 22:11:13 -0600 Subject: [PATCH 4/4] Switch scanner master prow job to apollo-ci stable tag Update scanner-test image to stable floating tag for the master branch config. Release branch configs keep their current pinned versions. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../config/stackrox/scanner/stackrox-scanner-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/stackrox/scanner/stackrox-scanner-master.yaml b/ci-operator/config/stackrox/scanner/stackrox-scanner-master.yaml index d9694af86c933..f93c873dc3e1d 100644 --- a/ci-operator/config/stackrox/scanner/stackrox-scanner-master.yaml +++ b/ci-operator/config/stackrox/scanner/stackrox-scanner-master.yaml @@ -7,7 +7,7 @@ build_root: image_stream_tag: name: apollo-ci namespace: stackrox - tag: scanner-test-0.4.9 + tag: scanner-test-stable resources: '*': requests: