From 31eee68b9b7a111f944da00ce2bac698381d904f Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 29 Apr 2026 08:47:27 -0500 Subject: [PATCH 01/11] INTEROP-8979: Wire Slack webhook into firewatch-report-issues step --- ...QE-firewatch-main__slack-webhook-test.yaml | 63 ++++++++++++++ .../RedHatQE-firewatch-main-periodics.yaml | 84 +++++++++++++++++++ .../cluster-profiles-config.yaml | 1 + .../firewatch-report-issues-commands.sh | 10 +++ .../firewatch-report-issues-ref.yaml | 6 ++ 5 files changed, 164 insertions(+) create mode 100644 ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml create mode 100644 ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-periodics.yaml diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml new file mode 100644 index 0000000000000..07855198c61ea --- /dev/null +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -0,0 +1,63 @@ +base_images: + cli: + name: "4.21" + namespace: ocp + tag: cli + firewatch-base: + name: firewatch + namespace: firewatch + tag: main +images: + items: + - dockerfile_literal: | + FROM firewatch-base + RUN pip install --no-cache-dir \ + "firewatch @ git+https://github.com/RedHatQE/firewatch.git@slack-webhook-notifications" \ + && echo "SUCCESS: firewatch PR branch (slack-webhook-notifications) injected at image build time" + from: firewatch-base + to: main +releases: + latest: + candidate: + product: ocp + stream: nightly + version: "4.21" +resources: + '*': + requests: + cpu: 200m + memory: 400Mi +tests: +- as: slack-webhook-test + cron: 0 23 31 2 * + steps: + cluster_profile: aws-cspi-qe + env: + BASE_DOMAIN: cspilp.interop.ccitredhat.com + FIREWATCH_CONFIG: | + { + "failure_rules": [ + {"step": "*", "failure_type": "all", "classification": "Test", "jira_project": "INTEROP", "slack_user": "mpruitt@redhat.com"} + ] + } + FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com + FIREWATCH_DEFAULT_JIRA_PROJECT: INTEROP + post: + - ref: firewatch-report-issues + pre: + - ref: ipi-conf + - ref: ipi-conf-aws + - chain: ipi-install + test: + - as: fail-to-trigger-webhook + commands: exit 1 + from: cli + resources: + requests: + cpu: 100m + memory: 100Mi +zz_generated_metadata: + branch: main + org: RedHatQE + repo: firewatch + variant: slack-webhook-test diff --git a/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-periodics.yaml b/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-periodics.yaml new file mode 100644 index 0000000000000..fcd0bd08185e8 --- /dev/null +++ b/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-periodics.yaml @@ -0,0 +1,84 @@ +periodics: +- agent: kubernetes + cluster: build10 + cron: 0 23 31 2 * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: main + org: RedHatQE + repo: firewatch + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe + ci-operator.openshift.io/variant: slack-webhook-test + ci.openshift.io/generator: prowgen + job-release: "4.21" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-RedHatQE-firewatch-main-slack-webhook-test-slack-webhook-test + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=slack-webhook-test + - --variant=slack-webhook-test + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/step-registry/cluster-profiles/cluster-profiles-config.yaml b/ci-operator/step-registry/cluster-profiles/cluster-profiles-config.yaml index 7983cb68e1c4d..56e1bb36bb829 100644 --- a/ci-operator/step-registry/cluster-profiles/cluster-profiles-config.yaml +++ b/ci-operator/step-registry/cluster-profiles/cluster-profiles-config.yaml @@ -1073,6 +1073,7 @@ - web-terminal-operator - org: RedHatQE repos: + - firewatch - interop-testing - org: rh-messaging-qe repos: diff --git a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh index 0597051b1c151..f13d74e300755 100644 --- a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh +++ b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh @@ -42,6 +42,16 @@ if [ -f "${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}" ]; then report_command+=" --additional-labels-file=${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}" fi +if [ -f /tmp/secrets/slack/slack_rule_notification_webhook_url ]; then + SLACK_WEBHOOK_URL=$(cat /tmp/secrets/slack/slack_rule_notification_webhook_url) + SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL%"${SLACK_WEBHOOK_URL##*[![:space:]]}"}" + if [ -z "${SLACK_WEBHOOK_URL}" ]; then + echo "ERROR: slack_rule_notification_webhook_url secret is present but empty" >&2 + exit 1 + fi + export SLACK_WEBHOOK_URL +fi + echo $report_command eval "$report_command" diff --git a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-ref.yaml b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-ref.yaml index f3da045038b39..be17fbad0df2d 100644 --- a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-ref.yaml +++ b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-ref.yaml @@ -12,6 +12,9 @@ ref: - namespace: test-credentials name: firewatch-tool-private-deck-credentials mount_path: /tmp/secrets/private-deck + - namespace: test-credentials + name: firewatch-slack-notify-webhook-url + mount_path: /tmp/secrets/slack resources: requests: cpu: 10m @@ -74,6 +77,9 @@ ref: - name: FIREWATCH_CONFIG_FILE_PATH default: "" documentation: The URL/file path of the Firewatch configuration (json file). + - name: SLACK_WEBHOOK_URL + default: "" + documentation: Slack incoming webhook URL. When set, firewatch posts notifications on issue creation and duplicate detection. documentation: |- Please see the CSPI-QE repository for more documentation on this tool: https://github.com/CSPI-QE/firewatch \ No newline at end of file From 70f3137c83ba23f725a930e098b9318f6b8362bb Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 29 Apr 2026 10:39:04 -0500 Subject: [PATCH 02/11] Fix CI config: add build_root and remove redundant from field --- ...QE-firewatch-main__slack-webhook-test.yaml | 6 +- .../RedHatQE-firewatch-main-presubmits.yaml | 58 +++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index 07855198c61ea..93e8e38837677 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -7,6 +7,11 @@ base_images: name: firewatch namespace: firewatch tag: main +build_root: + image_stream_tag: + name: release + namespace: openshift + tag: rhel-9-release-golang-1.19-openshift-4.15 images: items: - dockerfile_literal: | @@ -14,7 +19,6 @@ images: RUN pip install --no-cache-dir \ "firewatch @ git+https://github.com/RedHatQE/firewatch.git@slack-webhook-notifications" \ && echo "SUCCESS: firewatch PR branch (slack-webhook-notifications) injected at image build time" - from: firewatch-base to: main releases: latest: diff --git a/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-presubmits.yaml b/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-presubmits.yaml index 73de2c6a7b2d1..1284af8bebd7f 100644 --- a/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-presubmits.yaml +++ b/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-presubmits.yaml @@ -57,3 +57,61 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )image-images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build11 + context: ci/prow/slack-webhook-test-images + decorate: true + decoration_config: + skip_cloning: true + labels: + ci-operator.openshift.io/variant: slack-webhook-test + ci.openshift.io/generator: prowgen + job-release: "4.21" + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-RedHatQE-firewatch-main-slack-webhook-test-images + rerun_command: /test slack-webhook-test-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --variant=slack-webhook-test + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )slack-webhook-test-images,?($|\s.*) From 197488a4489d1e36e9ee3a975c882493d399ce79 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 29 Apr 2026 11:40:30 -0500 Subject: [PATCH 03/11] Restore from field for dockerfile_literal base image resolution --- .../firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index 93e8e38837677..1aa3dd022053f 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -19,6 +19,7 @@ images: RUN pip install --no-cache-dir \ "firewatch @ git+https://github.com/RedHatQE/firewatch.git@slack-webhook-notifications" \ && echo "SUCCESS: firewatch PR branch (slack-webhook-notifications) injected at image build time" + from: firewatch-base to: main releases: latest: From 6af2cf8480626205059a03c1b7be0204085b7697 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 29 Apr 2026 12:11:05 -0500 Subject: [PATCH 04/11] Fix pip install to use fork branch (amp-rh/firewatch@interop-8976/token-rotation-alerts) --- .../RedHatQE-firewatch-main__slack-webhook-test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index 1aa3dd022053f..ef5d4440ea7b0 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -17,8 +17,8 @@ images: - dockerfile_literal: | FROM firewatch-base RUN pip install --no-cache-dir \ - "firewatch @ git+https://github.com/RedHatQE/firewatch.git@slack-webhook-notifications" \ - && echo "SUCCESS: firewatch PR branch (slack-webhook-notifications) injected at image build time" + "firewatch @ git+https://github.com/amp-rh/firewatch.git@interop-8976/token-rotation-alerts" \ + && echo "SUCCESS: firewatch PR branch injected at image build time" from: firewatch-base to: main releases: From 170dff3d1d9d2df5126f2e42b3f632b0cef3034a Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Wed, 29 Apr 2026 13:59:08 -0500 Subject: [PATCH 05/11] Add debugging output to firewatch-report-issues step --- .../firewatch-report-issues-commands.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh index f13d74e300755..761aa40d75322 100644 --- a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh +++ b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh @@ -4,6 +4,12 @@ set -o nounset set -o errexit set -o pipefail +echo "=== Firewatch Debug Info ===" +firewatch --version 2>&1 || echo "WARNING: firewatch --version not supported" +pip show firewatch 2>/dev/null | grep -E '^(Name|Version|Location)' || true +python3 -c "import firewatch; print('firewatch package path:', firewatch.__file__)" 2>/dev/null || true +echo "=== End Debug Info ===" + jira_config_cmd="firewatch jira-config-gen --token-path ${FIREWATCH_JIRA_API_TOKEN_PATH} --server-url ${FIREWATCH_JIRA_SERVER}" if [ -f "${FIREWATCH_JIRA_EMAIL_PATH}" ]; then @@ -50,8 +56,18 @@ if [ -f /tmp/secrets/slack/slack_rule_notification_webhook_url ]; then exit 1 fi export SLACK_WEBHOOK_URL + echo "=== Slack Webhook ===" + echo "SLACK_WEBHOOK_URL is set (${#SLACK_WEBHOOK_URL} chars, starts with: ${SLACK_WEBHOOK_URL:0:30}...)" + echo "=== End Slack Webhook ===" +else + echo "=== Slack Webhook ===" + echo "WARNING: /tmp/secrets/slack/slack_rule_notification_webhook_url not found" + ls -la /tmp/secrets/slack/ 2>/dev/null || echo "WARNING: /tmp/secrets/slack/ directory does not exist" + echo "=== End Slack Webhook ===" fi +echo "=== Report Command ===" echo $report_command +echo "=== End Report Command ===" eval "$report_command" From 95fdca97c9f7e470c9abe66ec8881ccad1601cfe Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 1 May 2026 11:51:04 -0500 Subject: [PATCH 06/11] Cache-bust dockerfile_literal to force fresh pip install of fork branch --- .../firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index ef5d4440ea7b0..7747732b97a5d 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -18,7 +18,8 @@ images: FROM firewatch-base RUN pip install --no-cache-dir \ "firewatch @ git+https://github.com/amp-rh/firewatch.git@interop-8976/token-rotation-alerts" \ - && echo "SUCCESS: firewatch PR branch injected at image build time" + && firewatch --help > /dev/null \ + && echo "CACHE_BUST=2026-05-01a" from: firewatch-base to: main releases: From 66675bd3a94d8a97f0b506d227cc67ab73509d8b Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 1 May 2026 13:06:36 -0500 Subject: [PATCH 07/11] Use pipeline image for firewatch-report-issues step The step ref's from_image pulls the published firewatch:main imagestream directly, ignoring the custom-built pipeline image. Inline the step with from: main so it uses the image built from the dockerfile_literal containing the fork branch. --- ...QE-firewatch-main__slack-webhook-test.yaml | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index 7747732b97a5d..f52bca720e2b1 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -19,7 +19,7 @@ images: RUN pip install --no-cache-dir \ "firewatch @ git+https://github.com/amp-rh/firewatch.git@interop-8976/token-rotation-alerts" \ && firewatch --help > /dev/null \ - && echo "CACHE_BUST=2026-05-01a" + && echo "CACHE_BUST=2026-05-01b" from: firewatch-base to: main releases: @@ -49,7 +49,23 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_PROJECT: INTEROP post: - - ref: firewatch-report-issues + - as: firewatch-report-issues + from: main + commands: firewatch-report-issues-commands.sh + credentials: + - namespace: test-credentials + name: firewatch-tool-jira-credentials + mount_path: /tmp/secrets/jira + - namespace: test-credentials + name: firewatch-tool-private-deck-credentials + mount_path: /tmp/secrets/private-deck + - namespace: test-credentials + name: firewatch-slack-notify-webhook-url + mount_path: /tmp/secrets/slack + resources: + requests: + cpu: 10m + memory: 100Mi pre: - ref: ipi-conf - ref: ipi-conf-aws From 552d72e57a1841a23795bfc113ecf40d86863fa8 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 1 May 2026 14:33:52 -0500 Subject: [PATCH 08/11] Add env declarations to inlined firewatch-report-issues step --- ...QE-firewatch-main__slack-webhook-test.yaml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index f52bca720e2b1..ae2c8effda495 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -62,6 +62,46 @@ tests: - namespace: test-credentials name: firewatch-slack-notify-webhook-url mount_path: /tmp/secrets/slack + env: + - name: FIREWATCH_JIRA_SERVER + default: https://redhat.atlassian.net + - name: FIREWATCH_JIRA_API_TOKEN_PATH + default: /tmp/secrets/jira/access_token + - name: FIREWATCH_JIRA_EMAIL_PATH + default: /tmp/secrets/jira/email + - name: FIREWATCH_FAIL_WITH_TEST_FAILURES + default: "false" + - name: FIREWATCH_FAIL_WITH_POD_FAILURES + default: "false" + - name: FIREWATCH_CONFIG + default: "" + - name: FIREWATCH_PRIVATE_DECK + default: "false" + - name: FIREWATCH_DEFAULT_JIRA_PROJECT + - name: FIREWATCH_DEFAULT_JIRA_EPIC + default: "" + - name: FIREWATCH_DEFAULT_JIRA_COMPONENT + default: "" + - name: FIREWATCH_DEFAULT_JIRA_AFFECTS_VERSION + default: "" + - name: FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS + default: "" + - name: FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE + default: firewatch-additional-labels + - name: FIREWATCH_DEFAULT_JIRA_ASSIGNEE + default: "" + - name: FIREWATCH_DEFAULT_JIRA_PRIORITY + default: "" + - name: FIREWATCH_DEFAULT_JIRA_SECURITY_LEVEL + default: "" + - name: FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING + default: "false" + - name: FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING_LIMIT + default: "10" + - name: FIREWATCH_CONFIG_FILE_PATH + default: "" + - name: SLACK_WEBHOOK_URL + default: "" resources: requests: cpu: 10m From 9082f6c77671766518c93ae9056314b29328944c Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Mon, 4 May 2026 07:57:36 -0500 Subject: [PATCH 09/11] Inline commands for firewatch-report-issues step The commands field in an inline step is executed as a literal shell command, not resolved from the step registry. Replace the filename reference with the actual script content. --- ...QE-firewatch-main__slack-webhook-test.yaml | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index ae2c8effda495..f8d15bd690a6b 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -51,7 +51,67 @@ tests: post: - as: firewatch-report-issues from: main - commands: firewatch-report-issues-commands.sh + commands: | + #!/bin/bash + set -o nounset + set -o errexit + set -o pipefail + + echo "=== Firewatch Debug Info ===" + firewatch --version 2>&1 || echo "WARNING: firewatch --version not supported" + pip show firewatch 2>/dev/null | grep -E '^(Name|Version|Location)' || true + python3 -c "import firewatch; print('firewatch package path:', firewatch.__file__)" 2>/dev/null || true + echo "=== End Debug Info ===" + + jira_config_cmd="firewatch jira-config-gen --token-path ${FIREWATCH_JIRA_API_TOKEN_PATH} --server-url ${FIREWATCH_JIRA_SERVER}" + if [ -f "${FIREWATCH_JIRA_EMAIL_PATH}" ]; then + jira_config_cmd+=" --email $(cat "${FIREWATCH_JIRA_EMAIL_PATH}")" + fi + eval "${jira_config_cmd}" + + report_command="firewatch report" + if [ "${FIREWATCH_PRIVATE_DECK,,}" = "true" ]; then + report_command+=" --gcs-bucket qe-private-deck --gcs-creds-file /tmp/secrets/private-deck/creds.json" + fi + if [ "${FIREWATCH_FAIL_WITH_TEST_FAILURES,,}" = "true" ]; then + report_command+=" --fail-with-test-failures" + fi + if [ "${FIREWATCH_FAIL_WITH_POD_FAILURES,,}" = "true" ]; then + report_command+=" --fail-with-pod-failures" + fi + if [ "${FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING,,}" = "true" ]; then + report_command+=" --verbose-test-failure-reporting" + report_command+=" --verbose-test-failure-reporting-ticket-limit ${FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING_LIMIT}" + fi + if [ -n "${FIREWATCH_CONFIG_FILE_PATH}" ]; then + report_command+=" --firewatch-config-path=${FIREWATCH_CONFIG_FILE_PATH}" + fi + if [ -f "${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}" ]; then + report_command+=" --additional-labels-file=${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}" + fi + + if [ -f /tmp/secrets/slack/slack_rule_notification_webhook_url ]; then + SLACK_WEBHOOK_URL=$(cat /tmp/secrets/slack/slack_rule_notification_webhook_url) + SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL%"${SLACK_WEBHOOK_URL##*[![:space:]]}"}" + if [ -z "${SLACK_WEBHOOK_URL}" ]; then + echo "ERROR: slack_rule_notification_webhook_url secret is present but empty" >&2 + exit 1 + fi + export SLACK_WEBHOOK_URL + echo "=== Slack Webhook ===" + echo "SLACK_WEBHOOK_URL is set (${#SLACK_WEBHOOK_URL} chars, starts with: ${SLACK_WEBHOOK_URL:0:30}...)" + echo "=== End Slack Webhook ===" + else + echo "=== Slack Webhook ===" + echo "WARNING: /tmp/secrets/slack/slack_rule_notification_webhook_url not found" + ls -la /tmp/secrets/slack/ 2>/dev/null || echo "WARNING: /tmp/secrets/slack/ directory does not exist" + echo "=== End Slack Webhook ===" + fi + + echo "=== Report Command ===" + echo $report_command + echo "=== End Report Command ===" + eval "$report_command" credentials: - namespace: test-credentials name: firewatch-tool-jira-credentials From 1f40470cfeb5ba780de4af15bbe0891b9d1253d6 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Mon, 4 May 2026 09:36:13 -0500 Subject: [PATCH 10/11] Rename inline step to use pipeline image instead of published imagestream The step name firewatch-report-issues collided with the registered step ref, causing ci-operator to use the ref's from_image (published firewatch:main imagestream) instead of the pipeline image built from the fork branch. Renaming to firewatch-report-issues-fork breaks the ref association so from: main correctly resolves to the pipeline image. --- .../firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index f8d15bd690a6b..1e4ab491eff39 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -49,7 +49,7 @@ tests: FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com FIREWATCH_DEFAULT_JIRA_PROJECT: INTEROP post: - - as: firewatch-report-issues + - as: firewatch-report-issues-fork from: main commands: | #!/bin/bash From f33cb445e302f14a8fd69a9e10e9dc64002b2f46 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Mon, 4 May 2026 12:13:51 -0500 Subject: [PATCH 11/11] Add pipeline image diagnostic: marker file and Report method check Writes a marker file during image build and checks for it at step runtime. Also verifies Report._notify_failure_webhooks exists in the installed code. This will definitively show whether the step uses the pipeline image or the published imagestream. --- .../RedHatQE-firewatch-main__slack-webhook-test.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml index 1e4ab491eff39..331415c037dcb 100644 --- a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml +++ b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml @@ -19,7 +19,8 @@ images: RUN pip install --no-cache-dir \ "firewatch @ git+https://github.com/amp-rh/firewatch.git@interop-8976/token-rotation-alerts" \ && firewatch --help > /dev/null \ - && echo "CACHE_BUST=2026-05-01b" + && echo "PIPELINE_IMAGE_MARKER=2026-05-04" > /tmp/.pipeline-image-marker \ + && python3 -c "from src.report.report import Report; print('IMPORT CHECK: Report has _notify_failure_webhooks:', hasattr(Report, '_notify_failure_webhooks'))" from: firewatch-base to: main releases: @@ -59,8 +60,15 @@ tests: echo "=== Firewatch Debug Info ===" firewatch --version 2>&1 || echo "WARNING: firewatch --version not supported" - pip show firewatch 2>/dev/null | grep -E '^(Name|Version|Location)' || true + pip show firewatch 2>/dev/null | grep -E '^(Name|Version|Location|Requires)' || true python3 -c "import firewatch; print('firewatch package path:', firewatch.__file__)" 2>/dev/null || true + echo "=== Pipeline Image Check ===" + if [ -f /tmp/.pipeline-image-marker ]; then + echo "PIPELINE IMAGE CONFIRMED: $(cat /tmp/.pipeline-image-marker)" + else + echo "WARNING: NOT running pipeline image (marker file missing)" + fi + python3 -c "from src.report.report import Report; print('has _notify_failure_webhooks:', hasattr(Report, '_notify_failure_webhooks'))" 2>/dev/null || echo "WARNING: could not import Report" echo "=== End Debug Info ===" jira_config_cmd="firewatch jira-config-gen --token-path ${FIREWATCH_JIRA_API_TOKEN_PATH} --server-url ${FIREWATCH_JIRA_SERVER}"