openshift · amp-rh · Apr 29, 2026 · Apr 29, 2026 · Apr 29, 2026 · Apr 29, 2026
diff --git a/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml b/ci-operator/config/RedHatQE/firewatch/RedHatQE-firewatch-main__slack-webhook-test.yaml
@@ -0,0 +1,193 @@
+base_images:
+  cli:
+    name: "4.21"
+    namespace: ocp
+    tag: cli
+  firewatch-base:
+    name: firewatch
+    namespace: firewatch
+    tag: main
+build_root:
+  image_stream_tag:
+    name: release
+    namespace: openshift
+    tag: rhel-9-release-golang-1.19-openshift-4.15
+images:
+  items:
+  - dockerfile_literal: |
+      FROM firewatch-base
+      RUN pip install --no-cache-dir \
+          "firewatch @ git+https://github.com/amp-rh/firewatch.git@interop-8976/token-rotation-alerts" \
+          && firewatch --help > /dev/null \
+          && echo "PIPELINE_IMAGE_MARKER=2026-05-04" > /tmp/.pipeline-image-marker \
+          && python3 -c "from src.report.report import Report; print('IMPORT CHECK: Report has _notify_failure_webhooks:', hasattr(Report, '_notify_failure_webhooks'))"
+    from: firewatch-base
+    to: main
+releases:
+  latest:
+    candidate:
+      product: ocp
+      stream: nightly
+      version: "4.21"
+resources:
+  '*':
+    requests:
+      cpu: 200m
+      memory: 400Mi
+tests:
+- as: slack-webhook-test
+  cron: 0 23 31 2 *
+  steps:
+    cluster_profile: aws-cspi-qe
+    env:
+      BASE_DOMAIN: cspilp.interop.ccitredhat.com
+      FIREWATCH_CONFIG: |
+        {
+          "failure_rules": [
+            {"step": "*", "failure_type": "all", "classification": "Test", "jira_project": "INTEROP", "slack_user": "mpruitt@redhat.com"}
+          ]
+        }
+      FIREWATCH_DEFAULT_JIRA_ASSIGNEE: mpruitt@redhat.com
+      FIREWATCH_DEFAULT_JIRA_PROJECT: INTEROP
+    post:
+    - as: firewatch-report-issues-fork
+      from: main
+      commands: |
+        #!/bin/bash
+        set -o nounset
+        set -o errexit
+        set -o pipefail
+
+        echo "=== Firewatch Debug Info ==="
+        firewatch --version 2>&1 || echo "WARNING: firewatch --version not supported"
+        pip show firewatch 2>/dev/null | grep -E '^(Name|Version|Location|Requires)' || true
+        python3 -c "import firewatch; print('firewatch package path:', firewatch.__file__)" 2>/dev/null || true
+        echo "=== Pipeline Image Check ==="
+        if [ -f /tmp/.pipeline-image-marker ]; then
+            echo "PIPELINE IMAGE CONFIRMED: $(cat /tmp/.pipeline-image-marker)"
+        else
+            echo "WARNING: NOT running pipeline image (marker file missing)"
+        fi
+        python3 -c "from src.report.report import Report; print('has _notify_failure_webhooks:', hasattr(Report, '_notify_failure_webhooks'))" 2>/dev/null || echo "WARNING: could not import Report"
+        echo "=== End Debug Info ==="
+
+        jira_config_cmd="firewatch jira-config-gen --token-path ${FIREWATCH_JIRA_API_TOKEN_PATH} --server-url ${FIREWATCH_JIRA_SERVER}"
+        if [ -f "${FIREWATCH_JIRA_EMAIL_PATH}" ]; then
+            jira_config_cmd+=" --email $(cat "${FIREWATCH_JIRA_EMAIL_PATH}")"
+        fi
+        eval "${jira_config_cmd}"
+
+        report_command="firewatch report"
+        if [ "${FIREWATCH_PRIVATE_DECK,,}" = "true" ]; then
+            report_command+=" --gcs-bucket qe-private-deck --gcs-creds-file /tmp/secrets/private-deck/creds.json"
+        fi
+        if [ "${FIREWATCH_FAIL_WITH_TEST_FAILURES,,}" = "true" ]; then
+            report_command+=" --fail-with-test-failures"
+        fi
+        if [ "${FIREWATCH_FAIL_WITH_POD_FAILURES,,}" = "true" ]; then
+            report_command+=" --fail-with-pod-failures"
+        fi
+        if [ "${FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING,,}" = "true" ]; then
+            report_command+=" --verbose-test-failure-reporting"
+            report_command+=" --verbose-test-failure-reporting-ticket-limit ${FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING_LIMIT}"
+        fi
+        if [ -n "${FIREWATCH_CONFIG_FILE_PATH}" ]; then
+            report_command+=" --firewatch-config-path=${FIREWATCH_CONFIG_FILE_PATH}"
+        fi
+        if [ -f "${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}" ]; then
+            report_command+=" --additional-labels-file=${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}"
+        fi
+
+        if [ -f /tmp/secrets/slack/slack_rule_notification_webhook_url ]; then
+            SLACK_WEBHOOK_URL=$(cat /tmp/secrets/slack/slack_rule_notification_webhook_url)
+            SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL%"${SLACK_WEBHOOK_URL##*[![:space:]]}"}"
+            if [ -z "${SLACK_WEBHOOK_URL}" ]; then
+                echo "ERROR: slack_rule_notification_webhook_url secret is present but empty" >&2
+                exit 1
+            fi
+            export SLACK_WEBHOOK_URL
+            echo "=== Slack Webhook ==="
+            echo "SLACK_WEBHOOK_URL is set (${#SLACK_WEBHOOK_URL} chars, starts with: ${SLACK_WEBHOOK_URL:0:30}...)"
+            echo "=== End Slack Webhook ==="
+        else
+            echo "=== Slack Webhook ==="
+            echo "WARNING: /tmp/secrets/slack/slack_rule_notification_webhook_url not found"
+            ls -la /tmp/secrets/slack/ 2>/dev/null || echo "WARNING: /tmp/secrets/slack/ directory does not exist"
+            echo "=== End Slack Webhook ==="
+        fi
+
+        echo "=== Report Command ==="
+        echo $report_command
+        echo "=== End Report Command ==="
+        eval "$report_command"
+      credentials:
+      - namespace: test-credentials
+        name: firewatch-tool-jira-credentials
+        mount_path: /tmp/secrets/jira
+      - namespace: test-credentials
+        name: firewatch-tool-private-deck-credentials
+        mount_path: /tmp/secrets/private-deck
+      - namespace: test-credentials
+        name: firewatch-slack-notify-webhook-url
+        mount_path: /tmp/secrets/slack
+      env:
+      - name: FIREWATCH_JIRA_SERVER
+        default: https://redhat.atlassian.net
+      - name: FIREWATCH_JIRA_API_TOKEN_PATH
+        default: /tmp/secrets/jira/access_token
+      - name: FIREWATCH_JIRA_EMAIL_PATH
+        default: /tmp/secrets/jira/email
+      - name: FIREWATCH_FAIL_WITH_TEST_FAILURES
+        default: "false"
+      - name: FIREWATCH_FAIL_WITH_POD_FAILURES
+        default: "false"
+      - name: FIREWATCH_CONFIG
+        default: ""
+      - name: FIREWATCH_PRIVATE_DECK
+        default: "false"
+      - name: FIREWATCH_DEFAULT_JIRA_PROJECT
+      - name: FIREWATCH_DEFAULT_JIRA_EPIC
+        default: ""
+      - name: FIREWATCH_DEFAULT_JIRA_COMPONENT
+        default: ""
+      - name: FIREWATCH_DEFAULT_JIRA_AFFECTS_VERSION
+        default: ""
+      - name: FIREWATCH_DEFAULT_JIRA_ADDITIONAL_LABELS
+        default: ""
+      - name: FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE
+        default: firewatch-additional-labels
+      - name: FIREWATCH_DEFAULT_JIRA_ASSIGNEE
+        default: ""
+      - name: FIREWATCH_DEFAULT_JIRA_PRIORITY
+        default: ""
+      - name: FIREWATCH_DEFAULT_JIRA_SECURITY_LEVEL
+        default: ""
+      - name: FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING
+        default: "false"
+      - name: FIREWATCH_VERBOSE_TEST_FAILURE_REPORTING_LIMIT
+        default: "10"
+      - name: FIREWATCH_CONFIG_FILE_PATH
+        default: ""
+      - name: SLACK_WEBHOOK_URL
+        default: ""
+      resources:
+        requests:
+          cpu: 10m
+          memory: 100Mi
+    pre:
+    - ref: ipi-conf
+    - ref: ipi-conf-aws
+    - chain: ipi-install
+    test:
+    - as: fail-to-trigger-webhook
+      commands: exit 1
+      from: cli
+      resources:
+        requests:
+          cpu: 100m
+          memory: 100Mi
+zz_generated_metadata:
+  branch: main
+  org: RedHatQE
+  repo: firewatch
+  variant: slack-webhook-test
diff --git a/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-periodics.yaml b/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-periodics.yaml
@@ -0,0 +1,84 @@
+periodics:
+- agent: kubernetes
+  cluster: build10
+  cron: 0 23 31 2 *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: main
+    org: RedHatQE
+    repo: firewatch
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws-cspi-qe
+    ci-operator.openshift.io/variant: slack-webhook-test
+    ci.openshift.io/generator: prowgen
+    job-release: "4.21"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-RedHatQE-firewatch-main-slack-webhook-test-slack-webhook-test
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=slack-webhook-test
+      - --variant=slack-webhook-test
+      command:
+      - ci-operator
+      env:
+      - name: HTTP_SERVER_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      ports:
+      - containerPort: 8080
+        name: http
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
diff --git a/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-presubmits.yaml b/ci-operator/jobs/RedHatQE/firewatch/RedHatQE-firewatch-main-presubmits.yaml
@@ -57,3 +57,61 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )image-images,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build11
+    context: ci/prow/slack-webhook-test-images
+    decorate: true
+    decoration_config:
+      skip_cloning: true
+    labels:
+      ci-operator.openshift.io/variant: slack-webhook-test
+      ci.openshift.io/generator: prowgen
+      job-release: "4.21"
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-RedHatQE-firewatch-main-slack-webhook-test-images
+    rerun_command: /test slack-webhook-test-images
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --report-credentials-file=/etc/report/credentials
+        - --target=[images]
+        - --variant=slack-webhook-test
+        command:
+        - ci-operator
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )slack-webhook-test-images,?($|\s.*)
diff --git a/ci-operator/step-registry/cluster-profiles/cluster-profiles-config.yaml b/ci-operator/step-registry/cluster-profiles/cluster-profiles-config.yaml
@@ -1073,6 +1073,7 @@
         - web-terminal-operator
     - org: RedHatQE
       repos:
+        - firewatch
         - interop-testing
     - org: rh-messaging-qe
       repos:

diff --git a/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh b/ci-operator/step-registry/firewatch/report-issues/firewatch-report-issues-commands.sh
@@ -4,6 +4,12 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
+echo "=== Firewatch Debug Info ==="
+firewatch --version 2>&1 || echo "WARNING: firewatch --version not supported"
+pip show firewatch 2>/dev/null | grep -E '^(Name|Version|Location)' || true
+python3 -c "import firewatch; print('firewatch package path:', firewatch.__file__)" 2>/dev/null || true
+echo "=== End Debug Info ==="
+
 jira_config_cmd="firewatch jira-config-gen --token-path ${FIREWATCH_JIRA_API_TOKEN_PATH} --server-url ${FIREWATCH_JIRA_SERVER}"
 
 if [ -f "${FIREWATCH_JIRA_EMAIL_PATH}" ]; then
@@ -42,6 +48,26 @@ if [ -f "${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}" ]; then
     report_command+=" --additional-labels-file=${SHARED_DIR}/${FIREWATCH_JIRA_ADDITIONAL_LABELS_FILE}"
 fi
 
+if [ -f /tmp/secrets/slack/slack_rule_notification_webhook_url ]; then
+    SLACK_WEBHOOK_URL=$(cat /tmp/secrets/slack/slack_rule_notification_webhook_url)
+    SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL%"${SLACK_WEBHOOK_URL##*[![:space:]]}"}"
+    if [ -z "${SLACK_WEBHOOK_URL}" ]; then
+        echo "ERROR: slack_rule_notification_webhook_url secret is present but empty" >&2
+        exit 1
+    fi
+    export SLACK_WEBHOOK_URL
+    echo "=== Slack Webhook ==="
+    echo "SLACK_WEBHOOK_URL is set (${#SLACK_WEBHOOK_URL} chars, starts with: ${SLACK_WEBHOOK_URL:0:30}...)"
+    echo "=== End Slack Webhook ==="
+else
+    echo "=== Slack Webhook ==="
+    echo "WARNING: /tmp/secrets/slack/slack_rule_notification_webhook_url not found"
+    ls -la /tmp/secrets/slack/ 2>/dev/null || echo "WARNING: /tmp/secrets/slack/ directory does not exist"
+    echo "=== End Slack Webhook ==="
+fi
+
+echo "=== Report Command ==="
 echo $report_command
+echo "=== End Report Command ==="
 
 eval "$report_command"