From 9a2ba48bca88e2569b67a8ac6be25ddf28947708 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Sat, 6 Jun 2026 21:38:35 -0400 Subject: [PATCH 1/3] OCPEDGE-2727: Add eval harness CI jobs for cluster-diagnostic and threat-model skills Co-Authored-By: Claude Opus 4.6 --- .../openshift-eng-edge-tooling-main.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml index 05b9ec3101cf5..a1a6bbf76fd50 100644 --- a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml +++ b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml @@ -55,6 +55,24 @@ tests: clone: true from: root run_if_changed: (SKILL\.md|^scripts/lint-skills\.py|^Makefile|^plugins/.*/skills/) +- always_run: false + as: eval-cluster-diagnostic + optional: true + steps: + env: + EVAL_CONFIG: plugins/two-node/evals/cluster-diagnostic.yaml + EVAL_MODEL: claude-opus-4-6 + EVAL_PARALLELISM: "3" + workflow: openshift-claude-agent-eval +- always_run: false + as: eval-threat-model-tnf + optional: true + steps: + env: + EVAL_CONFIG: plugins/two-node/evals/threat-model-tnf.yaml + EVAL_MODEL: claude-opus-4-6 + EVAL_PARALLELISM: "3" + workflow: openshift-claude-agent-eval - as: ocp-ci-monitor cron: 0 7 * * 1-5 reporter_config: From a8bfc8790ede7282cbd5102d539a28463fd7c8db Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Tue, 16 Jun 2026 14:35:39 -0400 Subject: [PATCH 2/3] Added a line for run_if_changed --- .../edge-tooling/openshift-eng-edge-tooling-main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml index a1a6bbf76fd50..95f88719020c4 100644 --- a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml +++ b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml @@ -64,6 +64,7 @@ tests: EVAL_MODEL: claude-opus-4-6 EVAL_PARALLELISM: "3" workflow: openshift-claude-agent-eval + run_if_changed: ^plugins/two-node/(skills|evals) - always_run: false as: eval-threat-model-tnf optional: true From 3dbd3be8bd21f3fb67e18fe59bf836335fe44680 Mon Sep 17 00:00:00 2001 From: Douglas Hensel Date: Tue, 16 Jun 2026 15:05:02 -0400 Subject: [PATCH 3/3] Adding presubmit file and updating run_if_changed to both TNF skills --- .../openshift-eng-edge-tooling-main.yaml | 3 +- ...hift-eng-edge-tooling-main-presubmits.yaml | 154 ++++++++++++++++++ 2 files changed, 156 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml index 95f88719020c4..73ed6b1fe27b9 100644 --- a/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml +++ b/ci-operator/config/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main.yaml @@ -58,16 +58,17 @@ tests: - always_run: false as: eval-cluster-diagnostic optional: true + run_if_changed: ^plugins/two-node/(skills|evals) steps: env: EVAL_CONFIG: plugins/two-node/evals/cluster-diagnostic.yaml EVAL_MODEL: claude-opus-4-6 EVAL_PARALLELISM: "3" workflow: openshift-claude-agent-eval - run_if_changed: ^plugins/two-node/(skills|evals) - always_run: false as: eval-threat-model-tnf optional: true + run_if_changed: ^plugins/two-node/(skills|evals) steps: env: EVAL_CONFIG: plugins/two-node/evals/threat-model-tnf.yaml diff --git a/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml b/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml index 458e2d110f7b6..74d798c360f6f 100644 --- a/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-eng/edge-tooling/openshift-eng-edge-tooling-main-presubmits.yaml @@ -1,5 +1,159 @@ presubmits: openshift-eng/edge-tooling: + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build12 + context: ci/prow/eval-cluster-diagnostic + decorate: true + decoration_config: + sparse_checkout_files: + - images/Containerfile.ci + - images/Containerfile.markdownlint + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-edge-tooling-main-eval-cluster-diagnostic + optional: true + rerun_command: /test eval-cluster-diagnostic + run_if_changed: ^plugins/two-node/(skills|evals) + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-cluster-diagnostic + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-cluster-diagnostic,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build12 + context: ci/prow/eval-threat-model-tnf + decorate: true + decoration_config: + sparse_checkout_files: + - images/Containerfile.ci + - images/Containerfile.markdownlint + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-eng-edge-tooling-main-eval-threat-model-tnf + optional: true + rerun_command: /test eval-threat-model-tnf + run_if_changed: ^plugins/two-node/(skills|evals) + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=eval-threat-model-tnf + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )eval-threat-model-tnf,?($|\s.*) - agent: kubernetes always_run: true branches: