diff --git a/evals/tasks/openshift/plan-mustgather-audit-logs/plan-mustgather-audit-logs.yaml b/evals/tasks/openshift/plan-mustgather-audit-logs/plan-mustgather-audit-logs.yaml new file mode 100644 index 000000000..f1c778c32 --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-audit-logs/plan-mustgather-audit-logs.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-audit-logs + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "gather_audit_logs" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with the gather_command set to "/usr/bin/gather_audit_logs" to plan collecting audit logs from my OpenShift cluster. diff --git a/evals/tasks/openshift/plan-mustgather-custom-images/plan-mustgather-custom-images.yaml b/evals/tasks/openshift/plan-mustgather-custom-images/plan-mustgather-custom-images.yaml new file mode 100644 index 000000000..324ba5c6b --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-custom-images/plan-mustgather-custom-images.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-custom-images + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "ose-must-gather" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with images set to "registry.redhat.io/openshift4/ose-must-gather:v4.15,registry.redhat.io/openshift-logging/cluster-logging-rhel9-operator:latest" to plan a must-gather using custom images for both the platform and logging operator. diff --git a/evals/tasks/openshift/plan-mustgather-custom-namespace/plan-mustgather-custom-namespace.yaml b/evals/tasks/openshift/plan-mustgather-custom-namespace/plan-mustgather-custom-namespace.yaml new file mode 100644 index 000000000..2476e2797 --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-custom-namespace/plan-mustgather-custom-namespace.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-custom-namespace + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "my-debug-namespace" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with the namespace set to "my-debug-namespace" to plan a must-gather collection. diff --git a/evals/tasks/openshift/plan-mustgather-default/plan-mustgather-default.yaml b/evals/tasks/openshift/plan-mustgather-default/plan-mustgather-default.yaml new file mode 100644 index 000000000..e366f1626 --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-default/plan-mustgather-default.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-default + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "must-gather" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with default parameters to generate a must-gather collection plan for my OpenShift cluster. diff --git a/evals/tasks/openshift/plan-mustgather-host-network/plan-mustgather-host-network.yaml b/evals/tasks/openshift/plan-mustgather-host-network/plan-mustgather-host-network.yaml new file mode 100644 index 000000000..a79230aab --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-host-network/plan-mustgather-host-network.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-host-network + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "hostNetwork" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with host_network set to "true" and keep_resources set to "true" to plan a must-gather that uses host networking and keeps pod resources after collection. diff --git a/evals/tasks/openshift/plan-mustgather-node-selector/plan-mustgather-node-selector.yaml b/evals/tasks/openshift/plan-mustgather-node-selector/plan-mustgather-node-selector.yaml new file mode 100644 index 000000000..42ca5d0ff --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-node-selector/plan-mustgather-node-selector.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-node-selector + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "node-role.kubernetes.io/worker" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with a node_selector of "node-role.kubernetes.io/worker=true" to plan a must-gather that runs only on worker nodes. diff --git a/evals/tasks/openshift/plan-mustgather-timeout-since/plan-mustgather-timeout-since.yaml b/evals/tasks/openshift/plan-mustgather-timeout-since/plan-mustgather-timeout-since.yaml new file mode 100644 index 000000000..8f9d98cd8 --- /dev/null +++ b/evals/tasks/openshift/plan-mustgather-timeout-since/plan-mustgather-timeout-since.yaml @@ -0,0 +1,21 @@ +kind: Task +metadata: + labels: + suite: openshift + name: plan-mustgather-timeout-since + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No setup needed - plan_mustgather is a prompt that generates a plan + verify: + contains: "timeout" + cleanup: + inline: |- + #!/usr/bin/env bash + set -euo pipefail + # No cleanup needed - plan_mustgather only generates a plan without applying + prompt: + inline: Use the plan_mustgather prompt with a timeout of "30m" and since set to "2h" to plan a must-gather that only collects data from the last 2 hours and times out after 30 minutes.