From bc26dbe16ba146851f07ddef8922dc7a8c3cd492 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 17 Mar 2026 10:41:58 -0700 Subject: [PATCH 1/2] control-plane-operator/controllers/hostedcontrolplane/v2/cvo: Consume include.release.openshift.io/bootstrap-cluster-version-operator annotation The cluster-version operator has a complicated system for deciding whether a given release-image manifest should be managed in the current cluster [1,2]. Implementing that system here, or even using library-go and remembering to vendor-bump here, both seem like an annoying maintenance load. We could use the CVO's render command like the standalone installer [3,4], but that logic is fairly complicated because it needs to generate all the artifacts necessary for bootstrap MachineConfig rendering, or the production machine-config operator will complain about MachineConfigPools requesting rendered-... MachineConfig that don't exist. All we actually need out of the bootstrap container are the resources that the cluster-version operator needs to launch and run, which are labeled with the grep target since [5]. That avoids installing anything the cluster doesn't actually need here by mistake. Once the production CVO container starts, it will apply the remaining resources that the cluster actually needs. The new "is there a .status.history entry?" guard keeps this loop from running if we already have a functioning cluster-version operator (we don't want to be wrestling with the CVO over the state of the ClusterVersion CRD). The 'oc apply' (instead of 'oc create') gives us a clear "all of those exist now" exit code we can use to break out of the loop during the initial setup (because this init-container needs to complete before the long-running CVO container can start). I'm also dropping the openshift-config and openshift-config-managed Namespace creation. They are from a30db71271 (Refactor cluster-version-operator, 2024-11-18, #5125), but that commit doesn't explain why they were added or hint at where they lived before (if anywhere). I would expect the cluster-version operator to be able to create those Namespaces from the release-image manifests when they are needed, as with other cluster resources. I'm also shifting the ClusterVersion custom resource apply into the loop, to avoid attempting to apply before the ClusterVersion CRD exists and to more gracefully recover from temporary API hiccup sorts of things. I'm also adding some debugging echos and other output to make it easier to debug "hey, why is it applying these resources that I didn't expect it to?" or "... not applying the resources I did expect?". [1]: https://github.com/openshift/enhancements/blob/2b38513b8661632f08e64f4acc3b856e842f8669/dev-guide/cluster-version-operator/dev/operators.md#manifest-inclusion-annotations [2]: https://github.com/openshift/library-go/blob/ac826d10cb4081fe3034b027863c08953d95f602/pkg/manifest/manifest.go#L296-L376 [3]: https://github.com/openshift/installer/blob/a300d8c0e9d9d566a85740244a7da74d3d63e23c/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template#L189-L216 [4]: https://github.com/openshift/cluster-version-operator/blob/eaf28f5165bde27435b0f0c9a69458677034a58d/pkg/payload/render.go [5]: https://github.com/openshift/cluster-version-operator/pull/1352 --- .../assets/cluster-version-operator/deployment.yaml | 10 ++++------ .../hostedcontrolplane/v2/cvo/deployment.go | 12 ------------ 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-version-operator/deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-version-operator/deployment.yaml index f43b002ccda..2880ab4ef68 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-version-operator/deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/cluster-version-operator/deployment.yaml @@ -91,13 +91,11 @@ spec: cat > /tmp/clusterversion.json < /dev/null || oc create ns openshift-config - oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed - oc apply -f /var/payload/manifests/0000_00_cluster-version-operator_01_clusterversions* - oc apply -f /tmp/clusterversion.json - while true; do + echo 'Deciding whether bootstrap resources need to be applied...' + ls -l $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests) /tmp/clusterversion.json + while test -z "$(oc get -o jsonpath='{.status.history[0].version}' clusterversions.config.openshift.io version)"; do echo "Applying CVO bootstrap manifests..." - if oc apply -f /var/payload/manifests; then + if oc apply $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests | sed 's/^/-f /') -f /tmp/clusterversion.json; then echo "Bootstrap manifests applied successfully." break fi diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go index 56c175e9868..d74df2b1103 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cvo/deployment.go @@ -204,18 +204,6 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool, fmt.Sprintf("cp -R /release-manifests %s/", payloadDir), ) - // NOTE: We would need to leverage part of the manifest.Include logic (https://github.com/openshift/library-go/blob/0064ad7bd060b9fd52f7840972c1d3e72186d0f0/pkg/manifest/manifest.go#L190-L196) - // to properly evaluate which CVO manifests to select based on featureset. - // We only have access to bash, so we must filter based on the feature-set annotation in the manifests manually. - // Any file that has a feature-set annotation must have a value that matches the current feature set else it is removed. - // Files that do not have a feature-set annotation are included unconditionally. - stmts = append(stmts, fmt.Sprintf(`for file in $(find %s/manifests/ -name "*.yaml"); do - IFS=',' read -ra feature_sets <<< "$(cat $file | grep "release.openshift.io/feature-set:" | awk '{print $2}')" - if [[ "${#feature_sets[@]}" -gt 0 ]] && ! [[ " ${feature_sets[*]} " =~ "%s" ]]; then - rm -vf $file - fi -done`, payloadDir, featureSet)) - for _, manifest := range manifestsToOmit { if platformType == hyperv1.IBMCloudPlatform || platformType == hyperv1.PowerVSPlatform { if manifest == "0000_50_cluster-storage-operator_10_deployment-ibm-cloud-managed.yaml" || manifest == "0000_50_cluster-csi-snapshot-controller-operator_07_deployment-ibm-cloud-managed.yaml" { From 87457d8dd6683d3cbd5c6df64a4f96856a674c06 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 17 Mar 2026 12:45:48 -0700 Subject: [PATCH 2/2] control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator: Regenerate Regenerate with: $ UPDATE=true make test --- ...ents_cluster_version_operator_deployment.yaml | 16 ++++------------ ...ents_cluster_version_operator_deployment.yaml | 16 ++++------------ ...ents_cluster_version_operator_deployment.yaml | 16 ++++------------ ...ents_cluster_version_operator_deployment.yaml | 16 ++++------------ ...ents_cluster_version_operator_deployment.yaml | 16 ++++------------ 5 files changed, 20 insertions(+), 60 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/AROSwift/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/AROSwift/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml index 2d9b1f3026d..fb3ee8bbb26 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/AROSwift/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/AROSwift/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml @@ -137,12 +137,6 @@ spec: rm -f /var/payload/manifests/*_deployment.yaml rm -f /var/payload/manifests/*_servicemonitor.yaml cp -R /release-manifests /var/payload/ - for file in $(find /var/payload/manifests/ -name "*.yaml"); do - IFS=',' read -ra feature_sets <<< "$(cat $file | grep "release.openshift.io/feature-set:" | awk '{print $2}')" - if [[ "${#feature_sets[@]}" -gt 0 ]] && ! [[ " ${feature_sets[*]} " =~ "Default" ]]; then - rm -vf $file - fi - done rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-config.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-rbac.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-secret.yaml @@ -306,13 +300,11 @@ spec: cat > /tmp/clusterversion.json < /dev/null || oc create ns openshift-config - oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed - oc apply -f /var/payload/manifests/0000_00_cluster-version-operator_01_clusterversions* - oc apply -f /tmp/clusterversion.json - while true; do + echo 'Deciding whether bootstrap resources need to be applied...' + ls -l $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests) /tmp/clusterversion.json + while test -z "$(oc get -o jsonpath='{.status.history[0].version}' clusterversions.config.openshift.io version)"; do echo "Applying CVO bootstrap manifests..." - if oc apply -f /var/payload/manifests; then + if oc apply $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests | sed 's/^/-f /') -f /tmp/clusterversion.json; then echo "Bootstrap manifests applied successfully." break fi diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/GCP/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/GCP/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml index 1b3c7f44f78..0a5070f2c81 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/GCP/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/GCP/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml @@ -148,12 +148,6 @@ spec: rm -f /var/payload/manifests/*_deployment.yaml rm -f /var/payload/manifests/*_servicemonitor.yaml cp -R /release-manifests /var/payload/ - for file in $(find /var/payload/manifests/ -name "*.yaml"); do - IFS=',' read -ra feature_sets <<< "$(cat $file | grep "release.openshift.io/feature-set:" | awk '{print $2}')" - if [[ "${#feature_sets[@]}" -gt 0 ]] && ! [[ " ${feature_sets[*]} " =~ "TechPreviewNoUpgrade" ]]; then - rm -vf $file - fi - done rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-config.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-rbac.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-secret.yaml @@ -323,13 +317,11 @@ spec: cat > /tmp/clusterversion.json < /dev/null || oc create ns openshift-config - oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed - oc apply -f /var/payload/manifests/0000_00_cluster-version-operator_01_clusterversions* - oc apply -f /tmp/clusterversion.json - while true; do + echo 'Deciding whether bootstrap resources need to be applied...' + ls -l $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests) /tmp/clusterversion.json + while test -z "$(oc get -o jsonpath='{.status.history[0].version}' clusterversions.config.openshift.io version)"; do echo "Applying CVO bootstrap manifests..." - if oc apply -f /var/payload/manifests; then + if oc apply $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests | sed 's/^/-f /') -f /tmp/clusterversion.json; then echo "Bootstrap manifests applied successfully." break fi diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/IBMCloud/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/IBMCloud/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml index 553aad2bbe4..5d4a8451a8c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/IBMCloud/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/IBMCloud/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml @@ -137,12 +137,6 @@ spec: rm -f /var/payload/manifests/*_deployment.yaml rm -f /var/payload/manifests/*_servicemonitor.yaml cp -R /release-manifests /var/payload/ - for file in $(find /var/payload/manifests/ -name "*.yaml"); do - IFS=',' read -ra feature_sets <<< "$(cat $file | grep "release.openshift.io/feature-set:" | awk '{print $2}')" - if [[ "${#feature_sets[@]}" -gt 0 ]] && ! [[ " ${feature_sets[*]} " =~ "Default" ]]; then - rm -vf $file - fi - done rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-config.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-rbac.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-secret.yaml @@ -243,13 +237,11 @@ spec: cat > /tmp/clusterversion.json < /dev/null || oc create ns openshift-config - oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed - oc apply -f /var/payload/manifests/0000_00_cluster-version-operator_01_clusterversions* - oc apply -f /tmp/clusterversion.json - while true; do + echo 'Deciding whether bootstrap resources need to be applied...' + ls -l $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests) /tmp/clusterversion.json + while test -z "$(oc get -o jsonpath='{.status.history[0].version}' clusterversions.config.openshift.io version)"; do echo "Applying CVO bootstrap manifests..." - if oc apply -f /var/payload/manifests; then + if oc apply $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests | sed 's/^/-f /') -f /tmp/clusterversion.json; then echo "Bootstrap manifests applied successfully." break fi diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml index e1230b097c4..fb3ee8bbb26 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml @@ -137,12 +137,6 @@ spec: rm -f /var/payload/manifests/*_deployment.yaml rm -f /var/payload/manifests/*_servicemonitor.yaml cp -R /release-manifests /var/payload/ - for file in $(find /var/payload/manifests/ -name "*.yaml"); do - IFS=',' read -ra feature_sets <<< "$(cat $file | grep "release.openshift.io/feature-set:" | awk '{print $2}')" - if [[ "${#feature_sets[@]}" -gt 0 ]] && ! [[ " ${feature_sets[*]} " =~ "TechPreviewNoUpgrade" ]]; then - rm -vf $file - fi - done rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-config.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-rbac.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-secret.yaml @@ -306,13 +300,11 @@ spec: cat > /tmp/clusterversion.json < /dev/null || oc create ns openshift-config - oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed - oc apply -f /var/payload/manifests/0000_00_cluster-version-operator_01_clusterversions* - oc apply -f /tmp/clusterversion.json - while true; do + echo 'Deciding whether bootstrap resources need to be applied...' + ls -l $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests) /tmp/clusterversion.json + while test -z "$(oc get -o jsonpath='{.status.history[0].version}' clusterversions.config.openshift.io version)"; do echo "Applying CVO bootstrap manifests..." - if oc apply -f /var/payload/manifests; then + if oc apply $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests | sed 's/^/-f /') -f /tmp/clusterversion.json; then echo "Bootstrap manifests applied successfully." break fi diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml index 2d9b1f3026d..fb3ee8bbb26 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/cluster-version-operator/zz_fixture_TestControlPlaneComponents_cluster_version_operator_deployment.yaml @@ -137,12 +137,6 @@ spec: rm -f /var/payload/manifests/*_deployment.yaml rm -f /var/payload/manifests/*_servicemonitor.yaml cp -R /release-manifests /var/payload/ - for file in $(find /var/payload/manifests/ -name "*.yaml"); do - IFS=',' read -ra feature_sets <<< "$(cat $file | grep "release.openshift.io/feature-set:" | awk '{print $2}')" - if [[ "${#feature_sets[@]}" -gt 0 ]] && ! [[ " ${feature_sets[*]} " =~ "Default" ]]; then - rm -vf $file - fi - done rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-config.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-rbac.yaml rm -f /var/payload/release-manifests/0000_50_olm_00-pprof-secret.yaml @@ -306,13 +300,11 @@ spec: cat > /tmp/clusterversion.json < /dev/null || oc create ns openshift-config - oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed - oc apply -f /var/payload/manifests/0000_00_cluster-version-operator_01_clusterversions* - oc apply -f /tmp/clusterversion.json - while true; do + echo 'Deciding whether bootstrap resources need to be applied...' + ls -l $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests) /tmp/clusterversion.json + while test -z "$(oc get -o jsonpath='{.status.history[0].version}' clusterversions.config.openshift.io version)"; do echo "Applying CVO bootstrap manifests..." - if oc apply -f /var/payload/manifests; then + if oc apply $(grep -rl 'include.release.openshift.io/bootstrap-cluster-version-operator: .*hypershift' /var/payload/manifests | sed 's/^/-f /') -f /tmp/clusterversion.json; then echo "Bootstrap manifests applied successfully." break fi