From afc8ffaf99dd5f45c966ad32dacf53b95d56c278 Mon Sep 17 00:00:00 2001 From: Dhruv Gautam Date: Tue, 12 May 2026 16:52:17 +0530 Subject: [PATCH 1/3] feat(CPO): add kube-scheduler ServiceMonitor with CA-signed serving certs The kube-scheduler previously auto-generated self-signed serving certificates via --cert-dir. This change adds a CA-signed serving certificate, a Service, and a ServiceMonitor to enable Prometheus metrics scraping with proper mTLS authentication. --- .../hostedcontrolplane_controller.go | 7 +++++ .../hostedcontrolplane/manifests/pki.go | 2 ++ .../hostedcontrolplane/pki/scheduler.go | 19 ++++++++++++ .../v2/assets/kube-scheduler/deployment.yaml | 17 +++++++---- .../v2/assets/kube-scheduler/service.yaml | 18 ++++++++++++ .../assets/kube-scheduler/servicemonitor.yaml | 29 +++++++++++++++++++ .../v2/kube_scheduler/component.go | 4 +++ .../v2/kube_scheduler/servicemonitor.go | 19 ++++++++++++ support/metrics/sets.go | 12 ++++++++ 9 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 control-plane-operator/controllers/hostedcontrolplane/pki/scheduler.go create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 629ab0e2b60..c8f86e05a02 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -1587,6 +1587,13 @@ func (r *HostedControlPlaneReconciler) reconcileOLMAndMiscCerts(ctx context.Cont return fmt.Errorf("failed to reconcile olm operator serving cert: %w", err) } + schedulerServerSecret := manifests.SchedulerServerCertSecret(hcp.Namespace) + if _, err := createOrUpdate(ctx, r, schedulerServerSecret, func() error { + return pki.ReconcileSchedulerServerSecret(schedulerServerSecret, rootCASecret, p.OwnerRef) + }); err != nil { + return fmt.Errorf("failed to reconcile scheduler serving cert: %w", err) + } + cvoServerCert := manifests.ClusterVersionOperatorServerCertSecret(hcp.Namespace) if _, err := createOrUpdate(ctx, r, cvoServerCert, func() error { return pki.ReconcileCVOServerSecret(cvoServerCert, rootCASecret, p.OwnerRef) diff --git a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go index b6283061039..70ba112b823 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go +++ b/control-plane-operator/controllers/hostedcontrolplane/manifests/pki.go @@ -272,6 +272,8 @@ func KASMachineBootstrapClientCertSecret(ns string) *corev1.Secret { func KCMServerCertSecret(ns string) *corev1.Secret { return secretFor(ns, "kcm-server") } +func SchedulerServerCertSecret(ns string) *corev1.Secret { return secretFor(ns, "scheduler-server") } + func ServiceAccountSigningKeySecret(ns string) *corev1.Secret { return secretFor(ns, "sa-signing-key") } func OpenShiftAPIServerCertSecret(ns string) *corev1.Secret { diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/scheduler.go b/control-plane-operator/controllers/hostedcontrolplane/pki/scheduler.go new file mode 100644 index 00000000000..6798a732900 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/pki/scheduler.go @@ -0,0 +1,19 @@ +package pki + +import ( + "fmt" + + "github.com/openshift/hypershift/support/config" + + corev1 "k8s.io/api/core/v1" +) + +func ReconcileSchedulerServerSecret(secret, ca *corev1.Secret, ownerRef config.OwnerRef) error { + dnsNames := []string{ + fmt.Sprintf("kube-scheduler.%s.svc", secret.Namespace), + fmt.Sprintf("kube-scheduler.%s.svc.cluster.local", secret.Namespace), + "kube-scheduler", + "localhost", + } + return reconcileSignedCertWithAddresses(secret, ca, ownerRef, "kube-scheduler", []string{"openshift"}, X509UsageClientServerAuth, dnsNames, nil) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml index 7e72d26c0d0..6a189ad9506 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/deployment.yaml @@ -20,7 +20,8 @@ spec: containers: - args: - --config=/etc/kubernetes/config/config.json - - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -41,13 +42,17 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m memory: 150Mi volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work + - mountPath: /etc/kubernetes/certs + name: serving-cert - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config @@ -57,8 +62,10 @@ spec: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work + - name: serving-cert + secret: + defaultMode: 416 + secretName: scheduler-server - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml new file mode 100644 index 00000000000..8226c8fde1d --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml new file mode 100644 index 00000000000..2d9e9ec8004 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,29 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kube-scheduler + annotations: + hypershift.openshift.io/metrics-job: kube-scheduler + hypershift.openshift.io/metrics-namespace: openshift-kube-scheduler + hypershift.openshift.io/metrics-service: kube-scheduler + hypershift.openshift.io/metrics-endpoint: https +spec: + endpoints: + - scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: metrics-client + keySecret: + key: tls.key + name: metrics-client + serverName: kube-scheduler + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go index 3b4bdb2818f..3b95b7b1be5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component.go @@ -40,6 +40,10 @@ func NewComponent() component.ControlPlaneComponent { "kubeconfig.yaml", component.WithAdaptFunction(adaptKubeconfig), ). + WithManifestAdapter( + "servicemonitor.yaml", + component.WithAdaptFunction(adaptServiceMonitor), + ). InjectAvailabilityProberContainer(podspec.AvailabilityProberOpts{}). Build() } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go new file mode 100644 index 00000000000..e7a0470fce5 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor.go @@ -0,0 +1,19 @@ +package scheduler + +import ( + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/metrics" + "github.com/openshift/hypershift/support/util" + + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +func adaptServiceMonitor(cpContext component.WorkloadContext, sm *prometheusoperatorv1.ServiceMonitor) error { + sm.Spec.NamespaceSelector = prometheusoperatorv1.NamespaceSelector{ + MatchNames: []string{sm.Namespace}, + } + sm.Spec.Endpoints[0].MetricRelabelConfigs = metrics.SchedulerRelabelConfigs(cpContext.MetricsSet) + util.ApplyClusterIDLabel(&sm.Spec.Endpoints[0], cpContext.HCP.Spec.ClusterID) + + return nil +} diff --git a/support/metrics/sets.go b/support/metrics/sets.go index 9e2d9736f51..dc323d3a886 100644 --- a/support/metrics/sets.go +++ b/support/metrics/sets.go @@ -52,6 +52,7 @@ type MetricsSetConfig struct { OLM []prometheusoperatorv1.RelabelConfig `json:"olm,omitempty"` CatalogOperator []prometheusoperatorv1.RelabelConfig `json:"catalogOperator,omitempty"` RegistryOperator []prometheusoperatorv1.RelabelConfig `json:"registryOperator,omitempty"` + KubeScheduler []prometheusoperatorv1.RelabelConfig `json:"kubeScheduler,omitempty"` NodeTuningOperator []prometheusoperatorv1.RelabelConfig `json:"nodeTuningOperator,omitempty"` // HyperShift components @@ -225,6 +226,17 @@ func KASRelabelConfigs(set MetricsSet) []prometheusoperatorv1.RelabelConfig { } } +func SchedulerRelabelConfigs(set MetricsSet) []prometheusoperatorv1.RelabelConfig { + switch set { + case MetricsSetTelemetry: + return sreMetricsSetConfig.KubeScheduler + case MetricsSetSRE: + return sreMetricsSetConfig.KubeScheduler + default: + return nil + } +} + func KCMRelabelConfigs(set MetricsSet) []prometheusoperatorv1.RelabelConfig { switch set { case MetricsSetTelemetry: From 7d26c7b55891bc238560db94b9875f550bd9306d Mon Sep 17 00:00:00 2001 From: Dhruv Gautam Date: Wed, 13 May 2026 11:21:45 +0530 Subject: [PATCH 2/3] test(CPO): add unit tests and fixtures for kube-scheduler ServiceMonitor Add unit tests for scheduler component options, ServiceMonitor adapter, and server certificate reconciliation. Regenerate test fixtures after adding Service, ServiceMonitor, and CA-signed serving certificate support. --- .../hostedcontrolplane/pki/scheduler_test.go | 76 ++++++++++++ ..._kube_scheduler_controlplanecomponent.yaml | 6 + ...eComponents_kube_scheduler_deployment.yaml | 19 ++- ...laneComponents_kube_scheduler_service.yaml | 29 +++++ ...ponents_kube_scheduler_servicemonitor.yaml | 49 ++++++++ ..._kube_scheduler_controlplanecomponent.yaml | 6 + ...eComponents_kube_scheduler_deployment.yaml | 19 ++- ...laneComponents_kube_scheduler_service.yaml | 29 +++++ ...ponents_kube_scheduler_servicemonitor.yaml | 49 ++++++++ ..._kube_scheduler_controlplanecomponent.yaml | 6 + ...eComponents_kube_scheduler_deployment.yaml | 19 ++- ...laneComponents_kube_scheduler_service.yaml | 29 +++++ ...ponents_kube_scheduler_servicemonitor.yaml | 49 ++++++++ ..._kube_scheduler_controlplanecomponent.yaml | 6 + ...eComponents_kube_scheduler_deployment.yaml | 19 ++- ...laneComponents_kube_scheduler_service.yaml | 29 +++++ ...ponents_kube_scheduler_servicemonitor.yaml | 49 ++++++++ ..._kube_scheduler_controlplanecomponent.yaml | 6 + ...eComponents_kube_scheduler_deployment.yaml | 19 ++- ...laneComponents_kube_scheduler_service.yaml | 29 +++++ ...ponents_kube_scheduler_servicemonitor.yaml | 49 ++++++++ .../v2/kube_scheduler/component_test.go | 35 ++++++ .../v2/kube_scheduler/servicemonitor_test.go | 110 ++++++++++++++++++ 23 files changed, 706 insertions(+), 30 deletions(-) create mode 100644 control-plane-operator/controllers/hostedcontrolplane/pki/scheduler_test.go create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component_test.go create mode 100644 control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor_test.go diff --git a/control-plane-operator/controllers/hostedcontrolplane/pki/scheduler_test.go b/control-plane-operator/controllers/hostedcontrolplane/pki/scheduler_test.go new file mode 100644 index 00000000000..5b55ce7f94b --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/pki/scheduler_test.go @@ -0,0 +1,76 @@ +package pki + +import ( + "testing" + + . "github.com/onsi/gomega" + + "github.com/openshift/hypershift/support/certs" + "github.com/openshift/hypershift/support/config" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestReconcileSchedulerServerSecret(t *testing.T) { + t.Parallel() + + t.Run("When secret is empty it should generate a valid cert", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ca := createTestCA(t) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scheduler-server", + Namespace: "test-namespace", + }, + } + + err := ReconcileSchedulerServerSecret(secret, ca, config.OwnerRef{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(secret.Data[corev1.TLSCertKey]).ToNot(BeEmpty()) + g.Expect(secret.Data[corev1.TLSPrivateKeyKey]).ToNot(BeEmpty()) + }) + + t.Run("When secret already has a valid cert it should not regenerate", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ca := createTestCA(t) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "scheduler-server", + Namespace: "test-namespace", + }, + } + + err := ReconcileSchedulerServerSecret(secret, ca, config.OwnerRef{}) + g.Expect(err).ToNot(HaveOccurred()) + + initialCert := make([]byte, len(secret.Data[corev1.TLSCertKey])) + copy(initialCert, secret.Data[corev1.TLSCertKey]) + + err = ReconcileSchedulerServerSecret(secret, ca, config.OwnerRef{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(secret.Data[corev1.TLSCertKey]).To(Equal(initialCert)) + }) +} + +func createTestCA(t *testing.T) *corev1.Secret { + t.Helper() + g := NewWithT(t) + + caSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "root-ca", + Namespace: "test-namespace", + }, + } + err := ReconcileRootCA(caSecret, config.OwnerRef{}) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(caSecret.Data[certs.CASignerCertMapKey]).ToNot(BeEmpty()) + g.Expect(caSecret.Data[certs.CASignerKeyMapKey]).ToNot(BeEmpty()) + + return caSecret +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index 42ea13481bd..5aea35dd97a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -25,3 +25,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 43cad14d192..4cfcd4435ea 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -27,7 +27,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir component.hypershift.openshift.io/config-hash: 022a8a3ab4d55cfe hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 labels: @@ -78,7 +78,8 @@ spec: containers: - args: - --config=/etc/kubernetes/config/config.json - - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -199,6 +200,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -207,8 +212,8 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work + - mountPath: /etc/kubernetes/certs + name: serving-cert - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config @@ -241,8 +246,10 @@ spec: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work + - name: serving-cert + secret: + defaultMode: 416 + secretName: scheduler-server - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 00000000000..27d98cf99d2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 00000000000..7dcc95e5e47 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/AROSwift/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,49 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + hypershift.openshift.io/metrics-endpoint: https + hypershift.openshift.io/metrics-job: kube-scheduler + hypershift.openshift.io/metrics-namespace: openshift-kube-scheduler + hypershift.openshift.io/metrics-service: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - metricRelabelings: + - action: replace + replacement: "" + targetLabel: _id + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: metrics-client + keySecret: + key: tls.key + name: metrics-client + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index 42ea13481bd..5aea35dd97a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -25,3 +25,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 09c9b254637..c88e625cfbd 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -27,7 +27,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir component.hypershift.openshift.io/config-hash: 022a8a3ab4d55cfe hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 labels: @@ -78,7 +78,8 @@ spec: containers: - args: - --config=/etc/kubernetes/config/config.json - - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -198,6 +199,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -211,8 +216,8 @@ spec: runAsNonRoot: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work + - mountPath: /etc/kubernetes/certs + name: serving-cert - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config @@ -251,8 +256,10 @@ spec: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work + - name: serving-cert + secret: + defaultMode: 416 + secretName: scheduler-server - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 00000000000..27d98cf99d2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 00000000000..7dcc95e5e47 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/GCP/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,49 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + hypershift.openshift.io/metrics-endpoint: https + hypershift.openshift.io/metrics-job: kube-scheduler + hypershift.openshift.io/metrics-namespace: openshift-kube-scheduler + hypershift.openshift.io/metrics-service: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - metricRelabelings: + - action: replace + replacement: "" + targetLabel: _id + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: metrics-client + keySecret: + key: tls.key + name: metrics-client + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index 42ea13481bd..5aea35dd97a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -25,3 +25,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 0225d00a823..c210b35ea56 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -27,7 +27,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir component.hypershift.openshift.io/config-hash: 022a8a3a40ea5fe1 hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 labels: @@ -78,7 +78,8 @@ spec: containers: - args: - --config=/etc/kubernetes/config/config.json - - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -199,6 +200,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -207,8 +212,8 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work + - mountPath: /etc/kubernetes/certs + name: serving-cert - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config @@ -241,8 +246,10 @@ spec: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work + - name: serving-cert + secret: + defaultMode: 416 + secretName: scheduler-server - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 00000000000..27d98cf99d2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 00000000000..7dcc95e5e47 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/IBMCloud/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,49 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + hypershift.openshift.io/metrics-endpoint: https + hypershift.openshift.io/metrics-job: kube-scheduler + hypershift.openshift.io/metrics-namespace: openshift-kube-scheduler + hypershift.openshift.io/metrics-service: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - metricRelabelings: + - action: replace + replacement: "" + targetLabel: _id + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: metrics-client + keySecret: + key: tls.key + name: metrics-client + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index 42ea13481bd..5aea35dd97a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -25,3 +25,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 435e2113074..d0a45306e3d 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -27,7 +27,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir component.hypershift.openshift.io/config-hash: 022a8a3ab4d55cfe hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 labels: @@ -78,7 +78,8 @@ spec: containers: - args: - --config=/etc/kubernetes/config/config.json - - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -198,6 +199,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -206,8 +211,8 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work + - mountPath: /etc/kubernetes/certs + name: serving-cert - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config @@ -240,8 +245,10 @@ spec: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work + - name: serving-cert + secret: + defaultMode: 416 + secretName: scheduler-server - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 00000000000..27d98cf99d2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 00000000000..7dcc95e5e47 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,49 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + hypershift.openshift.io/metrics-endpoint: https + hypershift.openshift.io/metrics-job: kube-scheduler + hypershift.openshift.io/metrics-namespace: openshift-kube-scheduler + hypershift.openshift.io/metrics-service: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - metricRelabelings: + - action: replace + replacement: "" + targetLabel: _id + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: metrics-client + keySecret: + key: tls.key + name: metrics-client + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml index 42ea13481bd..5aea35dd97a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_controlplanecomponent.yaml @@ -25,3 +25,9 @@ status: - group: "" kind: Secret name: kube-scheduler-kubeconfig + - group: "" + kind: Service + name: kube-scheduler + - group: monitoring.coreos.com + kind: ServiceMonitor + name: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml index 43cad14d192..4cfcd4435ea 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_deployment.yaml @@ -27,7 +27,7 @@ spec: template: metadata: annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: cert-work,tmp-dir + cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp-dir component.hypershift.openshift.io/config-hash: 022a8a3ab4d55cfe hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64 labels: @@ -78,7 +78,8 @@ spec: containers: - args: - --config=/etc/kubernetes/config/config.json - - --cert-dir=/var/run/kubernetes + - --tls-cert-file=/etc/kubernetes/certs/tls.crt + - --tls-private-key-file=/etc/kubernetes/certs/tls.key - --secure-port=10259 - --authentication-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig - --authorization-kubeconfig=/etc/kubernetes/kubeconfig/kubeconfig @@ -199,6 +200,10 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: kube-scheduler + ports: + - containerPort: 10259 + name: client + protocol: TCP resources: requests: cpu: 25m @@ -207,8 +212,8 @@ spec: readOnlyRootFilesystem: true terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/run/kubernetes - name: cert-work + - mountPath: /etc/kubernetes/certs + name: serving-cert - mountPath: /etc/kubernetes/kubeconfig name: kubeconfig - mountPath: /etc/kubernetes/config @@ -241,8 +246,10 @@ spec: defaultMode: 420 name: kube-scheduler name: scheduler-config - - emptyDir: {} - name: cert-work + - name: serving-cert + secret: + defaultMode: 416 + secretName: scheduler-server - name: kubeconfig secret: defaultMode: 416 diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml new file mode 100644 index 00000000000..27d98cf99d2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_service.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + internalTrafficPolicy: Cluster + ipFamilyPolicy: PreferDualStack + ports: + - name: client + port: 10259 + protocol: TCP + targetPort: client + selector: + app: kube-scheduler + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml new file mode 100644 index 00000000000..7dcc95e5e47 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/kube-scheduler/zz_fixture_TestControlPlaneComponents_kube_scheduler_servicemonitor.yaml @@ -0,0 +1,49 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + hypershift.openshift.io/metrics-endpoint: https + hypershift.openshift.io/metrics-job: kube-scheduler + hypershift.openshift.io/metrics-namespace: openshift-kube-scheduler + hypershift.openshift.io/metrics-service: kube-scheduler + name: kube-scheduler + namespace: hcp-namespace + ownerReferences: + - apiVersion: hypershift.openshift.io/v1beta1 + blockOwnerDeletion: true + controller: true + kind: HostedControlPlane + name: hcp + uid: "" + resourceVersion: "1" +spec: + endpoints: + - metricRelabelings: + - action: replace + replacement: "" + targetLabel: _id + relabelings: + - action: replace + replacement: "" + targetLabel: _id + scheme: https + targetPort: client + tlsConfig: + ca: + configMap: + key: ca.crt + name: root-ca + cert: + secret: + key: tls.crt + name: metrics-client + keySecret: + key: tls.key + name: metrics-client + serverName: kube-scheduler + namespaceSelector: + matchNames: + - hcp-namespace + selector: + matchLabels: + app: kube-scheduler diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component_test.go new file mode 100644 index 00000000000..160f9a51ee4 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/component_test.go @@ -0,0 +1,35 @@ +package scheduler + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestKubeSchedulerOptions(t *testing.T) { + t.Parallel() + + t.Run("When IsRequestServing is called it should return false", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ks := &kubeScheduler{} + g.Expect(ks.IsRequestServing()).To(BeFalse()) + }) + + t.Run("When MultiZoneSpread is called it should return true", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ks := &kubeScheduler{} + g.Expect(ks.MultiZoneSpread()).To(BeTrue()) + }) + + t.Run("When NeedsManagementKASAccess is called it should return false", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + ks := &kubeScheduler{} + g.Expect(ks.NeedsManagementKASAccess()).To(BeFalse()) + }) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor_test.go new file mode 100644 index 00000000000..bacbc7d69f6 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kube_scheduler/servicemonitor_test.go @@ -0,0 +1,110 @@ +package scheduler + +import ( + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + component "github.com/openshift/hypershift/support/controlplane-component" + "github.com/openshift/hypershift/support/metrics" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + prometheusoperatorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" +) + +func TestAdaptServiceMonitor(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + metricsSet metrics.MetricsSet + clusterID string + validate func(*testing.T, *prometheusoperatorv1.ServiceMonitor, error) + }{ + { + name: "When service monitor is adapted it should set namespace selector", + metricsSet: metrics.MetricsSetTelemetry, + clusterID: "test-cluster-id", + validate: func(t *testing.T, sm *prometheusoperatorv1.ServiceMonitor, err error) { + g := NewWithT(t) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(sm.Spec.NamespaceSelector.MatchNames).To(Equal([]string{"test-namespace"})) + }, + }, + { + name: "When service monitor is adapted it should apply cluster ID label", + metricsSet: metrics.MetricsSetAll, + clusterID: "cluster-abc-123", + validate: func(t *testing.T, sm *prometheusoperatorv1.ServiceMonitor, err error) { + g := NewWithT(t) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(sm.Spec.Endpoints).To(HaveLen(1)) + + relabelConfigs := sm.Spec.Endpoints[0].RelabelConfigs + foundClusterIDLabel := false + for _, config := range relabelConfigs { + if config.TargetLabel == "_id" && config.Replacement != nil && *config.Replacement == "cluster-abc-123" { + foundClusterIDLabel = true + break + } + } + g.Expect(foundClusterIDLabel).To(BeTrue(), "cluster ID label should be applied") + }, + }, + { + name: "When metrics set is Telemetry it should not add metric relabel configs", + metricsSet: metrics.MetricsSetTelemetry, + clusterID: "test-cluster", + validate: func(t *testing.T, sm *prometheusoperatorv1.ServiceMonitor, err error) { + g := NewWithT(t) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(sm.Spec.Endpoints).To(HaveLen(1)) + + // SchedulerRelabelConfigs returns nil for Telemetry, so only the _id label should be present + g.Expect(sm.Spec.Endpoints[0].MetricRelabelConfigs).To(HaveLen(1)) + g.Expect(sm.Spec.Endpoints[0].MetricRelabelConfigs[0].TargetLabel).To(Equal("_id")) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: "test-namespace", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + ClusterID: tc.clusterID, + }, + } + + cpContext := component.WorkloadContext{ + Context: t.Context(), + HCP: hcp, + MetricsSet: tc.metricsSet, + } + + sm := &prometheusoperatorv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-scheduler", + Namespace: "test-namespace", + }, + Spec: prometheusoperatorv1.ServiceMonitorSpec{ + Endpoints: []prometheusoperatorv1.Endpoint{ + { + Port: "metrics", + }, + }, + }, + } + + err := adaptServiceMonitor(cpContext, sm) + tc.validate(t, sm, err) + }) + } +} From 0b0a67a2c532f2d48fc6a45935d910404c7241cc Mon Sep 17 00:00:00 2001 From: Dhruv Gautam Date: Fri, 5 Jun 2026 13:25:26 +0530 Subject: [PATCH 3/3] feat(HCCO): add RBAC for /metrics/resources endpoint The built-in system:monitoring ClusterRole only grants access to /metrics. This adds a ClusterRole and ClusterRoleBinding in the guest cluster to allow the metrics client to scrape /metrics/resources on kube-scheduler, which exposes kube_pod_resource_request and kube_pod_resource_limit metrics. --- .../controllers/resources/manifests/rbac.go | 16 ++++++++++++ .../controllers/resources/rbac/reconcile.go | 26 +++++++++++++++++++ .../controllers/resources/resources.go | 2 ++ 3 files changed, 44 insertions(+) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/rbac.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/rbac.go index d2dc5d0ee14..ba44f30b02c 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/rbac.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests/rbac.go @@ -137,6 +137,22 @@ func MetricsClientClusterRoleBinding() *rbacv1.ClusterRoleBinding { } } +func MetricsResourcesClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hypershift-metrics-resources-reader", + }, + } +} + +func MetricsResourcesClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hypershift-metrics-resources-reader", + }, + } +} + func AuthenticatedReaderForAuthenticatedUserRolebinding() *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/rbac/reconcile.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/rbac/reconcile.go index b93e482b0d4..1225984b570 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/rbac/reconcile.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/rbac/reconcile.go @@ -280,6 +280,32 @@ func ReconcileGenericMetricsClusterRoleBinding(cn string) func(*rbacv1.ClusterRo } } +func ReconcileMetricsResourcesClusterRole(r *rbacv1.ClusterRole) error { + r.Rules = []rbacv1.PolicyRule{ + { + NonResourceURLs: []string{"/metrics/resources"}, + Verbs: []string{"get"}, + }, + } + return nil +} + +func ReconcileMetricsResourcesClusterRoleBinding(r *rbacv1.ClusterRoleBinding) error { + r.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "ClusterRole", + Name: "hypershift-metrics-resources-reader", + } + r.Subjects = []rbacv1.Subject{ + { + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "User", + Name: "system:serviceaccount:hypershift:prometheus", + }, + } + return nil +} + func ReconcileAuthenticatedReaderForAuthenticatedUserRolebinding(r *rbacv1.RoleBinding) error { r.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.SchemeGroupVersion.Group, diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index 8e4170836d6..74881655b8b 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -1328,6 +1328,8 @@ func (r *reconciler) reconcileRBAC(ctx context.Context, hcp *hyperv1.HostedContr manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.NodeBootstrapperClusterRoleBinding, reconcile: rbac.ReconcileNodeBootstrapperClusterRoleBinding}, manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.CSRRenewalClusterRoleBinding, reconcile: rbac.ReconcileCSRRenewalClusterRoleBinding}, manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.MetricsClientClusterRoleBinding, reconcile: rbac.ReconcileGenericMetricsClusterRoleBinding("system:serviceaccount:hypershift:prometheus")}, + manifestAndReconcile[*rbacv1.ClusterRole]{manifest: manifests.MetricsResourcesClusterRole, reconcile: rbac.ReconcileMetricsResourcesClusterRole}, + manifestAndReconcile[*rbacv1.ClusterRoleBinding]{manifest: manifests.MetricsResourcesClusterRoleBinding, reconcile: rbac.ReconcileMetricsResourcesClusterRoleBinding}, manifestAndReconcile[*rbacv1.RoleBinding]{manifest: manifests.IngressToRouteControllerRoleBinding, reconcile: rbac.ReconcileIngressToRouteControllerRoleBinding},