From 3af97845bd8546fa0a8809759d93307e0306c32f Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Wed, 25 Mar 2026 15:52:15 +0100 Subject: [PATCH 1/7] Initial scafolding of cyborg controllers and crds Using operator-sdk command: operator-sdk create api --group cyborg --version v1beta1 --kind Cyborg --resource --controller operator-sdk create api --group cyborg --version v1beta1 --kind CyborgAPI --resource --controller operator-sdk create api --group cyborg --version v1beta1 --kind CyborgConductor --resource --controller Signed-off-by: Alfredo Moralejo --- PROJECT | 27 ++ .../cyborg.openstack.org_cyborgapis.yaml | 54 ++++ ...cyborg.openstack.org_cyborgconductors.yaml | 54 ++++ api/bases/cyborg.openstack.org_cyborgs.yaml | 54 ++++ api/cyborg/v1beta1/cyborg_types.go | 61 ++++ api/cyborg/v1beta1/cyborgapi_types.go | 59 ++++ api/cyborg/v1beta1/cyborgconductor_types.go | 64 ++++ api/cyborg/v1beta1/groupversion_info.go | 36 +++ api/cyborg/v1beta1/zz_generated.deepcopy.go | 292 ++++++++++++++++++ cmd/main.go | 28 +- .../cyborg.openstack.org_cyborgapis.yaml | 54 ++++ ...cyborg.openstack.org_cyborgconductors.yaml | 54 ++++ .../bases/cyborg.openstack.org_cyborgs.yaml | 54 ++++ config/crd/kustomization.yaml | 3 + .../nova-operator.clusterserviceversion.yaml | 15 + config/rbac/cyborg_cyborg_admin_role.yaml | 27 ++ config/rbac/cyborg_cyborg_editor_role.yaml | 33 ++ config/rbac/cyborg_cyborg_viewer_role.yaml | 29 ++ config/rbac/cyborg_cyborgapi_admin_role.yaml | 27 ++ config/rbac/cyborg_cyborgapi_editor_role.yaml | 33 ++ config/rbac/cyborg_cyborgapi_viewer_role.yaml | 29 ++ .../cyborg_cyborgconductor_admin_role.yaml | 27 ++ .../cyborg_cyborgconductor_editor_role.yaml | 33 ++ .../cyborg_cyborgconductor_viewer_role.yaml | 29 ++ config/rbac/kustomization.yaml | 9 + config/rbac/role.yaml | 32 ++ config/samples/cyborg_v1beta1_cyborg.yaml | 9 + config/samples/cyborg_v1beta1_cyborgapi.yaml | 9 + .../cyborg_v1beta1_cyborgconductor.yaml | 9 + config/samples/kustomization.yaml | 3 + .../controller/cyborg/cyborg_controller.go | 63 ++++ .../controller/cyborg/cyborgapi_controller.go | 63 ++++ .../cyborg/cyborgconductor_controller.go | 63 ++++ 33 files changed, 1435 insertions(+), 1 deletion(-) create mode 100644 api/bases/cyborg.openstack.org_cyborgapis.yaml create mode 100644 api/bases/cyborg.openstack.org_cyborgconductors.yaml create mode 100644 api/bases/cyborg.openstack.org_cyborgs.yaml create mode 100644 api/cyborg/v1beta1/cyborg_types.go create mode 100644 api/cyborg/v1beta1/cyborgapi_types.go create mode 100644 api/cyborg/v1beta1/cyborgconductor_types.go create mode 100644 api/cyborg/v1beta1/groupversion_info.go create mode 100644 api/cyborg/v1beta1/zz_generated.deepcopy.go create mode 100644 config/crd/bases/cyborg.openstack.org_cyborgapis.yaml create mode 100644 config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml create mode 100644 config/crd/bases/cyborg.openstack.org_cyborgs.yaml create mode 100644 config/rbac/cyborg_cyborg_admin_role.yaml create mode 100644 config/rbac/cyborg_cyborg_editor_role.yaml create mode 100644 config/rbac/cyborg_cyborg_viewer_role.yaml create mode 100644 config/rbac/cyborg_cyborgapi_admin_role.yaml create mode 100644 config/rbac/cyborg_cyborgapi_editor_role.yaml create mode 100644 config/rbac/cyborg_cyborgapi_viewer_role.yaml create mode 100644 config/rbac/cyborg_cyborgconductor_admin_role.yaml create mode 100644 config/rbac/cyborg_cyborgconductor_editor_role.yaml create mode 100644 config/rbac/cyborg_cyborgconductor_viewer_role.yaml create mode 100644 config/samples/cyborg_v1beta1_cyborg.yaml create mode 100644 config/samples/cyborg_v1beta1_cyborgapi.yaml create mode 100644 config/samples/cyborg_v1beta1_cyborgconductor.yaml create mode 100644 internal/controller/cyborg/cyborg_controller.go create mode 100644 internal/controller/cyborg/cyborgapi_controller.go create mode 100644 internal/controller/cyborg/cyborgconductor_controller.go diff --git a/PROJECT b/PROJECT index 641c5684a..cdad0b7e7 100644 --- a/PROJECT +++ b/PROJECT @@ -116,4 +116,31 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: cyborg + kind: Cyborg + path: github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: cyborg + kind: CyborgAPI + path: github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: cyborg + kind: CyborgConductor + path: github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1 + version: v1beta1 version: "3" diff --git a/api/bases/cyborg.openstack.org_cyborgapis.yaml b/api/bases/cyborg.openstack.org_cyborgapis.yaml new file mode 100644 index 000000000..3815a2751 --- /dev/null +++ b/api/bases/cyborg.openstack.org_cyborgapis.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: cyborgapis.cyborg.openstack.org +spec: + group: cyborg.openstack.org + names: + kind: CyborgAPI + listKind: CyborgAPIList + plural: cyborgapis + singular: cyborgapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CyborgAPI is the Schema for the cyborgapis API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CyborgAPISpec defines the desired state of CyborgAPI. + properties: + foo: + description: Foo is an example field of CyborgAPI. Edit cyborgapi_types.go + to remove/update + type: string + type: object + status: + description: CyborgAPIStatus defines the observed state of CyborgAPI. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/cyborg.openstack.org_cyborgconductors.yaml b/api/bases/cyborg.openstack.org_cyborgconductors.yaml new file mode 100644 index 000000000..59312e83c --- /dev/null +++ b/api/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: cyborgconductors.cyborg.openstack.org +spec: + group: cyborg.openstack.org + names: + kind: CyborgConductor + listKind: CyborgConductorList + plural: cyborgconductors + singular: cyborgconductor + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CyborgConductor is the Schema for the cyborgconductors API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CyborgConductorSpec defines the desired state of CyborgConductor. + properties: + foo: + description: Foo is an example field of CyborgConductor. Edit cyborgconductor_types.go + to remove/update + type: string + type: object + status: + description: CyborgConductorStatus defines the observed state of CyborgConductor. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/cyborg.openstack.org_cyborgs.yaml b/api/bases/cyborg.openstack.org_cyborgs.yaml new file mode 100644 index 000000000..8e22ea151 --- /dev/null +++ b/api/bases/cyborg.openstack.org_cyborgs.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: cyborgs.cyborg.openstack.org +spec: + group: cyborg.openstack.org + names: + kind: Cyborg + listKind: CyborgList + plural: cyborgs + singular: cyborg + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Cyborg is the Schema for the cyborgs API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CyborgSpec defines the desired state of Cyborg. + properties: + foo: + description: Foo is an example field of Cyborg. Edit cyborg_types.go + to remove/update + type: string + type: object + status: + description: CyborgStatus defines the observed state of Cyborg. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go new file mode 100644 index 000000000..f494b227f --- /dev/null +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -0,0 +1,61 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CyborgSpec defines the desired state of Cyborg. +type CyborgSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of Cyborg. Edit cyborg_types.go to remove/update + Foo string `json:"foo,omitempty"` +} + +// CyborgStatus defines the observed state of Cyborg. +type CyborgStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// Cyborg is the Schema for the cyborgs API. +type Cyborg struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CyborgSpec `json:"spec,omitempty"` + Status CyborgStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// CyborgList contains a list of Cyborg. +type CyborgList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Cyborg `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Cyborg{}, &CyborgList{}) +} diff --git a/api/cyborg/v1beta1/cyborgapi_types.go b/api/cyborg/v1beta1/cyborgapi_types.go new file mode 100644 index 000000000..c2d9b0872 --- /dev/null +++ b/api/cyborg/v1beta1/cyborgapi_types.go @@ -0,0 +1,59 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CyborgAPISpec defines the desired state of CyborgAPI. +type CyborgAPISpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of CyborgAPI. Edit cyborgapi_types.go to remove/update + Foo string `json:"foo,omitempty"` +} + +// CyborgAPIStatus defines the observed state of CyborgAPI. +type CyborgAPIStatus struct { +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// CyborgAPI is the Schema for the cyborgapis API. +type CyborgAPI struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CyborgAPISpec `json:"spec,omitempty"` + Status CyborgAPIStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// CyborgAPIList contains a list of CyborgAPI. +type CyborgAPIList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CyborgAPI `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CyborgAPI{}, &CyborgAPIList{}) +} diff --git a/api/cyborg/v1beta1/cyborgconductor_types.go b/api/cyborg/v1beta1/cyborgconductor_types.go new file mode 100644 index 000000000..7af1c2b85 --- /dev/null +++ b/api/cyborg/v1beta1/cyborgconductor_types.go @@ -0,0 +1,64 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// CyborgConductorSpec defines the desired state of CyborgConductor. +type CyborgConductorSpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Foo is an example field of CyborgConductor. Edit cyborgconductor_types.go to remove/update + Foo string `json:"foo,omitempty"` +} + +// CyborgConductorStatus defines the observed state of CyborgConductor. +type CyborgConductorStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// CyborgConductor is the Schema for the cyborgconductors API. +type CyborgConductor struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CyborgConductorSpec `json:"spec,omitempty"` + Status CyborgConductorStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// CyborgConductorList contains a list of CyborgConductor. +type CyborgConductorList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CyborgConductor `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CyborgConductor{}, &CyborgConductorList{}) +} diff --git a/api/cyborg/v1beta1/groupversion_info.go b/api/cyborg/v1beta1/groupversion_info.go new file mode 100644 index 000000000..00c01a872 --- /dev/null +++ b/api/cyborg/v1beta1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the cyborg v1beta1 API group. +// +kubebuilder:object:generate=true +// +groupName=cyborg.openstack.org +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "cyborg.openstack.org", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/cyborg/v1beta1/zz_generated.deepcopy.go b/api/cyborg/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 000000000..cfe17e260 --- /dev/null +++ b/api/cyborg/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,292 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Cyborg) DeepCopyInto(out *Cyborg) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cyborg. +func (in *Cyborg) DeepCopy() *Cyborg { + if in == nil { + return nil + } + out := new(Cyborg) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Cyborg) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgAPI) DeepCopyInto(out *CyborgAPI) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPI. +func (in *CyborgAPI) DeepCopy() *CyborgAPI { + if in == nil { + return nil + } + out := new(CyborgAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CyborgAPI) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgAPIList) DeepCopyInto(out *CyborgAPIList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CyborgAPI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPIList. +func (in *CyborgAPIList) DeepCopy() *CyborgAPIList { + if in == nil { + return nil + } + out := new(CyborgAPIList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CyborgAPIList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgAPISpec) DeepCopyInto(out *CyborgAPISpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPISpec. +func (in *CyborgAPISpec) DeepCopy() *CyborgAPISpec { + if in == nil { + return nil + } + out := new(CyborgAPISpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgAPIStatus) DeepCopyInto(out *CyborgAPIStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPIStatus. +func (in *CyborgAPIStatus) DeepCopy() *CyborgAPIStatus { + if in == nil { + return nil + } + out := new(CyborgAPIStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgConductor) DeepCopyInto(out *CyborgConductor) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductor. +func (in *CyborgConductor) DeepCopy() *CyborgConductor { + if in == nil { + return nil + } + out := new(CyborgConductor) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CyborgConductor) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgConductorList) DeepCopyInto(out *CyborgConductorList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CyborgConductor, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorList. +func (in *CyborgConductorList) DeepCopy() *CyborgConductorList { + if in == nil { + return nil + } + out := new(CyborgConductorList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CyborgConductorList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgConductorSpec) DeepCopyInto(out *CyborgConductorSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorSpec. +func (in *CyborgConductorSpec) DeepCopy() *CyborgConductorSpec { + if in == nil { + return nil + } + out := new(CyborgConductorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgConductorStatus) DeepCopyInto(out *CyborgConductorStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorStatus. +func (in *CyborgConductorStatus) DeepCopy() *CyborgConductorStatus { + if in == nil { + return nil + } + out := new(CyborgConductorStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgList) DeepCopyInto(out *CyborgList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Cyborg, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgList. +func (in *CyborgList) DeepCopy() *CyborgList { + if in == nil { + return nil + } + out := new(CyborgList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CyborgList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgSpec) DeepCopyInto(out *CyborgSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgSpec. +func (in *CyborgSpec) DeepCopy() *CyborgSpec { + if in == nil { + return nil + } + out := new(CyborgSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgStatus) DeepCopyInto(out *CyborgStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgStatus. +func (in *CyborgStatus) DeepCopy() *CyborgStatus { + if in == nil { + return nil + } + out := new(CyborgStatus) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go index e6a81938d..c1ce46e94 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -38,8 +38,11 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborgcontroller "github.com/openstack-k8s-operators/nova-operator/internal/controller/cyborg" "github.com/openstack-k8s-operators/nova-operator/internal/controller/nova" webhookv1beta1 "github.com/openstack-k8s-operators/nova-operator/internal/webhook/nova/v1beta1" + // +kubebuilder:scaffold:imports networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" @@ -48,11 +51,12 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/operator" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" - novav1 "github.com/openstack-k8s-operators/nova-operator/api/nova/v1beta1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client/config" + + novav1 "github.com/openstack-k8s-operators/nova-operator/api/nova/v1beta1" ) var ( @@ -71,6 +75,7 @@ func init() { utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(memcachedv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(cyborgv1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -302,6 +307,27 @@ func main() { checker = mgr.GetWebhookServer().StartedChecker() } + if err := (&cyborgcontroller.CyborgReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Cyborg") + os.Exit(1) + } + if err := (&cyborgcontroller.CyborgAPIReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CyborgAPI") + os.Exit(1) + } + if err := (&cyborgcontroller.CyborgConductorReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CyborgConductor") + os.Exit(1) + } // +kubebuilder:scaffold:builder if metricsCertWatcher != nil { diff --git a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml new file mode 100644 index 000000000..3815a2751 --- /dev/null +++ b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: cyborgapis.cyborg.openstack.org +spec: + group: cyborg.openstack.org + names: + kind: CyborgAPI + listKind: CyborgAPIList + plural: cyborgapis + singular: cyborgapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CyborgAPI is the Schema for the cyborgapis API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CyborgAPISpec defines the desired state of CyborgAPI. + properties: + foo: + description: Foo is an example field of CyborgAPI. Edit cyborgapi_types.go + to remove/update + type: string + type: object + status: + description: CyborgAPIStatus defines the observed state of CyborgAPI. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml new file mode 100644 index 000000000..59312e83c --- /dev/null +++ b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: cyborgconductors.cyborg.openstack.org +spec: + group: cyborg.openstack.org + names: + kind: CyborgConductor + listKind: CyborgConductorList + plural: cyborgconductors + singular: cyborgconductor + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CyborgConductor is the Schema for the cyborgconductors API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CyborgConductorSpec defines the desired state of CyborgConductor. + properties: + foo: + description: Foo is an example field of CyborgConductor. Edit cyborgconductor_types.go + to remove/update + type: string + type: object + status: + description: CyborgConductorStatus defines the observed state of CyborgConductor. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml new file mode 100644 index 000000000..8e22ea151 --- /dev/null +++ b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: cyborgs.cyborg.openstack.org +spec: + group: cyborg.openstack.org + names: + kind: Cyborg + listKind: CyborgList + plural: cyborgs + singular: cyborg + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Cyborg is the Schema for the cyborgs API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CyborgSpec defines the desired state of Cyborg. + properties: + foo: + description: Foo is an example field of Cyborg. Edit cyborg_types.go + to remove/update + type: string + type: object + status: + description: CyborgStatus defines the observed state of Cyborg. + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 6c51197bd..03ddb070e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,6 +10,9 @@ resources: - bases/nova.openstack.org_novacells.yaml - bases/nova.openstack.org_nova.yaml - bases/nova.openstack.org_novacomputes.yaml +- bases/cyborg.openstack.org_cyborgs.yaml +- bases/cyborg.openstack.org_cyborgapis.yaml +- bases/cyborg.openstack.org_cyborgconductors.yaml # +kubebuilder:scaffold:crdkustomizeresource patches: diff --git a/config/manifests/bases/nova-operator.clusterserviceversion.yaml b/config/manifests/bases/nova-operator.clusterserviceversion.yaml index 9ea1c400b..2caf4e9f8 100644 --- a/config/manifests/bases/nova-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/nova-operator.clusterserviceversion.yaml @@ -19,6 +19,21 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: + - description: CyborgAPI is the Schema for the cyborgapis API. + displayName: Cyborg API + kind: CyborgAPI + name: cyborgapis.cyborg.openstack.org + version: v1beta1 + - description: CyborgConductor is the Schema for the cyborgconductors API. + displayName: Cyborg Conductor + kind: CyborgConductor + name: cyborgconductors.cyborg.openstack.org + version: v1beta1 + - description: Cyborg is the Schema for the cyborgs API. + displayName: Cyborg + kind: Cyborg + name: cyborgs.cyborg.openstack.org + version: v1beta1 - description: NovaAPI is the Schema for the novaapis API displayName: Nova API kind: NovaAPI diff --git a/config/rbac/cyborg_cyborg_admin_role.yaml b/config/rbac/cyborg_cyborg_admin_role.yaml new file mode 100644 index 000000000..583d82521 --- /dev/null +++ b/config/rbac/cyborg_cyborg_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over cyborg.openstack.org. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborg-admin-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgs + verbs: + - '*' +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgs/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborg_editor_role.yaml b/config/rbac/cyborg_cyborg_editor_role.yaml new file mode 100644 index 000000000..979d13235 --- /dev/null +++ b/config/rbac/cyborg_cyborg_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the cyborg.openstack.org. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborg-editor-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgs/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborg_viewer_role.yaml b/config/rbac/cyborg_cyborg_viewer_role.yaml new file mode 100644 index 000000000..4040dc0b8 --- /dev/null +++ b/config/rbac/cyborg_cyborg_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to cyborg.openstack.org resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborg-viewer-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgs + verbs: + - get + - list + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgs/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborgapi_admin_role.yaml b/config/rbac/cyborg_cyborgapi_admin_role.yaml new file mode 100644 index 000000000..2ac18cb9f --- /dev/null +++ b/config/rbac/cyborg_cyborgapi_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over cyborg.openstack.org. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborgapi-admin-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis + verbs: + - '*' +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborgapi_editor_role.yaml b/config/rbac/cyborg_cyborgapi_editor_role.yaml new file mode 100644 index 000000000..6014780db --- /dev/null +++ b/config/rbac/cyborg_cyborgapi_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the cyborg.openstack.org. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborgapi-editor-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborgapi_viewer_role.yaml b/config/rbac/cyborg_cyborgapi_viewer_role.yaml new file mode 100644 index 000000000..cbc981517 --- /dev/null +++ b/config/rbac/cyborg_cyborgapi_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to cyborg.openstack.org resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborgapi-viewer-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis + verbs: + - get + - list + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborgconductor_admin_role.yaml b/config/rbac/cyborg_cyborgconductor_admin_role.yaml new file mode 100644 index 000000000..e7c67ec61 --- /dev/null +++ b/config/rbac/cyborg_cyborgconductor_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over cyborg.openstack.org. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborgconductor-admin-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgconductors + verbs: + - '*' +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgconductors/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborgconductor_editor_role.yaml b/config/rbac/cyborg_cyborgconductor_editor_role.yaml new file mode 100644 index 000000000..9cf5ee598 --- /dev/null +++ b/config/rbac/cyborg_cyborgconductor_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the cyborg.openstack.org. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborgconductor-editor-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgconductors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgconductors/status + verbs: + - get diff --git a/config/rbac/cyborg_cyborgconductor_viewer_role.yaml b/config/rbac/cyborg_cyborgconductor_viewer_role.yaml new file mode 100644 index 000000000..e6b058da1 --- /dev/null +++ b/config/rbac/cyborg_cyborgconductor_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project nova-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to cyborg.openstack.org resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-cyborgconductor-viewer-role +rules: +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgconductors + verbs: + - get + - list + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgconductors/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 63d4c550c..f91080738 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -22,6 +22,15 @@ resources: # default, aiding admins in cluster management. Those roles are # not used by the nova-operator itself. You can comment the following lines # if you do not want those helpers be installed with your Project. +- cyborg_cyborgconductor_admin_role.yaml +- cyborg_cyborgconductor_editor_role.yaml +- cyborg_cyborgconductor_viewer_role.yaml +- cyborg_cyborgapi_admin_role.yaml +- cyborg_cyborgapi_editor_role.yaml +- cyborg_cyborgapi_viewer_role.yaml +- cyborg_cyborg_admin_role.yaml +- cyborg_cyborg_editor_role.yaml +- cyborg_cyborg_viewer_role.yaml - novacompute_admin_role.yaml - novacompute_editor_role.yaml - novacompute_viewer_role.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 422793d58..9cd3467c7 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -54,6 +54,38 @@ rules: - patch - update - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis + - cyborgconductors + - cyborgs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis/finalizers + - cyborgconductors/finalizers + - cyborgs/finalizers + verbs: + - update +- apiGroups: + - cyborg.openstack.org + resources: + - cyborgapis/status + - cyborgconductors/status + - cyborgs/status + verbs: + - get + - patch + - update - apiGroups: - k8s.cni.cncf.io resources: diff --git a/config/samples/cyborg_v1beta1_cyborg.yaml b/config/samples/cyborg_v1beta1_cyborg.yaml new file mode 100644 index 000000000..224eddbfc --- /dev/null +++ b/config/samples/cyborg_v1beta1_cyborg.yaml @@ -0,0 +1,9 @@ +apiVersion: cyborg.openstack.org/v1beta1 +kind: Cyborg +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborg-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/cyborg_v1beta1_cyborgapi.yaml b/config/samples/cyborg_v1beta1_cyborgapi.yaml new file mode 100644 index 000000000..576a4fa53 --- /dev/null +++ b/config/samples/cyborg_v1beta1_cyborgapi.yaml @@ -0,0 +1,9 @@ +apiVersion: cyborg.openstack.org/v1beta1 +kind: CyborgAPI +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborgapi-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/cyborg_v1beta1_cyborgconductor.yaml b/config/samples/cyborg_v1beta1_cyborgconductor.yaml new file mode 100644 index 000000000..7de9c6695 --- /dev/null +++ b/config/samples/cyborg_v1beta1_cyborgconductor.yaml @@ -0,0 +1,9 @@ +apiVersion: cyborg.openstack.org/v1beta1 +kind: CyborgConductor +metadata: + labels: + app.kubernetes.io/name: nova-operator + app.kubernetes.io/managed-by: kustomize + name: cyborgconductor-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 84a35fd9b..42b7d3041 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -8,4 +8,7 @@ resources: - nova_v1beta1_novacell1-upcall.yaml - nova_v1beta1_nova.yaml - nova_v1beta1_novacompute-ironic.yaml +- cyborg_v1beta1_cyborg.yaml +- cyborg_v1beta1_cyborgapi.yaml +- cyborg_v1beta1_cyborgconductor.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/internal/controller/cyborg/cyborg_controller.go b/internal/controller/cyborg/cyborg_controller.go new file mode 100644 index 000000000..56baa8454 --- /dev/null +++ b/internal/controller/cyborg/cyborg_controller.go @@ -0,0 +1,63 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cyborg + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +// CyborgReconciler reconciles a Cyborg object +type CyborgReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgs/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the Cyborg object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile +func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = logf.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CyborgReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&cyborgv1beta1.Cyborg{}). + Named("cyborg-cyborg"). + Complete(r) +} diff --git a/internal/controller/cyborg/cyborgapi_controller.go b/internal/controller/cyborg/cyborgapi_controller.go new file mode 100644 index 000000000..fa8d38914 --- /dev/null +++ b/internal/controller/cyborg/cyborgapi_controller.go @@ -0,0 +1,63 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cyborg + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +// CyborgAPIReconciler reconciles a CyborgAPI object +type CyborgAPIReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the CyborgAPI object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile +func (r *CyborgAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = logf.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CyborgAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&cyborgv1beta1.CyborgAPI{}). + Named("cyborg-cyborgapi"). + Complete(r) +} diff --git a/internal/controller/cyborg/cyborgconductor_controller.go b/internal/controller/cyborg/cyborgconductor_controller.go new file mode 100644 index 000000000..61a5f967a --- /dev/null +++ b/internal/controller/cyborg/cyborgconductor_controller.go @@ -0,0 +1,63 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cyborg + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +// CyborgConductorReconciler reconciles a CyborgConductor object +type CyborgConductorReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the CyborgConductor object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile +func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = logf.FromContext(ctx) + + // TODO(user): your logic here + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CyborgConductorReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&cyborgv1beta1.CyborgConductor{}). + Named("cyborg-cyborgconductor"). + Complete(r) +} From c191cf476c4c93a58c65b0b8f976c3b7e5668e65 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Thu, 26 Mar 2026 11:14:55 +0100 Subject: [PATCH 2/7] cyborg: Add CRD type definitions and webhook validation Define CRD specs for Cyborg, CyborgAPI and CyborgConductor resources: - Add CyborgSpec with DB, RabbitMQ, Keystone and TLS configuration - Add CyborgAPISpec and CyborgConductorSpec with configSecret, replicas, resources, nodeSelector and TLS fields - Implement defaulting and validation webhooks for all three CRDs - Register CRDs in the operator scheme - Update CRD YAML manifests and CSV for OLM Reconcile and configuration logic will be created in next commits. Assisted-By: Claude Signed-off-by: Alfredo Moralejo --- .../cyborg.openstack.org_cyborgapis.yaml | 295 ++++++++- ...cyborg.openstack.org_cyborgconductors.yaml | 106 +++- api/bases/cyborg.openstack.org_cyborgs.yaml | 577 +++++++++++++++++- api/cyborg/v1beta1/common_types.go | 76 +++ api/cyborg/v1beta1/cyborg_types.go | 101 ++- api/cyborg/v1beta1/cyborg_webhook.go | 239 ++++++++ api/cyborg/v1beta1/cyborgapi_types.go | 62 +- api/cyborg/v1beta1/cyborgapi_webhook.go | 107 ++++ api/cyborg/v1beta1/cyborgconductor_types.go | 41 +- api/cyborg/v1beta1/cyborgconductor_webhook.go | 100 +++ api/cyborg/v1beta1/zz_generated.deepcopy.go | 257 +++++++- .../cyborg.openstack.org_cyborgapis.yaml | 295 ++++++++- ...cyborg.openstack.org_cyborgconductors.yaml | 106 +++- .../bases/cyborg.openstack.org_cyborgs.yaml | 577 +++++++++++++++++- .../nova-operator.clusterserviceversion.yaml | 17 + config/samples/cyborg_v1beta1_cyborgapi.yaml | 2 +- .../cyborg_v1beta1_cyborgconductor.yaml | 2 +- 17 files changed, 2923 insertions(+), 37 deletions(-) create mode 100644 api/cyborg/v1beta1/common_types.go create mode 100644 api/cyborg/v1beta1/cyborg_webhook.go create mode 100644 api/cyborg/v1beta1/cyborgapi_webhook.go create mode 100644 api/cyborg/v1beta1/cyborgconductor_webhook.go diff --git a/api/bases/cyborg.openstack.org_cyborgapis.yaml b/api/bases/cyborg.openstack.org_cyborgapis.yaml index 3815a2751..b8fb2cf09 100644 --- a/api/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/api/bases/cyborg.openstack.org_cyborgapis.yaml @@ -39,10 +39,299 @@ spec: spec: description: CyborgAPISpec defines the desired state of CyborgAPI. properties: - foo: - description: Foo is an example field of CyborgAPI. Edit cyborgapi_types.go - to remove/update + configSecret: + description: ConfigSecret - containing all the configuration needed + provided by Cyborg object type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - configSecret type: object status: description: CyborgAPIStatus defines the observed state of CyborgAPI. diff --git a/api/bases/cyborg.openstack.org_cyborgconductors.yaml b/api/bases/cyborg.openstack.org_cyborgconductors.yaml index 59312e83c..6f3f19f9a 100644 --- a/api/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/api/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -39,10 +39,110 @@ spec: spec: description: CyborgConductorSpec defines the desired state of CyborgConductor. properties: - foo: - description: Foo is an example field of CyborgConductor. Edit cyborgconductor_types.go - to remove/update + configSecret: + description: ConfigSecret - containing all the configuration needed + provided by Cyborg object type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - configSecret type: object status: description: CyborgConductorStatus defines the observed state of CyborgConductor. diff --git a/api/bases/cyborg.openstack.org_cyborgs.yaml b/api/bases/cyborg.openstack.org_cyborgs.yaml index 8e22ea151..cab317b3f 100644 --- a/api/bases/cyborg.openstack.org_cyborgs.yaml +++ b/api/bases/cyborg.openstack.org_cyborgs.yaml @@ -39,13 +39,584 @@ spec: spec: description: CyborgSpec defines the desired state of Cyborg. properties: - foo: - description: Foo is an example field of Cyborg. Edit cyborg_types.go - to remove/update + agentContainerImageURL: + description: AgentContainerImageURL type: string + apiContainerImageURL: + description: APIContainerImageURL + type: string + apiServiceTemplate: + default: + replicas: 1 + description: APIServiceTemplate - define the cyborg-api service + properties: + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + type: object + apiTimeout: + default: 60 + description: APITimeout for Route and Apache + minimum: 10 + type: integer + auth: + description: Auth - Parameters related to authentication (shared by + all Cyborg services) + properties: + applicationCredentialSecret: + description: |- + ApplicationCredentialSecret - the name of the k8s Secret that contains the + application credential data used for authentication + type: string + type: object + conductorContainerImageURL: + description: ConductorContainerImageURL + type: string + conductorServiceTemplate: + default: + replicas: 1 + description: ConductorServiceTemplate - define the cyborg-conductor + service + properties: + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + type: object + databaseAccount: + default: cyborg + description: DatabaseAccount - MariaDBAccount to use when accessing + the API DB + type: string + databaseInstance: + default: openstack + description: |- + DatabaseInstance is the name of the MariaDB CR to select the DB + Service instance used for the Cyborg API DB. + type: string + keystoneInstance: + default: keystone + description: |- + KeystoneInstance to name of the KeystoneAPI CR to select the Service + instance used by the Cyborg services to authenticate. + type: string + messagingBus: + description: MessagingBus configuration (username, vhost, and cluster) + properties: + cluster: + description: Name of the cluster + minLength: 1 + type: string + user: + description: User - RabbitMQ username + type: string + vhost: + description: Vhost - RabbitMQ vhost name + type: string + required: + - cluster + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelectors: + default: + service: CyborgPassword + description: |- + PasswordSelectors - Selectors to identify the DB and ServiceUser + passwords from the Secret + properties: + service: + default: CyborgPassword + description: |- + Service - Selector to get the keystone service user password from the + Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + secret: + description: |- + Secret is the name of the Secret instance containing password + information for cyborg like the keystone service password and DB passwords + type: string + serviceUser: + default: cyborg + description: ServiceUser - optional username used for this service + to register in keystone + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - agentContainerImageURL + - apiContainerImageURL + - conductorContainerImageURL + - secret type: object status: description: CyborgStatus defines the observed state of Cyborg. + properties: + apiServiceReadyCount: + description: APIServiceReadyCount defines the number or replicas ready + from cyborg-api + format: int32 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + conductorServiceReadyCount: + description: ConductorServiceReadyCount defines the number or replicas + ready from cyborg-conductor + format: int32 + type: integer + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes. + format: int64 + type: integer type: object type: object served: true diff --git a/api/cyborg/v1beta1/common_types.go b/api/cyborg/v1beta1/common_types.go new file mode 100644 index 000000000..816afea9c --- /dev/null +++ b/api/cyborg/v1beta1/common_types.go @@ -0,0 +1,76 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// Container image fall-back defaults +const ( + CyborgAPIContainerImage = "quay.io/openstack.kolla/cyborg-api:master-rocky-10" + CyborgConductorContainerImage = "quay.io/openstack.kolla/cyborg-conductor:master-rocky-10" + CyborgAgentContainerImage = "quay.io/openstack.kolla/cyborg-agent:master-rocky-10" +) + +// PasswordSelector to identify the DB and AdminUser password from the Secret +type PasswordSelector struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default="CyborgPassword" + // Service - Selector to get the keystone service user password from the + // Secret + Service string `json:"service"` +} + +// AuthSpec defines authentication parameters for Cyborg services +type AuthSpec struct { + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // ApplicationCredentialSecret - the name of the k8s Secret that contains the + // application credential data used for authentication + ApplicationCredentialSecret string `json:"applicationCredentialSecret,omitempty"` +} + +// CyborgImages defines container images used by top level Cyborg CR +type CyborgImages struct { + // +kubebuilder:validation:Required + // APIContainerImageURL + APIContainerImageURL string `json:"apiContainerImageURL"` + + // +kubebuilder:validation:Required + // ConductorContainerImageURL + ConductorContainerImageURL string `json:"conductorContainerImageURL"` + + // +kubebuilder:validation:Required + // AgentContainerImageURL + AgentContainerImageURL string `json:"agentContainerImageURL"` +} + +// SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) +func SetupDefaults() { + + // Acquire environmental defaults and initialize Cyborg defaults with them + cyborgDefaults := CyborgDefaults{ + APIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CYBORG_API_IMAGE_URL_DEFAULT", CyborgAPIContainerImage), + ConductorContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CYBORG_CONDUCTOR_IMAGE_URL_DEFAULT", CyborgConductorContainerImage), + AgentContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CYBORG_AGENT_IMAGE_URL_DEFAULT", CyborgAgentContainerImage), + APITimeout: 60, + } + + SetupCyborgDefaults(cyborgDefaults) + +} diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go index f494b227f..187f1b393 100644 --- a/api/cyborg/v1beta1/cyborg_types.go +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -17,22 +17,111 @@ limitations under the License. package v1beta1 import ( + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// CyborgSpecCore defines the template for CyborgSpec used in OpenStackControlPlane +type CyborgSpecCore struct { + + // +kubebuilder:validation:Optional + // +kubebuilder:default=keystone + // KeystoneInstance to name of the KeystoneAPI CR to select the Service + // instance used by the Cyborg services to authenticate. + KeystoneInstance *string `json:"keystoneInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=openstack + // DatabaseInstance is the name of the MariaDB CR to select the DB + // Service instance used for the Cyborg API DB. + DatabaseInstance *string `json:"databaseInstance"` + + // +kubebuilder:validation:Optional + // MessagingBus configuration (username, vhost, and cluster) + MessagingBus rabbitmqv1.RabbitMqConfig `json:"messagingBus,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="cyborg" + // ServiceUser - optional username used for this service to register in keystone + ServiceUser *string `json:"serviceUser"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={service: CyborgPassword} + // PasswordSelectors - Selectors to identify the DB and ServiceUser + // passwords from the Secret + PasswordSelectors *PasswordSelector `json:"passwordSelectors"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="cyborg" + // DatabaseAccount - MariaDBAccount to use when accessing the API DB + DatabaseAccount *string `json:"databaseAccount"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for Route and Apache + APITimeout *int `json:"apiTimeout"` + + // +kubebuilder:validation:Required + // Secret is the name of the Secret instance containing password + // information for cyborg like the keystone service password and DB passwords + Secret *string `json:"secret"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting + // NodeSelector here acts as a default value and can be overridden by service + // specific NodeSelector Settings. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={replicas:1} + // APIServiceTemplate - define the cyborg-api service + APIServiceTemplate CyborgAPITemplate `json:"apiServiceTemplate"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={replicas:1} + // ConductorServiceTemplate - define the cyborg-conductor service + ConductorServiceTemplate CyborgConductorTemplate `json:"conductorServiceTemplate"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // Auth - Parameters related to authentication (shared by all Cyborg services) + Auth AuthSpec `json:"auth,omitempty"` +} + // CyborgSpec defines the desired state of Cyborg. type CyborgSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file + CyborgSpecCore `json:",inline"` - // Foo is an example field of Cyborg. Edit cyborg_types.go to remove/update - Foo string `json:"foo,omitempty"` + CyborgImages `json:",inline"` } // CyborgStatus defines the observed state of Cyborg. type CyborgStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // APIServiceReadyCount defines the number or replicas ready from cyborg-api + APIServiceReadyCount int32 `json:"apiServiceReadyCount,omitempty"` + + // ConductorServiceReadyCount defines the number or replicas ready from cyborg-conductor + ConductorServiceReadyCount int32 `json:"conductorServiceReadyCount,omitempty"` + + //ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/cyborg/v1beta1/cyborg_webhook.go b/api/cyborg/v1beta1/cyborg_webhook.go new file mode 100644 index 000000000..b95014df6 --- /dev/null +++ b/api/cyborg/v1beta1/cyborg_webhook.go @@ -0,0 +1,239 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// CyborgDefaults - +type CyborgDefaults struct { + APIContainerImageURL string + ConductorContainerImageURL string + AgentContainerImageURL string + APITimeout int +} + +var cyborgDefaults CyborgDefaults + +// log is for logging in this package. +var cyborglog = logf.Log.WithName("cyborg-resource") + +// SetupCyborgDefaults - initialize Cyborg spec defaults for use with either internal or external webhooks +func SetupCyborgDefaults(defaults CyborgDefaults) { + cyborgDefaults = defaults + cyborglog.Info("Cyborg defaults initialized", "defaults", defaults) +} + +var _ webhook.Defaulter = &Cyborg{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *Cyborg) Default() { + cyborglog.Info("default", "name", r.Name) + + r.Spec.Default() +} + +// Default - set defaults for this Cyborg spec. +func (spec *CyborgSpec) Default() { + spec.CyborgImages.Default(cyborgDefaults) + spec.CyborgSpecCore.Default() +} + +func (images *CyborgImages) Default(defaults CyborgDefaults) { + if images.APIContainerImageURL == "" { + images.APIContainerImageURL = cyborgDefaults.APIContainerImageURL + } + if images.ConductorContainerImageURL == "" { + images.ConductorContainerImageURL = cyborgDefaults.ConductorContainerImageURL + } + if images.AgentContainerImageURL == "" { + images.AgentContainerImageURL = cyborgDefaults.AgentContainerImageURL + } +} + +// Default - set defaults for this CyborgSpecCore spec. Expected to be called from +// the higher level meta operator. +func (spec *CyborgSpecCore) Default() { + if spec.APITimeout == nil { + spec.APITimeout = ptr.To(cyborgDefaults.APITimeout) + } + + // Default MessagingBus.Cluster if not set + // Migration from deprecated fields is handled by openstack-operator + if spec.MessagingBus.Cluster == "" { + spec.MessagingBus.Cluster = "rabbitmq" + } +} + +var _ webhook.Validator = &Cyborg{} + +// ValidateAPIServiceTemplate - +func (spec *CyborgSpecCore) ValidateAPIServiceTemplate(basePath *field.Path, namespace string) field.ErrorList { + errors := field.ErrorList{} + + // validate the service override key is valid + errors = append(errors, + service.ValidateRoutedOverrides( + basePath.Child("apiServiceTemplate").Child("override").Child("service"), + spec.APIServiceTemplate.Override.Service)...) + + errors = append(errors, + spec.APIServiceTemplate.ValidateTopology( + basePath.Child("apiServiceTemplate"), + namespace)...) + + return errors +} + +// ValidateConductorServiceTemplate - +func (spec *CyborgSpecCore) ValidateConductorServiceTemplate(basePath *field.Path, namespace string) field.ErrorList { + errors := field.ErrorList{} + // validate the referenced TopologyRef + errors = append(errors, + spec.ConductorServiceTemplate.ValidateTopology( + basePath.Child("conductorServiceTemplate"), + namespace)...) + return errors +} + +// ValidateCreate validates the CyborgSpec during the webhook invocation. +func (spec *CyborgSpec) ValidateCreate(basePath *field.Path, namespace string) (admission.Warnings, field.ErrorList) { + return spec.CyborgSpecCore.ValidateCreate(basePath, namespace) +} + +// ValidateCreate validates the CyborgSpecCore during the webhook invocation. It is +// expected to be called by the validation webhook in the higher level meta +// operator +func (spec *CyborgSpecCore) ValidateCreate(basePath *field.Path, namespace string) (admission.Warnings, field.ErrorList) { + + errors := field.ErrorList{} + + errors = append(errors, spec.ValidateAPIServiceTemplate(basePath, namespace)...) + errors = append(errors, spec.ValidateConductorServiceTemplate(basePath, namespace)...) + + // validate top-level topology + errors = append(errors, + topologyv1.ValidateTopologyRef( + spec.TopologyRef, *basePath.Child("topologyRef"), namespace)...) + + return nil, errors +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *Cyborg) ValidateCreate() (admission.Warnings, error) { + cyborglog.Info("validate create", "name", r.Name) + + warnings, errors := r.Spec.ValidateCreate(field.NewPath("spec"), r.Namespace) + if len(errors) != 0 { + cyborglog.Info("validation failed", "name", r.Name) + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: "cyborg.openstack.org", Kind: "Cyborg"}, + r.Name, errors) + } + return warnings, nil +} + +// ValidateUpdate validates the CyborgSpec during the webhook invocation. +func (spec *CyborgSpec) ValidateUpdate(old CyborgSpec, basePath *field.Path, namespace string) (admission.Warnings, field.ErrorList) { + return spec.CyborgSpecCore.ValidateUpdate(old.CyborgSpecCore, basePath, namespace) +} + +// ValidateUpdate validates the CyborgSpecCore during the webhook invocation. It is +// expected to be called by the validation webhook in the higher level meta +// operator +func (spec *CyborgSpecCore) ValidateUpdate(old CyborgSpecCore, basePath *field.Path, namespace string) (admission.Warnings, field.ErrorList) { + var errors field.ErrorList + var warnings admission.Warnings + + // Validate top-level TopologyRef + errors = append(errors, topologyv1.ValidateTopologyRef( + spec.TopologyRef, *basePath.Child("topologyRef"), namespace)...) + + errors = append(errors, spec.ValidateAPIServiceTemplate(basePath, namespace)...) + errors = append(errors, spec.ValidateConductorServiceTemplate(basePath, namespace)...) + + return warnings, errors +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *Cyborg) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cyborglog.Info("validate update", "name", r.Name) + oldCyborg, ok := old.(*Cyborg) + if !ok || oldCyborg == nil { + return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert existing object")) + } + + cyborglog.Info("validate update", "diff", cmp.Diff(oldCyborg, r)) + + warnings, errors := r.Spec.ValidateUpdate(oldCyborg.Spec, field.NewPath("spec"), r.Namespace) + if len(errors) != 0 { + cyborglog.Info("validation failed", "name", r.Name) + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: "cyborg.openstack.org", Kind: "Cyborg"}, + r.Name, errors) + } + return warnings, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *Cyborg) ValidateDelete() (admission.Warnings, error) { + cyborglog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} + +// SetDefaultRouteAnnotations sets HAProxy timeout values of the route +// NOTE: it is used by ctlplane webhook on openstack-operator +func (spec *CyborgSpecCore) SetDefaultRouteAnnotations(annotations map[string]string) { + const haProxyAnno = "haproxy.router.openshift.io/timeout" + // Use a custom annotation to flag when the operator has set the default HAProxy timeout + // With the annotation func determines when to overwrite existing HAProxy timeout with the APITimeout + const cyborgAnno = "api.cyborg.openstack.org/timeout" + + valCyborg, okCyborg := annotations[cyborgAnno] + valHAProxy, okHAProxy := annotations[haProxyAnno] + + // Human operator set the HAProxy timeout manually + if !okCyborg && okHAProxy { + return + } + + // Human operator modified the HAProxy timeout manually without removing the Cyborg flag + if okCyborg && okHAProxy && valCyborg != valHAProxy { + delete(annotations, cyborgAnno) + return + } + + timeout := fmt.Sprintf("%ds", *spec.APITimeout) + annotations[cyborgAnno] = timeout + annotations[haProxyAnno] = timeout +} diff --git a/api/cyborg/v1beta1/cyborgapi_types.go b/api/cyborg/v1beta1/cyborgapi_types.go index c2d9b0872..699f4a0b8 100644 --- a/api/cyborg/v1beta1/cyborgapi_types.go +++ b/api/cyborg/v1beta1/cyborgapi_types.go @@ -17,16 +17,70 @@ limitations under the License. package v1beta1 import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// CyborgAPITemplate defines the input parameters specified by the user to +// create a CyborgAPI via higher level CRDs. +type CyborgAPITemplate struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Maximum=32 + // +kubebuilder:validation:Minimum=0 + // Replicas of the service to run + Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting here overrides + // any global NodeSelector settings within the Cyborg CR. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as custom.conf file. + CustomServiceConfig string `json:"customServiceConfig"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` +} + // CyborgAPISpec defines the desired state of CyborgAPI. type CyborgAPISpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file + // +kubebuilder:validation:Optional + // API - define the cyborg-api service + CyborgAPITemplate `json:",inline"` - // Foo is an example field of CyborgAPI. Edit cyborgapi_types.go to remove/update - Foo string `json:"foo,omitempty"` + // +kubebuilder:validation:Required + // ConfigSecret - containing all the configuration needed provided by Cyborg object + ConfigSecret *string `json:"configSecret"` } // CyborgAPIStatus defines the observed state of CyborgAPI. diff --git a/api/cyborg/v1beta1/cyborgapi_webhook.go b/api/cyborg/v1beta1/cyborgapi_webhook.go new file mode 100644 index 000000000..d57bfa18f --- /dev/null +++ b/api/cyborg/v1beta1/cyborgapi_webhook.go @@ -0,0 +1,107 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" +) + +// log is for logging in this package. +var cyborgapilog = logf.Log.WithName("cyborgapi-resource") + +var _ webhook.Validator = &CyborgAPI{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *CyborgAPI) ValidateCreate() (admission.Warnings, error) { + cyborgapilog.Info("validate create", "name", r.Name) + + errors := field.ErrorList{} + basePath := field.NewPath("spec") + + // validate the service override key is valid + errors = append(errors, service.ValidateRoutedOverrides(basePath.Child("override").Child("service"), r.Spec.Override.Service)...) + + errors = append(errors, topologyv1.ValidateTopologyRef( + r.Spec.TopologyRef, *basePath.Child("").Child("topologyRef"), r.Namespace)...) + + if len(errors) != 0 { + cyborgapilog.Info("validation failed", "name", r.Name) + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "cyborg.openstack.org", Kind: "CyborgAPI"}, + r.Name, errors) + } + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *CyborgAPI) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cyborgapilog.Info("validate update", "name", r.Name) + oldCyborgAPI, ok := old.(*CyborgAPI) + if !ok || oldCyborgAPI == nil { + return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert existing object")) + } + + cyborgapilog.Info("validate update", "diff", cmp.Diff(oldCyborgAPI, r)) + + errors := field.ErrorList{} + basePath := field.NewPath("spec") + + // validate the service override key is valid + errors = append(errors, service.ValidateRoutedOverrides(basePath.Child("override").Child("service"), r.Spec.Override.Service)...) + + errors = append(errors, r.Spec.ValidateTopology(basePath.Child(""), r.Namespace)...) + + if len(errors) != 0 { + cyborgapilog.Info("validation failed", "name", r.Name) + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "cyborg.openstack.org", Kind: "CyborgAPI"}, + r.Name, errors) + } + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *CyborgAPI) ValidateDelete() (admission.Warnings, error) { + cyborgapilog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} + +// ValidateTopology validates the referenced TopoRef.Namespace. +func (r *CyborgAPITemplate) ValidateTopology( + basePath *field.Path, + namespace string, +) field.ErrorList { + return topologyv1.ValidateTopologyRef( + r.TopologyRef, + *basePath.Child("topologyRef"), + namespace, + ) +} diff --git a/api/cyborg/v1beta1/cyborgconductor_types.go b/api/cyborg/v1beta1/cyborgconductor_types.go index 7af1c2b85..d58ed24e1 100644 --- a/api/cyborg/v1beta1/cyborgconductor_types.go +++ b/api/cyborg/v1beta1/cyborgconductor_types.go @@ -17,19 +17,56 @@ limitations under the License. package v1beta1 import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// CyborgConductorTemplate defines the input parameters specified by the user to +// create a CyborgConductor via higher level CRDs. +type CyborgConductorTemplate struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Maximum=32 + // +kubebuilder:validation:Minimum=0 + // Replicas of the service to run + Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting here overrides + // any global NodeSelector settings within the Cyborg CR. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as custom.conf file. + CustomServiceConfig string `json:"customServiceConfig"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + // CyborgConductorSpec defines the desired state of CyborgConductor. type CyborgConductorSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - // Foo is an example field of CyborgConductor. Edit cyborgconductor_types.go to remove/update - Foo string `json:"foo,omitempty"` + CyborgConductorTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // ConfigSecret - containing all the configuration needed provided by Cyborg object + ConfigSecret string `json:"configSecret"` } // CyborgConductorStatus defines the observed state of CyborgConductor. diff --git a/api/cyborg/v1beta1/cyborgconductor_webhook.go b/api/cyborg/v1beta1/cyborgconductor_webhook.go new file mode 100644 index 000000000..7a19ee743 --- /dev/null +++ b/api/cyborg/v1beta1/cyborgconductor_webhook.go @@ -0,0 +1,100 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" +) + +// log is for logging in this package. +var cyborgconductorlog = logf.Log.WithName("cyborgconductor-resource") + +var _ webhook.Validator = &CyborgConductor{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *CyborgConductor) ValidateCreate() (admission.Warnings, error) { + cyborgconductorlog.Info("validate create", "name", r.Name) + + errors := field.ErrorList{} + basePath := field.NewPath("spec") + + errors = append(errors, topologyv1.ValidateTopologyRef( + r.Spec.TopologyRef, *basePath.Child("").Child("topologyRef"), r.Namespace)...) + + if len(errors) != 0 { + cyborgconductorlog.Info("validation failed", "name", r.Name) + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "cyborg.openstack.org", Kind: "CyborgConductor"}, + r.Name, errors) + } + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *CyborgConductor) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cyborgconductorlog.Info("validate update", "name", r.Name) + oldCyborgConductor, ok := old.(*CyborgConductor) + if !ok || oldCyborgConductor == nil { + return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert existing object")) + } + + cyborgconductorlog.Info("validate update", "diff", cmp.Diff(oldCyborgConductor, r)) + + errors := field.ErrorList{} + basePath := field.NewPath("spec") + + errors = append(errors, r.Spec.ValidateTopology(basePath.Child(""), r.Namespace)...) + + if len(errors) != 0 { + cyborgconductorlog.Info("validation failed", "name", r.Name) + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "cyborg.openstack.org", Kind: "CyborgConductor"}, + r.Name, errors) + } + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *CyborgConductor) ValidateDelete() (admission.Warnings, error) { + cyborgconductorlog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} + +// ValidateTopology validates the referenced TopoRef.Namespace. +func (r *CyborgConductorTemplate) ValidateTopology( + basePath *field.Path, + namespace string, +) field.ErrorList { + return topologyv1.ValidateTopologyRef( + r.TopologyRef, + *basePath.Child("topologyRef"), + namespace, + ) +} diff --git a/api/cyborg/v1beta1/zz_generated.deepcopy.go b/api/cyborg/v1beta1/zz_generated.deepcopy.go index cfe17e260..64d8b68cc 100644 --- a/api/cyborg/v1beta1/zz_generated.deepcopy.go +++ b/api/cyborg/v1beta1/zz_generated.deepcopy.go @@ -21,16 +21,56 @@ limitations under the License. package v1beta1 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + topologyv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIOverrideSpec) DeepCopyInto(out *APIOverrideSpec) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = make(map[service.Endpoint]service.RoutedOverrideSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIOverrideSpec. +func (in *APIOverrideSpec) DeepCopy() *APIOverrideSpec { + if in == nil { + return nil + } + out := new(APIOverrideSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthSpec) DeepCopyInto(out *AuthSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthSpec. +func (in *AuthSpec) DeepCopy() *AuthSpec { + if in == nil { + return nil + } + out := new(AuthSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cyborg) DeepCopyInto(out *Cyborg) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cyborg. @@ -56,7 +96,7 @@ func (in *CyborgAPI) DeepCopyInto(out *CyborgAPI) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } @@ -113,6 +153,12 @@ func (in *CyborgAPIList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgAPISpec) DeepCopyInto(out *CyborgAPISpec) { *out = *in + in.CyborgAPITemplate.DeepCopyInto(&out.CyborgAPITemplate) + if in.ConfigSecret != nil { + in, out := &in.ConfigSecret, &out.ConfigSecret + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPISpec. @@ -140,12 +186,51 @@ func (in *CyborgAPIStatus) DeepCopy() *CyborgAPIStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgAPITemplate) DeepCopyInto(out *CyborgAPITemplate) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + in.Resources.DeepCopyInto(&out.Resources) + in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPITemplate. +func (in *CyborgAPITemplate) DeepCopy() *CyborgAPITemplate { + if in == nil { + return nil + } + out := new(CyborgAPITemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgConductor) DeepCopyInto(out *CyborgConductor) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } @@ -202,6 +287,7 @@ func (in *CyborgConductorList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgConductorSpec) DeepCopyInto(out *CyborgConductorSpec) { *out = *in + in.CyborgConductorTemplate.DeepCopyInto(&out.CyborgConductorTemplate) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorSpec. @@ -229,6 +315,73 @@ func (in *CyborgConductorStatus) DeepCopy() *CyborgConductorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgConductorTemplate) DeepCopyInto(out *CyborgConductorTemplate) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorTemplate. +func (in *CyborgConductorTemplate) DeepCopy() *CyborgConductorTemplate { + if in == nil { + return nil + } + out := new(CyborgConductorTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgDefaults) DeepCopyInto(out *CyborgDefaults) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgDefaults. +func (in *CyborgDefaults) DeepCopy() *CyborgDefaults { + if in == nil { + return nil + } + out := new(CyborgDefaults) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgImages) DeepCopyInto(out *CyborgImages) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgImages. +func (in *CyborgImages) DeepCopy() *CyborgImages { + if in == nil { + return nil + } + out := new(CyborgImages) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgList) DeepCopyInto(out *CyborgList) { *out = *in @@ -264,6 +417,8 @@ func (in *CyborgList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgSpec) DeepCopyInto(out *CyborgSpec) { *out = *in + in.CyborgSpecCore.DeepCopyInto(&out.CyborgSpecCore) + out.CyborgImages = in.CyborgImages } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgSpec. @@ -276,9 +431,86 @@ func (in *CyborgSpec) DeepCopy() *CyborgSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CyborgSpecCore) DeepCopyInto(out *CyborgSpecCore) { + *out = *in + if in.KeystoneInstance != nil { + in, out := &in.KeystoneInstance, &out.KeystoneInstance + *out = new(string) + **out = **in + } + if in.DatabaseInstance != nil { + in, out := &in.DatabaseInstance, &out.DatabaseInstance + *out = new(string) + **out = **in + } + out.MessagingBus = in.MessagingBus + if in.ServiceUser != nil { + in, out := &in.ServiceUser, &out.ServiceUser + *out = new(string) + **out = **in + } + if in.PasswordSelectors != nil { + in, out := &in.PasswordSelectors, &out.PasswordSelectors + *out = new(PasswordSelector) + **out = **in + } + if in.DatabaseAccount != nil { + in, out := &in.DatabaseAccount, &out.DatabaseAccount + *out = new(string) + **out = **in + } + if in.APITimeout != nil { + in, out := &in.APITimeout, &out.APITimeout + *out = new(int) + **out = **in + } + if in.Secret != nil { + in, out := &in.Secret, &out.Secret + *out = new(string) + **out = **in + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + in.APIServiceTemplate.DeepCopyInto(&out.APIServiceTemplate) + in.ConductorServiceTemplate.DeepCopyInto(&out.ConductorServiceTemplate) + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } + out.Auth = in.Auth +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgSpecCore. +func (in *CyborgSpecCore) DeepCopy() *CyborgSpecCore { + if in == nil { + return nil + } + out := new(CyborgSpecCore) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgStatus) DeepCopyInto(out *CyborgStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgStatus. @@ -290,3 +522,18 @@ func (in *CyborgStatus) DeepCopy() *CyborgStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PasswordSelector) DeepCopyInto(out *PasswordSelector) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PasswordSelector. +func (in *PasswordSelector) DeepCopy() *PasswordSelector { + if in == nil { + return nil + } + out := new(PasswordSelector) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml index 3815a2751..b8fb2cf09 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml @@ -39,10 +39,299 @@ spec: spec: description: CyborgAPISpec defines the desired state of CyborgAPI. properties: - foo: - description: Foo is an example field of CyborgAPI. Edit cyborgapi_types.go - to remove/update + configSecret: + description: ConfigSecret - containing all the configuration needed + provided by Cyborg object type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - configSecret type: object status: description: CyborgAPIStatus defines the observed state of CyborgAPI. diff --git a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml index 59312e83c..6f3f19f9a 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -39,10 +39,110 @@ spec: spec: description: CyborgConductorSpec defines the desired state of CyborgConductor. properties: - foo: - description: Foo is an example field of CyborgConductor. Edit cyborgconductor_types.go - to remove/update + configSecret: + description: ConfigSecret - containing all the configuration needed + provided by Cyborg object type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - configSecret type: object status: description: CyborgConductorStatus defines the observed state of CyborgConductor. diff --git a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml index 8e22ea151..cab317b3f 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml @@ -39,13 +39,584 @@ spec: spec: description: CyborgSpec defines the desired state of Cyborg. properties: - foo: - description: Foo is an example field of Cyborg. Edit cyborg_types.go - to remove/update + agentContainerImageURL: + description: AgentContainerImageURL type: string + apiContainerImageURL: + description: APIContainerImageURL + type: string + apiServiceTemplate: + default: + replicas: 1 + description: APIServiceTemplate - define the cyborg-api service + properties: + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + type: object + apiTimeout: + default: 60 + description: APITimeout for Route and Apache + minimum: 10 + type: integer + auth: + description: Auth - Parameters related to authentication (shared by + all Cyborg services) + properties: + applicationCredentialSecret: + description: |- + ApplicationCredentialSecret - the name of the k8s Secret that contains the + application credential data used for authentication + type: string + type: object + conductorContainerImageURL: + description: ConductorContainerImageURL + type: string + conductorServiceTemplate: + default: + replicas: 1 + description: ConductorServiceTemplate - define the cyborg-conductor + service + properties: + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the Cyborg CR. + type: object + replicas: + default: 1 + description: Replicas of the service to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + type: object + databaseAccount: + default: cyborg + description: DatabaseAccount - MariaDBAccount to use when accessing + the API DB + type: string + databaseInstance: + default: openstack + description: |- + DatabaseInstance is the name of the MariaDB CR to select the DB + Service instance used for the Cyborg API DB. + type: string + keystoneInstance: + default: keystone + description: |- + KeystoneInstance to name of the KeystoneAPI CR to select the Service + instance used by the Cyborg services to authenticate. + type: string + messagingBus: + description: MessagingBus configuration (username, vhost, and cluster) + properties: + cluster: + description: Name of the cluster + minLength: 1 + type: string + user: + description: User - RabbitMQ username + type: string + vhost: + description: Vhost - RabbitMQ vhost name + type: string + required: + - cluster + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelectors: + default: + service: CyborgPassword + description: |- + PasswordSelectors - Selectors to identify the DB and ServiceUser + passwords from the Secret + properties: + service: + default: CyborgPassword + description: |- + Service - Selector to get the keystone service user password from the + Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + secret: + description: |- + Secret is the name of the Secret instance containing password + information for cyborg like the keystone service password and DB passwords + type: string + serviceUser: + default: cyborg + description: ServiceUser - optional username used for this service + to register in keystone + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - agentContainerImageURL + - apiContainerImageURL + - conductorContainerImageURL + - secret type: object status: description: CyborgStatus defines the observed state of Cyborg. + properties: + apiServiceReadyCount: + description: APIServiceReadyCount defines the number or replicas ready + from cyborg-api + format: int32 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + conductorServiceReadyCount: + description: ConductorServiceReadyCount defines the number or replicas + ready from cyborg-conductor + format: int32 + type: integer + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes. + format: int64 + type: integer type: object type: object served: true diff --git a/config/manifests/bases/nova-operator.clusterserviceversion.yaml b/config/manifests/bases/nova-operator.clusterserviceversion.yaml index 2caf4e9f8..934e436ce 100644 --- a/config/manifests/bases/nova-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/nova-operator.clusterserviceversion.yaml @@ -23,6 +23,10 @@ spec: displayName: Cyborg API kind: CyborgAPI name: cyborgapis.cyborg.openstack.org + specDescriptors: + - description: TLS - Parameters related to the TLS + displayName: TLS + path: tls version: v1beta1 - description: CyborgConductor is the Schema for the cyborgconductors API. displayName: Cyborg Conductor @@ -33,6 +37,19 @@ spec: displayName: Cyborg kind: Cyborg name: cyborgs.cyborg.openstack.org + specDescriptors: + - description: TLS - Parameters related to the TLS + displayName: TLS + path: apiServiceTemplate.tls + - description: Auth - Parameters related to authentication (shared by all Cyborg + services) + displayName: Auth + path: auth + - description: |- + ApplicationCredentialSecret - the name of the k8s Secret that contains the + application credential data used for authentication + displayName: Application Credential Secret + path: auth.applicationCredentialSecret version: v1beta1 - description: NovaAPI is the Schema for the novaapis API displayName: Nova API diff --git a/config/samples/cyborg_v1beta1_cyborgapi.yaml b/config/samples/cyborg_v1beta1_cyborgapi.yaml index 576a4fa53..16857c160 100644 --- a/config/samples/cyborg_v1beta1_cyborgapi.yaml +++ b/config/samples/cyborg_v1beta1_cyborgapi.yaml @@ -6,4 +6,4 @@ metadata: app.kubernetes.io/managed-by: kustomize name: cyborgapi-sample spec: - # TODO(user): Add fields here + configSecret: cyborg-sample-config diff --git a/config/samples/cyborg_v1beta1_cyborgconductor.yaml b/config/samples/cyborg_v1beta1_cyborgconductor.yaml index 7de9c6695..437e36792 100644 --- a/config/samples/cyborg_v1beta1_cyborgconductor.yaml +++ b/config/samples/cyborg_v1beta1_cyborgconductor.yaml @@ -6,4 +6,4 @@ metadata: app.kubernetes.io/managed-by: kustomize name: cyborgconductor-sample spec: - # TODO(user): Add fields here + configSecret: cyborg-sample-config From ce45604604c4c559dd64527d036e3b697ff8e0c3 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Thu, 26 Mar 2026 18:16:10 +0100 Subject: [PATCH 3/7] cyborg: Implement Cyborg top-level controller reconcile loop Add full reconcile logic for the Cyborg CR: - Manage RBAC resources (ServiceAccount, Role, RoleBinding) - Validate input password secret and RabbitMQ TransportURL secret - Create MariaDB database and run DB sync job via a batch Job - Register Cyborg service in Keystone - Create a sub-level secret aggregating DB credentials, transport URL and service password to be consumed by CyborgAPI and CyborgConductor - Track readiness via structured conditions on CyborgStatus - Add functional tests covering the full reconcile flow Assisted-By: Claude Signed-off-by: Alfredo Moralejo --- api/bases/cyborg.openstack.org_cyborgs.yaml | 18 +- api/cyborg/v1beta1/conditions.go | 55 ++ api/cyborg/v1beta1/cyborg_types.go | 33 +- api/cyborg/v1beta1/zz_generated.deepcopy.go | 7 + cmd/main.go | 54 +- .../bases/cyborg.openstack.org_cyborgs.yaml | 18 +- config/webhook/manifests.yaml | 80 ++ hack/clean_local_webhook.sh | 4 + hack/run_with_local_webhook.sh | 112 +++ internal/controller/cyborg/common.go | 137 +++ .../controller/cyborg/cyborg_controller.go | 920 +++++++++++++++++- .../controller/cyborg/cyborgapi_controller.go | 3 + .../cyborg/cyborgconductor_controller.go | 4 +- internal/cyborg/constants.go | 53 + internal/cyborg/dbsync.go | 150 +++ .../webhook/cyborg/v1beta1/cyborg_webhook.go | 107 ++ .../cyborg/v1beta1/cyborgapi_webhook.go | 80 ++ .../cyborg/v1beta1/cyborgconductor_webhook.go | 80 ++ templates/cyborg/00-default.conf | 9 + .../cyborg/cyborg/cyborg-dbsync-config.json | 3 + test/functional/cyborg/base_test.go | 157 +++ .../cyborg/cyborg_controller_test.go | 533 ++++++++++ test/functional/cyborg/suite_test.go | 271 ++++++ 23 files changed, 2837 insertions(+), 51 deletions(-) create mode 100644 api/cyborg/v1beta1/conditions.go create mode 100644 internal/controller/cyborg/common.go create mode 100644 internal/cyborg/constants.go create mode 100644 internal/cyborg/dbsync.go create mode 100644 internal/webhook/cyborg/v1beta1/cyborg_webhook.go create mode 100644 internal/webhook/cyborg/v1beta1/cyborgapi_webhook.go create mode 100644 internal/webhook/cyborg/v1beta1/cyborgconductor_webhook.go create mode 100644 templates/cyborg/00-default.conf create mode 100644 templates/cyborg/cyborg/cyborg-dbsync-config.json create mode 100644 test/functional/cyborg/base_test.go create mode 100644 test/functional/cyborg/cyborg_controller_test.go create mode 100644 test/functional/cyborg/suite_test.go diff --git a/api/bases/cyborg.openstack.org_cyborgs.yaml b/api/bases/cyborg.openstack.org_cyborgs.yaml index cab317b3f..9eb8f4851 100644 --- a/api/bases/cyborg.openstack.org_cyborgs.yaml +++ b/api/bases/cyborg.openstack.org_cyborgs.yaml @@ -524,6 +524,7 @@ spec: e.g. to check logs type: boolean secret: + default: osp-secret description: |- Secret is the name of the Secret instance containing password information for cyborg like the keystone service password and DB passwords @@ -554,7 +555,6 @@ spec: - agentContainerImageURL - apiContainerImageURL - conductorContainerImageURL - - secret type: object status: description: CyborgStatus defines the observed state of Cyborg. @@ -611,12 +611,22 @@ spec: ready from cyborg-conductor format: int32 type: integer + hash: + additionalProperties: + type: string + description: Hash - Map of hashes to track e.g. job status + type: object observedGeneration: - description: ObservedGeneration - the most recent generation observed - for this service. If the observed generation is less than the spec - generation, then the controller has not processed the latest changes. + description: |- + ObservedGeneration - the most recent generation observed for this + service. If the observed generation is less than the spec generation, + then the controller has not processed the latest changes. format: int64 type: integer + serviceID: + description: ServiceID - The ID of the cyborg service registered in + keystone + type: string type: object type: object served: true diff --git a/api/cyborg/v1beta1/conditions.go b/api/cyborg/v1beta1/conditions.go new file mode 100644 index 000000000..5fbb387a9 --- /dev/null +++ b/api/cyborg/v1beta1/conditions.go @@ -0,0 +1,55 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + +const ( + // DbSyncHash hash + DbSyncHash = "dbsync" +) + +const ( + // CyborgRabbitMQTransportURLReadyCondition indicates whether the Cyborg RabbitMQ TransportURL is ready + CyborgRabbitMQTransportURLReadyCondition condition.Type = "CyborgRabbitMQTransportURLReady" + + // CyborgAPIReadyCondition indicates whether the CyborgAPI is ready + CyborgAPIReadyCondition condition.Type = "CyborgAPIReady" + + // CyborgConductorReadyCondition indicates whether the CyborgConductor is ready + CyborgConductorReadyCondition condition.Type = "CyborgConductorReady" +) + +const ( + // CyborgRabbitMQTransportURLReadyRunningMessage - + CyborgRabbitMQTransportURLReadyRunningMessage = "CyborgRabbitMQTransportURL creation in progress" + + // CyborgRabbitMQTransportURLReadyMessage - + CyborgRabbitMQTransportURLReadyMessage = "CyborgRabbitMQTransportURL successfully created" + + // CyborgRabbitMQTransportURLReadyErrorMessage - + CyborgRabbitMQTransportURLReadyErrorMessage = "CyborgRabbitMQTransportURL error occurred %s" + + // CyborgAPIReadyInitMessage - + CyborgAPIReadyInitMessage = "CyborgAPI not started" + + // CyborgConductorReadyInitMessage - + CyborgConductorReadyInitMessage = "CyborgConductor not started" + + // CyborgApplicationCredentialSecretErrorMessage - + CyborgApplicationCredentialSecretErrorMessage = "Error with application credential secret" +) diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go index 187f1b393..eada70a2b 100644 --- a/api/cyborg/v1beta1/cyborg_types.go +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -64,7 +64,8 @@ type CyborgSpecCore struct { // APITimeout for Route and Apache APITimeout *int `json:"apiTimeout"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional + // +kubebuilder:default=osp-secret // Secret is the name of the Secret instance containing password // information for cyborg like the keystone service password and DB passwords Secret *string `json:"secret"` @@ -114,13 +115,21 @@ type CyborgStatus struct { // Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + // ServiceID - The ID of the cyborg service registered in keystone + ServiceID string `json:"serviceID,omitempty"` + + // Hash - Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + // APIServiceReadyCount defines the number or replicas ready from cyborg-api APIServiceReadyCount int32 `json:"apiServiceReadyCount,omitempty"` // ConductorServiceReadyCount defines the number or replicas ready from cyborg-conductor ConductorServiceReadyCount int32 `json:"conductorServiceReadyCount,omitempty"` - //ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes. + // ObservedGeneration - the most recent generation observed for this + // service. If the observed generation is less than the spec generation, + // then the controller has not processed the latest changes. ObservedGeneration int64 `json:"observedGeneration,omitempty"` } @@ -145,6 +154,26 @@ type CyborgList struct { Items []Cyborg `json:"items"` } +// RbacConditionsSet sets the conditions for the RBAC reconciliation +func (instance Cyborg) RbacConditionsSet(c *condition.Condition) { + instance.Status.Conditions.Set(c) +} + +// RbacNamespace returns the namespace +func (instance Cyborg) RbacNamespace() string { + return instance.Namespace +} + +// RbacResourceName returns the name to be used for RBAC objects (serviceaccount, role, rolebinding) +func (instance Cyborg) RbacResourceName() string { + return "cyborg-" + instance.Name +} + +// IsReady returns true if the ReadyCondition is true +func (instance *Cyborg) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.ReadyCondition) +} + func init() { SchemeBuilder.Register(&Cyborg{}, &CyborgList{}) } diff --git a/api/cyborg/v1beta1/zz_generated.deepcopy.go b/api/cyborg/v1beta1/zz_generated.deepcopy.go index 64d8b68cc..0065cd66b 100644 --- a/api/cyborg/v1beta1/zz_generated.deepcopy.go +++ b/api/cyborg/v1beta1/zz_generated.deepcopy.go @@ -511,6 +511,13 @@ func (in *CyborgStatus) DeepCopyInto(out *CyborgStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgStatus. diff --git a/cmd/main.go b/cmd/main.go index c1ce46e94..f9ad4c45e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -38,9 +38,10 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborgv1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" cyborgcontroller "github.com/openstack-k8s-operators/nova-operator/internal/controller/cyborg" - "github.com/openstack-k8s-operators/nova-operator/internal/controller/nova" + controller "github.com/openstack-k8s-operators/nova-operator/internal/controller/nova" + cyborgwebhookv1beta1 "github.com/openstack-k8s-operators/nova-operator/internal/webhook/cyborg/v1beta1" webhookv1beta1 "github.com/openstack-k8s-operators/nova-operator/internal/webhook/nova/v1beta1" // +kubebuilder:scaffold:imports @@ -75,7 +76,7 @@ func init() { utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(memcachedv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) - utilruntime.Must(cyborgv1beta1.AddToScheme(scheme)) + utilruntime.Must(cyborgv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -267,6 +268,16 @@ func main() { // Acquire environmental defaults and initialize operator defaults with them novav1.SetupDefaults() + if os.Getenv("ENABLE_CYBORG") == "true" { + cyborgreconcilers := cyborgcontroller.NewReconcilers(mgr, kclient) + err = cyborgreconcilers.Setup(mgr, setupLog) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Cyborg") + os.Exit(1) + } + cyborgv1.SetupDefaults() + } + // nolint:goconst checker := healthz.Ping if os.Getenv("ENABLE_WEBHOOKS") != "false" { @@ -304,30 +315,23 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "NovaCompute") os.Exit(1) } + if os.Getenv("ENABLE_CYBORG") == "true" { + if err := cyborgwebhookv1beta1.SetupCyborgWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Cyborg") + os.Exit(1) + } + if err := cyborgwebhookv1beta1.SetupCyborgAPIWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "CyborgAPI") + os.Exit(1) + } + if err := cyborgwebhookv1beta1.SetupCyborgConductorWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "CyborgConductor") + os.Exit(1) + } + } checker = mgr.GetWebhookServer().StartedChecker() - - } - if err := (&cyborgcontroller.CyborgReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Cyborg") - os.Exit(1) - } - if err := (&cyborgcontroller.CyborgAPIReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "CyborgAPI") - os.Exit(1) - } - if err := (&cyborgcontroller.CyborgConductorReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "CyborgConductor") - os.Exit(1) } + // +kubebuilder:scaffold:builder if metricsCertWatcher != nil { diff --git a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml index cab317b3f..9eb8f4851 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml @@ -524,6 +524,7 @@ spec: e.g. to check logs type: boolean secret: + default: osp-secret description: |- Secret is the name of the Secret instance containing password information for cyborg like the keystone service password and DB passwords @@ -554,7 +555,6 @@ spec: - agentContainerImageURL - apiContainerImageURL - conductorContainerImageURL - - secret type: object status: description: CyborgStatus defines the observed state of Cyborg. @@ -611,12 +611,22 @@ spec: ready from cyborg-conductor format: int32 type: integer + hash: + additionalProperties: + type: string + description: Hash - Map of hashes to track e.g. job status + type: object observedGeneration: - description: ObservedGeneration - the most recent generation observed - for this service. If the observed generation is less than the spec - generation, then the controller has not processed the latest changes. + description: |- + ObservedGeneration - the most recent generation observed for this + service. If the observed generation is less than the spec generation, + then the controller has not processed the latest changes. format: int64 type: integer + serviceID: + description: ServiceID - The ID of the cyborg service registered in + keystone + type: string type: object type: object served: true diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index ebdcbaaaf..800731f56 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -4,6 +4,26 @@ kind: MutatingWebhookConfiguration metadata: name: mutating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-cyborg-openstack-org-v1beta1-cyborg + failurePolicy: Fail + name: mcyborg-v1beta1.kb.io + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgs + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -170,6 +190,66 @@ kind: ValidatingWebhookConfiguration metadata: name: validating-webhook-configuration webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-cyborg-openstack-org-v1beta1-cyborg + failurePolicy: Fail + name: vcyborg-v1beta1.kb.io + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgs + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-cyborg-openstack-org-v1beta1-cyborgapi + failurePolicy: Fail + name: vcyborgapi-v1beta1.kb.io + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgapis + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-cyborg-openstack-org-v1beta1-cyborgconductor + failurePolicy: Fail + name: vcyborgconductor-v1beta1.kb.io + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgconductors + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/hack/clean_local_webhook.sh b/hack/clean_local_webhook.sh index db457995a..4552e8c3e 100755 --- a/hack/clean_local_webhook.sh +++ b/hack/clean_local_webhook.sh @@ -17,3 +17,7 @@ oc delete validatingwebhookconfiguration/vnovascheduler.kb.io --ignore-not-found oc delete mutatingwebhookconfiguration/mnovascheduler.kb.io --ignore-not-found oc delete validatingwebhookconfiguration/vnovacompute.kb.io --ignore-not-found oc delete mutatingwebhookconfiguration/mnovacompute.kb.io --ignore-not-found +oc delete validatingwebhookconfiguration/vcyborg.kb.io --ignore-not-found +oc delete mutatingwebhookconfiguration/mcyborg.kb.io --ignore-not-found +oc delete validatingwebhookconfiguration/vcyborgapi.kb.io --ignore-not-found +oc delete validatingwebhookconfiguration/vcyborgconductor.kb.io --ignore-not-found diff --git a/hack/run_with_local_webhook.sh b/hack/run_with_local_webhook.sh index edf927594..e980d2c10 100755 --- a/hack/run_with_local_webhook.sh +++ b/hack/run_with_local_webhook.sh @@ -490,6 +490,118 @@ webhooks: scope: '*' sideEffects: None timeoutSeconds: 10 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: vcyborg.kb.io +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: ${CA_BUNDLE} + url: https://${CRC_IP}:${WEBHOOK_PORT}/validate-cyborg-openstack-org-v1beta1-cyborg + failurePolicy: Fail + matchPolicy: Equivalent + name: vcyborg.kb.io + objectSelector: {} + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgs + scope: '*' + sideEffects: None + timeoutSeconds: 10 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mcyborg.kb.io +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: ${CA_BUNDLE} + url: https://${CRC_IP}:${WEBHOOK_PORT}/mutate-cyborg-openstack-org-v1beta1-cyborg + failurePolicy: Fail + matchPolicy: Equivalent + name: mcyborg.kb.io + objectSelector: {} + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgs + scope: '*' + sideEffects: None + timeoutSeconds: 10 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: vcyborgapi.kb.io +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: ${CA_BUNDLE} + url: https://${CRC_IP}:${WEBHOOK_PORT}/validate-cyborg-openstack-org-v1beta1-cyborgapi + failurePolicy: Fail + matchPolicy: Equivalent + name: vcyborgapi.kb.io + objectSelector: {} + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgapis + scope: '*' + sideEffects: None + timeoutSeconds: 10 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: vcyborgconductor.kb.io +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: ${CA_BUNDLE} + url: https://${CRC_IP}:${WEBHOOK_PORT}/validate-cyborg-openstack-org-v1beta1-cyborgconductor + failurePolicy: Fail + matchPolicy: Equivalent + name: vcyborgconductor.kb.io + objectSelector: {} + rules: + - apiGroups: + - cyborg.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cyborgconductors + scope: '*' + sideEffects: None + timeoutSeconds: 10 EOF_CAT oc apply -n openstack -f ${TMPDIR}/patch_webhook_configurations.yaml diff --git a/internal/controller/cyborg/common.go b/internal/controller/cyborg/common.go new file mode 100644 index 000000000..1ced69c21 --- /dev/null +++ b/internal/controller/cyborg/common.go @@ -0,0 +1,137 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cyborg implements the controllers for the Cyborg accelerator lifecycle management service. +package cyborg + +import ( + "errors" + "time" + + "github.com/go-logr/logr" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + passwordSecretField = ".spec.secret" + authAppCredSecretField = ".spec.auth.applicationCredentialSecret" //nolint:gosec + + // TransportURLSelector is the key for the transport URL in secrets + TransportURLSelector = "transport_url" + // QuorumQueuesSelector is the key for quorum queues in TransportURL secrets + QuorumQueuesSelector = "quorumqueues" + // DatabaseAccount is the key for the database account name + DatabaseAccount = "database_account" + // DatabaseUsername is the key for the database username + DatabaseUsername = "database_username" + // DatabasePassword is the key for the database password + DatabasePassword = "database_password" + // DatabaseHostname is the key for the database hostname + DatabaseHostname = "database_hostname" +) + +var ( + cyborgWatchFields = []string{ + passwordSecretField, + authAppCredSecretField, + } + + // ErrRetrievingSecretData indicates an error retrieving required data from a secret + ErrRetrievingSecretData = errors.New("error retrieving required data from secret") + // ErrRetrievingTransportURLSecretData indicates an error retrieving transport URL secret data + ErrRetrievingTransportURLSecretData = errors.New("error retrieving required data from transporturl secret") + // ErrTransportURLFieldMissing indicates the TransportURL secret is missing the transport_url field + ErrTransportURLFieldMissing = errors.New("the TransportURL secret does not have 'transport_url' field") + // ErrSecretFieldNotFound indicates a required field was not found in a secret + ErrSecretFieldNotFound = errors.New("field not found in secret") + // ErrACSecretNotFound indicates the ApplicationCredential secret was not found + ErrACSecretNotFound = errors.New("ApplicationCredential secret not found") + // ErrACSecretMissingKeys indicates the ApplicationCredential secret is missing required keys + ErrACSecretMissingKeys = errors.New("ApplicationCredential secret missing required keys") +) + +// ReconcilerBase provides a common set of clients scheme and loggers for all reconcilers. +type ReconcilerBase struct { + Client client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme + RequeueTimeout time.Duration +} + +// Manageable all types that conform to this interface can be setup with a controller-runtime manager. +type Manageable interface { + SetupWithManager(mgr ctrl.Manager) error +} + +// Reconciler represents a generic interface for all Reconciler objects in nova +type Reconciler interface { + Manageable + SetRequeueTimeout(timeout time.Duration) +} + +// NewReconcilerBase constructs a ReconcilerBase given a manager and Kclient. +func NewReconcilerBase( + mgr ctrl.Manager, kclient kubernetes.Interface, +) ReconcilerBase { + return ReconcilerBase{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + RequeueTimeout: time.Duration(5) * time.Second, + } +} + +// SetRequeueTimeout overrides the default RequeueTimeout of the Reconciler +func (r *ReconcilerBase) SetRequeueTimeout(timeout time.Duration) { + r.RequeueTimeout = timeout +} + +// Reconcilers holds all the Reconciler objects of the nova-operator to +// allow generic management of them. +type Reconcilers struct { + reconcilers map[string]Reconciler +} + +// NewReconcilers constructs all nova Reconciler objects +func NewReconcilers(mgr ctrl.Manager, kclient *kubernetes.Clientset) *Reconcilers { + return &Reconcilers{ + reconcilers: map[string]Reconciler{ + "Cyborg": &CyborgReconciler{ + ReconcilerBase: NewReconcilerBase(mgr, kclient), + }, + }} +} + +// Setup starts the reconcilers by connecting them to the Manager +func (r *Reconcilers) Setup(mgr ctrl.Manager, setupLog logr.Logger) error { + var err error + for name, controller := range r.reconcilers { + if err = controller.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", name) + return err + } + } + return nil +} + +type conditionUpdater interface { + Set(c *condition.Condition) + MarkTrue(t condition.Type, messageFormat string, messageArgs ...any) +} diff --git a/internal/controller/cyborg/cyborg_controller.go b/internal/controller/cyborg/cyborg_controller.go index 56baa8454..26c152aeb 100644 --- a/internal/controller/cyborg/cyborg_controller.go +++ b/internal/controller/cyborg/cyborg_controller.go @@ -18,46 +18,936 @@ package cyborg import ( "context" + "fmt" + "time" - "k8s.io/apimachinery/pkg/runtime" + "github.com/go-logr/logr" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/job" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborgservice "github.com/openstack-k8s-operators/nova-operator/internal/cyborg" ) -// CyborgReconciler reconciles a Cyborg object +// CyborgReconciler reconciles a Cyborg object. +// +//nolint:revive type CyborgReconciler struct { - client.Client - Scheme *runtime.Scheme + ReconcilerBase +} + +// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields +func (r *CyborgReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("Cyborg") } // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgs/finalizers,verbs=update +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis/finalizers,verbs=update +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors/finalizers,verbs=update +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts/finalizers,verbs=update +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Cyborg object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = logf.FromContext(ctx) +func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + instance := &cyborgv1beta1.Cyborg{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + Log.Info("Cyborg instance not found, probably deleted before reconciled. Nothing to do.") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + Log.Error(err, "Failed to read the Cyborg instance.") + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciling Cyborg instance '%s'", instance.Name)) + + h, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + Log.Error(err, "Failed to create lib-common Helper") + return ctrl.Result{}, err + } + + serviceLabels := map[string]string{ + common.AppSelector: cyborgservice.ServiceName, + } + + isNewInstance := instance.Status.Conditions == nil + savedConditions := instance.Status.Conditions.DeepCopy() + + defer func() { + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + + // Update the Ready condition based on the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + // something is not ready so reset the Ready condition + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + // and recalculate it based on the state of the rest of the conditions + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + err := h.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Initialize the status of the instance, including the conditions, hash, and observed generation. + err = r.initStatus(instance) + if err != nil { + return ctrl.Result{}, err + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, h) + } + + err = r.ensureRbac(ctx, h, instance) + if err != nil { + return ctrl.Result{}, err + } + + if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, h.GetFinalizer()) || isNewInstance { + return ctrl.Result{}, nil + } + + // + // Create the DB and required DB account + // + db, result, err := r.ensureDB(ctx, h, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + + // + // Create RabbitMQ TransportURL + // + transportURL, op, err := r.ensureMQ(ctx, instance, h, instance.Name+"-cyborg-transport", instance.Spec.MessagingBus, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + cyborgv1beta1.CyborgRabbitMQTransportURLReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if transportURL == nil { + Log.Info(fmt.Sprintf("Waiting for TransportURL for %s to be created", instance.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + cyborgv1beta1.CyborgRabbitMQTransportURLReadyRunningMessage)) + return ctrl.Result{RequeueAfter: time.Duration(10) * time.Second}, nil + } + + instance.Status.Conditions.MarkTrue( + cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, + cyborgv1beta1.CyborgRabbitMQTransportURLReadyMessage) + + _ = op + + // + // Validate input secret (password secret) + // + hash, _, inputSecret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: *instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.Service, + }, + h.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if err != nil || hash == "" { + return ctrl.Result{}, ErrRetrievingSecretData + } + + // TransportURL Secret + hashTransporturl, _, transporturlSecret, err := ensureSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: transportURL.Status.SecretName}, + []string{ + TransportURLSelector, + }, + h.GetClient(), + &instance.Status.Conditions, + r.RequeueTimeout, + ) + if err != nil || hashTransporturl == "" { + return ctrl.Result{}, ErrRetrievingTransportURLSecretData + } + + // + // Handle Application Credentials + // + var acData *keystonev1.ApplicationCredentialData + if instance.Spec.Auth.ApplicationCredentialSecret != "" { + acSecretObj, _, err := secret.GetSecret(ctx, h, instance.Spec.Auth.ApplicationCredentialSecret, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info("ApplicationCredential secret not found, waiting", "secret", instance.Spec.Auth.ApplicationCredentialSecret) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + cyborgv1beta1.CyborgApplicationCredentialSecretErrorMessage)) + return ctrl.Result{}, fmt.Errorf("%w: %s", ErrACSecretNotFound, instance.Spec.Auth.ApplicationCredentialSecret) + } + Log.Error(err, "Failed to get ApplicationCredential secret", "secret", instance.Spec.Auth.ApplicationCredentialSecret) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + cyborgv1beta1.CyborgApplicationCredentialSecretErrorMessage)) + return ctrl.Result{}, err + } + acID, okID := acSecretObj.Data[keystonev1.ACIDSecretKey] + acSecretData, okSecret := acSecretObj.Data[keystonev1.ACSecretSecretKey] + if okID && len(acID) > 0 && okSecret && len(acSecretData) > 0 { + acData = &keystonev1.ApplicationCredentialData{ + ID: string(acID), + Secret: string(acSecretData), + } + Log.Info("Using ApplicationCredentials auth", "secret", instance.Spec.Auth.ApplicationCredentialSecret) + } else { + Log.Error(nil, "ApplicationCredential secret missing required keys", "secret", instance.Spec.Auth.ApplicationCredentialSecret) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + cyborgv1beta1.CyborgApplicationCredentialSecretErrorMessage)) + return ctrl.Result{}, fmt.Errorf("%w: %s", ErrACSecretMissingKeys, instance.Spec.Auth.ApplicationCredentialSecret) + } + } + + // + // Create sub-level secret with required configuration + // + _, err = r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData) + if err != nil { + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // Create Keystone Service + // + _, err = r.ensureKeystoneSvc(ctx, h, instance, serviceLabels) + if err != nil { + return ctrl.Result{}, err + } + + // + // Generate config for dbsync + // + configVars := make(map[string]env.Setter) + + err = r.generateServiceConfig(ctx, instance, db, h, &configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // Create dbsync job + // + ctrlResult, err := r.ensureDBSync(ctx, h, instance, serviceLabels) + if err != nil { + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // Remove finalizers from unused MariaDBAccount records + // + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers( + ctx, h, cyborgservice.DatabaseCRName, + *instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } + + return ctrl.Result{}, nil +} + +func (r *CyborgReconciler) initStatus(instance *cyborgv1beta1.Cyborg) error { + err := r.initConditions(instance) + if err != nil { + return err + } + + instance.Status.ObservedGeneration = instance.Generation + + if instance.Status.Hash == nil { + instance.Status.Hash = make(map[string]string) + } + + return nil +} + +func (r *CyborgReconciler) initConditions(instance *cyborgv1beta1.Cyborg) error { + if instance.Status.Conditions == nil { + instance.Status.Conditions = condition.Conditions{} + } + + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), + condition.UnknownCondition( + cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, + condition.InitReason, + condition.RabbitMqTransportURLReadyInitMessage), + condition.UnknownCondition( + condition.InputReadyCondition, + condition.InitReason, + condition.InputReadyInitMessage), + condition.UnknownCondition( + condition.KeystoneServiceReadyCondition, + condition.InitReason, + "Service registration not started"), + condition.UnknownCondition( + condition.ServiceAccountReadyCondition, + condition.InitReason, + condition.ServiceAccountReadyInitMessage), + condition.UnknownCondition( + condition.RoleReadyCondition, + condition.InitReason, + condition.RoleReadyInitMessage), + condition.UnknownCondition( + condition.RoleBindingReadyCondition, + condition.InitReason, + condition.RoleBindingReadyInitMessage), + condition.UnknownCondition( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition( + condition.DBSyncReadyCondition, + condition.InitReason, + condition.DBSyncReadyInitMessage), + ) + + instance.Status.Conditions.Init(&cl) + + return nil +} + +func (r *CyborgReconciler) ensureRbac( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, +) error { + rbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"create", "get", "list", "watch", "update", "patch", "delete"}, + }, + } + + _, err := common_rbac.ReconcileRbac(ctx, h, instance, rbacRules) + if err != nil { + return err + } + + return nil +} + +func (r *CyborgReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, +) (*mariadbv1.Database, ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling the DB instance for '%s'", instance.Name)) + + _, _, err := mariadbv1.EnsureMariaDBAccount( + ctx, h, *instance.Spec.DatabaseAccount, + instance.Namespace, false, cyborgservice.DatabaseUsernamePrefix, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBAccountReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + mariadbv1.MariaDBAccountNotReadyMessage, + err.Error())) + return nil, ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBAccountReadyMessage) + + db := mariadbv1.NewDatabaseForAccount( + *instance.Spec.DatabaseInstance, + cyborgservice.DatabaseName, + cyborgservice.DatabaseCRName, + *instance.Spec.DatabaseAccount, + instance.Namespace, + ) + + ctrlResult, err := db.CreateOrPatchAll(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + ctrlResult, err = db.WaitForDBCreatedWithTimeout(ctx, h, r.RequeueTimeout) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + + return db, ctrl.Result{}, err +} + +func (r *CyborgReconciler) ensureMQ( + ctx context.Context, + instance *cyborgv1beta1.Cyborg, + h *helper.Helper, + transportURLName string, + rabbitMqConfig rabbitmqv1.RabbitMqConfig, + serviceLabels map[string]string, +) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling the RabbitMQ TransportURL '%s' for '%s'", transportURLName, instance.Name)) - // TODO(user): your logic here + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: transportURLName, + Namespace: instance.Namespace, + Labels: serviceLabels, + }, + } + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = rabbitMqConfig.Cluster + transportURL.Spec.Username = rabbitMqConfig.User + transportURL.Spec.Vhost = rabbitMqConfig.Vhost + + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + + if err != nil && !k8s_errors.IsNotFound(err) { + return nil, op, util.WrapErrorForObject( + fmt.Sprintf("error creating or updating TransportURL object %s", transportURLName), + transportURL, + err, + ) + } + + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + if !transportURL.IsReady() || transportURL.Status.SecretName == "" { + Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) + return nil, op, nil + } + + secretName := types.NamespacedName{Namespace: instance.Namespace, Name: transportURL.Status.SecretName} + transportSecret := &corev1.Secret{} + err = h.GetClient().Get(ctx, secretName, transportSecret) + if err != nil { + return nil, op, err + } + + _, ok := transportSecret.Data[TransportURLSelector] + if !ok { + return nil, op, fmt.Errorf("%w: %s", ErrTransportURLFieldMissing, transportURL.Status.SecretName) + } + + return transportURL, op, nil +} + +func (r *CyborgReconciler) ensureKeystoneSvc( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling the Keystone Service for '%s'", instance.Name)) + + ksSvcSpec := keystonev1.KeystoneServiceSpec{ + ServiceType: cyborgservice.ServiceType, + ServiceName: cyborgservice.ServiceName, + ServiceDescription: "Cyborg Accelerator Lifecycle Management Service", + Enabled: true, + ServiceUser: *instance.Spec.ServiceUser, + Secret: *instance.Spec.Secret, + PasswordSelector: instance.Spec.PasswordSelectors.Service, + } + + ksSvc := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, time.Duration(10)*time.Second) + ctrlResult, err := ksSvc.CreateOrPatch(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.KeystoneServiceReadyCondition, + condition.CreationFailedReason, + condition.SeverityError, + "Error while creating Keystone Service for Cyborg")) + return ctrlResult, err + } + + c := ksSvc.GetConditions().Mirror(condition.KeystoneServiceReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.ServiceID = ksSvc.GetServiceID() + + return ctrlResult, nil +} + +func (r *CyborgReconciler) createSubLevelSecret( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, + transportURLSecret corev1.Secret, + inputSecret corev1.Secret, + db *mariadbv1.Database, + acData *keystonev1.ApplicationCredentialData, +) (string, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Creating SubCR Level Secret for '%s'", instance.Name)) + + databaseAccount := db.GetAccount() + databaseSecret := db.GetSecret() + + data := map[string]string{ + instance.Spec.PasswordSelectors.Service: string(inputSecret.Data[instance.Spec.PasswordSelectors.Service]), + TransportURLSelector: string(transportURLSecret.Data[TransportURLSelector]), + QuorumQueuesSelector: string(transportURLSecret.Data[QuorumQueuesSelector]), + DatabaseAccount: databaseAccount.Name, + DatabaseUsername: databaseAccount.Spec.UserName, + DatabasePassword: string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]), + DatabaseHostname: db.GetDatabaseHostname(), + } + + if acData != nil { + data["ACID"] = acData.ID + data["ACSecret"] = acData.Secret + } + + secretName := instance.Name + serviceLabels := labels.GetLabels(instance, labels.GetGroupLabel(cyborgservice.ServiceName), map[string]string{}) + + template := util.Template{ + Name: secretName, + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + InstanceType: instance.GetObjectKind().GroupVersionKind().Kind, + Labels: serviceLabels, + CustomData: data, + } + + err := secret.EnsureSecrets(ctx, h, instance, []util.Template{template}, nil) + + return secretName, err +} + +func (r *CyborgReconciler) generateServiceConfig( + ctx context.Context, + instance *cyborgv1beta1.Cyborg, + db *mariadbv1.Database, + h *helper.Helper, + envVars *map[string]env.Setter, +) error { + Log := r.GetLogger(ctx) + Log.Info("generateServiceConfig - reconciling config for Cyborg CR") + + var tlsCfg *tls.Service + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + databaseAccount := db.GetAccount() + databaseSecret := db.GetSecret() + + templateParameters := map[string]any{ + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + databaseAccount.Spec.UserName, + string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]), + db.GetDatabaseHostname(), + cyborgservice.DatabaseName, + ), + } + + customData := map[string]string{ + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), + } + + serviceLabels := labels.GetLabels(instance, labels.GetGroupLabel(cyborgservice.ServiceName), map[string]string{}) + + cms := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.GetName()), + Namespace: instance.GetNamespace(), + Type: util.TemplateTypeConfig, + InstanceType: instance.GetObjectKind().GroupVersionKind().Kind, + ConfigOptions: templateParameters, + CustomData: customData, + Labels: serviceLabels, + AdditionalTemplate: map[string]string{ + "00-default.conf": "/cyborg/00-default.conf", + "cyborg-dbsync-config.json": "/cyborg/cyborg/cyborg-dbsync-config.json", + }, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, cms, envVars) +} + +func (r *CyborgReconciler) ensureDBSync( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling the DB Sync for '%s'", instance.Name)) + + dbSyncHash := instance.Status.Hash[cyborgv1beta1.DbSyncHash] + jobDef := cyborgservice.DbSyncJob(instance, serviceLabels, nil) + + dbSyncjob := job.NewJob( + jobDef, + cyborgv1beta1.DbSyncHash, + instance.Spec.PreserveJobs, + time.Duration(5)*time.Second, + dbSyncHash, + ) + + ctrlResult, err := dbSyncjob.DoJob(ctx, h) + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[cyborgv1beta1.DbSyncHash] = dbSyncjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[cyborgv1beta1.DbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + return ctrlResult, nil +} + +func (r *CyborgReconciler) reconcileDelete( + ctx context.Context, + instance *cyborgv1beta1.Cyborg, + h *helper.Helper, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconcile Service '%s' delete started", instance.Name)) + + err := mariadbv1.DeleteDatabaseAndAccountFinalizers(ctx, h, cyborgservice.DatabaseCRName, *instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + Log.Info("Removed finalizer from MariaDBDatabase CR", "MariaDBDatabase name", cyborgservice.DatabaseCRName) + + keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, h, cyborgservice.ServiceName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + if controllerutil.RemoveFinalizer(keystoneService, h.GetFinalizer()) { + err = h.GetClient().Update(ctx, keystoneService) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(h, "Removed finalizer from our KeystoneService", instance) + } + } + + // Successfully cleaned up everything. So as the final step let's remove the + // finalizer from ourselves to allow the deletion of Nova CR itself + updated := controllerutil.RemoveFinalizer(instance, h.GetFinalizer()) + if updated { + Log.Info("Removed finalizer from ourselves") + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *CyborgReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &cyborgv1beta1.Cyborg{}, passwordSecretField, func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.Cyborg) + if cr.Spec.Secret == nil || *cr.Spec.Secret == "" { + return nil + } + return []string{*cr.Spec.Secret} + }); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &cyborgv1beta1.Cyborg{}, authAppCredSecretField, func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.Cyborg) + if cr.Spec.Auth.ApplicationCredentialSecret == "" { + return nil + } + return []string{cr.Spec.Auth.ApplicationCredentialSecret} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&cyborgv1beta1.Cyborg{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&mariadbv1.MariaDBAccount{}). + Owns(&rabbitmqv1.TransportURL{}). + Owns(&keystonev1.KeystoneService{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + Owns(&batchv1.Job{}). + Owns(&corev1.Secret{}). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Named("cyborg-cyborg"). Complete(r) } + +func (r *CyborgReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + var requests []reconcile.Request + Log := r.GetLogger(ctx) + + for _, field := range cyborgWatchFields { + crList := &cyborgv1beta1.CyborgList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + Log.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + Log.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func ensureSecret( + ctx context.Context, + secretName types.NamespacedName, + expectedFields []string, + reader client.Reader, + conditionUpdater conditionUpdater, + requeueTimeout time.Duration, +) (string, ctrl.Result, corev1.Secret, error) { + s := &corev1.Secret{} + err := reader.Get(ctx, secretName, s) + if err != nil { + if k8s_errors.IsNotFound(err) { + log.FromContext(ctx).Info(fmt.Sprintf("secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyWaitingMessage)) + return "", + ctrl.Result{RequeueAfter: requeueTimeout}, + *s, + nil + } + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return "", ctrl.Result{}, *s, err + } + + var values [][]byte + for _, field := range expectedFields { + val, ok := s.Data[field] + if !ok { + err := fmt.Errorf("%w: '%s' in secret/%s", ErrSecretFieldNotFound, field, secretName.Name) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return "", ctrl.Result{}, *s, err + } + values = append(values, val) + } + + hash, err := util.ObjectHash(values) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return "", ctrl.Result{}, *s, err + } + + return hash, ctrl.Result{}, *s, nil +} diff --git a/internal/controller/cyborg/cyborgapi_controller.go b/internal/controller/cyborg/cyborgapi_controller.go index fa8d38914..ec61ecb39 100644 --- a/internal/controller/cyborg/cyborgapi_controller.go +++ b/internal/controller/cyborg/cyborgapi_controller.go @@ -28,6 +28,8 @@ import ( ) // CyborgAPIReconciler reconciles a CyborgAPI object +// +//nolint:revive type CyborgAPIReconciler struct { client.Client Scheme *runtime.Scheme @@ -49,6 +51,7 @@ type CyborgAPIReconciler struct { func (r *CyborgAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { _ = logf.FromContext(ctx) + _ = req // TODO(user): your logic here return ctrl.Result{}, nil diff --git a/internal/controller/cyborg/cyborgconductor_controller.go b/internal/controller/cyborg/cyborgconductor_controller.go index 61a5f967a..2b811503a 100644 --- a/internal/controller/cyborg/cyborgconductor_controller.go +++ b/internal/controller/cyborg/cyborgconductor_controller.go @@ -28,6 +28,8 @@ import ( ) // CyborgConductorReconciler reconciles a CyborgConductor object +// +//nolint:revive type CyborgConductorReconciler struct { client.Client Scheme *runtime.Scheme @@ -46,7 +48,7 @@ type CyborgConductorReconciler struct { // // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { _ = logf.FromContext(ctx) // TODO(user): your logic here diff --git a/internal/cyborg/constants.go b/internal/cyborg/constants.go new file mode 100644 index 000000000..b67f641a8 --- /dev/null +++ b/internal/cyborg/constants.go @@ -0,0 +1,53 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cyborg provides constants and utilities for OpenStack Cyborg service functionality +package cyborg + +const ( + // ServiceName identifies the cyborg service + ServiceName = "cyborg" + + // ServiceType is the Keystone service type for Cyborg + ServiceType = "accelerator" + + // DatabaseName is the name of the database used in CREATE DATABASE + DatabaseName = "cyborg" + + // DatabaseCRName is the CR name for the MariaDBDatabase + DatabaseCRName = "cyborg" + + // DatabaseUsernamePrefix is used by EnsureMariaDBAccount for new usernames + DatabaseUsernamePrefix = "cyborg" + + // CyborgPublicPort is the default public port for the cyborg-api service + CyborgPublicPort int32 = 6666 + + // CyborgInternalPort is the default internal port for the cyborg-api service + CyborgInternalPort int32 = 6666 + + // ConfigVolume is the default volume name used to mount service config + ConfigVolume = "config-data" + + // DefaultsConfigFileName is the file name with default configuration + DefaultsConfigFileName = "00-default.conf" + + // DBSyncCommand is the kolla command to run the dbsync job + DBSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" + + // CyborgLogPath is the default path for the cyborg service logs + CyborgLogPath = "/var/log/cyborg/" +) diff --git a/internal/cyborg/dbsync.go b/internal/cyborg/dbsync.go new file mode 100644 index 000000000..8a103aeb7 --- /dev/null +++ b/internal/cyborg/dbsync.go @@ -0,0 +1,150 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cyborg + +import ( + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DbSyncJob creates the Job definition for running the Cyborg database migration +func DbSyncJob( + instance *cyborgv1beta1.Cyborg, + labels map[string]string, + annotations map[string]string, +) *batchv1.Job { + runAsUser := int64(0) + completions := int32(1) + parallelism := int32(1) + var config0644AccessMode int32 = 0644 + + dbSyncVolume := []corev1.Volume{ + { + Name: "db-sync-config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + Items: []corev1.KeyToPath{ + { + Key: DefaultsConfigFileName, + Path: DefaultsConfigFileName, + }, + }, + }, + }, + }, + } + + dbSyncMounts := []corev1.VolumeMount{ + { + Name: "db-sync-config-data", + MountPath: "/etc/cyborg/cyborg.conf.d", + ReadOnly: true, + }, + { + Name: ConfigVolume, + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "cyborg-dbsync-config.json", + ReadOnly: true, + }, + } + + volumes := []corev1.Volume{ + { + Name: ConfigVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + }, + }, + }, + } + volumes = append(volumes, dbSyncVolume...) + + volumeMounts := []corev1.VolumeMount{ + { + Name: ConfigVolume, + MountPath: "/var/lib/config-data/default", + ReadOnly: true, + }, + { + Name: ConfigVolume, + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, + } + volumeMounts = append(volumeMounts, dbSyncMounts...) + + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.APIServiceTemplate.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.APIServiceTemplate.TLS.CreateVolumeMounts(nil)...) + } + + args := []string{"-c", DBSyncCommand} + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-db-sync", + Namespace: instance.Namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: batchv1.JobSpec{ + Completions: &completions, + Parallelism: ¶llelism, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: "cyborg-" + instance.Name, + Containers: []corev1.Container{ + { + Name: "cyborg-db-sync", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ConductorContainerImageURL, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + return job +} diff --git a/internal/webhook/cyborg/v1beta1/cyborg_webhook.go b/internal/webhook/cyborg/v1beta1/cyborg_webhook.go new file mode 100644 index 000000000..deb3a420b --- /dev/null +++ b/internal/webhook/cyborg/v1beta1/cyborg_webhook.go @@ -0,0 +1,107 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 implements webhooks for Cyborg v1beta1 API +package v1beta1 + +import ( + "context" + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +// nolint:unused +var cyborglog = logf.Log.WithName("cyborg-resource") + +var ( + errUnexpectedCyborgObjectType = errors.New("unexpected object type") +) + +// SetupCyborgWebhookWithManager registers the webhook for Cyborg in the manager. +func SetupCyborgWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&cyborgv1beta1.Cyborg{}). + WithValidator(&CyborgCustomValidator{}). + WithDefaulter(&CyborgCustomDefaulter{}). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-cyborg-openstack-org-v1beta1-cyborg,mutating=true,failurePolicy=fail,sideEffects=None,groups=cyborg.openstack.org,resources=cyborgs,verbs=create;update,versions=v1beta1,name=mcyborg-v1beta1.kb.io,admissionReviewVersions=v1 + +// CyborgCustomDefaulter implements webhook.CustomDefaulter for the Cyborg type. +type CyborgCustomDefaulter struct{} + +var _ webhook.CustomDefaulter = &CyborgCustomDefaulter{} + +// Default implements webhook.CustomDefaulter so a webhook will be registered for the Kind Cyborg. +func (d *CyborgCustomDefaulter) Default(_ context.Context, obj runtime.Object) error { + cyborg, ok := obj.(*cyborgv1beta1.Cyborg) + if !ok { + return fmt.Errorf("%w: expected a Cyborg object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborglog.Info("Defaulting for Cyborg", "name", cyborg.GetName()) + + cyborg.Default() + + return nil +} + +// +kubebuilder:webhook:path=/validate-cyborg-openstack-org-v1beta1-cyborg,mutating=false,failurePolicy=fail,sideEffects=None,groups=cyborg.openstack.org,resources=cyborgs,verbs=create;update,versions=v1beta1,name=vcyborg-v1beta1.kb.io,admissionReviewVersions=v1 + +// CyborgCustomValidator implements webhook.CustomValidator for the Cyborg type. +type CyborgCustomValidator struct{} + +var _ webhook.CustomValidator = &CyborgCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type Cyborg. +func (v *CyborgCustomValidator) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + cyborg, ok := obj.(*cyborgv1beta1.Cyborg) + if !ok { + return nil, fmt.Errorf("%w: expected a Cyborg object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborglog.Info("Validation for Cyborg upon creation", "name", cyborg.GetName()) + + return cyborg.ValidateCreate() +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type Cyborg. +func (v *CyborgCustomValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + cyborg, ok := newObj.(*cyborgv1beta1.Cyborg) + if !ok { + return nil, fmt.Errorf("%w: expected a Cyborg object for the newObj but got %T", errUnexpectedCyborgObjectType, newObj) + } + cyborglog.Info("Validation for Cyborg upon update", "name", cyborg.GetName()) + + return cyborg.ValidateUpdate(oldObj) +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type Cyborg. +func (v *CyborgCustomValidator) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + cyborg, ok := obj.(*cyborgv1beta1.Cyborg) + if !ok { + return nil, fmt.Errorf("%w: expected a Cyborg object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborglog.Info("Validation for Cyborg upon deletion", "name", cyborg.GetName()) + + return cyborg.ValidateDelete() +} diff --git a/internal/webhook/cyborg/v1beta1/cyborgapi_webhook.go b/internal/webhook/cyborg/v1beta1/cyborgapi_webhook.go new file mode 100644 index 000000000..c9e86363a --- /dev/null +++ b/internal/webhook/cyborg/v1beta1/cyborgapi_webhook.go @@ -0,0 +1,80 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +// nolint:unused +var cyborgapilog = logf.Log.WithName("cyborgapi-resource") + +// SetupCyborgAPIWebhookWithManager registers the webhook for CyborgAPI in the manager. +func SetupCyborgAPIWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&cyborgv1beta1.CyborgAPI{}). + WithValidator(&CyborgAPICustomValidator{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-cyborg-openstack-org-v1beta1-cyborgapi,mutating=false,failurePolicy=fail,sideEffects=None,groups=cyborg.openstack.org,resources=cyborgapis,verbs=create;update,versions=v1beta1,name=vcyborgapi-v1beta1.kb.io,admissionReviewVersions=v1 + +// CyborgAPICustomValidator implements webhook.CustomValidator for the CyborgAPI type. +type CyborgAPICustomValidator struct{} + +var _ webhook.CustomValidator = &CyborgAPICustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type CyborgAPI. +func (v *CyborgAPICustomValidator) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + cyborgapi, ok := obj.(*cyborgv1beta1.CyborgAPI) + if !ok { + return nil, fmt.Errorf("%w: expected a CyborgAPI object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborgapilog.Info("Validation for CyborgAPI upon creation", "name", cyborgapi.GetName()) + + return cyborgapi.ValidateCreate() +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type CyborgAPI. +func (v *CyborgAPICustomValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + cyborgapi, ok := newObj.(*cyborgv1beta1.CyborgAPI) + if !ok { + return nil, fmt.Errorf("%w: expected a CyborgAPI object for the newObj but got %T", errUnexpectedCyborgObjectType, newObj) + } + cyborgapilog.Info("Validation for CyborgAPI upon update", "name", cyborgapi.GetName()) + + return cyborgapi.ValidateUpdate(oldObj) +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type CyborgAPI. +func (v *CyborgAPICustomValidator) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + cyborgapi, ok := obj.(*cyborgv1beta1.CyborgAPI) + if !ok { + return nil, fmt.Errorf("%w: expected a CyborgAPI object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborgapilog.Info("Validation for CyborgAPI upon deletion", "name", cyborgapi.GetName()) + + return cyborgapi.ValidateDelete() +} diff --git a/internal/webhook/cyborg/v1beta1/cyborgconductor_webhook.go b/internal/webhook/cyborg/v1beta1/cyborgconductor_webhook.go new file mode 100644 index 000000000..167e53040 --- /dev/null +++ b/internal/webhook/cyborg/v1beta1/cyborgconductor_webhook.go @@ -0,0 +1,80 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +// nolint:unused +var cyborgconductorlog = logf.Log.WithName("cyborgconductor-resource") + +// SetupCyborgConductorWebhookWithManager registers the webhook for CyborgConductor in the manager. +func SetupCyborgConductorWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&cyborgv1beta1.CyborgConductor{}). + WithValidator(&CyborgConductorCustomValidator{}). + Complete() +} + +// +kubebuilder:webhook:path=/validate-cyborg-openstack-org-v1beta1-cyborgconductor,mutating=false,failurePolicy=fail,sideEffects=None,groups=cyborg.openstack.org,resources=cyborgconductors,verbs=create;update,versions=v1beta1,name=vcyborgconductor-v1beta1.kb.io,admissionReviewVersions=v1 + +// CyborgConductorCustomValidator implements webhook.CustomValidator for the CyborgConductor type. +type CyborgConductorCustomValidator struct{} + +var _ webhook.CustomValidator = &CyborgConductorCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type CyborgConductor. +func (v *CyborgConductorCustomValidator) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + cyborgconductor, ok := obj.(*cyborgv1beta1.CyborgConductor) + if !ok { + return nil, fmt.Errorf("%w: expected a CyborgConductor object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborgconductorlog.Info("Validation for CyborgConductor upon creation", "name", cyborgconductor.GetName()) + + return cyborgconductor.ValidateCreate() +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type CyborgConductor. +func (v *CyborgConductorCustomValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + cyborgconductor, ok := newObj.(*cyborgv1beta1.CyborgConductor) + if !ok { + return nil, fmt.Errorf("%w: expected a CyborgConductor object for the newObj but got %T", errUnexpectedCyborgObjectType, newObj) + } + cyborgconductorlog.Info("Validation for CyborgConductor upon update", "name", cyborgconductor.GetName()) + + return cyborgconductor.ValidateUpdate(oldObj) +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type CyborgConductor. +func (v *CyborgConductorCustomValidator) ValidateDelete(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + cyborgconductor, ok := obj.(*cyborgv1beta1.CyborgConductor) + if !ok { + return nil, fmt.Errorf("%w: expected a CyborgConductor object but got %T", errUnexpectedCyborgObjectType, obj) + } + cyborgconductorlog.Info("Validation for CyborgConductor upon deletion", "name", cyborgconductor.GetName()) + + return cyborgconductor.ValidateDelete() +} diff --git a/templates/cyborg/00-default.conf b/templates/cyborg/00-default.conf new file mode 100644 index 000000000..baa2fa9cc --- /dev/null +++ b/templates/cyborg/00-default.conf @@ -0,0 +1,9 @@ +[DEFAULT] +state_path = /var/lib/ +debug = True +{{ if (index . "LogFile") }} +log_file = {{ .LogFile }} +{{ end }} + +[database] +connection = {{ .DatabaseConnection }} diff --git a/templates/cyborg/cyborg/cyborg-dbsync-config.json b/templates/cyborg/cyborg/cyborg-dbsync-config.json new file mode 100644 index 000000000..5eea7cb8c --- /dev/null +++ b/templates/cyborg/cyborg/cyborg-dbsync-config.json @@ -0,0 +1,3 @@ +{ + "command": "cyborg-dbsync --config-dir /etc/cyborg/cyborg.conf.d/ upgrade" +} diff --git a/test/functional/cyborg/base_test.go b/test/functional/cyborg/base_test.go new file mode 100644 index 000000000..14a6ce562 --- /dev/null +++ b/test/functional/cyborg/base_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cyborg_test + +import ( + "time" + + . "github.com/onsi/gomega" //revive:disable:dot-imports + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + SecretName = "external-secret" + ContainerImage = "test://nova" + timeout = 45 * time.Second + // have maximum 100 retries before the timeout hits + interval = timeout / 100 +) + +func GetCronJob(name types.NamespacedName) *batchv1.CronJob { + cron := &batchv1.CronJob{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, cron)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + + return cron +} + +// GetSampleTopologySpec - An opinionated Topology Spec sample used to +// test Nova components. It returns both the user input representation +// in the form of map[string]string, and the Golang expected representation +// used in the test asserts. +func GetSampleTopologySpec(label string) (map[string]any, []corev1.TopologySpreadConstraint) { + // Build the topology Spec yaml representation + topologySpec := map[string]any{ + "topologySpreadConstraints": []map[string]any{ + { + "maxSkew": 1, + "topologyKey": corev1.LabelHostname, + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": map[string]any{ + "matchLabels": map[string]any{ + "service": label, + }, + }, + }, + }, + } + // Build the topologyObj representation + topologySpecObj := []corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: corev1.LabelHostname, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "service": label, + }, + }, + }, + } + return topologySpec, topologySpecObj +} + +func GetDefaultCyborgSpec() map[string]any { + return map[string]any{ + "apiContainerImageURL": CyborgContainerImage, + "conductorContainerImageURL": CyborgConductorImage, + "agentContainerImageURL": CyborgAgentImage, + } +} + +func GetCyborgSpecWithTLSAndAppCred(apiTLSSecretName, caBundleSecretName, appCredSecretName string) map[string]any { + spec := GetDefaultCyborgSpec() + spec["auth"] = map[string]any{ + "applicationCredentialSecret": appCredSecretName, + } + spec["apiServiceTemplate"] = map[string]any{ + "tls": map[string]any{ + "api": map[string]any{ + "public": map[string]any{ + "secretName": apiTLSSecretName, + }, + }, + "caBundleSecretName": caBundleSecretName, + }, + } + return spec +} + +func CreateCyborg(name types.NamespacedName, spec map[string]any) client.Object { + raw := map[string]any{ + "apiVersion": "cyborg.openstack.org/v1beta1", + "kind": "Cyborg", + "metadata": map[string]any{ + "name": name.Name, + "namespace": name.Namespace, + }, + "spec": spec, + } + return th.CreateUnstructured(raw) +} + +func GetCyborg(name types.NamespacedName) *cyborgv1beta1.Cyborg { + instance := &cyborgv1beta1.Cyborg{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, instance)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + return instance +} + +func CyborgConditionGetter(name types.NamespacedName) condition.Conditions { + instance := GetCyborg(name) + return instance.Status.Conditions +} + +func CreateCyborgSecret(namespace string) *corev1.Secret { + return th.CreateSecret( + types.NamespacedName{Namespace: namespace, Name: CyborgSecretName}, + map[string][]byte{ + CyborgPasswordSelectorValue: []byte("cyborg-service-password"), + }, + ) +} + +func CreateCyborgMessageBusSecret(names CyborgNames) *corev1.Secret { + return th.CreateSecret( + types.NamespacedName{ + Namespace: names.TransportURLName.Namespace, + Name: "rabbitmq-secret", + }, + map[string][]byte{ + "transport_url": []byte("rabbit://cyborg/fake"), + }, + ) +} diff --git a/test/functional/cyborg/cyborg_controller_test.go b/test/functional/cyborg/cyborg_controller_test.go new file mode 100644 index 000000000..b1aafc3ea --- /dev/null +++ b/test/functional/cyborg/cyborg_controller_test.go @@ -0,0 +1,533 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cyborg_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports + . "github.com/onsi/gomega" //revive:disable:dot-imports + + //revive:disable-next-line:dot-imports + . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + common_tls "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +const ( + CyborgSecretName = "osp-secret" + CyborgContainerImage = "test://cyborg-api" + CyborgConductorImage = "test://cyborg-conductor" + CyborgAgentImage = "test://cyborg-agent" + CyborgPasswordSelectorValue = "CyborgPassword" +) + +type CyborgNames struct { + CyborgName types.NamespacedName + MariaDBServiceName types.NamespacedName + MariaDBDatabaseName types.NamespacedName + MariaDBAccountName types.NamespacedName + TransportURLName types.NamespacedName + KeystoneServiceName types.NamespacedName + DBSyncJobName types.NamespacedName + ConfigDataName types.NamespacedName + SubLevelSecretName types.NamespacedName + ServiceAccountName types.NamespacedName + RoleName types.NamespacedName + RoleBindingName types.NamespacedName +} + +func GetCyborgNames(cyborgName types.NamespacedName) CyborgNames { + return CyborgNames{ + CyborgName: cyborgName, + MariaDBServiceName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "openstack", + }, + MariaDBDatabaseName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg", + }, + MariaDBAccountName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg", + }, + TransportURLName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-cyborg-transport", + }, + KeystoneServiceName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg", + }, + DBSyncJobName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-db-sync", + }, + ConfigDataName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-config-data", + }, + SubLevelSecretName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name, + }, + ServiceAccountName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg-" + cyborgName.Name, + }, + RoleName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg-" + cyborgName.Name + "-role", + }, + RoleBindingName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg-" + cyborgName.Name + "-rolebinding", + }, + } +} + +var _ = Describe("Cyborg controller", func() { + When("a Cyborg CR is created with minimal spec", func() { + BeforeEach(func() { + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + }) + + It("initializes status conditions", func() { + Eventually(func(g Gomega) { + cyborg := GetCyborg(cyborgNames.CyborgName) + g.Expect(cyborg.Status.Conditions).NotTo(BeNil()) + g.Expect(cyborg.Status.Conditions.Has(condition.ReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(condition.DBReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(condition.InputReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(condition.ServiceConfigReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(condition.DBSyncReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(condition.KeystoneServiceReadyCondition)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("creates the RBAC resources", func() { + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.ServiceAccountReadyCondition, + corev1.ConditionTrue, + ) + sa := th.GetServiceAccount(cyborgNames.ServiceAccountName) + Expect(sa).NotTo(BeNil()) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.RoleReadyCondition, + corev1.ConditionTrue, + ) + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.RoleBindingReadyCondition, + corev1.ConditionTrue, + ) + role := th.GetRole(cyborgNames.RoleName) + Expect(role.Rules).To(HaveLen(2)) + binding := th.GetRoleBinding(cyborgNames.RoleBindingName) + Expect(binding.RoleRef.Name).To(Equal(role.Name)) + Expect(binding.Subjects).To(HaveLen(1)) + Expect(binding.Subjects[0].Name).To(Equal(sa.Name)) + }) + + It("sets the ObservedGeneration on the status", func() { + Eventually(func(g Gomega) { + cyborg := GetCyborg(cyborgNames.CyborgName) + g.Expect(cyborg.Status.ObservedGeneration).To(Equal(cyborg.Generation)) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("a Cyborg CR is created with all required prerequisites", func() { + BeforeEach(func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + + account, secret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, secret) + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + }) + + It("creates a MariaDBDatabase and MariaDBAccount", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.DBReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates a RabbitMQ TransportURL", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates a sub-level secret with the required data", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + + Eventually(func(g Gomega) { + secret := th.GetSecret(cyborgNames.SubLevelSecretName) + g.Expect(secret.Data).To(HaveKey(CyborgPasswordSelectorValue)) + g.Expect(secret.Data).To(HaveKey("transport_url")) + g.Expect(secret.Data).To(HaveKey("database_account")) + g.Expect(secret.Data).To(HaveKey("database_username")) + g.Expect(secret.Data).To(HaveKey("database_password")) + g.Expect(secret.Data).To(HaveKey("database_hostname")) + }, timeout, interval).Should(Succeed()) + }) + + It("creates a config data secret for dbsync", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) + + Eventually(func(g Gomega) { + configSecret := th.GetSecret(cyborgNames.ConfigDataName) + g.Expect(configSecret.Data).To(HaveKey("00-default.conf")) + g.Expect(configSecret.Data).To(HaveKey("my.cnf")) + + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("[database]")) + g.Expect(defaultConf).To(ContainSubstring("connection = mysql+pymysql://")) + }, timeout, interval).Should(Succeed()) + }) + + It("reaches Ready when all dependencies are resolved", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.DBReadyCondition, + corev1.ConditionTrue, + ) + + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, + corev1.ConditionTrue, + ) + + keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.KeystoneServiceReadyCondition, + corev1.ConditionTrue, + ) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.DBSyncReadyCondition, + corev1.ConditionFalse, + ) + + th.SimulateJobSuccess(cyborgNames.DBSyncJobName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.DBSyncReadyCondition, + corev1.ConditionTrue, + ) + }) + }) + + When("Cyborg CR is created with TLS and ApplicationCredentials", func() { + const ( + apiTLSSecretName = "cyborg-api-tls" //nolint:gosec + caBundleSecretName = "cyborg-test-ca-bundle" //nolint:gosec + appCredSecretName = "cyborg-app-cred-secret" //nolint:gosec + appCredID = "test-cyborg-appcred-id" //nolint:gosec + appCredSecretValue = "test-cyborg-appcred-secret" //nolint:gosec + ) + + BeforeEach(func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + + account, dbSecret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, dbSecret) + + apiTLSSecret := th.CreateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: apiTLSSecretName}, + map[string][]byte{ + common_tls.CertKey: []byte("dummy-tls-cert"), + common_tls.PrivateKey: []byte("dummy-tls-key"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, apiTLSSecret) + + caBundleSecret := th.CreateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: caBundleSecretName}, + map[string][]byte{ + common_tls.CABundleKey: []byte("dummy-ca-bundle"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, caBundleSecret) + + appCredSecret := th.CreateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: appCredSecretName}, + map[string][]byte{ + keystonev1.ACIDSecretKey: []byte(appCredID), + keystonev1.ACSecretSecretKey: []byte(appCredSecretValue), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, appCredSecret) + + DeferCleanup( + th.DeleteInstance, + CreateCyborg( + cyborgNames.CyborgName, + GetCyborgSpecWithTLSAndAppCred(apiTLSSecretName, caBundleSecretName, appCredSecretName), + ), + ) + }) + + It("creates dbsync job, TLS-aware config secret, and application credential data in the sub-level secret", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBTLSDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) + + Eventually(func(_ Gomega) { + _ = th.GetJob(cyborgNames.DBSyncJobName) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + configSecret := th.GetSecret(cyborgNames.ConfigDataName) + myCnf := string(configSecret.Data["my.cnf"]) + g.Expect(myCnf).To(ContainSubstring("ssl-ca=")) + g.Expect(myCnf).To(ContainSubstring("ssl=1")) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + subSecret := th.GetSecret(cyborgNames.SubLevelSecretName) + g.Expect(subSecret.Data).To(HaveKey("ACID")) + g.Expect(subSecret.Data).To(HaveKey("ACSecret")) + g.Expect(string(subSecret.Data["ACID"])).To(Equal(appCredID)) + g.Expect(string(subSecret.Data["ACSecret"])).To(Equal(appCredSecretValue)) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("Cyborg CR is deleted", func() { + It("cleans up finalizers", func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + + account, secret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, secret) + + cyborg := CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()) + + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) + th.SimulateJobSuccess(cyborgNames.DBSyncJobName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.DBSyncReadyCondition, + corev1.ConditionTrue, + ) + + th.DeleteInstance(cyborg) + + Eventually(func(g Gomega) { + instance := &cyborgv1beta1.Cyborg{} + err := k8sClient.Get(ctx, cyborgNames.CyborgName, instance) + g.Expect(err).To(HaveOccurred()) + }, timeout, interval).Should(Succeed()) + }) + }) +}) + +var _ = Describe("Cyborg defaults", func() { + It("sets all expected kubebuilder and webhook defaults", func() { + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + + cyborg := GetCyborg(cyborgNames.CyborgName) + + // kubebuilder defaults for CyborgSpecCore pointer fields + Expect(cyborg.Spec.KeystoneInstance).NotTo(BeNil()) + Expect(*cyborg.Spec.KeystoneInstance).To(Equal("keystone")) + Expect(cyborg.Spec.DatabaseInstance).NotTo(BeNil()) + Expect(*cyborg.Spec.DatabaseInstance).To(Equal("openstack")) + Expect(cyborg.Spec.ServiceUser).NotTo(BeNil()) + Expect(*cyborg.Spec.ServiceUser).To(Equal("cyborg")) + Expect(cyborg.Spec.PasswordSelectors).NotTo(BeNil()) + Expect(cyborg.Spec.PasswordSelectors.Service).To(Equal("CyborgPassword")) + Expect(cyborg.Spec.DatabaseAccount).NotTo(BeNil()) + Expect(*cyborg.Spec.DatabaseAccount).To(Equal("cyborg")) + Expect(cyborg.Spec.APITimeout).NotTo(BeNil()) + Expect(*cyborg.Spec.APITimeout).To(Equal(60)) + Expect(cyborg.Spec.Secret).NotTo(BeNil()) + Expect(*cyborg.Spec.Secret).To(Equal("osp-secret")) + + // kubebuilder defaults for non-pointer fields + Expect(cyborg.Spec.PreserveJobs).To(BeFalse()) + + // kubebuilder defaults for sub-template replicas + Expect(cyborg.Spec.APIServiceTemplate.Replicas).NotTo(BeNil()) + Expect(*cyborg.Spec.APIServiceTemplate.Replicas).To(Equal(int32(1))) + Expect(cyborg.Spec.ConductorServiceTemplate.Replicas).NotTo(BeNil()) + Expect(*cyborg.Spec.ConductorServiceTemplate.Replicas).To(Equal(int32(1))) + + // webhook default for messagingBus.Cluster + Expect(cyborg.Spec.MessagingBus.Cluster).To(Equal("rabbitmq")) + }) +}) + +var _ = Describe("Cyborg webhook validation", func() { + It("rejects Cyborg with wrong service override endpoint type in apiServiceTemplate", func() { + spec := GetDefaultCyborgSpec() + spec["apiServiceTemplate"] = map[string]any{ + "override": map[string]any{ + "service": map[string]any{ + "internal": map[string]any{}, + "wrooong": map[string]any{}, + }, + }, + } + raw := map[string]any{ + "apiVersion": "cyborg.openstack.org/v1beta1", + "kind": "Cyborg", + "metadata": map[string]any{ + "name": cyborgNames.CyborgName.Name, + "namespace": cyborgNames.CyborgName.Namespace, + }, + "spec": spec, + } + + unstructuredObj := &unstructured.Unstructured{Object: raw} + _, err := controllerutil.CreateOrPatch( + ctx, k8sClient, unstructuredObj, func() error { return nil }) + + Expect(err).To(HaveOccurred()) + var statusError *k8s_errors.StatusError + Expect(errors.As(err, &statusError)).To(BeTrue()) + Expect(statusError.ErrStatus.Details.Kind).To(Equal("Cyborg")) + Expect(statusError.ErrStatus.Message).To( + ContainSubstring( + "invalid: spec.apiServiceTemplate.override.service[wrooong]: " + + "Invalid value: \"wrooong\": invalid endpoint type: wrooong", + ), + ) + }) + + It("accepts Cyborg with a correct spec", func() { + raw := map[string]any{ + "apiVersion": "cyborg.openstack.org/v1beta1", + "kind": "Cyborg", + "metadata": map[string]any{ + "name": cyborgNames.CyborgName.Name, + "namespace": cyborgNames.CyborgName.Namespace, + }, + "spec": GetDefaultCyborgSpec(), + } + + unstructuredObj := &unstructured.Unstructured{Object: raw} + _, err := controllerutil.CreateOrPatch( + ctx, k8sClient, unstructuredObj, func() error { return nil }) + + Expect(err).Should(Succeed()) + DeferCleanup(th.DeleteInstance, unstructuredObj) + }) +}) diff --git a/test/functional/cyborg/suite_test.go b/test/functional/cyborg/suite_test.go new file mode 100644 index 000000000..e0f531773 --- /dev/null +++ b/test/functional/cyborg/suite_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cyborg_test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports + . "github.com/onsi/gomega" //revive:disable:dot-imports + "go.uber.org/zap/zapcore" + + admissionv1beta1 "k8s.io/api/admission/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/types" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + test "github.com/openstack-k8s-operators/lib-common/modules/test" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + controllers "github.com/openstack-k8s-operators/nova-operator/internal/controller/cyborg" + cyborgwebhookv1beta1 "github.com/openstack-k8s-operators/nova-operator/internal/webhook/cyborg/v1beta1" + + infra_test "github.com/openstack-k8s-operators/infra-operator/apis/test/helpers" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + keystone_test "github.com/openstack-k8s-operators/keystone-operator/api/test/helpers" + common_test "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + mariadb_test "github.com/openstack-k8s-operators/mariadb-operator/api/test/helpers" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + cfg *rest.Config + k8sClient client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc + logger logr.Logger + th *common_test.TestHelper + keystone *keystone_test.TestHelper + mariadb *mariadb_test.TestHelper + infra *infra_test.TestHelper + cyborgNames CyborgNames +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), func(o *zap.Options) { + o.Development = true + o.TimeEncoder = zapcore.ISO8601TimeEncoder + })) + + ctx, cancel = context.WithCancel(context.TODO()) + + const gomod = "../../../go.mod" + + keystoneCRDs, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/keystone-operator/api", gomod, "bases") + Expect(err).ShouldNot(HaveOccurred()) + mariadbCRDs, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/mariadb-operator/api", gomod, "bases") + Expect(err).ShouldNot(HaveOccurred()) + rabbitCRDs, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/infra-operator/apis", gomod, "bases") + Expect(err).ShouldNot(HaveOccurred()) + // NOTE(gibi): there are packages where the CRD directory has other + // yamls files as well, then we need to specify the exact file to load + networkv1CRD, err := test.GetCRDDirFromModule( + "github.com/k8snetworkplumbingwg/network-attachment-definition-client", gomod, "artifacts/networks-crd.yaml") + Expect(err).ShouldNot(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + // NOTE(gibi): we need to list all the external CRDs our operator depends on + mariadbCRDs, + keystoneCRDs, + rabbitCRDs, + }, + // Increase this to 60 or 120 seconds for the single-core run + ControlPlaneStartTimeout: 120 * time.Second, + // Give it plenty of time to wind down (e.g., 60-120 seconds) + ControlPlaneStopTimeout: 120 * time.Second, + CRDInstallOptions: envtest.CRDInstallOptions{ + Paths: []string{ + networkv1CRD, + }, + }, + ErrorIfCRDPathMissing: true, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")}, + // NOTE(gibi): if localhost is resolved to ::1 (ipv6) then starting + // the webhook fails as it try to parse the address as ipv4 and + // failing on the colons in ::1 + LocalServingHost: "127.0.0.1", + }, + ControlPlane: envtest.ControlPlane{ + APIServer: &envtest.APIServer{ + Args: []string{ + "--service-cluster-ip-range=10.0.0.0/12", // 65k+ IPs + "--disable-admission-plugins=ResourceQuota,ServiceAccount,NamespaceLifecycle", + }, + }, + }, + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + // NOTE(gibi): Need to add all API schemas our operator can own. + // this includes external scheme like mariadb otherwise the + // reconciler loop will silently not start + // TODO(sean): factor this out to a common function. + err = cyborgv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = mariadbv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = keystonev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = corev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = appsv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = rabbitmqv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = rbacv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = admissionv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = topologyv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + logger = ctrl.Log.WithName("---Test---") + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + th = common_test.NewTestHelper(ctx, k8sClient, timeout, interval, logger) + Expect(th).NotTo(BeNil()) + keystone = keystone_test.NewTestHelper(ctx, k8sClient, timeout, interval, logger) + Expect(keystone).NotTo(BeNil()) + mariadb = mariadb_test.NewTestHelper(ctx, k8sClient, timeout, interval, logger) + Expect(mariadb).NotTo(BeNil()) + infra = infra_test.NewTestHelper(ctx, k8sClient, timeout, interval, logger) + Expect(infra).NotTo(BeNil()) + + // Start the controller-manager in a goroutine + webhookInstallOptions := &testEnv.WebhookInstallOptions + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{ + BindAddress: "0", + }, + WebhookServer: webhook.NewServer( + webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + }) + Expect(err).ToNot(HaveOccurred()) + + kclient, err := kubernetes.NewForConfig(cfg) + Expect(err).ToNot(HaveOccurred(), "failed to create kclient") + + reconcilers := controllers.NewReconcilers(k8sManager, kclient) + err = reconcilers.Setup(k8sManager, ctrl.Log.WithName("testSetup")) + Expect(err).ToNot(HaveOccurred()) + + // Acquire environmental defaults and initialize operator defaults with them + cyborgv1beta1.SetupDefaults() + + err = cyborgwebhookv1beta1.SetupCyborgWebhookWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + err = cyborgwebhookv1beta1.SetupCyborgAPIWebhookWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + err = cyborgwebhookv1beta1.SetupCyborgConductorWebhookWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + err = k8sManager.Start(ctx) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Duration(10) * time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) // #nosec G402 + if err != nil { + return err + } + _ = conn.Close() // Ignore close error in test + return nil + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = BeforeEach(func() { + // NOTE(gibi): We need to create a unique namespace for each test run + // as namespaces cannot be deleted in a locally running envtest. See + // https://book.kubebuilder.io/reference/envtest.html#namespace-usage-limitation + namespace := uuid.New().String() + th.CreateNamespace(namespace) + // We still request the delete of the Namespace to properly cleanup if + // we run the test in an existing cluster. + DeferCleanup(th.DeleteNamespace, namespace) + + cyborgName := types.NamespacedName{ + Namespace: namespace, + Name: "cyborg-" + uuid.New().String()[:10], + } + cyborgNames = GetCyborgNames(cyborgName) +}) From 92b50876ab96fd59a375c3b4c4941812f56521f6 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Fri, 27 Mar 2026 15:35:45 +0100 Subject: [PATCH 4/7] cyborg: Implement CyborgConductor controller reconcile loop Add full reconcile logic for the CyborgConductor CR: - Validate input from the config secret created by the Cyborg controller - Generate conductor config from templates (00-default.conf) - Create a StatefulSet to run cyborg-conductor pods - Track readiness (ReadyCount, conditions, hash, topology) - Expose IsReady and topology helpers on CyborgConductor type - Update CyborgConductorStatus with structured conditions and hash - Extend Cyborg controller to propagate conductor and check readiness upwards - Add functional tests for the conductor reconcile loop Assisted-By: Claude Signed-off-by: Alfredo Moralejo --- ...cyborg.openstack.org_cyborgconductors.yaml | 103 ++- api/cyborg/v1beta1/conditions.go | 3 + api/cyborg/v1beta1/cyborgconductor_types.go | 62 +- api/cyborg/v1beta1/zz_generated.deepcopy.go | 22 +- ...cyborg.openstack.org_cyborgconductors.yaml | 103 ++- .../nova-operator.clusterserviceversion.yaml | 4 + internal/controller/cyborg/common.go | 62 ++ .../controller/cyborg/cyborg_controller.go | 144 ++++- .../cyborg/cyborgconductor_controller.go | 455 +++++++++++++- internal/cyborg/conductor/statefulset.go | 175 ++++++ templates/cyborg/00-default.conf | 82 +++ .../conductor/cyborg-conductor-config.json | 25 + test/functional/cyborg/base_test.go | 50 ++ .../cyborg/cyborg_controller_test.go | 225 ++++++- .../cyborg/cyborgconductor_controller_test.go | 589 ++++++++++++++++++ 15 files changed, 2051 insertions(+), 53 deletions(-) create mode 100644 internal/cyborg/conductor/statefulset.go create mode 100644 templates/cyborg/conductor/cyborg-conductor-config.json create mode 100644 test/functional/cyborg/cyborgconductor_controller_test.go diff --git a/api/bases/cyborg.openstack.org_cyborgconductors.yaml b/api/bases/cyborg.openstack.org_cyborgconductors.yaml index 6f3f19f9a..f5ccb5d8e 100644 --- a/api/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/api/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -14,7 +14,16 @@ spec: singular: cyborgconductor scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 schema: openAPIV3Schema: description: CyborgConductor is the Schema for the cyborgconductors API. @@ -40,8 +49,12 @@ spec: description: CyborgConductorSpec defines the desired state of CyborgConductor. properties: configSecret: - description: ConfigSecret - containing all the configuration needed - provided by Cyborg object + description: |- + Secret is the name of the sub-level secret containing all required data + (transport URL, DB creds, keystone auth, service password, etc.) + type: string + containerImage: + description: ContainerImage is the container image URL for cyborg-conductor type: string customServiceConfig: description: |- @@ -124,6 +137,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceAccount: + description: ServiceAccount used by the conductor pods + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced @@ -143,9 +167,82 @@ spec: type: object required: - configSecret + - containerImage + - serviceAccount type: object status: description: CyborgConductorStatus defines the observed state of CyborgConductor. + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Hash - Map of hashes to track config changes + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + observedGeneration: + description: ObservedGeneration - the most recent generation observed + format: int64 + type: integer + readyCount: + description: ReadyCount defines the number of replicas ready + format: int32 + type: integer type: object type: object served: true diff --git a/api/cyborg/v1beta1/conditions.go b/api/cyborg/v1beta1/conditions.go index 5fbb387a9..64fe7314e 100644 --- a/api/cyborg/v1beta1/conditions.go +++ b/api/cyborg/v1beta1/conditions.go @@ -50,6 +50,9 @@ const ( // CyborgConductorReadyInitMessage - CyborgConductorReadyInitMessage = "CyborgConductor not started" + // CyborgConductorReadyErrorMessage - + CyborgConductorReadyErrorMessage = "CyborgConductor error occurred %s" + // CyborgApplicationCredentialSecretErrorMessage - CyborgApplicationCredentialSecretErrorMessage = "Error with application credential secret" ) diff --git a/api/cyborg/v1beta1/cyborgconductor_types.go b/api/cyborg/v1beta1/cyborgconductor_types.go index d58ed24e1..72d547da0 100644 --- a/api/cyborg/v1beta1/cyborgconductor_types.go +++ b/api/cyborg/v1beta1/cyborgconductor_types.go @@ -18,13 +18,12 @@ package v1beta1 import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - // CyborgConductorTemplate defines the input parameters specified by the user to // create a CyborgConductor via higher level CRDs. type CyborgConductorTemplate struct { @@ -59,24 +58,49 @@ type CyborgConductorTemplate struct { // CyborgConductorSpec defines the desired state of CyborgConductor. type CyborgConductorSpec struct { - // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster - // Important: Run "make" to regenerate code after modifying this file - CyborgConductorTemplate `json:",inline"` // +kubebuilder:validation:Required - // ConfigSecret - containing all the configuration needed provided by Cyborg object + // Secret is the name of the sub-level secret containing all required data + // (transport URL, DB creds, keystone auth, service password, etc.) ConfigSecret string `json:"configSecret"` + + // +kubebuilder:validation:Required + // ContainerImage is the container image URL for cyborg-conductor + ContainerImage string `json:"containerImage"` + + // +kubebuilder:validation:Required + // ServiceAccount used by the conductor pods + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // CyborgConductorStatus defines the observed state of CyborgConductor. type CyborgConductorStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + // ReadyCount defines the number of replicas ready + ReadyCount int32 `json:"readyCount,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ObservedGeneration - the most recent generation observed + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Hash - Map of hashes to track config changes + Hash map[string]string `json:"hash,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" // CyborgConductor is the Schema for the cyborgconductors API. type CyborgConductor struct { @@ -96,6 +120,26 @@ type CyborgConductorList struct { Items []CyborgConductor `json:"items"` } +// IsReady returns true if the ReadyCondition is true +func (instance *CyborgConductor) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.ReadyCondition) +} + +// GetSpecTopologyRef returns the TopologyRef defined in the Spec +func (instance *CyborgConductor) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology returns the LastAppliedTopology from the Status +func (instance *CyborgConductor) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology sets the LastAppliedTopology value in the Status +func (instance *CyborgConductor) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} + func init() { SchemeBuilder.Register(&CyborgConductor{}, &CyborgConductorList{}) } diff --git a/api/cyborg/v1beta1/zz_generated.deepcopy.go b/api/cyborg/v1beta1/zz_generated.deepcopy.go index 0065cd66b..c51cf80f4 100644 --- a/api/cyborg/v1beta1/zz_generated.deepcopy.go +++ b/api/cyborg/v1beta1/zz_generated.deepcopy.go @@ -231,7 +231,7 @@ func (in *CyborgConductor) DeepCopyInto(out *CyborgConductor) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductor. @@ -288,6 +288,7 @@ func (in *CyborgConductorList) DeepCopyObject() runtime.Object { func (in *CyborgConductorSpec) DeepCopyInto(out *CyborgConductorSpec) { *out = *in in.CyborgConductorTemplate.DeepCopyInto(&out.CyborgConductorTemplate) + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorSpec. @@ -303,6 +304,25 @@ func (in *CyborgConductorSpec) DeepCopy() *CyborgConductorSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgConductorStatus) DeepCopyInto(out *CyborgConductorStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgConductorStatus. diff --git a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml index 6f3f19f9a..f5ccb5d8e 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgconductors.yaml @@ -14,7 +14,16 @@ spec: singular: cyborgconductor scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 schema: openAPIV3Schema: description: CyborgConductor is the Schema for the cyborgconductors API. @@ -40,8 +49,12 @@ spec: description: CyborgConductorSpec defines the desired state of CyborgConductor. properties: configSecret: - description: ConfigSecret - containing all the configuration needed - provided by Cyborg object + description: |- + Secret is the name of the sub-level secret containing all required data + (transport URL, DB creds, keystone auth, service password, etc.) + type: string + containerImage: + description: ContainerImage is the container image URL for cyborg-conductor type: string customServiceConfig: description: |- @@ -124,6 +137,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceAccount: + description: ServiceAccount used by the conductor pods + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced @@ -143,9 +167,82 @@ spec: type: object required: - configSecret + - containerImage + - serviceAccount type: object status: description: CyborgConductorStatus defines the observed state of CyborgConductor. + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Hash - Map of hashes to track config changes + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + observedGeneration: + description: ObservedGeneration - the most recent generation observed + format: int64 + type: integer + readyCount: + description: ReadyCount defines the number of replicas ready + format: int32 + type: integer type: object type: object served: true diff --git a/config/manifests/bases/nova-operator.clusterserviceversion.yaml b/config/manifests/bases/nova-operator.clusterserviceversion.yaml index 934e436ce..ebb393ad0 100644 --- a/config/manifests/bases/nova-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/nova-operator.clusterserviceversion.yaml @@ -32,6 +32,10 @@ spec: displayName: Cyborg Conductor kind: CyborgConductor name: cyborgconductors.cyborg.openstack.org + specDescriptors: + - description: TLS - Parameters related to the TLS + displayName: TLS + path: tls version: v1beta1 - description: Cyborg is the Schema for the cyborgs API. displayName: Cyborg diff --git a/internal/controller/cyborg/common.go b/internal/controller/cyborg/common.go index 1ced69c21..1348fd955 100644 --- a/internal/controller/cyborg/common.go +++ b/internal/controller/cyborg/common.go @@ -18,11 +18,16 @@ limitations under the License. package cyborg import ( + "context" "errors" + "fmt" "time" "github.com/go-logr/logr" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -116,9 +121,19 @@ func NewReconcilers(mgr ctrl.Manager, kclient *kubernetes.Clientset) *Reconciler "Cyborg": &CyborgReconciler{ ReconcilerBase: NewReconcilerBase(mgr, kclient), }, + "CyborgConductor": &CyborgConductorReconciler{ + ReconcilerBase: NewReconcilerBase(mgr, kclient), + }, }} } +// OverrideRequeueTimeout overrides the default RequeueTimeout of our reconcilers +func (r *Reconcilers) OverrideRequeueTimeout(timeout time.Duration) { + for _, reconciler := range r.reconcilers { + reconciler.SetRequeueTimeout(timeout) + } +} + // Setup starts the reconcilers by connecting them to the Manager func (r *Reconcilers) Setup(mgr ctrl.Manager, setupLog logr.Logger) error { var err error @@ -135,3 +150,50 @@ type conditionUpdater interface { Set(c *condition.Condition) MarkTrue(t condition.Type, messageFormat string, messageArgs ...any) } + +type topologyHandler interface { + GetSpecTopologyRef() *topologyv1.TopoRef + GetLastAppliedTopology() *topologyv1.TopoRef + SetLastAppliedTopology(t *topologyv1.TopoRef) +} + +// ensureTopology - when a Topology CR is referenced, remove the +// finalizer from a previous referenced Topology (if any), and retrieve the +// newly referenced topology object +func ensureTopology( + ctx context.Context, + h *helper.Helper, + instance topologyHandler, + finalizer string, + condUpdater conditionUpdater, + defaultLabelSelector metav1.LabelSelector, +) (*topologyv1.Topology, error) { + topology, err := topologyv1.EnsureServiceTopology( + ctx, + h, + instance.GetSpecTopologyRef(), + instance.GetLastAppliedTopology(), + finalizer, + defaultLabelSelector, + ) + if err != nil { + condUpdater.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return nil, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + tr := instance.GetSpecTopologyRef() + instance.SetLastAppliedTopology(tr) + + if tr != nil { + condUpdater.MarkTrue( + condition.TopologyReadyCondition, + condition.TopologyReadyMessage, + ) + } + return topology, nil +} diff --git a/internal/controller/cyborg/cyborg_controller.go b/internal/controller/cyborg/cyborg_controller.go index 26c152aeb..08f36ade1 100644 --- a/internal/controller/cyborg/cyborg_controller.go +++ b/internal/controller/cyborg/cyborg_controller.go @@ -42,6 +42,7 @@ import ( keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/job" @@ -295,10 +296,24 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res } } + // + // Get keystone internal auth URL and region for service configuration + // + keystoneAuthURL, keystoneRegion, err := r.getKeystoneAuthURL(ctx, h, instance) + if err != nil { + Log.Info("Keystone internal endpoint not available yet, waiting") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: r.RequeueTimeout}, nil + } + // // Create sub-level secret with required configuration // - _, err = r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData) + _, err = r.createSubLevelSecret(ctx, h, instance, transporturlSecret, inputSecret, db, acData, keystoneAuthURL, keystoneRegion) if err != nil { return ctrl.Result{}, err } @@ -351,6 +366,16 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return ctrl.Result{}, err } + // + // Create CyborgConductor sub-CR + // + ctrlResult, err = r.ensureConductor(ctx, h, instance, serviceLabels) + if err != nil { + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + if instance.Status.Conditions.AllSubConditionIsTrue() { instance.Status.Conditions.MarkTrue( condition.ReadyCondition, condition.ReadyMessage) @@ -414,6 +439,10 @@ func (r *CyborgReconciler) initConditions(instance *cyborgv1beta1.Cyborg) error condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition( + cyborgv1beta1.CyborgConductorReadyCondition, + condition.InitReason, + cyborgv1beta1.CyborgConductorReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -636,6 +665,8 @@ func (r *CyborgReconciler) createSubLevelSecret( inputSecret corev1.Secret, db *mariadbv1.Database, acData *keystonev1.ApplicationCredentialData, + keystoneAuthURL string, + keystoneRegion string, ) (string, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Creating SubCR Level Secret for '%s'", instance.Name)) @@ -643,8 +674,14 @@ func (r *CyborgReconciler) createSubLevelSecret( databaseAccount := db.GetAccount() databaseSecret := db.GetSecret() + servicePassword := string(inputSecret.Data[instance.Spec.PasswordSelectors.Service]) + data := map[string]string{ - instance.Spec.PasswordSelectors.Service: string(inputSecret.Data[instance.Spec.PasswordSelectors.Service]), + instance.Spec.PasswordSelectors.Service: servicePassword, + "ServicePassword": servicePassword, + "ServiceUser": *instance.Spec.ServiceUser, + "KeystoneAuthURL": keystoneAuthURL, + "Region": keystoneRegion, TransportURLSelector: string(transportURLSecret.Data[TransportURLSelector]), QuorumQueuesSelector: string(transportURLSecret.Data[QuorumQueuesSelector]), DatabaseAccount: databaseAccount.Name, @@ -816,6 +853,108 @@ func (r *CyborgReconciler) reconcileDelete( return ctrl.Result{}, nil } +func (r *CyborgReconciler) getKeystoneAuthURL( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, +) (string, string, error) { + // returns internalAuthURL, region, error + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + if err != nil { + return "", "", err + } + + internalAuthURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return "", "", err + } + + region := normalizeRegion(keystoneAPI.GetRegion()) + + return internalAuthURL, region, nil +} + +func normalizeRegion(region string) string { + if region == "" { + return "regionOne" + } + return region +} + +func (r *CyborgReconciler) ensureConductor( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling CyborgConductor for '%s'", instance.Name)) + + conductorName := fmt.Sprintf("%s-conductor", instance.Name) + + conductorTemplate := *instance.Spec.ConductorServiceTemplate.DeepCopy() + if conductorTemplate.TopologyRef == nil && instance.Spec.TopologyRef != nil { + conductorTemplate.TopologyRef = instance.Spec.TopologyRef.DeepCopy() + } + + conductorSpec := cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: conductorTemplate, + ConfigSecret: instance.Name, + ContainerImage: instance.Spec.ConductorContainerImageURL, + ServiceAccount: instance.RbacResourceName(), + } + + if instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName != "" { + conductorSpec.TLS.CaBundleSecretName = instance.Spec.APIServiceTemplate.TLS.CaBundleSecretName + } + + conductor := &cyborgv1beta1.CyborgConductor{ + ObjectMeta: metav1.ObjectMeta{ + Name: conductorName, + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrPatch(ctx, h.GetClient(), conductor, func() error { + conductor.Spec = conductorSpec + conductor.Labels = serviceLabels + + err := controllerutil.SetControllerReference(instance, conductor, r.Scheme) + return err + }) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + cyborgv1beta1.CyborgConductorReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + cyborgv1beta1.CyborgConductorReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("CyborgConductor CR %s - %s", conductorName, op)) + } + + conductorObj := &cyborgv1beta1.CyborgConductor{} + err = h.GetClient().Get(ctx, types.NamespacedName{Name: conductorName, Namespace: instance.Namespace}, conductorObj) + if err != nil { + return ctrl.Result{}, err + } + + if conductorObj.IsReady() { + instance.Status.ConductorServiceReadyCount = conductorObj.Status.ReadyCount + instance.Status.Conditions.MarkTrue(cyborgv1beta1.CyborgConductorReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + cyborgv1beta1.CyborgConductorReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *CyborgReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField(context.Background(), &cyborgv1beta1.Cyborg{}, passwordSecretField, func(rawObj client.Object) []string { @@ -844,6 +983,7 @@ func (r *CyborgReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&mariadbv1.MariaDBAccount{}). Owns(&rabbitmqv1.TransportURL{}). Owns(&keystonev1.KeystoneService{}). + Owns(&cyborgv1beta1.CyborgConductor{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). diff --git a/internal/controller/cyborg/cyborgconductor_controller.go b/internal/controller/cyborg/cyborgconductor_controller.go index 2b811503a..36a1b3042 100644 --- a/internal/controller/cyborg/cyborgconductor_controller.go +++ b/internal/controller/cyborg/cyborgconductor_controller.go @@ -1,5 +1,5 @@ /* -Copyright 2022. +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,48 +18,469 @@ package cyborg import ( "context" + "fmt" - "k8s.io/apimachinery/pkg/runtime" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborgservice "github.com/openstack-k8s-operators/nova-operator/internal/cyborg" + cyborgconductor "github.com/openstack-k8s-operators/nova-operator/internal/cyborg/conductor" +) + +const ( + conductorConfigSecretField = ".spec.configSecret" //nolint:gosec + conductorTopologyField = ".spec.topologyRef.Name" ) // CyborgConductorReconciler reconciles a CyborgConductor object // //nolint:revive type CyborgConductorReconciler struct { - client.Client - Scheme *runtime.Scheme + ReconcilerBase +} + +// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields +func (r *CyborgConductorReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CyborgConductor") } // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgconductors/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete; -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the CyborgConductor object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { - _ = logf.FromContext(ctx) +// Reconcile is part of the main kubernetes reconciliation loop for CyborgConductor resources. +func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + instance := &cyborgv1beta1.CyborgConductor{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info("CyborgConductor instance not found, probably deleted before reconciled. Nothing to do.") + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciling CyborgConductor '%s'", instance.Name)) + + h, err := helper.NewHelper(instance, r.Client, r.Kclient, r.Scheme, Log) + if err != nil { + return ctrl.Result{}, err + } + + isNewInstance := instance.Status.Conditions == nil + savedConditions := instance.Status.Conditions.DeepCopy() + + defer func() { + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + err := h.PatchInstance(ctx, instance) + if err != nil { + _err = err + } + }() + + r.initStatus(instance) + + if !instance.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.reconcileDelete(ctx, h, instance) + } + + if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, h.GetFinalizer()) || isNewInstance { + return ctrl.Result{}, nil + } + + // Read the sub-level secret created by the Cyborg controller + subSecret := &corev1.Secret{} + secretName := types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.ConfigSecret} + err = r.Client.Get(ctx, secretName, subSecret) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info("Secret not found, waiting", "secret", instance.Spec.ConfigSecret) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: r.RequeueTimeout}, nil + } + return ctrl.Result{}, err + } + + // Hash the input secret so we detect changes to passwords, transport URL, etc. + inputHashes := make(map[string]env.Setter) + secretHash, err := util.ObjectHash(subSecret.Data) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error calculating input secret hash: %w", err) + } + inputHashes["input"] = env.SetValue(secretHash) + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - // TODO(user): your logic here + // Generate config + configVars := make(map[string]env.Setter) + err = r.generateServiceConfig(ctx, instance, subSecret, h, &configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + for key, hashVal := range configVars { + inputHashes[key] = hashVal + } + + // Compute a combined hash of all inputs (secret + generated config). + // This hash is set as CONFIG_HASH env var in the pod template so that + // any change in the input secret or generated config triggers a rollout. + inputHash, err := util.HashOfInputHashes(inputHashes) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error calculating combined input hash: %w", err) + } + instance.Status.Hash[common.InputHashName] = inputHash + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + serviceLabels := map[string]string{ + common.AppSelector: cyborgconductor.ComponentName, + } + + topology, err := ensureTopology( + ctx, + h, + instance, + instance.Name, + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // Create or update the StatefulSet + ssDef := cyborgconductor.StatefulSet(instance, inputHash, serviceLabels, topology) + + ss := statefulset.NewStatefulSet(ssDef, r.RequeueTimeout) + ctrlResult, err := ss.CreateOrPatch(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + + ssObj := ss.GetStatefulSet() + instance.Status.ReadyCount = ssObj.Status.ReadyReplicas + if ssObj.Status.ReadyReplicas == ssObj.Status.Replicas && ssObj.Generation == ssObj.Status.ObservedGeneration { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + + instance.Status.ObservedGeneration = instance.Generation return ctrl.Result{}, nil } +func (r *CyborgConductorReconciler) initStatus(instance *cyborgv1beta1.CyborgConductor) { + if instance.Status.Conditions == nil { + instance.Status.Conditions = condition.Conditions{} + } + + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + ) + + if instance.Spec.TopologyRef != nil { + cl.Set(condition.UnknownCondition( + condition.TopologyReadyCondition, + condition.InitReason, + condition.TopologyReadyInitMessage, + )) + } + + instance.Status.Conditions.Init(&cl) + + if instance.Status.Hash == nil { + instance.Status.Hash = make(map[string]string) + } +} + +func (r *CyborgConductorReconciler) generateServiceConfig( + ctx context.Context, + instance *cyborgv1beta1.CyborgConductor, + subSecret *corev1.Secret, + h *helper.Helper, + envVars *map[string]env.Setter, +) error { + Log := r.GetLogger(ctx) + Log.Info("generateServiceConfig - reconciling config for CyborgConductor") + + var tlsCfg *tls.Service + if instance.Spec.TLS.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + databaseConnection := fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + string(subSecret.Data[DatabaseUsername]), + string(subSecret.Data[DatabasePassword]), + string(subSecret.Data[DatabaseHostname]), + cyborgservice.DatabaseName, + ) + + templateParameters := map[string]any{ + "DatabaseConnection": databaseConnection, + "TransportURL": string(subSecret.Data[TransportURLSelector]), + } + + if quorumQueues, ok := subSecret.Data[QuorumQueuesSelector]; ok && string(quorumQueues) == "true" { + templateParameters["QuorumQueues"] = true + } + + if keystoneURL, ok := subSecret.Data["KeystoneAuthURL"]; ok && len(keystoneURL) > 0 { + templateParameters["KeystoneAuthURL"] = string(keystoneURL) + } + + if serviceUser, ok := subSecret.Data["ServiceUser"]; ok && len(serviceUser) > 0 { + templateParameters["ServiceUser"] = string(serviceUser) + } + + if servicePassword, ok := subSecret.Data["ServicePassword"]; ok && len(servicePassword) > 0 { + templateParameters["ServicePassword"] = string(servicePassword) + } + + if region, ok := subSecret.Data["Region"]; ok && len(region) > 0 { + templateParameters["Region"] = string(region) + } + + if acid, ok := subSecret.Data["ACID"]; ok && len(acid) > 0 { + templateParameters["ACID"] = string(acid) + templateParameters["ACSecret"] = string(subSecret.Data["ACSecret"]) + } + + if instance.Spec.TLS.CaBundleSecretName != "" { + templateParameters["CaFilePath"] = tls.DownstreamTLSCABundlePath + } + + customData := map[string]string{ + "my.cnf": generateMyCnf(tlsCfg), + } + + serviceLabels := labels.GetLabels(instance, labels.GetGroupLabel(cyborgservice.ServiceName), map[string]string{}) + + if instance.Spec.CustomServiceConfig != "" { + customData["01-service-custom.conf"] = instance.Spec.CustomServiceConfig + } + + cms := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.GetName()), + Namespace: instance.GetNamespace(), + Type: util.TemplateTypeConfig, + InstanceType: instance.GetObjectKind().GroupVersionKind().Kind, + ConfigOptions: templateParameters, + CustomData: customData, + Labels: serviceLabels, + AdditionalTemplate: map[string]string{ + "00-default.conf": "/cyborg/00-default.conf", + "cyborg-conductor-config.json": "/cyborg/conductor/cyborg-conductor-config.json", + }, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, cms, envVars) +} + +func generateMyCnf(tlsCfg *tls.Service) string { + if tlsCfg != nil { + return fmt.Sprintf("[client]\nssl-ca=%s\nssl=1\n", tls.DownstreamTLSCABundlePath) + } + return "[client]\n" +} + // SetupWithManager sets up the controller with the Manager. func (r *CyborgConductorReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgConductor{}, + conductorConfigSecretField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgConductor) + if cr.Spec.ConfigSecret == "" { + return nil + } + return []string{cr.Spec.ConfigSecret} + }, + ); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgConductor{}, + conductorTopologyField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgConductor) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }, + ); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&cyborgv1beta1.CyborgConductor{}). + Owns(&corev1.Secret{}). + Owns(&appsv1.StatefulSet{}). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findConductorsForSecret), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches( + &topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findConductorsForTopology), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). Named("cyborg-cyborgconductor"). Complete(r) } + +func (r *CyborgConductorReconciler) findConductorsForTopology(ctx context.Context, src client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + crList := &cyborgv1beta1.CyborgConductorList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(conductorTopologyField, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + Log.Error(err, "listing CyborgConductors for topology change") + return nil + } + + requests := make([]reconcile.Request, 0, len(crList.Items)) + for _, item := range crList.Items { + Log.Info(fmt.Sprintf("Topology %s changed, reconciling CyborgConductor %s", src.GetName(), item.GetName())) + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + return requests +} + +func (r *CyborgConductorReconciler) findConductorsForSecret(ctx context.Context, src client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + crList := &cyborgv1beta1.CyborgConductorList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(conductorConfigSecretField, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + Log.Error(err, "listing CyborgConductors for secret change") + return nil + } + + requests := make([]reconcile.Request, 0, len(crList.Items)) + for _, item := range crList.Items { + Log.Info(fmt.Sprintf("Secret %s changed, reconciling CyborgConductor %s", src.GetName(), item.GetName())) + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + return requests +} + +func (r *CyborgConductorReconciler) reconcileDelete( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.CyborgConductor, +) error { + Log := r.GetLogger(ctx) + + Log.Info("Reconciling delete") + + // Remove finalizer from the referenced Topology CR + if _, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + h, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return err + } + + // Successfully cleaned up everything. So as the final step let's remove the + // finalizer from ourselves to allow the deletion of CyborgConductor CR itself + updated := controllerutil.RemoveFinalizer(instance, h.GetFinalizer()) + if updated { + Log.Info("Removed finalizer from ourselves") + } + + Log.Info("Reconciled delete successfully") + return nil +} diff --git a/internal/cyborg/conductor/statefulset.go b/internal/cyborg/conductor/statefulset.go new file mode 100644 index 000000000..38558549d --- /dev/null +++ b/internal/cyborg/conductor/statefulset.go @@ -0,0 +1,175 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package conductor provides helpers for the CyborgConductor StatefulSet. +package conductor + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborg "github.com/openstack-k8s-operators/nova-operator/internal/cyborg" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const ( + // ComponentName is the name used for the conductor container + ComponentName = "cyborg-conductor" + + // KollaServiceCommand is the kolla start command for the conductor + KollaServiceCommand = "/usr/local/bin/kolla_start" +) + +// StatefulSet creates a StatefulSet for the cyborg-conductor service +func StatefulSet( + instance *cyborgv1beta1.CyborgConductor, + configHash string, + labels map[string]string, + topology *topologyv1.Topology, +) *appsv1.StatefulSet { + var config0644AccessMode int32 = 0644 + runAsUser := int64(0) + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + args := []string{"-c", KollaServiceCommand} + + startupProbe := &corev1.Probe{ + FailureThreshold: 6, + PeriodSeconds: 10, + } + livenessProbe := &corev1.Probe{ + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + readinessProbe := &corev1.Probe{ + TimeoutSeconds: 5, + PeriodSeconds: 5, + } + + probeCmd := &corev1.ExecAction{ + Command: []string{ + "/usr/bin/pgrep", "-f", "-r", "DRST", ComponentName, + }, + } + startupProbe.Exec = probeCmd + livenessProbe.Exec = probeCmd + readinessProbe.Exec = probeCmd + + volumes := []corev1.Volume{ + { + Name: cyborg.ConfigVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + }, + }, + }, + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: cyborg.ConfigVolume, + MountPath: "/var/lib/config-data/default", + ReadOnly: true, + }, + { + Name: cyborg.ConfigVolume, + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "cyborg-conductor-config.json", + ReadOnly: true, + }, + { + Name: cyborg.ConfigVolume, + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, + } + + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: instance.Spec.Replicas, + PodManagementPolicy: appsv1.ParallelPodManagement, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(runAsUser), + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + StartupProbe: startupProbe, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + statefulset.Spec.Template.Spec.Affinity = affinity.DistributePods( + common.AppSelector, + []string{instance.Name}, + corev1.LabelHostname, + ) + } + + return statefulset +} diff --git a/templates/cyborg/00-default.conf b/templates/cyborg/00-default.conf index baa2fa9cc..d6fae27e1 100644 --- a/templates/cyborg/00-default.conf +++ b/templates/cyborg/00-default.conf @@ -1,9 +1,91 @@ [DEFAULT] state_path = /var/lib/ debug = True +{{ if (index . "TransportURL") }} +transport_url = {{ .TransportURL }} +{{ end }} {{ if (index . "LogFile") }} log_file = {{ .LogFile }} {{ end }} [database] connection = {{ .DatabaseConnection }} + +{{ if (index . "TransportURL") }} +[oslo_messaging_rabbit] +{{- if (index . "QuorumQueues") }} +rabbit_quorum_queue=true +rabbit_transient_quorum_queue=true +amqp_durable_queues=true +{{- else }} +amqp_durable_queues=false +amqp_auto_delete=false +heartbeat_in_pthread=false +{{- end }} +{{ if (index . "CaFilePath") }} +ssl_ca_file = {{ .CaFilePath }} +{{ end }} +{{ end }} + +{{ if (index . "KeystoneAuthURL") }} +[keystone_authtoken] +{{ if (index . "ACID") }} +auth_type = v3applicationcredential +application_credential_id = {{ .ACID }} +application_credential_secret = {{ .ACSecret }} +{{ else }} +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_type = password +{{ end }} +auth_url = {{ .KeystoneAuthURL }} +interface = internal +{{ if (index . "Region") }}region_name = {{ index . "Region" }}{{ end }} +{{ if (index . "CaFilePath") }} +cafile = {{ .CaFilePath }} +{{ end }} + +[placement] +{{ if (index . "ACID") }} +auth_type = v3applicationcredential +application_credential_id = {{ .ACID }} +application_credential_secret = {{ .ACSecret }} +{{ else }} +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_type = password +{{ end }} +auth_url = {{ .KeystoneAuthURL }} +interface = internal +{{ if (index . "CaFilePath") }} +cafile = {{ .CaFilePath }} +{{ end }} + +[nova] +{{ if (index . "ACID") }} +auth_type = v3applicationcredential +application_credential_id = {{ .ACID }} +application_credential_secret = {{ .ACSecret }} +{{ else }} +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_type = password +{{ end }} +auth_url = {{ .KeystoneAuthURL }} +interface = internal +{{ if (index . "CaFilePath") }} +cafile = {{ .CaFilePath }} +{{ end }} + +[agent] +enabled_drivers = fake_driver +{{ end }} diff --git a/templates/cyborg/conductor/cyborg-conductor-config.json b/templates/cyborg/conductor/cyborg-conductor-config.json new file mode 100644 index 000000000..0022c0e2e --- /dev/null +++ b/templates/cyborg/conductor/cyborg-conductor-config.json @@ -0,0 +1,25 @@ +{ + "command": "cyborg-conductor --config-dir /etc/cyborg/cyborg.conf.d", + "config_files": [ + { + "source": "/var/lib/config-data/default/00-default.conf", + "dest": "/etc/cyborg/cyborg.conf.d/00-default.conf", + "owner": "cyborg", + "perm": "0600" + }, + { + "source": "/var/lib/config-data/default/01-service-custom.conf", + "dest": "/etc/cyborg/cyborg.conf.d/01-service-custom.conf", + "owner": "cyborg", + "perm": "0600", + "optional": true + } + ], + "permissions": [ + { + "path": "/var/log/cyborg", + "owner": "cyborg:cyborg", + "recurse": true + } + ] +} diff --git a/test/functional/cyborg/base_test.go b/test/functional/cyborg/base_test.go index 14a6ce562..2ad655aab 100644 --- a/test/functional/cyborg/base_test.go +++ b/test/functional/cyborg/base_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -155,3 +156,52 @@ func CreateCyborgMessageBusSecret(names CyborgNames) *corev1.Secret { }, ) } + +func GetCyborgConductor(name types.NamespacedName) *cyborgv1beta1.CyborgConductor { + instance := &cyborgv1beta1.CyborgConductor{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, instance)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + return instance +} + +func CyborgConductorConditionGetter(name types.NamespacedName) condition.Conditions { + instance := GetCyborgConductor(name) + return instance.Status.Conditions +} + +func CreateKeystoneAPIForCyborg(namespace string) types.NamespacedName { + keystoneAPIName := keystone.CreateKeystoneAPI(namespace) + keystoneAPI := keystone.GetKeystoneAPI(keystoneAPIName) + keystoneAPI.Spec.Region = "regionOne" + Expect(k8sClient.Update(ctx, keystoneAPI)).To(Succeed()) + Eventually(func(g Gomega) { + ks := keystone.GetKeystoneAPI(keystoneAPIName) + ks.Status.Region = "regionOne" + g.Expect(k8sClient.Status().Update(ctx, ks)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + return keystoneAPIName +} + +func SimulateCyborgPrerequisitesReady(names CyborgNames) { + mariadb.SimulateMariaDBAccountCompleted(names.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName) + infra.SimulateTransportURLReady(names.TransportURLName) + keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName) + th.SimulateJobSuccess(names.DBSyncJobName) +} + +func CreateCyborgTopology(namespace, name string) *topologyv1.Topology { + topology := &topologyv1.Topology{} + topology.Name = name + topology.Namespace = namespace + topology.Spec.TopologySpreadConstraints = &[]corev1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: "kubernetes.io/hostname", + WhenUnsatisfiable: corev1.ScheduleAnyway, + }, + } + Expect(k8sClient.Create(ctx, topology)).To(Succeed()) + return topology +} diff --git a/test/functional/cyborg/cyborg_controller_test.go b/test/functional/cyborg/cyborg_controller_test.go index b1aafc3ea..82c0862ff 100644 --- a/test/functional/cyborg/cyborg_controller_test.go +++ b/test/functional/cyborg/cyborg_controller_test.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -47,18 +48,22 @@ const ( ) type CyborgNames struct { - CyborgName types.NamespacedName - MariaDBServiceName types.NamespacedName - MariaDBDatabaseName types.NamespacedName - MariaDBAccountName types.NamespacedName - TransportURLName types.NamespacedName - KeystoneServiceName types.NamespacedName - DBSyncJobName types.NamespacedName - ConfigDataName types.NamespacedName - SubLevelSecretName types.NamespacedName - ServiceAccountName types.NamespacedName - RoleName types.NamespacedName - RoleBindingName types.NamespacedName + CyborgName types.NamespacedName + MariaDBServiceName types.NamespacedName + MariaDBDatabaseName types.NamespacedName + MariaDBAccountName types.NamespacedName + TransportURLName types.NamespacedName + KeystoneServiceName types.NamespacedName + KeystoneAPIName types.NamespacedName + DBSyncJobName types.NamespacedName + ConfigDataName types.NamespacedName + SubLevelSecretName types.NamespacedName + ServiceAccountName types.NamespacedName + RoleName types.NamespacedName + RoleBindingName types.NamespacedName + ConductorName types.NamespacedName + ConductorStatefulSetName types.NamespacedName + ConductorConfigDataName types.NamespacedName } func GetCyborgNames(cyborgName types.NamespacedName) CyborgNames { @@ -108,6 +113,18 @@ func GetCyborgNames(cyborgName types.NamespacedName) CyborgNames { Namespace: cyborgName.Namespace, Name: "cyborg-" + cyborgName.Name + "-rolebinding", }, + ConductorName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-conductor", + }, + ConductorStatefulSetName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-conductor", + }, + ConductorConfigDataName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-conductor-config-data", + }, } } @@ -191,6 +208,9 @@ var _ = Describe("Cyborg controller", func() { DeferCleanup(k8sClient.Delete, ctx, account) DeferCleanup(k8sClient.Delete, ctx, secret) + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + DeferCleanup( th.DeleteInstance, CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), @@ -255,7 +275,7 @@ var _ = Describe("Cyborg controller", func() { }, timeout, interval).Should(Succeed()) }) - It("reaches Ready when all dependencies are resolved", func() { + It("reaches Ready when all dependencies are resolved including conductor", func() { mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) @@ -299,6 +319,23 @@ var _ = Describe("Cyborg controller", func() { condition.DBSyncReadyCondition, corev1.ConditionTrue, ) + + // After dbsync, the conductor CR is created + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgConductorReadyCondition, + corev1.ConditionFalse, + ) + + th.SimulateStatefulSetReplicaReady(cyborgNames.ConductorStatefulSetName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgConductorReadyCondition, + corev1.ConditionTrue, + ) }) }) @@ -329,6 +366,9 @@ var _ = Describe("Cyborg controller", func() { DeferCleanup(k8sClient.Delete, ctx, account) DeferCleanup(k8sClient.Delete, ctx, dbSecret) + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + apiTLSSecret := th.CreateSecret( types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: apiTLSSecretName}, map[string][]byte{ @@ -410,13 +450,12 @@ var _ = Describe("Cyborg controller", func() { DeferCleanup(k8sClient.Delete, ctx, account) DeferCleanup(k8sClient.Delete, ctx, secret) + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + cyborg := CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()) - mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) - mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) - infra.SimulateTransportURLReady(cyborgNames.TransportURLName) - keystone.SimulateKeystoneServiceReady(cyborgNames.KeystoneServiceName) - th.SimulateJobSuccess(cyborgNames.DBSyncJobName) + SimulateCyborgPrerequisitesReady(cyborgNames) th.ExpectCondition( cyborgNames.CyborgName, @@ -531,3 +570,153 @@ var _ = Describe("Cyborg webhook validation", func() { DeferCleanup(th.DeleteInstance, unstructuredObj) }) }) + +var _ = Describe("Cyborg controller creates CyborgConductor", func() { + BeforeEach(func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + account, secret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, secret) + + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + }) + + It("creates a CyborgConductor sub-CR with default values after dbsync", func() { + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(cyborgNames.ConductorName) + g.Expect(conductor.Spec.ContainerImage).To(Equal(CyborgConductorImage)) + g.Expect(conductor.Spec.ServiceAccount).To(Equal("cyborg-" + cyborgNames.CyborgName.Name)) + g.Expect(conductor.Spec.ConfigSecret).To(Equal(cyborgNames.CyborgName.Name)) + g.Expect(conductor.Spec.Replicas).NotTo(BeNil()) + g.Expect(*conductor.Spec.Replicas).To(Equal(int32(1))) + }, timeout, interval).Should(Succeed()) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgConductorReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("inherits the top-level TopologyRef when conductor template has none", func() { + spec := GetDefaultCyborgSpec() + spec["topologyRef"] = map[string]any{"name": "cyborg-topology"} + + topologyObj := CreateCyborgTopology(cyborgNames.CyborgName.Namespace, "cyborg-topology") + DeferCleanup(k8sClient.Delete, ctx, topologyObj) + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(cyborgNames.ConductorName) + g.Expect(conductor.Spec.TopologyRef).NotTo(BeNil()) + g.Expect(conductor.Spec.TopologyRef.Name).To(Equal("cyborg-topology")) + }, timeout, interval).Should(Succeed()) + }) + + It("uses conductor-specific TopologyRef when defined", func() { + spec := GetDefaultCyborgSpec() + spec["topologyRef"] = map[string]any{"name": "global-topology"} + spec["conductorServiceTemplate"] = map[string]any{ + "topologyRef": map[string]any{"name": "conductor-topology"}, + } + + globalTopo := CreateCyborgTopology(cyborgNames.CyborgName.Namespace, "global-topology") + DeferCleanup(k8sClient.Delete, ctx, globalTopo) + conductorTopo := CreateCyborgTopology(cyborgNames.CyborgName.Namespace, "conductor-topology") + DeferCleanup(k8sClient.Delete, ctx, conductorTopo) + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(cyborgNames.ConductorName) + g.Expect(conductor.Spec.TopologyRef).NotTo(BeNil()) + g.Expect(conductor.Spec.TopologyRef.Name).To(Equal("conductor-topology")) + }, timeout, interval).Should(Succeed()) + }) + + It("passes custom resources to the CyborgConductor", func() { + spec := GetDefaultCyborgSpec() + spec["conductorServiceTemplate"] = map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{ + "memory": "256Mi", + "cpu": "500m", + }, + "limits": map[string]any{ + "memory": "512Mi", + "cpu": "1", + }, + }, + } + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(cyborgNames.ConductorName) + g.Expect(conductor.Spec.Resources.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse("256Mi"))) + g.Expect(conductor.Spec.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) + g.Expect(conductor.Spec.Resources.Limits[corev1.ResourceMemory]).To(Equal(resource.MustParse("512Mi"))) + g.Expect(conductor.Spec.Resources.Limits[corev1.ResourceCPU]).To(Equal(resource.MustParse("1"))) + }, timeout, interval).Should(Succeed()) + }) + + It("passes TLS CaBundleSecretName to the CyborgConductor", func() { + const caBundleSecretName = "cyborg-ca-bundle-for-conductor" + + caBundleSecret := th.CreateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: caBundleSecretName}, + map[string][]byte{common_tls.CABundleKey: []byte("dummy-ca-bundle")}, + ) + DeferCleanup(k8sClient.Delete, ctx, caBundleSecret) + + spec := GetDefaultCyborgSpec() + spec["apiServiceTemplate"] = map[string]any{ + "tls": map[string]any{ + "caBundleSecretName": caBundleSecretName, + }, + } + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(cyborgNames.ConductorName) + g.Expect(conductor.Spec.TLS.CaBundleSecretName).To(Equal(caBundleSecretName)) + }, timeout, interval).Should(Succeed()) + }) +}) diff --git a/test/functional/cyborg/cyborgconductor_controller_test.go b/test/functional/cyborg/cyborgconductor_controller_test.go new file mode 100644 index 000000000..dcbe7dec4 --- /dev/null +++ b/test/functional/cyborg/cyborgconductor_controller_test.go @@ -0,0 +1,589 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cyborg_test + +import ( + . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports + . "github.com/onsi/gomega" //revive:disable:dot-imports + + //revive:disable-next-line:dot-imports + . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +const ( + ConductorTestImage = "test://cyborg-conductor" + ConductorTestSA = "cyborg-test-sa" +) + +type CyborgConductorNames struct { + ConductorName types.NamespacedName + ConfigSecretName types.NamespacedName + StatefulSetName types.NamespacedName + ConfigDataName types.NamespacedName +} + +func GetCyborgConductorNames(namespace string, name string) CyborgConductorNames { + return CyborgConductorNames{ + ConductorName: types.NamespacedName{ + Namespace: namespace, + Name: name, + }, + ConfigSecretName: types.NamespacedName{ + Namespace: namespace, + Name: name + "-input-secret", + }, + StatefulSetName: types.NamespacedName{ + Namespace: namespace, + Name: name, + }, + ConfigDataName: types.NamespacedName{ + Namespace: namespace, + Name: name + "-config-data", + }, + } +} + +func CreateCyborgConductorConfigSecret(name types.NamespacedName) *corev1.Secret { + return th.CreateSecret(name, map[string][]byte{ + "transport_url": []byte("rabbit://user:pass@rabbitmq:5672/"), + "database_account": []byte("cyborg"), + "database_username": []byte("cyborg_user"), + "database_password": []byte("cyborg_pass"), + "database_hostname": []byte("openstack.openstack.svc"), + "ServiceUser": []byte("cyborg"), + "ServicePassword": []byte("service-pass"), + "KeystoneAuthURL": []byte("https://keystone-internal.openstack.svc:5000"), + "Region": []byte("regionOne"), + }) +} + +func CreateCyborgConductorConfigSecretWithAppCred(name types.NamespacedName, acid, acSecret string) *corev1.Secret { + return th.CreateSecret(name, map[string][]byte{ + "transport_url": []byte("rabbit://user:pass@rabbitmq:5672/?ssl=1"), + "database_account": []byte("cyborg"), + "database_username": []byte("cyborg_user"), + "database_password": []byte("cyborg_pass"), + "database_hostname": []byte("openstack.openstack.svc"), + "ServiceUser": []byte("cyborg"), + "ServicePassword": []byte("service-pass"), + "KeystoneAuthURL": []byte("https://keystone-internal.openstack.svc:5000"), + "Region": []byte("regionOne"), + "ACID": []byte(acid), + "ACSecret": []byte(acSecret), + }) +} + +func CreateCyborgConductorCR(name types.NamespacedName, spec cyborgv1beta1.CyborgConductorSpec) client.Object { + conductor := &cyborgv1beta1.CyborgConductor{ + ObjectMeta: metav1.ObjectMeta{ + Name: name.Name, + Namespace: name.Namespace, + }, + Spec: spec, + } + Expect(k8sClient.Create(ctx, conductor)).To(Succeed()) + return conductor +} + +var _ = Describe("CyborgConductor controller", func() { + var conductorNames CyborgConductorNames + + BeforeEach(func() { + conductorNames = GetCyborgConductorNames( + cyborgNames.CyborgName.Namespace, + "cyborg-conductor-test", + ) + }) + + When("CyborgConductor is created with a missing config secret", func() { + It("sets InputReady to False while waiting for the secret", func() { + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: "nonexistent-secret", + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + ) + }) + }) + + When("CyborgConductor is created with default config", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("initializes all expected status conditions", func() { + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(conductorNames.ConductorName) + g.Expect(conductor.Status.Conditions).NotTo(BeNil()) + g.Expect(conductor.Status.Conditions.Has(condition.ReadyCondition)).To(BeTrue()) + g.Expect(conductor.Status.Conditions.Has(condition.InputReadyCondition)).To(BeTrue()) + g.Expect(conductor.Status.Conditions.Has(condition.ServiceConfigReadyCondition)).To(BeTrue()) + g.Expect(conductor.Status.Conditions.Has(condition.DeploymentReadyCondition)).To(BeTrue()) + // No topology => no TopologyReadyCondition + g.Expect(conductor.Status.Conditions.Has(condition.TopologyReadyCondition)).To(BeFalse()) + }, timeout, interval).Should(Succeed()) + }) + + It("marks InputReady as True", func() { + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.InputReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("generates a config secret with all expected keys and config sections", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(conductorNames.ConfigDataName) + g.Expect(configSecret.Data).To(HaveKey("00-default.conf")) + g.Expect(configSecret.Data).To(HaveKey("my.cnf")) + g.Expect(configSecret.Data).To(HaveKey("cyborg-conductor-config.json")) + + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("[database]")) + g.Expect(defaultConf).To(ContainSubstring("connection = mysql+pymysql://cyborg_user:cyborg_pass@openstack.openstack.svc/cyborg")) + g.Expect(defaultConf).To(ContainSubstring("[oslo_messaging_rabbit]")) + g.Expect(defaultConf).To(ContainSubstring("transport_url")) + g.Expect(defaultConf).To(ContainSubstring("[keystone_authtoken]")) + g.Expect(defaultConf).To(ContainSubstring("[placement]")) + g.Expect(defaultConf).To(ContainSubstring("[nova]")) + g.Expect(defaultConf).To(ContainSubstring("[agent]")) + g.Expect(defaultConf).To(ContainSubstring("auth_type = password")) + g.Expect(defaultConf).To(ContainSubstring("username = cyborg")) + g.Expect(defaultConf).To(ContainSubstring("region_name = regionOne")) + + // No TLS => my.cnf should be minimal + myCnf := string(configSecret.Data["my.cnf"]) + g.Expect(myCnf).To(Equal("[client]\n")) + }, timeout, interval).Should(Succeed()) + + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.ServiceConfigReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates a StatefulSet with the correct spec", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(conductorNames.StatefulSetName) + g.Expect(ss.Spec.Replicas).NotTo(BeNil()) + g.Expect(*ss.Spec.Replicas).To(Equal(int32(1))) + g.Expect(ss.Spec.Template.Spec.ServiceAccountName).To(Equal(ConductorTestSA)) + g.Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(1)) + + container := ss.Spec.Template.Spec.Containers[0] + g.Expect(container.Name).To(Equal("cyborg-conductor")) + g.Expect(container.Image).To(Equal(ConductorTestImage)) + + hasConfigHash := false + hasKollaStrategy := false + for _, envVar := range container.Env { + if envVar.Name == "CONFIG_HASH" { + hasConfigHash = true + g.Expect(envVar.Value).NotTo(BeEmpty()) + } + if envVar.Name == "KOLLA_CONFIG_STRATEGY" { + hasKollaStrategy = true + g.Expect(envVar.Value).To(Equal("COPY_ALWAYS")) + } + } + g.Expect(hasConfigHash).To(BeTrue()) + g.Expect(hasKollaStrategy).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("sets the hash in the status", func() { + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(conductorNames.ConductorName) + g.Expect(conductor.Status.Hash).NotTo(BeNil()) + g.Expect(conductor.Status.Hash).To(HaveKey("input")) + }, timeout, interval).Should(Succeed()) + }) + + It("reaches Ready when the StatefulSet replicas are ready", func() { + th.SimulateStatefulSetReplicaReady(conductorNames.StatefulSetName) + + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.DeploymentReadyCondition, + corev1.ConditionTrue, + ) + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + conductor := GetCyborgConductor(conductorNames.ConductorName) + Expect(conductor.Status.ObservedGeneration).To(Equal(conductor.Generation)) + }) + }) + + When("CyborgConductor is created with custom resources", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(2)), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("256Mi"), + corev1.ResourceCPU: resource.MustParse("500m"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("512Mi"), + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("applies resources and replicas to the StatefulSet", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(conductorNames.StatefulSetName) + g.Expect(*ss.Spec.Replicas).To(Equal(int32(2))) + + container := ss.Spec.Template.Spec.Containers[0] + g.Expect(container.Resources.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse("256Mi"))) + g.Expect(container.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) + g.Expect(container.Resources.Limits[corev1.ResourceMemory]).To(Equal(resource.MustParse("512Mi"))) + g.Expect(container.Resources.Limits[corev1.ResourceCPU]).To(Equal(resource.MustParse("1"))) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgConductor is created with a TopologyRef", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + topologyObj := CreateCyborgTopology(conductorNames.ConductorName.Namespace, "conductor-test-topo") + DeferCleanup(k8sClient.Delete, ctx, topologyObj) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + TopologyRef: &topologyv1.TopoRef{ + Name: "conductor-test-topo", + }, + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("initializes TopologyReadyCondition", func() { + Eventually(func(g Gomega) { + conductor := GetCyborgConductor(conductorNames.ConductorName) + g.Expect(conductor.Status.Conditions.Has(condition.TopologyReadyCondition)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("sets TopologyReady to True once the Topology is resolved", func() { + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.TopologyReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates a StatefulSet and reaches Ready", func() { + th.SimulateStatefulSetReplicaReady(conductorNames.StatefulSetName) + + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) + + When("CyborgConductor is created with TLS CA bundle", func() { + const caBundleSecretName = "conductor-test-ca-bundle" + + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + caBundleSecret := th.CreateSecret( + types.NamespacedName{ + Namespace: conductorNames.ConductorName.Namespace, + Name: caBundleSecretName, + }, + map[string][]byte{tls.CABundleKey: []byte("dummy-ca-bundle")}, + ) + DeferCleanup(k8sClient.Delete, ctx, caBundleSecret) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + TLS: tls.Ca{CaBundleSecretName: caBundleSecretName}, + }, + )) + }) + + It("generates my.cnf with TLS settings", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(conductorNames.ConfigDataName) + myCnf := string(configSecret.Data["my.cnf"]) + g.Expect(myCnf).To(ContainSubstring("ssl-ca=")) + g.Expect(myCnf).To(ContainSubstring("ssl=1")) + }, timeout, interval).Should(Succeed()) + }) + + It("includes cafile in the generated default config", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(conductorNames.ConfigDataName) + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("cafile = " + tls.DownstreamTLSCABundlePath)) + }, timeout, interval).Should(Succeed()) + }) + + It("mounts the CA bundle volume in the StatefulSet", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(conductorNames.StatefulSetName) + + hasCAVolume := false + for _, v := range ss.Spec.Template.Spec.Volumes { + if v.Secret != nil && v.Secret.SecretName == caBundleSecretName { + hasCAVolume = true + } + } + g.Expect(hasCAVolume).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgConductor is created with application credentials", func() { + const ( + appCredID = "test-acid-123" //nolint:gosec + appCredSecret = "test-acsecret-456" //nolint:gosec + ) + + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecretWithAppCred( + conductorNames.ConfigSecretName, appCredID, appCredSecret), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("generates config with v3applicationcredential auth instead of password", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(conductorNames.ConfigDataName) + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("auth_type = v3applicationcredential")) + g.Expect(defaultConf).To(ContainSubstring("application_credential_id = " + appCredID)) + g.Expect(defaultConf).To(ContainSubstring("application_credential_secret = " + appCredSecret)) + g.Expect(defaultConf).NotTo(ContainSubstring("auth_type = password")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgConductor is created with custom service config", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + CustomServiceConfig: "[DEFAULT]\nmy_custom_key = my_custom_value\n", + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("includes the custom config in the generated config secret", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(conductorNames.ConfigDataName) + g.Expect(configSecret.Data).To(HaveKey("01-service-custom.conf")) + customConf := string(configSecret.Data["01-service-custom.conf"]) + g.Expect(customConf).To(ContainSubstring("my_custom_key = my_custom_value")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgConductor is created with a NodeSelector", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + NodeSelector: &map[string]string{ + "disktype": "ssd", + }, + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("applies the nodeSelector to the StatefulSet pod template", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(conductorNames.StatefulSetName) + g.Expect(ss.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("disktype", "ssd")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("the config secret is updated", func() { + It("updates the CONFIG_HASH in the StatefulSet", func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgConductorConfigSecret(conductorNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + + var originalHash string + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(conductorNames.StatefulSetName) + for _, envVar := range ss.Spec.Template.Spec.Containers[0].Env { + if envVar.Name == "CONFIG_HASH" { + originalHash = envVar.Value + } + } + g.Expect(originalHash).NotTo(BeEmpty()) + }, timeout, interval).Should(Succeed()) + + // Update the secret with new data + Eventually(func(g Gomega) { + secret := &corev1.Secret{} + g.Expect(k8sClient.Get(ctx, conductorNames.ConfigSecretName, secret)).To(Succeed()) + secret.Data["ServicePassword"] = []byte("new-password") + g.Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(conductorNames.StatefulSetName) + for _, envVar := range ss.Spec.Template.Spec.Containers[0].Env { + if envVar.Name == "CONFIG_HASH" { + g.Expect(envVar.Value).NotTo(Equal(originalHash)) + } + } + }, timeout, interval).Should(Succeed()) + }) + }) +}) From 289eab9d63f7a7f2d16dca2cf743f0d480e7c765 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Fri, 10 Apr 2026 18:09:24 +0200 Subject: [PATCH 5/7] cyborg: Implement CyborgAPI controller reconcile loop Add full reconcile logic for the CyborgAPI CR: - Validate input from config secret provided by the Cyborg controller - Render WSGI/httpd and cyborg-api configuration templates - Create a StatefulSet for cyborg-api pods with TLS support - Register Keystone endpoints (public and internal) for the API - Track readiness (ReadyCount, conditions, hash, topology) - Expose IsReady and topology helpers on CyborgAPI type - Extend Cyborg controller to create CyborgAPI and check readiness upwards - Add functional tests covering the full API reconcile flow Assisted-By: Claude Signed-off-by: Alfredo Moralejo --- .../cyborg.openstack.org_cyborgapis.yaml | 95 +- api/cyborg/v1beta1/conditions.go | 3 + api/cyborg/v1beta1/cyborg_types.go | 2 +- api/cyborg/v1beta1/cyborgapi_types.go | 55 +- api/cyborg/v1beta1/zz_generated.deepcopy.go | 27 +- .../cyborg.openstack.org_cyborgapis.yaml | 95 +- internal/controller/cyborg/common.go | 10 +- .../controller/cyborg/cyborg_controller.go | 97 +- .../controller/cyborg/cyborgapi_controller.go | 783 ++++++++++++- .../cyborg/cyborgconductor_controller.go | 3 +- internal/cyborg/api/statefulset.go | 237 ++++ internal/cyborg/conductor/statefulset.go | 2 +- internal/cyborg/constants.go | 3 + internal/cyborg/dbsync.go | 2 +- templates/cyborg/00-default.conf | 2 - templates/cyborg/api/10-cyborg-wsgi-main.conf | 43 + templates/cyborg/api/cyborg-api-config.json | 66 ++ templates/cyborg/api/httpd.conf | 47 + templates/cyborg/api/ssl.conf | 21 + test/functional/cyborg/base_test.go | 13 + .../cyborg/cyborg_controller_test.go | 328 +++++- .../cyborg/cyborgapi_controller_test.go | 1037 +++++++++++++++++ .../cyborg/cyborgconductor_controller_test.go | 53 +- 23 files changed, 2977 insertions(+), 47 deletions(-) create mode 100644 internal/cyborg/api/statefulset.go create mode 100644 templates/cyborg/api/10-cyborg-wsgi-main.conf create mode 100644 templates/cyborg/api/cyborg-api-config.json create mode 100644 templates/cyborg/api/httpd.conf create mode 100644 templates/cyborg/api/ssl.conf create mode 100644 test/functional/cyborg/cyborgapi_controller_test.go diff --git a/api/bases/cyborg.openstack.org_cyborgapis.yaml b/api/bases/cyborg.openstack.org_cyborgapis.yaml index b8fb2cf09..b8c7d87ce 100644 --- a/api/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/api/bases/cyborg.openstack.org_cyborgapis.yaml @@ -14,7 +14,16 @@ spec: singular: cyborgapi scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 schema: openAPIV3Schema: description: CyborgAPI is the Schema for the cyborgapis API. @@ -39,10 +48,18 @@ spec: spec: description: CyborgAPISpec defines the desired state of CyborgAPI. properties: + apiTimeout: + default: 60 + description: APITimeout for Route and Apache + minimum: 10 + type: integer configSecret: description: ConfigSecret - containing all the configuration needed provided by Cyborg object type: string + containerImage: + description: ContainerImage is the container image URL for cyborg-api + type: string customServiceConfig: description: |- CustomServiceConfig - customize the service config using this parameter to change service defaults, @@ -283,6 +300,9 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceAccount: + description: ServiceAccount used by the api pods + type: string tls: description: TLS - Parameters related to the TLS properties: @@ -332,9 +352,82 @@ spec: type: object required: - configSecret + - containerImage + - serviceAccount type: object status: description: CyborgAPIStatus defines the observed state of CyborgAPI. + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Hash - Map of hashes to track config changes + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + observedGeneration: + description: ObservedGeneration - the most recent generation observed + format: int64 + type: integer + readyCount: + description: ReadyCount defines the number of replicas ready + format: int32 + type: integer type: object type: object served: true diff --git a/api/cyborg/v1beta1/conditions.go b/api/cyborg/v1beta1/conditions.go index 64fe7314e..842f40e22 100644 --- a/api/cyborg/v1beta1/conditions.go +++ b/api/cyborg/v1beta1/conditions.go @@ -47,6 +47,9 @@ const ( // CyborgAPIReadyInitMessage - CyborgAPIReadyInitMessage = "CyborgAPI not started" + // CyborgAPIReadyErrorMessage - + CyborgAPIReadyErrorMessage = "CyborgAPI error occurred %s" + // CyborgConductorReadyInitMessage - CyborgConductorReadyInitMessage = "CyborgConductor not started" diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go index eada70a2b..0b26a8ced 100644 --- a/api/cyborg/v1beta1/cyborg_types.go +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -19,7 +19,7 @@ package v1beta1 import ( rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/api/cyborg/v1beta1/cyborgapi_types.go b/api/cyborg/v1beta1/cyborgapi_types.go index 699f4a0b8..270dfb351 100644 --- a/api/cyborg/v1beta1/cyborgapi_types.go +++ b/api/cyborg/v1beta1/cyborgapi_types.go @@ -18,7 +18,8 @@ package v1beta1 import ( topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,15 +81,45 @@ type CyborgAPISpec struct { // +kubebuilder:validation:Required // ConfigSecret - containing all the configuration needed provided by Cyborg object - ConfigSecret *string `json:"configSecret"` + ConfigSecret string `json:"configSecret"` + + // +kubebuilder:validation:Required + // ContainerImage is the container image URL for cyborg-api + ContainerImage string `json:"containerImage"` + + // +kubebuilder:validation:Required + // ServiceAccount used by the api pods + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for Route and Apache + APITimeout *int `json:"apiTimeout"` } // CyborgAPIStatus defines the observed state of CyborgAPI. type CyborgAPIStatus struct { + // ReadyCount defines the number of replicas ready + ReadyCount int32 `json:"readyCount,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ObservedGeneration - the most recent generation observed + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Hash - Map of hashes to track config changes + Hash map[string]string `json:"hash,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" // CyborgAPI is the Schema for the cyborgapis API. type CyborgAPI struct { @@ -108,6 +139,26 @@ type CyborgAPIList struct { Items []CyborgAPI `json:"items"` } +// IsReady returns true if the ReadyCondition is true +func (instance *CyborgAPI) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.ReadyCondition) +} + +// GetSpecTopologyRef returns the TopologyRef defined in the Spec +func (instance *CyborgAPI) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology returns the LastAppliedTopology from the Status +func (instance *CyborgAPI) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology sets the LastAppliedTopology value in the Status +func (instance *CyborgAPI) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} + func init() { SchemeBuilder.Register(&CyborgAPI{}, &CyborgAPIList{}) } diff --git a/api/cyborg/v1beta1/zz_generated.deepcopy.go b/api/cyborg/v1beta1/zz_generated.deepcopy.go index c51cf80f4..e6cb26107 100644 --- a/api/cyborg/v1beta1/zz_generated.deepcopy.go +++ b/api/cyborg/v1beta1/zz_generated.deepcopy.go @@ -97,7 +97,7 @@ func (in *CyborgAPI) DeepCopyInto(out *CyborgAPI) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPI. @@ -154,9 +154,9 @@ func (in *CyborgAPIList) DeepCopyObject() runtime.Object { func (in *CyborgAPISpec) DeepCopyInto(out *CyborgAPISpec) { *out = *in in.CyborgAPITemplate.DeepCopyInto(&out.CyborgAPITemplate) - if in.ConfigSecret != nil { - in, out := &in.ConfigSecret, &out.ConfigSecret - *out = new(string) + if in.APITimeout != nil { + in, out := &in.APITimeout, &out.APITimeout + *out = new(int) **out = **in } } @@ -174,6 +174,25 @@ func (in *CyborgAPISpec) DeepCopy() *CyborgAPISpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CyborgAPIStatus) DeepCopyInto(out *CyborgAPIStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CyborgAPIStatus. diff --git a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml index b8fb2cf09..b8c7d87ce 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgapis.yaml @@ -14,7 +14,16 @@ spec: singular: cyborgapi scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 schema: openAPIV3Schema: description: CyborgAPI is the Schema for the cyborgapis API. @@ -39,10 +48,18 @@ spec: spec: description: CyborgAPISpec defines the desired state of CyborgAPI. properties: + apiTimeout: + default: 60 + description: APITimeout for Route and Apache + minimum: 10 + type: integer configSecret: description: ConfigSecret - containing all the configuration needed provided by Cyborg object type: string + containerImage: + description: ContainerImage is the container image URL for cyborg-api + type: string customServiceConfig: description: |- CustomServiceConfig - customize the service config using this parameter to change service defaults, @@ -283,6 +300,9 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + serviceAccount: + description: ServiceAccount used by the api pods + type: string tls: description: TLS - Parameters related to the TLS properties: @@ -332,9 +352,82 @@ spec: type: object required: - configSecret + - containerImage + - serviceAccount type: object status: description: CyborgAPIStatus defines the observed state of CyborgAPI. + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Hash - Map of hashes to track config changes + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + observedGeneration: + description: ObservedGeneration - the most recent generation observed + format: int64 + type: integer + readyCount: + description: ReadyCount defines the number of replicas ready + format: int32 + type: integer type: object type: object served: true diff --git a/internal/controller/cyborg/common.go b/internal/controller/cyborg/common.go index 1348fd955..2c7f38d42 100644 --- a/internal/controller/cyborg/common.go +++ b/internal/controller/cyborg/common.go @@ -35,8 +35,11 @@ import ( ) const ( - passwordSecretField = ".spec.secret" - authAppCredSecretField = ".spec.auth.applicationCredentialSecret" //nolint:gosec + passwordSecretField = ".spec.secret" + authAppCredSecretField = ".spec.auth.applicationCredentialSecret" //nolint:gosec + caBundleSecretNameField = ".spec.tls.caBundleSecretName" //nolint:gosec + tlsAPIInternalField = ".spec.tls.api.internal.secretName" + tlsAPIPublicField = ".spec.tls.api.public.secretName" // TransportURLSelector is the key for the transport URL in secrets TransportURLSelector = "transport_url" @@ -124,6 +127,9 @@ func NewReconcilers(mgr ctrl.Manager, kclient *kubernetes.Clientset) *Reconciler "CyborgConductor": &CyborgConductorReconciler{ ReconcilerBase: NewReconcilerBase(mgr, kclient), }, + "CyborgAPI": &CyborgAPIReconciler{ + ReconcilerBase: NewReconcilerBase(mgr, kclient), + }, }} } diff --git a/internal/controller/cyborg/cyborg_controller.go b/internal/controller/cyborg/cyborg_controller.go index 08f36ade1..d7ad2ee02 100644 --- a/internal/controller/cyborg/cyborg_controller.go +++ b/internal/controller/cyborg/cyborg_controller.go @@ -194,7 +194,7 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // // Create RabbitMQ TransportURL // - transportURL, op, err := r.ensureMQ(ctx, instance, h, instance.Name+"-cyborg-transport", instance.Spec.MessagingBus, serviceLabels) + transportURL, _, err := r.ensureMQ(ctx, instance, h, instance.Name+"-cyborg-transport", instance.Spec.MessagingBus, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, @@ -219,8 +219,6 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res cyborgv1beta1.CyborgRabbitMQTransportURLReadyCondition, cyborgv1beta1.CyborgRabbitMQTransportURLReadyMessage) - _ = op - // // Validate input secret (password secret) // @@ -366,6 +364,16 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return ctrl.Result{}, err } + // + // Create CyborgAPI sub-CR + // + ctrlResult, err = r.ensureCyborgAPI(ctx, h, instance, serviceLabels) + if err != nil { + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + // // Create CyborgConductor sub-CR // @@ -381,6 +389,7 @@ func (r *CyborgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res condition.ReadyCondition, condition.ReadyMessage) } + Log.Info("Successfully reconciled") return ctrl.Result{}, nil } @@ -439,6 +448,10 @@ func (r *CyborgReconciler) initConditions(instance *cyborgv1beta1.Cyborg) error condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition( + cyborgv1beta1.CyborgAPIReadyCondition, + condition.InitReason, + cyborgv1beta1.CyborgAPIReadyInitMessage), condition.UnknownCondition( cyborgv1beta1.CyborgConductorReadyCondition, condition.InitReason, @@ -470,11 +483,7 @@ func (r *CyborgReconciler) ensureRbac( } _, err := common_rbac.ReconcileRbac(ctx, h, instance, rbacRules) - if err != nil { - return err - } - - return nil + return err } func (r *CyborgReconciler) ensureDB( @@ -881,6 +890,77 @@ func normalizeRegion(region string) string { return region } +func (r *CyborgReconciler) ensureCyborgAPI( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.Cyborg, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling CyborgAPI for '%s'", instance.Name)) + + apiName := fmt.Sprintf("%s-api", instance.Name) + + apiTemplate := *instance.Spec.APIServiceTemplate.DeepCopy() + if apiTemplate.TopologyRef == nil && instance.Spec.TopologyRef != nil { + apiTemplate.TopologyRef = instance.Spec.TopologyRef.DeepCopy() + } + + apiSpec := cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: apiTemplate, + ConfigSecret: instance.Name, + ContainerImage: instance.Spec.APIContainerImageURL, + ServiceAccount: instance.RbacResourceName(), + APITimeout: instance.Spec.APITimeout, + } + + api := &cyborgv1beta1.CyborgAPI{ + ObjectMeta: metav1.ObjectMeta{ + Name: apiName, + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrPatch(ctx, h.GetClient(), api, func() error { + api.Spec = apiSpec + api.Labels = serviceLabels + + err := controllerutil.SetControllerReference(instance, api, r.Scheme) + return err + }) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + cyborgv1beta1.CyborgAPIReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + cyborgv1beta1.CyborgAPIReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("CyborgAPI CR %s - %s", apiName, op)) + } + + apiObj := &cyborgv1beta1.CyborgAPI{} + err = h.GetClient().Get(ctx, types.NamespacedName{Name: apiName, Namespace: instance.Namespace}, apiObj) + if err != nil { + return ctrl.Result{}, err + } + + if apiObj.IsReady() { + instance.Status.APIServiceReadyCount = apiObj.Status.ReadyCount + instance.Status.Conditions.MarkTrue(cyborgv1beta1.CyborgAPIReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + cyborgv1beta1.CyborgAPIReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + + return ctrl.Result{}, nil +} + func (r *CyborgReconciler) ensureConductor( ctx context.Context, h *helper.Helper, @@ -983,6 +1063,7 @@ func (r *CyborgReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&mariadbv1.MariaDBAccount{}). Owns(&rabbitmqv1.TransportURL{}). Owns(&keystonev1.KeystoneService{}). + Owns(&cyborgv1beta1.CyborgAPI{}). Owns(&cyborgv1beta1.CyborgConductor{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). diff --git a/internal/controller/cyborg/cyborgapi_controller.go b/internal/controller/cyborg/cyborgapi_controller.go index ec61ecb39..b86cd64d1 100644 --- a/internal/controller/cyborg/cyborgapi_controller.go +++ b/internal/controller/cyborg/cyborgapi_controller.go @@ -1,5 +1,5 @@ /* -Copyright 2022. +Copyright 2024. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,49 +18,796 @@ package cyborg import ( "context" + "fmt" - "k8s.io/apimachinery/pkg/runtime" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + libservice "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborgservice "github.com/openstack-k8s-operators/nova-operator/internal/cyborg" + cyborgapi "github.com/openstack-k8s-operators/nova-operator/internal/cyborg/api" +) + +const ( + apiConfigSecretField = ".spec.configSecret" // #nosec G101 + apiTopologyField = ".spec.topologyRef.Name" ) +var apiSecretWatchFields = []string{ + apiConfigSecretField, + caBundleSecretNameField, + tlsAPIInternalField, + tlsAPIPublicField, +} + // CyborgAPIReconciler reconciles a CyborgAPI object // //nolint:revive type CyborgAPIReconciler struct { - client.Client - Scheme *runtime.Scheme + ReconcilerBase +} + +// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields +func (r *CyborgAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CyborgAPI") } // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis/status,verbs=get;update;patch // +kubebuilder:rbac:groups=cyborg.openstack.org,resources=cyborgapis/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the CyborgAPI object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.21.0/pkg/reconcile -func (r *CyborgAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = logf.FromContext(ctx) +// Reconcile is part of the main kubernetes reconciliation loop +func (r *CyborgAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + instance := &cyborgv1beta1.CyborgAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info("CyborgAPI instance not found, probably deleted before reconciled. Nothing to do.") + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciling CyborgAPI '%s'", instance.Name)) + + h, err := helper.NewHelper(instance, r.Client, r.Kclient, r.Scheme, Log) + if err != nil { + return ctrl.Result{}, err + } + + isNewInstance := instance.Status.Conditions == nil + savedConditions := instance.Status.Conditions.DeepCopy() + + defer func() { + // Don't update the status, if Reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("Panic during reconcile %v\n", r)) + panic(r) + } + // update the Ready condition based on the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + err := h.PatchInstance(ctx, instance) + if err != nil { + _err = err + } + }() + + r.initStatus(instance) + + if instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, h.GetFinalizer()) || isNewInstance { + return ctrl.Result{}, nil + } + + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, h) + } + + // Read the sub-level secret created by the Cyborg controller + subSecret := &corev1.Secret{} + secretName := types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.ConfigSecret} + err = r.Client.Get(ctx, secretName, subSecret) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info("Secret not found, waiting", "secret", instance.Spec.ConfigSecret) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ctrl.Result{RequeueAfter: r.RequeueTimeout}, nil + } + return ctrl.Result{}, err + } + + // Hash the input secret so we detect changes to passwords, transport URL, etc. + inputHashes := make(map[string]env.Setter) + secretHash, err := util.ObjectHash(subSecret.Data) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error calculating input secret hash: %w", err) + } + inputHashes["input"] = env.SetValue(secretHash) + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + configVars := make(map[string]env.Setter) + + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + h.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName, + )) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API cert secrets + certsHash, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, h, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.TLSInputReadyWaitingMessage, err.Error(), + )) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars[tls.TLSHashName] = env.SetValue(certsHash) + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // Generate config + err = r.generateServiceConfig(ctx, instance, subSecret, h, &configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + for key, hashVal := range configVars { + inputHashes[key] = hashVal + } + + // Compute a combined hash of all inputs (secret + generated config). + // This hash is set as CONFIG_HASH env var in the pod template so that + // any change in the input secret or generated config triggers a rollout. + inputHash, err := util.HashOfInputHashes(inputHashes) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error calculating combined input hash: %w", err) + } + instance.Status.Hash[common.InputHashName] = inputHash + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + serviceLabels := map[string]string{ + common.AppSelector: cyborgapi.ComponentName, + } + + topology, err := ensureTopology( + ctx, + h, + instance, + instance.Name, + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // Create or update the StatefulSet + ssDef := cyborgapi.StatefulSet(instance, inputHash, serviceLabels, topology) + + ss := statefulset.NewStatefulSet(ssDef, r.RequeueTimeout) + ctrlResult, err := ss.CreateOrPatch(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrlResult, nil + } + + ssObj := ss.GetStatefulSet() + instance.Status.ReadyCount = ssObj.Status.ReadyReplicas + if statefulset.IsReady(ssObj) { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + } + + if instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) { + apiEndpoints, ctrlResult, err := r.ensureServiceExposed(ctx, h, instance, serviceLabels) + if err != nil || (ctrlResult != ctrl.Result{}) { + return ctrl.Result{}, err + } + + ctrlResult, err = r.ensureKeystoneEndpoint(ctx, h, instance, apiEndpoints, serviceLabels) + if err != nil || (ctrlResult != ctrl.Result{}) { + return ctrlResult, err + } + } + + instance.Status.ObservedGeneration = instance.Generation + + Log.Info("Successfully reconciled") + return ctrl.Result{}, nil +} + +func (r *CyborgAPIReconciler) initStatus(instance *cyborgv1beta1.CyborgAPI) { + if instance.Status.Conditions == nil { + instance.Status.Conditions = condition.Conditions{} + } + + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), + condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, "KeystoneEndpoint not created"), + ) + + if instance.Spec.TopologyRef != nil { + cl.Set(condition.UnknownCondition( + condition.TopologyReadyCondition, + condition.InitReason, + condition.TopologyReadyInitMessage, + )) + } + + instance.Status.Conditions.Init(&cl) + + if instance.Status.Hash == nil { + instance.Status.Hash = make(map[string]string) + } +} + +func (r *CyborgAPIReconciler) generateServiceConfig( + ctx context.Context, + instance *cyborgv1beta1.CyborgAPI, + subSecret *corev1.Secret, + h *helper.Helper, + envVars *map[string]env.Setter, +) error { + Log := r.GetLogger(ctx) + Log.Info("generateServiceConfig - reconciling config for CyborgAPI") + + var tlsCfg *tls.Service + if instance.Spec.TLS.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + databaseConnection := fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + string(subSecret.Data[DatabaseUsername]), + string(subSecret.Data[DatabasePassword]), + string(subSecret.Data[DatabaseHostname]), + cyborgservice.DatabaseName, + ) + + templateParameters := map[string]any{ + "DatabaseConnection": databaseConnection, + "TransportURL": string(subSecret.Data[TransportURLSelector]), + "APIPublicPort": cyborgservice.CyborgPublicPort, + "LogFile": cyborgservice.CyborgLogPath + instance.Name + ".log", + } + + if quorumQueues, ok := subSecret.Data[QuorumQueuesSelector]; ok && string(quorumQueues) == "true" { + templateParameters["QuorumQueues"] = true + } - _ = req - // TODO(user): your logic here + if keystoneURL, ok := subSecret.Data["KeystoneAuthURL"]; ok && len(keystoneURL) > 0 { + templateParameters["KeystoneAuthURL"] = string(keystoneURL) + } + + if serviceUser, ok := subSecret.Data["ServiceUser"]; ok && len(serviceUser) > 0 { + templateParameters["ServiceUser"] = string(serviceUser) + } + + if servicePassword, ok := subSecret.Data["ServicePassword"]; ok && len(servicePassword) > 0 { + templateParameters["ServicePassword"] = string(servicePassword) + } + + if region, ok := subSecret.Data["Region"]; ok && len(region) > 0 { + templateParameters["Region"] = string(region) + } + + if acid, ok := subSecret.Data["ACID"]; ok && len(acid) > 0 { + templateParameters["ACID"] = string(acid) + templateParameters["ACSecret"] = string(subSecret.Data["ACSecret"]) + } + + if instance.Spec.TLS.CaBundleSecretName != "" { + templateParameters["CaFilePath"] = tls.DownstreamTLSCABundlePath + } + + // Build per-endpoint VirtualHost parameters for the Apache WSGI config + httpdVhostConfig := map[string]any{} + for _, endpt := range []libservice.Endpoint{libservice.EndpointInternal, libservice.EndpointPublic} { + endptConfig := map[string]any{ + "ServerName": fmt.Sprintf("%s-%s.%s.svc", cyborgservice.ServiceName, endpt.String(), instance.Namespace), + "Port": cyborgservice.CyborgPublicPort, + "TLS": false, + "TimeOut": *instance.Spec.APITimeout, + } + if instance.Spec.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + + customData := map[string]string{ + "my.cnf": generateMyCnf(tlsCfg), + } + + serviceLabels := labels.GetLabels(instance, labels.GetGroupLabel(cyborgservice.ServiceName), map[string]string{}) + + if instance.Spec.CustomServiceConfig != "" { + customData["01-service-custom.conf"] = instance.Spec.CustomServiceConfig + } + + cms := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.GetName()), + Namespace: instance.GetNamespace(), + Type: util.TemplateTypeConfig, + InstanceType: instance.GetObjectKind().GroupVersionKind().Kind, + ConfigOptions: templateParameters, + CustomData: customData, + Labels: serviceLabels, + AdditionalTemplate: map[string]string{ + "00-default.conf": "/cyborg/00-default.conf", + "cyborg-api-config.json": "/cyborg/api/cyborg-api-config.json", + "httpd.conf": "/cyborg/api/httpd.conf", + "ssl.conf": "/cyborg/api/ssl.conf", + "10-cyborg-wsgi-main.conf": "/cyborg/api/10-cyborg-wsgi-main.conf", + }, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, cms, envVars) +} + +func (r *CyborgAPIReconciler) ensureServiceExposed( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.CyborgAPI, + serviceLabels map[string]string, +) (map[string]string, ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Exposing CyborgAPI services for '%s'", instance.Name)) + + ports := map[libservice.Endpoint]endpoint.Data{ + libservice.EndpointPublic: { + Port: cyborgservice.CyborgPublicPort, + Path: "/v2", + }, + libservice.EndpointInternal: { + Port: cyborgservice.CyborgInternalPort, + Path: "/v2", + }, + } + + apiEndpoints := make(map[string]string) + + for endpointType, data := range ports { + endpointTypeStr := string(endpointType) + endpointName := cyborgservice.ServiceName + "-" + endpointTypeStr + svcOverride := instance.Spec.Override.Service[endpointType] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &libservice.EmbeddedLabelsAnnotations{} + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + libservice.AnnotationEndpointKey: endpointTypeStr, + }, + ) + + svc, err := libservice.NewService( + libservice.GenericService(&libservice.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: libservice.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error(), + )) + return nil, ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + libservice.AnnotationEndpointKey: endpointTypeStr, + }) + + if endpointType == libservice.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + libservice.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + libservice.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + libservice.AnnotationHostnameKey: svc.GetServiceHostname(), + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error(), + )) + return nil, ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.CreateServiceReadyRunningMessage, + )) + return nil, ctrlResult, nil + } + + if instance.Spec.TLS.API.Enabled(endpointType) { + data.Protocol = ptr.To(libservice.ProtocolHTTPS) + } + + apiEndpoints[endpointTypeStr], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, + data.Protocol, + data.Path, + ) + if err != nil { + return nil, ctrl.Result{}, err + } + } + + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) + + return apiEndpoints, ctrl.Result{}, nil +} + +func (r *CyborgAPIReconciler) ensureKeystoneEndpoint( + ctx context.Context, + h *helper.Helper, + instance *cyborgv1beta1.CyborgAPI, + apiEndpoints map[string]string, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconciling CyborgAPI KeystoneEndpoint for '%s'", instance.Name)) + + endpointSpec := keystonev1.KeystoneEndpointSpec{ + ServiceName: cyborgservice.ServiceName, + Endpoints: apiEndpoints, + } + + ksEndpoint := keystonev1.NewKeystoneEndpoint( + cyborgservice.ServiceName, + instance.Namespace, + endpointSpec, + serviceLabels, + r.RequeueTimeout, + ) + + ctrlResult, err := ksEndpoint.CreateOrPatch(ctx, h) + if err != nil { + return ctrlResult, err + } + + c := ksEndpoint.GetConditions().Mirror(condition.KeystoneEndpointReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + return ctrlResult, nil +} + +func (r *CyborgAPIReconciler) reconcileDelete( + ctx context.Context, + instance *cyborgv1beta1.CyborgAPI, + h *helper.Helper, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Reconcile CyborgAPI '%s' delete started", instance.Name)) + + // Remove the finalizer from the KeystoneEndpoint so the keystone operator + // can clean it up without being blocked by our finalizer. + ksEndpoint, err := keystonev1.GetKeystoneEndpointWithName(ctx, h, cyborgservice.ServiceName, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil { + if controllerutil.RemoveFinalizer(ksEndpoint, h.GetFinalizer()) { + err = h.GetClient().Update(ctx, ksEndpoint) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + Log.Info("Removed finalizer from CyborgAPI KeystoneEndpoint") + } + } + + // Remove finalizer from the referenced Topology CR + if _, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + h, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrl.Result{}, err + } + + controllerutil.RemoveFinalizer(instance, h.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled CyborgAPI '%s' delete successfully", instance.Name)) return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *CyborgAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgAPI{}, + apiConfigSecretField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgAPI) + if cr.Spec.ConfigSecret == "" { + return nil + } + return []string{cr.Spec.ConfigSecret} + }, + ); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgAPI{}, + apiTopologyField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgAPI) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }, + ); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgAPI{}, + tlsAPIInternalField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }, + ); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgAPI{}, + tlsAPIPublicField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }, + ); err != nil { + return err + } + + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &cyborgv1beta1.CyborgAPI{}, + caBundleSecretNameField, + func(rawObj client.Object) []string { + cr := rawObj.(*cyborgv1beta1.CyborgAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }, + ); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&cyborgv1beta1.CyborgAPI{}). + Owns(&corev1.Secret{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). + Owns(&keystonev1.KeystoneEndpoint{}). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findAPIsForSecret), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches( + &topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findAPIsForTopology), + builder.WithPredicates(predicate.GenerationChangedPredicate{}), + ). Named("cyborg-cyborgapi"). Complete(r) } + +func (r *CyborgAPIReconciler) findAPIsForTopology(ctx context.Context, src client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + crList := &cyborgv1beta1.CyborgAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(apiTopologyField, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + Log.Error(err, "listing CyborgAPIs for topology change") + return nil + } + + requests := make([]reconcile.Request, 0, len(crList.Items)) + for _, item := range crList.Items { + Log.Info(fmt.Sprintf("Topology %s changed, reconciling CyborgAPI %s", src.GetName(), item.GetName())) + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + return requests +} + +func (r *CyborgAPIReconciler) findAPIsForSecret(ctx context.Context, src client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + var requests []reconcile.Request + + for _, field := range apiSecretWatchFields { + crList := &cyborgv1beta1.CyborgAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + if err := r.Client.List(ctx, crList, listOps); err != nil { + Log.Error(err, "listing CyborgAPIs for secret change", "field", field) + continue + } + for _, item := range crList.Items { + Log.Info(fmt.Sprintf("Secret %s changed, reconciling CyborgAPI %s", src.GetName(), item.GetName())) + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }) + } + } + return requests +} diff --git a/internal/controller/cyborg/cyborgconductor_controller.go b/internal/controller/cyborg/cyborgconductor_controller.go index 36a1b3042..7ce527c0b 100644 --- a/internal/controller/cyborg/cyborgconductor_controller.go +++ b/internal/controller/cyborg/cyborgconductor_controller.go @@ -219,7 +219,7 @@ func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Requ ssObj := ss.GetStatefulSet() instance.Status.ReadyCount = ssObj.Status.ReadyReplicas - if ssObj.Status.ReadyReplicas == ssObj.Status.Replicas && ssObj.Generation == ssObj.Status.ObservedGeneration { + if statefulset.IsReady(ssObj) { instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) } else { instance.Status.Conditions.Set(condition.FalseCondition( @@ -231,6 +231,7 @@ func (r *CyborgConductorReconciler) Reconcile(ctx context.Context, req ctrl.Requ instance.Status.ObservedGeneration = instance.Generation + Log.Info("Successfully reconciled") return ctrl.Result{}, nil } diff --git a/internal/cyborg/api/statefulset.go b/internal/cyborg/api/statefulset.go new file mode 100644 index 000000000..2655ba2be --- /dev/null +++ b/internal/cyborg/api/statefulset.go @@ -0,0 +1,237 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package api provides the StatefulSet for the cyborg-api service +// +// revive:disable:var-naming +package api + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + libservice "github.com/openstack-k8s-operators/lib-common/modules/common/service" + + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" + cyborg "github.com/openstack-k8s-operators/nova-operator/internal/cyborg" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +const ( + // ComponentName is the name used for the api container + ComponentName = "cyborg-api" + + // KollaServiceCommand is the kolla start command for cyborg-api + KollaServiceCommand = "/usr/local/bin/kolla_start" +) + +// StatefulSet creates a StatefulSet for the cyborg-api service +func StatefulSet( + instance *cyborgv1beta1.CyborgAPI, + configHash string, + labels map[string]string, + topology *topologyv1.Topology, +) *appsv1.StatefulSet { + var config0644AccessMode int32 = 0644 + runAsUser := int64(0) + + envVars := make(map[string]env.Setter) + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + args := []string{"-c", KollaServiceCommand} + + probeHandler := corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt32(cyborg.CyborgInternalPort), + }, + } + + startupProbe := &corev1.Probe{ + FailureThreshold: 6, + PeriodSeconds: 10, + ProbeHandler: probeHandler, + } + livenessProbe := &corev1.Probe{ + TimeoutSeconds: 10, + PeriodSeconds: 10, + ProbeHandler: probeHandler, + } + readinessProbe := &corev1.Probe{ + TimeoutSeconds: 5, + PeriodSeconds: 5, + ProbeHandler: probeHandler, + } + + logVolumeMount := corev1.VolumeMount{ + Name: cyborg.LogVolume, + MountPath: "/var/log/cyborg", + ReadOnly: false, + } + + volumes := []corev1.Volume{ + { + Name: cyborg.ConfigVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: instance.Name + "-config-data", + }, + }, + }, + { + Name: cyborg.LogVolume, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, + }, + }, + } + + volumeMounts := []corev1.VolumeMount{ + { + Name: cyborg.ConfigVolume, + MountPath: "/var/lib/config-data/default", + ReadOnly: true, + }, + { + Name: cyborg.ConfigVolume, + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: "cyborg-api-config.json", + ReadOnly: true, + }, + { + Name: cyborg.ConfigVolume, + MountPath: "/etc/my.cnf", + SubPath: "my.cnf", + ReadOnly: true, + }, + logVolumeMount, + } + + // Add CA bundle volume if set + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + // Add API TLS cert volumes for each enabled endpoint + for _, endpt := range []libservice.Endpoint{libservice.EndpointInternal, libservice.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + switch endpt { + case libservice.EndpointPublic: + svc, err := instance.Spec.TLS.API.Public.ToService() + if err == nil { + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + case libservice.EndpointInternal: + svc, err := instance.Spec.TLS.API.Internal.ToService() + if err == nil { + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + } + } + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: instance.Spec.Replicas, + PodManagementPolicy: appsv1.ParallelPodManagement, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + { + Name: ComponentName + "-log", + Command: []string{ + "/usr/bin/dumb-init", + }, + Args: []string{ + "--single-child", + "--", + "/usr/bin/tail", + "-n+1", + "-F", + cyborg.CyborgLogPath + instance.Name + ".log", + }, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(runAsUser), + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: []corev1.VolumeMount{logVolumeMount}, + Resources: instance.Spec.Resources, + StartupProbe: startupProbe, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(runAsUser), + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + StartupProbe: startupProbe, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + statefulset.Spec.Template.Spec.Affinity = affinity.DistributePods( + common.AppSelector, + []string{instance.Name}, + corev1.LabelHostname, + ) + } + + return statefulset +} diff --git a/internal/cyborg/conductor/statefulset.go b/internal/cyborg/conductor/statefulset.go index 38558549d..74ce5ffb2 100644 --- a/internal/cyborg/conductor/statefulset.go +++ b/internal/cyborg/conductor/statefulset.go @@ -50,7 +50,7 @@ func StatefulSet( var config0644AccessMode int32 = 0644 runAsUser := int64(0) - envVars := map[string]env.Setter{} + envVars := make(map[string]env.Setter) envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) args := []string{"-c", KollaServiceCommand} diff --git a/internal/cyborg/constants.go b/internal/cyborg/constants.go index b67f641a8..2cf730252 100644 --- a/internal/cyborg/constants.go +++ b/internal/cyborg/constants.go @@ -50,4 +50,7 @@ const ( // CyborgLogPath is the default path for the cyborg service logs CyborgLogPath = "/var/log/cyborg/" + + // LogVolume is the name of the EmptyDir volume used for log streaming + LogVolume = "logs" ) diff --git a/internal/cyborg/dbsync.go b/internal/cyborg/dbsync.go index 8a103aeb7..7a1d805af 100644 --- a/internal/cyborg/dbsync.go +++ b/internal/cyborg/dbsync.go @@ -103,7 +103,7 @@ func DbSyncJob( args := []string{"-c", DBSyncCommand} - envVars := map[string]env.Setter{} + envVars := make(map[string]env.Setter) envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") diff --git a/templates/cyborg/00-default.conf b/templates/cyborg/00-default.conf index d6fae27e1..63972a232 100644 --- a/templates/cyborg/00-default.conf +++ b/templates/cyborg/00-default.conf @@ -86,6 +86,4 @@ interface = internal cafile = {{ .CaFilePath }} {{ end }} -[agent] -enabled_drivers = fake_driver {{ end }} diff --git a/templates/cyborg/api/10-cyborg-wsgi-main.conf b/templates/cyborg/api/10-cyborg-wsgi-main.conf new file mode 100644 index 000000000..55b295333 --- /dev/null +++ b/templates/cyborg/api/10-cyborg-wsgi-main.conf @@ -0,0 +1,43 @@ +{{ if (index . "VHosts") }} +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration + + ServerName {{ $vhost.ServerName }} + + ## Vhost docroot + DocumentRoot "/var/www/cgi-bin" + + # Set the timeout for the cyborg-api + TimeOut {{ $vhost.TimeOut }} + + ## Directories, there should at least be a declaration for /var/www/cgi-bin + + Options -Indexes +FollowSymLinks +MultiViews + AllowOverride None + Require all granted + + + ## Logging + ErrorLog /dev/stdout + ServerSignature Off + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + LogLevel info + +{{- if $vhost.TLS }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" +{{- end }} + + ## WSGI configuration + WSGIApplicationGroup %{GLOBAL} + WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cyborg processes=2 threads=1 user=cyborg + WSGIProcessGroup {{ $endpt }} + WSGIScriptAlias / "/var/lib/kolla/venv/lib/python3.12/site-packages/cyborg/wsgi/api.py" + +{{ end }} +{{ end }} diff --git a/templates/cyborg/api/cyborg-api-config.json b/templates/cyborg/api/cyborg-api-config.json new file mode 100644 index 000000000..baed33843 --- /dev/null +++ b/templates/cyborg/api/cyborg-api-config.json @@ -0,0 +1,66 @@ +{ + "command": "/usr/sbin/httpd -DFOREGROUND", + "config_files": [ + { + "source": "/var/lib/config-data/default/00-default.conf", + "dest": "/etc/cyborg/cyborg.conf.d/00-default.conf", + "owner": "cyborg", + "perm": "0600" + }, + { + "source": "/var/lib/config-data/default/01-service-custom.conf", + "dest": "/etc/cyborg/cyborg.conf.d/01-service-custom.conf", + "owner": "cyborg", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/config-data/default/10-cyborg-wsgi-main.conf", + "dest": "/etc/httpd/conf.d/10-cyborg-wsgi-main.conf", + "owner": "apache", + "perm": "0644", + "optional": true + }, + { + "source": "/var/lib/config-data/default/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "apache", + "perm": "0644", + "optional": true + }, + { + "source": "/var/lib/config-data/default/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "apache", + "perm": "0444" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cyborg", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cyborg", + "perm": "0600", + "optional": true, + "merge": true + } + ], + "permissions": [ + { + "path": "/var/log/cyborg", + "owner": "cyborg:cyborg", + "recurse": true + }, + { + "path": "/etc/httpd/run/", + "owner": "cyborg:apache", + "recurse": true + } + ] +} diff --git a/templates/cyborg/api/httpd.conf b/templates/cyborg/api/httpd.conf new file mode 100644 index 000000000..e794d84ba --- /dev/null +++ b/templates/cyborg/api/httpd.conf @@ -0,0 +1,47 @@ +ServerTokens Prod +ServerSignature Off +TraceEnable Off + +ServerName "cyborg.openstack.svc" +ServerRoot "/etc/httpd" + +PidFile run/httpd.pid +Timeout 90 +KeepAlive On +MaxKeepAliveRequests 100 +KeepAliveTimeout 15 +LimitRequestFieldSize 8190 +LimitRequestFields 100 + +User apache +Group apache +Listen {{ .APIPublicPort }} + +AccessFileName .htaccess + + Require all denied + + + + Options FollowSymLinks + AllowOverride None + + +HostnameLookups Off +EnableSendfile On + +TypesConfig /etc/mime.types +Include "/etc/httpd/conf.modules.d/*.conf" +Include "/etc/httpd/conf.d/*.conf" + +LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%a %l %u %t \"%r\" %>s %b" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"" forwarded + +ErrorLog /dev/stderr +TransferLog /dev/stdout +CustomLog /dev/stdout combined env=!forwarded +CustomLog /dev/stdout proxy env=!forwarded +LogLevel info diff --git a/templates/cyborg/api/ssl.conf b/templates/cyborg/api/ssl.conf new file mode 100644 index 000000000..e3da4ecb2 --- /dev/null +++ b/templates/cyborg/api/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + diff --git a/test/functional/cyborg/base_test.go b/test/functional/cyborg/base_test.go index 2ad655aab..8f94bca4e 100644 --- a/test/functional/cyborg/base_test.go +++ b/test/functional/cyborg/base_test.go @@ -170,6 +170,19 @@ func CyborgConductorConditionGetter(name types.NamespacedName) condition.Conditi return instance.Status.Conditions } +func GetCyborgAPI(name types.NamespacedName) *cyborgv1beta1.CyborgAPI { + instance := &cyborgv1beta1.CyborgAPI{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, instance)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + return instance +} + +func CyborgAPIConditionGetter(name types.NamespacedName) condition.Conditions { + instance := GetCyborgAPI(name) + return instance.Status.Conditions +} + func CreateKeystoneAPIForCyborg(namespace string) types.NamespacedName { keystoneAPIName := keystone.CreateKeystoneAPI(namespace) keystoneAPI := keystone.GetKeystoneAPI(keystoneAPIName) diff --git a/test/functional/cyborg/cyborg_controller_test.go b/test/functional/cyborg/cyborg_controller_test.go index 82c0862ff..847d7dc15 100644 --- a/test/functional/cyborg/cyborg_controller_test.go +++ b/test/functional/cyborg/cyborg_controller_test.go @@ -35,7 +35,7 @@ import ( common_tls "github.com/openstack-k8s-operators/lib-common/modules/common/tls" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" - condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" ) @@ -55,12 +55,15 @@ type CyborgNames struct { TransportURLName types.NamespacedName KeystoneServiceName types.NamespacedName KeystoneAPIName types.NamespacedName + KeystoneEndpointName types.NamespacedName DBSyncJobName types.NamespacedName ConfigDataName types.NamespacedName SubLevelSecretName types.NamespacedName ServiceAccountName types.NamespacedName RoleName types.NamespacedName RoleBindingName types.NamespacedName + APIName types.NamespacedName + APIStatefulSetName types.NamespacedName ConductorName types.NamespacedName ConductorStatefulSetName types.NamespacedName ConductorConfigDataName types.NamespacedName @@ -89,6 +92,10 @@ func GetCyborgNames(cyborgName types.NamespacedName) CyborgNames { Namespace: cyborgName.Namespace, Name: "cyborg", }, + KeystoneEndpointName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: "cyborg", + }, DBSyncJobName: types.NamespacedName{ Namespace: cyborgName.Namespace, Name: cyborgName.Name + "-db-sync", @@ -113,6 +120,14 @@ func GetCyborgNames(cyborgName types.NamespacedName) CyborgNames { Namespace: cyborgName.Namespace, Name: "cyborg-" + cyborgName.Name + "-rolebinding", }, + APIName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-api", + }, + APIStatefulSetName: types.NamespacedName{ + Namespace: cyborgName.Namespace, + Name: cyborgName.Name + "-api", + }, ConductorName: types.NamespacedName{ Namespace: cyborgName.Namespace, Name: cyborgName.Name + "-conductor", @@ -148,6 +163,7 @@ var _ = Describe("Cyborg controller", func() { g.Expect(cyborg.Status.Conditions.Has(condition.ServiceConfigReadyCondition)).To(BeTrue()) g.Expect(cyborg.Status.Conditions.Has(condition.DBSyncReadyCondition)).To(BeTrue()) g.Expect(cyborg.Status.Conditions.Has(condition.KeystoneServiceReadyCondition)).To(BeTrue()) + g.Expect(cyborg.Status.Conditions.Has(cyborgv1beta1.CyborgAPIReadyCondition)).To(BeTrue()) }, timeout, interval).Should(Succeed()) }) @@ -275,7 +291,7 @@ var _ = Describe("Cyborg controller", func() { }, timeout, interval).Should(Succeed()) }) - It("reaches Ready when all dependencies are resolved including conductor", func() { + It("reaches Ready when all dependencies are resolved including API and conductor", func() { mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) @@ -320,7 +336,24 @@ var _ = Describe("Cyborg controller", func() { corev1.ConditionTrue, ) - // After dbsync, the conductor CR is created + // After dbsync, the API and conductor CRs are created + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgAPIReadyCondition, + corev1.ConditionFalse, + ) + + th.SimulateStatefulSetReplicaReady(cyborgNames.APIStatefulSetName) + keystone.SimulateKeystoneEndpointReady(cyborgNames.KeystoneEndpointName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgAPIReadyCondition, + corev1.ConditionTrue, + ) + th.ExpectCondition( cyborgNames.CyborgName, ConditionGetterFunc(CyborgConditionGetter), @@ -431,6 +464,100 @@ var _ = Describe("Cyborg controller", func() { }) }) + When("a Cyborg CR references an ApplicationCredential secret that does not exist", func() { + BeforeEach(func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + + account, secret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, secret) + + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + + spec := GetDefaultCyborgSpec() + spec["auth"] = map[string]any{ + "applicationCredentialSecret": "nonexistent-appcred-secret", + } + DeferCleanup(th.DeleteInstance, CreateCyborg(cyborgNames.CyborgName, spec)) + }) + + It("sets InputReady to False", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + ) + }) + }) + + When("a Cyborg CR references an ApplicationCredential secret with missing required keys", func() { + const badAppCredSecretName = "cyborg-bad-appcred-secret" // #nosec G101 + + BeforeEach(func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + + account, secret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, secret) + + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + + badSecret := th.CreateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: badAppCredSecretName}, + map[string][]byte{"wrong-key": []byte("wrong-value")}, + ) + DeferCleanup(k8sClient.Delete, ctx, badSecret) + + spec := GetDefaultCyborgSpec() + spec["auth"] = map[string]any{ + "applicationCredentialSecret": badAppCredSecretName, + } + DeferCleanup(th.DeleteInstance, CreateCyborg(cyborgNames.CyborgName, spec)) + }) + + It("sets InputReady to False", func() { + mariadb.SimulateMariaDBAccountCompleted(cyborgNames.MariaDBAccountName) + mariadb.SimulateMariaDBDatabaseCompleted(cyborgNames.MariaDBDatabaseName) + infra.SimulateTransportURLReady(cyborgNames.TransportURLName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + ) + }) + }) + When("Cyborg CR is deleted", func() { It("cleans up finalizers", func() { serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} @@ -720,3 +847,198 @@ var _ = Describe("Cyborg controller creates CyborgConductor", func() { }, timeout, interval).Should(Succeed()) }) }) + +var _ = Describe("Cyborg controller creates CyborgAPI", func() { + BeforeEach(func() { + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgSecret(cyborgNames.CyborgName.Namespace)) + DeferCleanup(k8sClient.Delete, ctx, CreateCyborgMessageBusSecret(cyborgNames)) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + cyborgNames.MariaDBServiceName.Namespace, + cyborgNames.MariaDBServiceName.Name, + serviceSpec, + ), + ) + account, secret := mariadb.CreateMariaDBAccountAndSecret( + cyborgNames.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, account) + DeferCleanup(k8sClient.Delete, ctx, secret) + + cyborgNames.KeystoneAPIName = CreateKeystoneAPIForCyborg(cyborgNames.CyborgName.Namespace) + DeferCleanup(keystone.DeleteKeystoneAPI, cyborgNames.KeystoneAPIName) + }) + + It("creates a CyborgAPI sub-CR with default values after dbsync", func() { + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + api := GetCyborgAPI(cyborgNames.APIName) + g.Expect(api.Spec.ContainerImage).To(Equal(CyborgContainerImage)) + g.Expect(api.Spec.ServiceAccount).To(Equal("cyborg-" + cyborgNames.CyborgName.Name)) + g.Expect(api.Spec.ConfigSecret).To(Equal(cyborgNames.CyborgName.Name)) + g.Expect(api.Spec.Replicas).NotTo(BeNil()) + g.Expect(*api.Spec.Replicas).To(Equal(int32(1))) + }, timeout, interval).Should(Succeed()) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgAPIReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("updates ConductorServiceReadyCount in Cyborg status when the Conductor StatefulSet is ready", func() { + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + th.SimulateStatefulSetReplicaReady(cyborgNames.ConductorStatefulSetName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgConductorReadyCondition, + corev1.ConditionTrue, + ) + + Eventually(func(g Gomega) { + cyborg := GetCyborg(cyborgNames.CyborgName) + g.Expect(cyborg.Status.ConductorServiceReadyCount).To(BeNumerically(">", 0)) + }, timeout, interval).Should(Succeed()) + }) + + It("updates APIServiceReadyCount in Cyborg status when the API StatefulSet is ready", func() { + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, GetDefaultCyborgSpec()), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + th.SimulateStatefulSetReplicaReady(cyborgNames.APIStatefulSetName) + keystone.SimulateKeystoneEndpointReady(cyborgNames.KeystoneEndpointName) + + th.ExpectCondition( + cyborgNames.CyborgName, + ConditionGetterFunc(CyborgConditionGetter), + cyborgv1beta1.CyborgAPIReadyCondition, + corev1.ConditionTrue, + ) + + Eventually(func(g Gomega) { + cyborg := GetCyborg(cyborgNames.CyborgName) + g.Expect(cyborg.Status.APIServiceReadyCount).To(BeNumerically(">", 0)) + }, timeout, interval).Should(Succeed()) + }) + + It("inherits the top-level TopologyRef when API template has none", func() { + spec := GetDefaultCyborgSpec() + spec["topologyRef"] = map[string]any{"name": "cyborg-api-topology"} + + topologyObj := CreateCyborgTopology(cyborgNames.CyborgName.Namespace, "cyborg-api-topology") + DeferCleanup(k8sClient.Delete, ctx, topologyObj) + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + api := GetCyborgAPI(cyborgNames.APIName) + g.Expect(api.Spec.TopologyRef).NotTo(BeNil()) + g.Expect(api.Spec.TopologyRef.Name).To(Equal("cyborg-api-topology")) + }, timeout, interval).Should(Succeed()) + }) + + It("uses API-specific TopologyRef when defined", func() { + spec := GetDefaultCyborgSpec() + spec["topologyRef"] = map[string]any{"name": "global-topology"} + spec["apiServiceTemplate"] = map[string]any{ + "topologyRef": map[string]any{"name": "api-topology"}, + } + + globalTopo := CreateCyborgTopology(cyborgNames.CyborgName.Namespace, "global-topology") + DeferCleanup(k8sClient.Delete, ctx, globalTopo) + apiTopo := CreateCyborgTopology(cyborgNames.CyborgName.Namespace, "api-topology") + DeferCleanup(k8sClient.Delete, ctx, apiTopo) + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + api := GetCyborgAPI(cyborgNames.APIName) + g.Expect(api.Spec.TopologyRef).NotTo(BeNil()) + g.Expect(api.Spec.TopologyRef.Name).To(Equal("api-topology")) + }, timeout, interval).Should(Succeed()) + }) + + It("passes custom resources to the CyborgAPI", func() { + spec := GetDefaultCyborgSpec() + spec["apiServiceTemplate"] = map[string]any{ + "resources": map[string]any{ + "requests": map[string]any{ + "memory": "512Mi", + "cpu": "250m", + }, + "limits": map[string]any{ + "memory": "1Gi", + "cpu": "500m", + }, + }, + } + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + api := GetCyborgAPI(cyborgNames.APIName) + g.Expect(api.Spec.Resources.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse("512Mi"))) + g.Expect(api.Spec.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("250m"))) + g.Expect(api.Spec.Resources.Limits[corev1.ResourceMemory]).To(Equal(resource.MustParse("1Gi"))) + g.Expect(api.Spec.Resources.Limits[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) + }, timeout, interval).Should(Succeed()) + }) + + It("passes TLS CaBundleSecretName to the CyborgAPI", func() { + const caBundleSecretName = "cyborg-ca-bundle-for-api" + + caBundleSecret := th.CreateSecret( + types.NamespacedName{Namespace: cyborgNames.CyborgName.Namespace, Name: caBundleSecretName}, + map[string][]byte{common_tls.CABundleKey: []byte("dummy-ca-bundle")}, + ) + DeferCleanup(k8sClient.Delete, ctx, caBundleSecret) + + spec := GetDefaultCyborgSpec() + spec["apiServiceTemplate"] = map[string]any{ + "tls": map[string]any{ + "caBundleSecretName": caBundleSecretName, + }, + } + + DeferCleanup( + th.DeleteInstance, + CreateCyborg(cyborgNames.CyborgName, spec), + ) + SimulateCyborgPrerequisitesReady(cyborgNames) + + Eventually(func(g Gomega) { + api := GetCyborgAPI(cyborgNames.APIName) + g.Expect(api.Spec.TLS.CaBundleSecretName).To(Equal(caBundleSecretName)) + }, timeout, interval).Should(Succeed()) + }) +}) diff --git a/test/functional/cyborg/cyborgapi_controller_test.go b/test/functional/cyborg/cyborgapi_controller_test.go new file mode 100644 index 000000000..89daea647 --- /dev/null +++ b/test/functional/cyborg/cyborgapi_controller_test.go @@ -0,0 +1,1037 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cyborg_test + +import ( + . "github.com/onsi/ginkgo/v2" //revive:disable:dot-imports + . "github.com/onsi/gomega" //revive:disable:dot-imports + + //revive:disable-next-line:dot-imports + . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" +) + +const ( + APITestImage = "test://cyborg-api" + APITestSA = "cyborg-test-api-sa" +) + +type CyborgAPINames struct { + APIName types.NamespacedName + ConfigSecretName types.NamespacedName + StatefulSetName types.NamespacedName + ConfigDataName types.NamespacedName + KeystoneEndpointName types.NamespacedName + PublicServiceName types.NamespacedName + InternalServiceName types.NamespacedName + CaBundleSecretName types.NamespacedName + InternalCertSecretName types.NamespacedName + PublicCertSecretName types.NamespacedName +} + +func GetCyborgAPINames(namespace string, name string) CyborgAPINames { + return CyborgAPINames{ + APIName: types.NamespacedName{ + Namespace: namespace, + Name: name, + }, + ConfigSecretName: types.NamespacedName{ + Namespace: namespace, + Name: name + "-input-secret", + }, + StatefulSetName: types.NamespacedName{ + Namespace: namespace, + Name: name, + }, + ConfigDataName: types.NamespacedName{ + Namespace: namespace, + Name: name + "-config-data", + }, + KeystoneEndpointName: types.NamespacedName{ + Namespace: namespace, + Name: "cyborg", + }, + PublicServiceName: types.NamespacedName{ + Namespace: namespace, + Name: "cyborg-public", + }, + InternalServiceName: types.NamespacedName{ + Namespace: namespace, + Name: "cyborg-internal", + }, + CaBundleSecretName: types.NamespacedName{ + Namespace: namespace, + Name: "combined-ca-bundle", + }, + InternalCertSecretName: types.NamespacedName{ + Namespace: namespace, + Name: "cert-cyborg-internal-svc", + }, + PublicCertSecretName: types.NamespacedName{ + Namespace: namespace, + Name: "cert-cyborg-public-svc", + }, + } +} + +func CreateCyborgAPIConfigSecret(name types.NamespacedName) *corev1.Secret { + return th.CreateSecret(name, map[string][]byte{ + "transport_url": []byte("rabbit://user:pass@rabbitmq:5672/"), + "database_account": []byte("cyborg"), + "database_username": []byte("cyborg_user"), + "database_password": []byte("cyborg_pass"), + "database_hostname": []byte("openstack.openstack.svc"), + "ServiceUser": []byte("cyborg"), + "ServicePassword": []byte("service-pass"), + "KeystoneAuthURL": []byte("https://keystone-internal.openstack.svc:5000"), + "Region": []byte("regionOne"), + }) +} + +func CreateCyborgAPIConfigSecretWithAppCred(name types.NamespacedName, acid, acSecret string) *corev1.Secret { + return th.CreateSecret(name, map[string][]byte{ + "transport_url": []byte("rabbit://user:pass@rabbitmq:5672/?ssl=1"), + "database_account": []byte("cyborg"), + "database_username": []byte("cyborg_user"), + "database_password": []byte("cyborg_pass"), + "database_hostname": []byte("openstack.openstack.svc"), + "ServiceUser": []byte("cyborg"), + "ServicePassword": []byte("service-pass"), + "KeystoneAuthURL": []byte("https://keystone-internal.openstack.svc:5000"), + "Region": []byte("regionOne"), + "ACID": []byte(acid), + "ACSecret": []byte(acSecret), + }) +} + +func CreateCyborgAPICR(name types.NamespacedName, spec cyborgv1beta1.CyborgAPISpec) client.Object { + api := &cyborgv1beta1.CyborgAPI{ + ObjectMeta: metav1.ObjectMeta{ + Name: name.Name, + Namespace: name.Namespace, + }, + Spec: spec, + } + Expect(k8sClient.Create(ctx, api)).To(Succeed()) + return api +} + +var _ = Describe("CyborgAPI controller", func() { + var apiNames CyborgAPINames + + BeforeEach(func() { + apiNames = GetCyborgAPINames( + cyborgNames.CyborgName.Namespace, + "cyborg-api-test", + ) + }) + + When("CyborgAPI is created with a missing config secret", func() { + It("sets InputReady to False while waiting for the secret", func() { + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: "nonexistent-secret", + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.InputReadyCondition, + corev1.ConditionFalse, + ) + }) + }) + + When("CyborgAPI is created with default config", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("initializes all expected status conditions", func() { + Eventually(func(g Gomega) { + api := GetCyborgAPI(apiNames.APIName) + g.Expect(api.Status.Conditions).NotTo(BeNil()) + g.Expect(api.Status.Conditions.Has(condition.ReadyCondition)).To(BeTrue()) + g.Expect(api.Status.Conditions.Has(condition.InputReadyCondition)).To(BeTrue()) + g.Expect(api.Status.Conditions.Has(condition.TLSInputReadyCondition)).To(BeTrue()) + g.Expect(api.Status.Conditions.Has(condition.ServiceConfigReadyCondition)).To(BeTrue()) + g.Expect(api.Status.Conditions.Has(condition.DeploymentReadyCondition)).To(BeTrue()) + g.Expect(api.Status.Conditions.Has(condition.CreateServiceReadyCondition)).To(BeTrue()) + g.Expect(api.Status.Conditions.Has(condition.KeystoneEndpointReadyCondition)).To(BeTrue()) + // No topology => no TopologyReadyCondition + g.Expect(api.Status.Conditions.Has(condition.TopologyReadyCondition)).To(BeFalse()) + }, timeout, interval).Should(Succeed()) + }) + + It("marks InputReady as True", func() { + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.InputReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("generates a config secret with all expected keys and config sections", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + g.Expect(configSecret.Data).To(HaveKey("00-default.conf")) + g.Expect(configSecret.Data).To(HaveKey("my.cnf")) + g.Expect(configSecret.Data).To(HaveKey("cyborg-api-config.json")) + g.Expect(configSecret.Data).To(HaveKey("httpd.conf")) + g.Expect(configSecret.Data).To(HaveKey("ssl.conf")) + g.Expect(configSecret.Data).To(HaveKey("10-cyborg-wsgi-main.conf")) + + wsgiConf := string(configSecret.Data["10-cyborg-wsgi-main.conf"]) + g.Expect(wsgiConf).To(ContainSubstring("TimeOut 60")) + g.Expect(wsgiConf).To(ContainSubstring("WSGIScriptAlias / \"/var/lib/kolla/venv/lib/python3.12/site-packages/cyborg/wsgi/api.py\"")) + + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("[database]")) + g.Expect(defaultConf).To(ContainSubstring("connection = mysql+pymysql://cyborg_user:cyborg_pass@openstack.openstack.svc/cyborg")) + g.Expect(defaultConf).To(ContainSubstring("[oslo_messaging_rabbit]")) + g.Expect(defaultConf).To(ContainSubstring("transport_url")) + g.Expect(defaultConf).To(ContainSubstring("[keystone_authtoken]")) + g.Expect(defaultConf).To(ContainSubstring("[placement]")) + g.Expect(defaultConf).To(ContainSubstring("[nova]")) + g.Expect(defaultConf).To(ContainSubstring("auth_type = password")) + g.Expect(defaultConf).To(ContainSubstring("username = cyborg")) + g.Expect(defaultConf).To(ContainSubstring("region_name = regionOne")) + g.Expect(defaultConf).To(ContainSubstring("log_file = /var/log/cyborg/cyborg-api-test.log")) + + // No TLS => my.cnf should be minimal + myCnf := string(configSecret.Data["my.cnf"]) + g.Expect(myCnf).To(Equal("[client]\n")) + }, timeout, interval).Should(Succeed()) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.ServiceConfigReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates a StatefulSet with the correct spec", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(apiNames.StatefulSetName) + g.Expect(ss.Spec.Replicas).NotTo(BeNil()) + g.Expect(*ss.Spec.Replicas).To(Equal(int32(1))) + g.Expect(ss.Spec.Template.Spec.ServiceAccountName).To(Equal(APITestSA)) + g.Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(2)) + + // Log sidecar container (index 0) + logContainer := ss.Spec.Template.Spec.Containers[0] + g.Expect(logContainer.Name).To(Equal("cyborg-api-log")) + g.Expect(logContainer.Image).To(Equal(APITestImage)) + g.Expect(logContainer.Command).To(Equal([]string{"/usr/bin/dumb-init"})) + g.Expect(logContainer.Args).To(ContainElement("/var/log/cyborg/cyborg-api-test.log")) + hasLogVolumeMount := false + for _, vm := range logContainer.VolumeMounts { + if vm.MountPath == "/var/log/cyborg" { + hasLogVolumeMount = true + } + } + g.Expect(hasLogVolumeMount).To(BeTrue()) + + // Main API container (index 1) + container := ss.Spec.Template.Spec.Containers[1] + g.Expect(container.Name).To(Equal("cyborg-api")) + g.Expect(container.Image).To(Equal(APITestImage)) + + hasConfigHash := false + hasKollaStrategy := false + for _, envVar := range container.Env { + if envVar.Name == "CONFIG_HASH" { + hasConfigHash = true + g.Expect(envVar.Value).NotTo(BeEmpty()) + } + if envVar.Name == "KOLLA_CONFIG_STRATEGY" { + hasKollaStrategy = true + g.Expect(envVar.Value).To(Equal("COPY_ALWAYS")) + } + } + g.Expect(hasConfigHash).To(BeTrue()) + g.Expect(hasKollaStrategy).To(BeTrue()) + + // Log volume (EmptyDir) must be present + hasLogVolume := false + for _, v := range ss.Spec.Template.Spec.Volumes { + if v.Name == "logs" && v.EmptyDir != nil { + hasLogVolume = true + } + } + g.Expect(hasLogVolume).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("sets the hash in the status", func() { + Eventually(func(g Gomega) { + api := GetCyborgAPI(apiNames.APIName) + g.Expect(api.Status.Hash).NotTo(BeNil()) + g.Expect(api.Status.Hash).To(HaveKey("input")) + }, timeout, interval).Should(Succeed()) + }) + + It("does not expose services until the StatefulSet is ready", func() { + // Services should not exist before deployment is ready + Eventually(func(g Gomega) { + // Config secret should be created (StatefulSet exists) + _ = th.GetStatefulSet(apiNames.StatefulSetName) + // But services should not exist yet + svcList := &corev1.ServiceList{} + g.Expect(k8sClient.List(ctx, svcList, client.InNamespace(apiNames.APIName.Namespace))).To(Succeed()) + names := []string{} + for _, svc := range svcList.Items { + names = append(names, svc.Name) + } + g.Expect(names).NotTo(ContainElement("cyborg-public")) + g.Expect(names).NotTo(ContainElement("cyborg-internal")) + }, timeout, interval).Should(Succeed()) + }) + + It("exposes internal and public services after deployment is ready", func() { + th.SimulateStatefulSetReplicaReady(apiNames.StatefulSetName) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.CreateServiceReadyCondition, + corev1.ConditionTrue, + ) + + _ = th.GetService(apiNames.PublicServiceName) + _ = th.GetService(apiNames.InternalServiceName) + }) + + It("creates the KeystoneEndpoint with correct URLs after deployment is ready", func() { + th.SimulateStatefulSetReplicaReady(apiNames.StatefulSetName) + keystone.SimulateKeystoneEndpointReady(apiNames.KeystoneEndpointName) + + ksEndpoint := keystone.GetKeystoneEndpoint(apiNames.KeystoneEndpointName) + endpoints := ksEndpoint.Spec.Endpoints + Expect(endpoints).To(HaveKeyWithValue("public", + "http://cyborg-public."+apiNames.APIName.Namespace+".svc:6666/v2")) + Expect(endpoints).To(HaveKeyWithValue("internal", + "http://cyborg-internal."+apiNames.APIName.Namespace+".svc:6666/v2")) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.KeystoneEndpointReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("reaches Ready when StatefulSet and KeystoneEndpoint are ready", func() { + // DeploymentReady is False while the StatefulSet replicas are not yet ready + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.DeploymentReadyCondition, + corev1.ConditionFalse, + ) + + th.SimulateStatefulSetReplicaReady(apiNames.StatefulSetName) + keystone.SimulateKeystoneEndpointReady(apiNames.KeystoneEndpointName) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.DeploymentReadyCondition, + corev1.ConditionTrue, + ) + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + api := GetCyborgAPI(apiNames.APIName) + Expect(api.Status.ObservedGeneration).To(Equal(api.Generation)) + }) + }) + + When("CyborgAPI is created with custom resources", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(2)), + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("256Mi"), + corev1.ResourceCPU: resource.MustParse("500m"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("512Mi"), + corev1.ResourceCPU: resource.MustParse("1"), + }, + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("applies resources and replicas to the StatefulSet", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(apiNames.StatefulSetName) + g.Expect(*ss.Spec.Replicas).To(Equal(int32(2))) + + // API container is at index 1 (log sidecar is at index 0) + container := ss.Spec.Template.Spec.Containers[1] + g.Expect(container.Resources.Requests[corev1.ResourceMemory]).To(Equal(resource.MustParse("256Mi"))) + g.Expect(container.Resources.Requests[corev1.ResourceCPU]).To(Equal(resource.MustParse("500m"))) + g.Expect(container.Resources.Limits[corev1.ResourceMemory]).To(Equal(resource.MustParse("512Mi"))) + g.Expect(container.Resources.Limits[corev1.ResourceCPU]).To(Equal(resource.MustParse("1"))) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with a TopologyRef", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + topologyObj := CreateCyborgTopology(apiNames.APIName.Namespace, "api-test-topo") + DeferCleanup(k8sClient.Delete, ctx, topologyObj) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TopologyRef: &topologyv1.TopoRef{ + Name: "api-test-topo", + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("initializes TopologyReadyCondition", func() { + Eventually(func(g Gomega) { + api := GetCyborgAPI(apiNames.APIName) + g.Expect(api.Status.Conditions.Has(condition.TopologyReadyCondition)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("sets TopologyReady to True once the Topology is resolved", func() { + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.TopologyReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates a StatefulSet and reaches Ready", func() { + th.SimulateStatefulSetReplicaReady(apiNames.StatefulSetName) + keystone.SimulateKeystoneEndpointReady(apiNames.KeystoneEndpointName) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) + + When("CyborgAPI is created with TLS CA bundle", func() { + const caBundleSecretName = "api-test-ca-bundle" // #nosec G101 + + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + caBundleSecret := th.CreateSecret( + types.NamespacedName{ + Namespace: apiNames.APIName.Namespace, + Name: caBundleSecretName, + }, + map[string][]byte{tls.CABundleKey: []byte("dummy-ca-bundle")}, + ) + DeferCleanup(k8sClient.Delete, ctx, caBundleSecret) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TLS: tls.API{Ca: tls.Ca{CaBundleSecretName: caBundleSecretName}}, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("generates my.cnf with TLS settings", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + myCnf := string(configSecret.Data["my.cnf"]) + g.Expect(myCnf).To(ContainSubstring("ssl-ca=")) + g.Expect(myCnf).To(ContainSubstring("ssl=1")) + }, timeout, interval).Should(Succeed()) + }) + + It("includes cafile in the generated default config", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("cafile = " + tls.DownstreamTLSCABundlePath)) + }, timeout, interval).Should(Succeed()) + }) + + It("mounts the CA bundle volume in the StatefulSet", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(apiNames.StatefulSetName) + + hasCAVolume := false + for _, v := range ss.Spec.Template.Spec.Volumes { + if v.Secret != nil && v.Secret.SecretName == caBundleSecretName { + hasCAVolume = true + } + } + g.Expect(hasCAVolume).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with application credentials", func() { + const ( + appCredID = "test-api-acid-123" //nolint:gosec + appCredSecret = "test-api-acsecret-456" //nolint:gosec + ) + + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecretWithAppCred( + apiNames.ConfigSecretName, appCredID, appCredSecret), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("generates config with v3applicationcredential auth instead of password", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("auth_type = v3applicationcredential")) + g.Expect(defaultConf).To(ContainSubstring("application_credential_id = " + appCredID)) + g.Expect(defaultConf).To(ContainSubstring("application_credential_secret = " + appCredSecret)) + g.Expect(defaultConf).NotTo(ContainSubstring("auth_type = password")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with quorum queues enabled", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + th.CreateSecret(apiNames.ConfigSecretName, map[string][]byte{ + "transport_url": []byte("rabbit://user:pass@rabbitmq:5672/"), + "database_account": []byte("cyborg"), + "database_username": []byte("cyborg_user"), + "database_password": []byte("cyborg_pass"), + "database_hostname": []byte("openstack.openstack.svc"), + "ServiceUser": []byte("cyborg"), + "ServicePassword": []byte("service-pass"), + "KeystoneAuthURL": []byte("https://keystone-internal.openstack.svc:5000"), + "Region": []byte("regionOne"), + "quorumqueues": []byte("true"), + }), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("generates config with quorum queue settings", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("rabbit_quorum_queue=true")) + g.Expect(defaultConf).To(ContainSubstring("rabbit_transient_quorum_queue=true")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with only a CA bundle (no API certs)", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(apiNames.CaBundleSecretName)) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TLS: tls.API{Ca: tls.Ca{CaBundleSecretName: apiNames.CaBundleSecretName.Name}}, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("sets TLSInputReady to True without API cert secrets", func() { + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("generates my.cnf with TLS settings and cafile in default config", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + g.Expect(string(configSecret.Data["my.cnf"])).To(ContainSubstring("ssl-ca=")) + g.Expect(string(configSecret.Data["00-default.conf"])).To(ContainSubstring( + "cafile = " + tls.DownstreamTLSCABundlePath)) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with custom service config", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + CustomServiceConfig: "[DEFAULT]\nmy_api_key = my_api_value\n", + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("includes the custom config in the generated config secret", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(apiNames.ConfigDataName) + g.Expect(configSecret.Data).To(HaveKey("01-service-custom.conf")) + customConf := string(configSecret.Data["01-service-custom.conf"]) + g.Expect(customConf).To(ContainSubstring("my_api_key = my_api_value")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with a NodeSelector", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + NodeSelector: &map[string]string{ + "disktype": "ssd", + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("applies the nodeSelector to the StatefulSet pod template", func() { + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(apiNames.StatefulSetName) + g.Expect(ss.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("disktype", "ssd")) + }, timeout, interval).Should(Succeed()) + }) + }) + + When("the config secret is updated", func() { + It("updates the CONFIG_HASH in the StatefulSet", func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + + var originalHash string + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(apiNames.StatefulSetName) + for _, envVar := range ss.Spec.Template.Spec.Containers[1].Env { + if envVar.Name == "CONFIG_HASH" { + originalHash = envVar.Value + } + } + g.Expect(originalHash).NotTo(BeEmpty()) + }, timeout, interval).Should(Succeed()) + + // Update the secret with new data + Eventually(func(g Gomega) { + secret := &corev1.Secret{} + g.Expect(k8sClient.Get(ctx, apiNames.ConfigSecretName, secret)).To(Succeed()) + secret.Data["ServicePassword"] = []byte("new-api-password") + g.Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + }, timeout, interval).Should(Succeed()) + + Eventually(func(g Gomega) { + ss := th.GetStatefulSet(apiNames.StatefulSetName) + for _, envVar := range ss.Spec.Template.Spec.Containers[1].Env { + if envVar.Name == "CONFIG_HASH" { + g.Expect(envVar.Value).NotTo(Equal(originalHash)) + } + } + }, timeout, interval).Should(Succeed()) + }) + }) + + When("CyborgAPI is created with a missing CA bundle secret", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TLS: tls.API{ + Ca: tls.Ca{CaBundleSecretName: apiNames.CaBundleSecretName.Name}, + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("sets TLSInputReady to False while waiting for the CA bundle secret", func() { + th.ExpectConditionWithDetails( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + "TLSInput is missing: "+apiNames.CaBundleSecretName.Name, + ) + }) + }) + + When("CyborgAPI is created with an invalid CA bundle secret", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + // Secret exists but is missing the required tls-ca-bundle.pem key + DeferCleanup(k8sClient.Delete, ctx, th.CreateSecret( + apiNames.CaBundleSecretName, + map[string][]byte{"wrong-key": []byte("some-data")}, + )) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TLS: tls.API{ + Ca: tls.Ca{CaBundleSecretName: apiNames.CaBundleSecretName.Name}, + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("sets TLSInputReady to False with a missing key error", func() { + th.ExpectConditionWithDetails( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + "TLSInput error occured in TLS sources field not found in Secret: "+ //nolint:misspell // codespell:ignore occured + "field tls-ca-bundle.pem not found in Secret "+ + apiNames.CaBundleSecretName.Namespace+"/"+apiNames.CaBundleSecretName.Name, + ) + }) + }) + + When("CyborgAPI is created with TLS certs but an invalid cert secret", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(apiNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(apiNames.PublicCertSecretName)) + // Internal cert secret exists but is missing the required tls.key field + DeferCleanup(k8sClient.Delete, ctx, th.CreateSecret( + apiNames.InternalCertSecretName, + map[string][]byte{"wrong-key": []byte("some-data")}, + )) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TLS: tls.API{ + Ca: tls.Ca{CaBundleSecretName: apiNames.CaBundleSecretName.Name}, + API: tls.APIService{ + Internal: tls.GenericService{SecretName: &apiNames.InternalCertSecretName.Name}, + Public: tls.GenericService{SecretName: &apiNames.PublicCertSecretName.Name}, + }, + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("sets TLSInputReady to False with a missing key error", func() { + th.ExpectConditionWithDetails( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + "TLSInput error occured in TLS sources field not found in Secret: "+ //nolint:misspell // codespell:ignore occured + "field tls.key not found in Secret "+ + apiNames.InternalCertSecretName.Namespace+"/"+apiNames.InternalCertSecretName.Name, + ) + }) + }) + + When("CyborgAPI is created with valid TLS certs", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(apiNames.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(apiNames.InternalCertSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(apiNames.PublicCertSecretName)) + + DeferCleanup(th.DeleteInstance, CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + TLS: tls.API{ + Ca: tls.Ca{CaBundleSecretName: apiNames.CaBundleSecretName.Name}, + API: tls.APIService{ + Internal: tls.GenericService{SecretName: &apiNames.InternalCertSecretName.Name}, + Public: tls.GenericService{SecretName: &apiNames.PublicCertSecretName.Name}, + }, + }, + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + )) + }) + + It("sets TLSInputReady to True", func() { + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("creates https endpoints in the KeystoneEndpoint", func() { + th.SimulateStatefulSetReplicaReady(apiNames.StatefulSetName) + keystone.SimulateKeystoneEndpointReady(apiNames.KeystoneEndpointName) + + ksEndpoint := keystone.GetKeystoneEndpoint(apiNames.KeystoneEndpointName) + endpoints := ksEndpoint.Spec.Endpoints + Expect(endpoints).To(HaveKeyWithValue("public", + "https://cyborg-public."+apiNames.APIName.Namespace+".svc:6666/v2")) + Expect(endpoints).To(HaveKeyWithValue("internal", + "https://cyborg-internal."+apiNames.APIName.Namespace+".svc:6666/v2")) + }) + }) + + When("CyborgAPI CR is deleted", func() { + It("removes the finalizer from the KeystoneEndpoint", func() { + DeferCleanup( + k8sClient.Delete, ctx, + CreateCyborgAPIConfigSecret(apiNames.ConfigSecretName), + ) + + api := CreateCyborgAPICR( + apiNames.APIName, + cyborgv1beta1.CyborgAPISpec{ + CyborgAPITemplate: cyborgv1beta1.CyborgAPITemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: apiNames.ConfigSecretName.Name, + ContainerImage: APITestImage, + ServiceAccount: APITestSA, + APITimeout: ptr.To(60), + }, + ) + + // Drive to fully ready + th.SimulateStatefulSetReplicaReady(apiNames.StatefulSetName) + keystone.SimulateKeystoneEndpointReady(apiNames.KeystoneEndpointName) + + th.ExpectCondition( + apiNames.APIName, + ConditionGetterFunc(CyborgAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + + // The KeystoneEndpoint should carry our finalizer + ksEndpoint := keystone.GetKeystoneEndpoint(apiNames.KeystoneEndpointName) + Expect(ksEndpoint.Finalizers).To(ContainElement("openstack.org/cyborgapi")) + + // Delete the CyborgAPI and confirm the finalizer is removed + th.DeleteInstance(api) + + Eventually(func(g Gomega) { + ksEndpoint := keystone.GetKeystoneEndpoint(apiNames.KeystoneEndpointName) + g.Expect(ksEndpoint.Finalizers).NotTo(ContainElement("openstack.org/cyborgapi")) + }, timeout, interval).Should(Succeed()) + }) + }) +}) diff --git a/test/functional/cyborg/cyborgconductor_controller_test.go b/test/functional/cyborg/cyborgconductor_controller_test.go index dcbe7dec4..0028081bb 100644 --- a/test/functional/cyborg/cyborgconductor_controller_test.go +++ b/test/functional/cyborg/cyborgconductor_controller_test.go @@ -30,7 +30,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" cyborgv1beta1 "github.com/openstack-k8s-operators/nova-operator/api/cyborg/v1beta1" ) @@ -200,7 +201,6 @@ var _ = Describe("CyborgConductor controller", func() { g.Expect(defaultConf).To(ContainSubstring("[keystone_authtoken]")) g.Expect(defaultConf).To(ContainSubstring("[placement]")) g.Expect(defaultConf).To(ContainSubstring("[nova]")) - g.Expect(defaultConf).To(ContainSubstring("[agent]")) g.Expect(defaultConf).To(ContainSubstring("auth_type = password")) g.Expect(defaultConf).To(ContainSubstring("username = cyborg")) g.Expect(defaultConf).To(ContainSubstring("region_name = regionOne")) @@ -256,6 +256,14 @@ var _ = Describe("CyborgConductor controller", func() { }) It("reaches Ready when the StatefulSet replicas are ready", func() { + // DeploymentReady is False while the StatefulSet replicas are not yet ready + th.ExpectCondition( + conductorNames.ConductorName, + ConditionGetterFunc(CyborgConductorConditionGetter), + condition.DeploymentReadyCondition, + corev1.ConditionFalse, + ) + th.SimulateStatefulSetReplicaReady(conductorNames.StatefulSetName) th.ExpectCondition( @@ -476,6 +484,47 @@ var _ = Describe("CyborgConductor controller", func() { }) }) + When("CyborgConductor is created with quorum queues enabled", func() { + BeforeEach(func() { + DeferCleanup( + k8sClient.Delete, ctx, + th.CreateSecret(conductorNames.ConfigSecretName, map[string][]byte{ + "transport_url": []byte("rabbit://user:pass@rabbitmq:5672/"), + "database_account": []byte("cyborg"), + "database_username": []byte("cyborg_user"), + "database_password": []byte("cyborg_pass"), + "database_hostname": []byte("openstack.openstack.svc"), + "ServiceUser": []byte("cyborg"), + "ServicePassword": []byte("service-pass"), + "KeystoneAuthURL": []byte("https://keystone-internal.openstack.svc:5000"), + "Region": []byte("regionOne"), + "quorumqueues": []byte("true"), + }), + ) + + DeferCleanup(th.DeleteInstance, CreateCyborgConductorCR( + conductorNames.ConductorName, + cyborgv1beta1.CyborgConductorSpec{ + CyborgConductorTemplate: cyborgv1beta1.CyborgConductorTemplate{ + Replicas: ptr.To(int32(1)), + }, + ConfigSecret: conductorNames.ConfigSecretName.Name, + ContainerImage: ConductorTestImage, + ServiceAccount: ConductorTestSA, + }, + )) + }) + + It("generates config with quorum queue settings", func() { + Eventually(func(g Gomega) { + configSecret := th.GetSecret(conductorNames.ConfigDataName) + defaultConf := string(configSecret.Data["00-default.conf"]) + g.Expect(defaultConf).To(ContainSubstring("rabbit_quorum_queue=true")) + g.Expect(defaultConf).To(ContainSubstring("rabbit_transient_quorum_queue=true")) + }, timeout, interval).Should(Succeed()) + }) + }) + When("CyborgConductor is created with custom service config", func() { BeforeEach(func() { DeferCleanup( From 9ba68e16774eed76eed03c122e0c80e2c01202b0 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Tue, 14 Apr 2026 09:18:38 +0200 Subject: [PATCH 6/7] cyborg: Add kuttl integration tests for Cyborg deployment Add an end-to-end kuttl test suite for the Cyborg operator: - Cleanup step to delete any pre-existing Cyborg CR before the test - Deploy step creating a full Cyborg CR (cyborg-kuttl) - Assert step verifying all conditions are True on Cyborg, CyborgAPI, CyborgConductor and MariaDBDatabase CRs - Error step covering missing-dependency failure scenarios - Register cyborg container images (api, conductor, agent) as default RELATED_IMAGE env vars in the manager deployment - Enable ENABLE_CYBORG=true in the CI webhook deploy script Assisted-By: Claude Signed-off-by: Alfredo Moralejo --- Makefile | 1 + ci/nova-operator-kuttl/deploy_webhooks.yaml | 1 + ci/olm.sh | 6 + config/default/manager_default_images.yaml | 6 + .../cyborg-tests/00-cleanup-cyborg.yaml | 7 + .../default/cyborg-tests/01-assert.yaml | 505 ++++++++++++++++++ .../cyborg-tests/01-deploy-cyborg.yaml | 8 + .../cyborg-tests/02-cleanup-cyborg.yaml | 7 + .../default/cyborg-tests/02-errors.yaml | 61 +++ .../default/deps/kustomization.yaml | 1 + 10 files changed, 603 insertions(+) create mode 100644 test/kuttl/test-suites/default/cyborg-tests/00-cleanup-cyborg.yaml create mode 100644 test/kuttl/test-suites/default/cyborg-tests/01-assert.yaml create mode 100644 test/kuttl/test-suites/default/cyborg-tests/01-deploy-cyborg.yaml create mode 100644 test/kuttl/test-suites/default/cyborg-tests/02-cleanup-cyborg.yaml create mode 100644 test/kuttl/test-suites/default/cyborg-tests/02-errors.yaml diff --git a/Makefile b/Makefile index f8100d0cb..83e1e3c4c 100644 --- a/Makefile +++ b/Makefile @@ -447,6 +447,7 @@ crd-schema-check: manifests .PHONY: run_with_olm run_with_olm: export CATALOG_IMG=${CATALOG_IMAGE} +run_with_olm: export ENABLE_CYBORG?=false run_with_olm: ## Install nova operator via olm # explicitly to delete any running nova-operator deployments from openstack-operator here as # label selectors can change and installing a service catalog/index like this alongside diff --git a/ci/nova-operator-kuttl/deploy_webhooks.yaml b/ci/nova-operator-kuttl/deploy_webhooks.yaml index 84923debd..9ac4f9351 100644 --- a/ci/nova-operator-kuttl/deploy_webhooks.yaml +++ b/ci/nova-operator-kuttl/deploy_webhooks.yaml @@ -19,3 +19,4 @@ script: make run_with_olm extra_args: CATALOG_IMAGE: "{{ nova_catalog_image }}" + ENABLE_CYBORG: "true" diff --git a/ci/olm.sh b/ci/olm.sh index bf05c96d9..41dba5ded 100644 --- a/ci/olm.sh +++ b/ci/olm.sh @@ -1,3 +1,5 @@ +ENABLE_CYBORG=${ENABLE_CYBORG:-false} + cat > ci/olm.yaml < Date: Wed, 15 Apr 2026 16:26:06 +0200 Subject: [PATCH 7/7] Print Status and Message on Cyborg objects Similar to for CyborgAPI and CyborgConductor and other OpenStack CRDs. Signed-off-by: Alfredo Moralejo --- api/bases/cyborg.openstack.org_cyborgs.yaml | 11 ++++++++++- api/cyborg/v1beta1/cyborg_types.go | 2 ++ config/crd/bases/cyborg.openstack.org_cyborgs.yaml | 11 ++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/api/bases/cyborg.openstack.org_cyborgs.yaml b/api/bases/cyborg.openstack.org_cyborgs.yaml index 9eb8f4851..4baa09b2b 100644 --- a/api/bases/cyborg.openstack.org_cyborgs.yaml +++ b/api/bases/cyborg.openstack.org_cyborgs.yaml @@ -14,7 +14,16 @@ spec: singular: cyborg scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 schema: openAPIV3Schema: description: Cyborg is the Schema for the cyborgs API. diff --git a/api/cyborg/v1beta1/cyborg_types.go b/api/cyborg/v1beta1/cyborg_types.go index 0b26a8ced..5a146611a 100644 --- a/api/cyborg/v1beta1/cyborg_types.go +++ b/api/cyborg/v1beta1/cyborg_types.go @@ -135,6 +135,8 @@ type CyborgStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" // Cyborg is the Schema for the cyborgs API. type Cyborg struct { diff --git a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml index 9eb8f4851..4baa09b2b 100644 --- a/config/crd/bases/cyborg.openstack.org_cyborgs.yaml +++ b/config/crd/bases/cyborg.openstack.org_cyborgs.yaml @@ -14,7 +14,16 @@ spec: singular: cyborg scope: Namespaced versions: - - name: v1beta1 + - additionalPrinterColumns: + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 schema: openAPIV3Schema: description: Cyborg is the Schema for the cyborgs API.