diff --git a/api/bases/ovn.openstack.org_ovncontrollers.yaml b/api/bases/ovn.openstack.org_ovncontrollers.yaml index 85ccbfe9..6d332395 100644 --- a/api/bases/ovn.openstack.org_ovncontrollers.yaml +++ b/api/bases/ovn.openstack.org_ovncontrollers.yaml @@ -150,6 +150,13 @@ spec: description: Image used for the ovn-controller container (will be set to environmental default if empty) type: string + ovnIssuerName: + description: |- + OvnIssuerName - The name of the cert-manager Issuer used to sign + per-node ovn-controller certificates. When set, the controller + creates cert-manager Certificate resources for each node with + CN matching the chassis system-id for OVN RBAC. + type: string ovnLogLevel: default: info description: OVNLogLevel - Set log level off, emer, err, warn, info diff --git a/api/bases/ovn.openstack.org_ovndbclusters.yaml b/api/bases/ovn.openstack.org_ovndbclusters.yaml index 75c312ad..1402c7c3 100644 --- a/api/bases/ovn.openstack.org_ovndbclusters.yaml +++ b/api/bases/ovn.openstack.org_ovndbclusters.yaml @@ -438,6 +438,11 @@ spec: description: InternalDBAddress - DB IP address used by other Pods in the cluster type: string + internalDbAddressRbacFullAccess: + description: |- + InternalDBAddressRbacFullAccess - DB IP address for full-access (non-RBAC) + connections, used by ovn-northd when RBAC is enabled on SB DB + type: string lastAppliedTopology: description: LastAppliedTopology - the last applied Topology properties: diff --git a/api/v1beta1/client.go b/api/v1beta1/client.go index b42f2c64..9250425c 100644 --- a/api/v1beta1/client.go +++ b/api/v1beta1/client.go @@ -72,6 +72,27 @@ func GetOVNController( return nil, nil } +// GetOVNNorthd - return OVNNorthd instance in the given namespace +func GetOVNNorthd( + ctx context.Context, + h *helper.Helper, + namespace string, +) (*OVNNorthd, error) { + ovnNorthdList := &OVNNorthdList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + err := h.GetClient().List(ctx, ovnNorthdList, listOpts...) + if err != nil { + return nil, err + } + if len(ovnNorthdList.Items) > 0 { + return &ovnNorthdList.Items[0], nil + } + + return nil, nil +} + // GetDBClusterByType - return OVNDBCluster for the given dbType func GetDBClusterByType( ctx context.Context, diff --git a/api/v1beta1/ovncontroller_types.go b/api/v1beta1/ovncontroller_types.go index 35afaf06..c4664cc9 100644 --- a/api/v1beta1/ovncontroller_types.go +++ b/api/v1beta1/ovncontroller_types.go @@ -124,6 +124,13 @@ type OVNControllerSpecCore struct { // +kubebuilder:validation:Optional // MetricsTLS - Parameters related to TLS for metrics daemonset MetricsTLS tls.SimpleService `json:"metricsTLS,omitempty"` + + // +kubebuilder:validation:Optional + // OvnIssuerName - The name of the cert-manager Issuer used to sign + // per-node ovn-controller certificates. When set, the controller + // creates cert-manager Certificate resources for each node with + // CN matching the chassis system-id for OVN RBAC. + OvnIssuerName string `json:"ovnIssuerName,omitempty"` } // OVNControllerStatus defines the observed state of OVNController diff --git a/api/v1beta1/ovndbcluster_types.go b/api/v1beta1/ovndbcluster_types.go index 1ec60e35..b3e3c8ca 100644 --- a/api/v1beta1/ovndbcluster_types.go +++ b/api/v1beta1/ovndbcluster_types.go @@ -171,6 +171,10 @@ type OVNDBClusterStatus struct { // InternalDBAddress - DB IP address used by other Pods in the cluster InternalDBAddress string `json:"internalDbAddress,omitempty"` + // InternalDBAddressRbacFullAccess - DB IP address for full-access (non-RBAC) + // connections, used by ovn-northd when RBAC is enabled on SB DB + InternalDBAddressRbacFullAccess string `json:"internalDbAddressRbacFullAccess,omitempty"` + // NetworkAttachments status of the deployment pods NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` @@ -239,6 +243,18 @@ func (instance OVNDBCluster) GetInternalEndpoint() (string, error) { return instance.Status.InternalDBAddress, nil } +// GetInternalEndpointRbacFullAccess - return the full-access (non-RBAC) internal endpoint for SB DB. +// Falls back to GetInternalEndpoint if the DB is not SB or TLS is not enabled. +func (instance OVNDBCluster) GetInternalEndpointRbacFullAccess() (string, error) { + if instance.Spec.DBType != SBDBType || !instance.Spec.TLS.Enabled() { + return instance.GetInternalEndpoint() + } + if instance.Status.InternalDBAddressRbacFullAccess == "" { + return "", fmt.Errorf("internal RBAC full-access DBEndpoint not ready yet for %s", instance.Spec.DBType) + } + return instance.Status.InternalDBAddressRbacFullAccess, nil +} + // GetExternalEndpoint - return the DNS that openstack dnsmasq can resolve func (instance OVNDBCluster) GetExternalEndpoint() (string, error) { if (instance.Spec.NetworkAttachment != "" || instance.Spec.Override.Service != nil) && instance.Status.DBAddress == "" { diff --git a/cmd/main.go b/cmd/main.go index b17be762..d44c6033 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -46,6 +46,7 @@ import ( // +kubebuilder:scaffold:imports + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" @@ -66,6 +67,7 @@ func init() { utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(infranetworkv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(certmgrv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml b/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml index 85ccbfe9..6d332395 100644 --- a/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml +++ b/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml @@ -150,6 +150,13 @@ spec: description: Image used for the ovn-controller container (will be set to environmental default if empty) type: string + ovnIssuerName: + description: |- + OvnIssuerName - The name of the cert-manager Issuer used to sign + per-node ovn-controller certificates. When set, the controller + creates cert-manager Certificate resources for each node with + CN matching the chassis system-id for OVN RBAC. + type: string ovnLogLevel: default: info description: OVNLogLevel - Set log level off, emer, err, warn, info diff --git a/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml b/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml index 75c312ad..1402c7c3 100644 --- a/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml +++ b/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml @@ -438,6 +438,11 @@ spec: description: InternalDBAddress - DB IP address used by other Pods in the cluster type: string + internalDbAddressRbacFullAccess: + description: |- + InternalDBAddressRbacFullAccess - DB IP address for full-access (non-RBAC) + connections, used by ovn-northd when RBAC is enabled on SB DB + type: string lastAppliedTopology: description: LastAppliedTopology - the last applied Topology properties: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4e6e5293..19df9e46 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -65,6 +65,26 @@ rules: - patch - update - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - issuers + verbs: + - get + - list + - watch - apiGroups: - k8s.cni.cncf.io resources: diff --git a/go.mod b/go.mod index 0879a253..c1056ca5 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/openstack-k8s-operators/ovn-operator go 1.24.4 require ( + github.com/cert-manager/cert-manager v1.16.5 github.com/go-logr/logr v1.4.3 github.com/google/uuid v1.6.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 github.com/onsi/ginkgo/v2 v2.28.2 github.com/onsi/gomega v1.40.0 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260506154724-30a976ba8ef0 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 github.com/openstack-k8s-operators/ovn-operator/api v0.0.0-20230418071801-b5843d9e05fb @@ -65,7 +67,7 @@ require ( github.com/prometheus/procfs v0.16.1 // indirect github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.7 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect @@ -103,6 +105,7 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250902184714-7fc278399c7f // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect + sigs.k8s.io/gateway-api v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect diff --git a/go.sum b/go.sum index 6612f306..f74de97d 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cert-manager/cert-manager v1.16.5 h1:XIhKoS4zQV9RHXAkqQW0NLivvoxAnWzbPsy9BG6cPVc= +github.com/cert-manager/cert-manager v1.16.5/go.mod h1:0DwmIGjMOreiP7/6gAqnjaBRJ+yHCfZ5DP7NNqKV+tY= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -118,6 +120,8 @@ github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyU github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260506154724-30a976ba8ef0 h1:Dck0sbRnRRm0JZ9zdbKoJADZqk37pEv7sFUZ8sZwqDA= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:+0PCJgchLUfBcZM3pebsN0pN+d5a22NiC2vy2cpGfRs= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs= github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 h1:mG3QhS/QWv9Y/AkZZ5OzO6hu6+l5oDXnI/Q5ZUbj6Zs= @@ -143,11 +147,15 @@ github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wx github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -254,9 +262,9 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.31.14 h1:xYn/S/WFJsksI7dk/5uBRd3Umm/D8W5g7sRnd4csotA= @@ -281,6 +289,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.7 h1:DLABZfMr20A+AwCZOHhcbcu+TqBXnJZaVBri9K3EO48= sigs.k8s.io/controller-runtime v0.19.7/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM= +sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/internal/common/const.go b/internal/common/const.go index 1158bac4..273a4601 100644 --- a/internal/common/const.go +++ b/internal/common/const.go @@ -14,6 +14,14 @@ const ( // OVNMetricsKeyPath is the path to the metrics private key file OVNMetricsKeyPath string = "/etc/pki/tls/private/ovnmetrics.key" + // OVNRbacCertMountPath is the mount path for the per-node RBAC certificate in config jobs + OVNRbacCertMountPath string = "/tmp/ovn-rbac-cert" + + // OVNControllerCertPath is the destination path for the RBAC client certificate on the host + OVNControllerCertPath string = "/etc/openvswitch/ovn-controller-cert.pem" + // OVNControllerKeyPath is the destination path for the RBAC client key on the host + OVNControllerKeyPath string = "/etc/openvswitch/ovn-controller-privkey.pem" + // MetricsPort is the port used for metrics MetricsPort int32 = 1981 ) diff --git a/internal/controller/ovncontroller_controller.go b/internal/controller/ovncontroller_controller.go index 5eb75860..4dbb993d 100644 --- a/internal/controller/ovncontroller_controller.go +++ b/internal/controller/ovncontroller_controller.go @@ -37,7 +37,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + certmanager "github.com/openstack-k8s-operators/lib-common/modules/certmanager" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" @@ -96,6 +98,9 @@ func (r *OVNControllerReconciler) GetClient() client.Client { // +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid;privileged,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update +// cert-manager permissions for per-node RBAC certificates +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch // Reconcile reconciles the OVNController CR to deploy OVN controller pods func (r *OVNControllerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { @@ -266,7 +271,9 @@ func (r *OVNControllerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&certmgrv1.Certificate{}). Watches(&ovnv1.OVNDBCluster{}, handler.EnqueueRequestsFromMapFunc(ovnv1.OVNCRNamespaceMapFunc(crs, mgr.GetClient()))). + Watches(&ovnv1.OVNNorthd{}, handler.EnqueueRequestsFromMapFunc(ovnv1.OVNCRNamespaceMapFunc(crs, mgr.GetClient()))). Watches( &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), @@ -641,6 +648,26 @@ func (r *OVNControllerReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) } + // When TLS is enabled, RBAC is enforced on the SB database. The RBAC + // permission tables are populated by ovn-northd, so ovn-controller must + // not start until northd is running; otherwise all SB operations are + // denied with "permission error". + if instance.Spec.TLS.Enabled() { + northd, err := ovnv1.GetOVNNorthd(ctx, helper, instance.Namespace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to look up OVNNorthd: %w", err) + } + if northd == nil || northd.Status.ReadyCount == 0 { + Log.Info("OVNNorthd is not ready yet, waiting before deploying OVNController...") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + } + // Define a new DaemonSet object for OVNController dset := daemonset.NewDaemonSet( ovncontroller.CreateOVNDaemonSet(instance, inputHash, ovnServiceLabels, topology), @@ -848,12 +875,70 @@ func (r *OVNControllerReconciler) reconcileNormal(ctx context.Context, instance } // create OVN Config Job - start - // Waits for OVS pods to run the configJob which basically will set config into OVS database - if instance.Status.OVSNumberReady != instance.Status.DesiredNumberScheduled { - Log.Info("OVS DaemonSet not ready yet. Configuration job cannot be started.") - return ctrl.Result{Requeue: true}, nil + + // Create per-node RBAC certificates if configured + nodeSystemIDs := map[string]string{} + if instance.Spec.OvnIssuerName != "" { + ovnPods, podErr := ovncontroller.GetOVNControllerPods(ctx, r.Client, instance) + if podErr != nil { + return ctrl.Result{}, podErr + } + + issuer := &certmgrv1.Issuer{} + if err := helper.GetClient().Get(ctx, types.NamespacedName{ + Name: instance.Spec.OvnIssuerName, + Namespace: instance.Namespace, + }, issuer); err != nil { + return ctrl.Result{}, fmt.Errorf("error getting issuer %s/%s - %w", instance.Spec.OvnIssuerName, instance.Namespace, err) + } + + durationString := certmanager.CertDefaultDuration + if d, ok := issuer.Annotations[certmanager.CertDurationAnnotation]; ok && d != "" { + durationString = d + } + duration, err := time.ParseDuration(durationString) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error parsing certificate duration %s - %w", durationString, err) + } + + var renewBefore *time.Duration + if r, ok := issuer.Annotations[certmanager.CertRenewBeforeAnnotation]; ok && r != "" { + rb, err := time.ParseDuration(r) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error parsing certificate renewBefore %s - %w", r, err) + } + renewBefore = &rb + } + + for _, pod := range ovnPods.Items { + nodeName := pod.Spec.NodeName + if nodeName == "" { + continue + } + systemID := ovncontroller.ComputeSystemID(nodeName) + certName := ovncontroller.RbacCertName(nodeName) + nodeSystemIDs[nodeName] = systemID + + _, certResult, certErr := certmanager.EnsureCert(ctx, helper, certmanager.CertificateRequest{ + IssuerName: instance.Spec.OvnIssuerName, + CertName: certName, + CommonName: &systemID, + Labels: ovnServiceLabels, + Usages: []certmgrv1.KeyUsage{certmgrv1.UsageClientAuth, certmgrv1.UsageDigitalSignature}, + Duration: &duration, + RenewBefore: renewBefore, + }, instance) + if certErr != nil { + return certResult, certErr + } + if (certResult != ctrl.Result{}) { + Log.Info("Waiting for RBAC certificate", "node", nodeName, "cert", certName) + return certResult, nil + } + } } - jobsDef, err := ovncontroller.ConfigJob(ctx, r.Client, instance, sbCluster, ovnServiceLabels) + + jobsDef, err := ovncontroller.ConfigJob(ctx, r.Client, instance, sbCluster, ovnServiceLabels, nodeSystemIDs) if err != nil { Log.Error(err, "Failed to create OVN controller configuration Job") return ctrl.Result{}, err diff --git a/internal/controller/ovndbcluster_controller.go b/internal/controller/ovndbcluster_controller.go index 69f6ce8b..dae9aded 100644 --- a/internal/controller/ovndbcluster_controller.go +++ b/internal/controller/ovndbcluster_controller.go @@ -740,6 +740,7 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance * instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) internalDbAddress := []string{} + internalDbAddressRbacFullAccess := []string{} var svcPort int32 scheme := "tcp" if instance.Spec.TLS.Enabled() { @@ -755,6 +756,9 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance * // TODO: Watch operator.openshift.io resource once cluster domain is customizable clusterDomain := clusterdns.GetDNSClusterDomain() internalDbAddress = append(internalDbAddress, fmt.Sprintf("%s:%s.%s.svc.%s:%d", scheme, svc.Name, svc.Namespace, clusterDomain, svcPort)) + if instance.Spec.DBType == ovnv1.SBDBType && instance.Spec.TLS.Enabled() { + internalDbAddressRbacFullAccess = append(internalDbAddressRbacFullAccess, fmt.Sprintf("%s:%s.%s.svc.%s:%d", scheme, svc.Name, svc.Namespace, clusterDomain, ovndbcluster.DbPortSBRBACFullAccess)) + } } // Note setting this to the singular headless service address (e.g ssl:ovsdbserver-sb...) "works" but will not @@ -764,6 +768,9 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance * // Set DB Address instance.Status.InternalDBAddress = strings.Join(internalDbAddress, ",") + if len(internalDbAddressRbacFullAccess) > 0 { + instance.Status.InternalDBAddressRbacFullAccess = strings.Join(internalDbAddressRbacFullAccess, ",") + } if instance.Spec.DBType == ovnv1.SBDBType && (instance.Spec.NetworkAttachment != "" || instance.Spec.Override.Service != nil) { // This config map will populate the sb db address to edpm, can't use the nb // If there's no networkAttachments the configMap is not needed @@ -1290,6 +1297,11 @@ func (r *OVNDBClusterReconciler) generateServiceConfigMaps( templateParameters["OVN_METRICS_CERT_PATH"] = ovn_common.OVNMetricsCertPath templateParameters["OVN_METRICS_KEY_PATH"] = ovn_common.OVNMetricsKeyPath templateParameters["OVN_RUNDIR"] = "/etc/ovn" + if instance.Spec.DBType == ovnv1.SBDBType && instance.Spec.TLS.Enabled() { + templateParameters["DB_PORT_FULL_ACCESS"] = ovndbcluster.DbPortSBRBACFullAccess + } else { + templateParameters["DB_PORT_FULL_ACCESS"] = templateParameters["DB_PORT"] + } cms := []util.Template{ // ScriptsConfigMap diff --git a/internal/controller/ovnnorthd_controller.go b/internal/controller/ovnnorthd_controller.go index 8651514c..5ef530e5 100644 --- a/internal/controller/ovnnorthd_controller.go +++ b/internal/controller/ovnnorthd_controller.go @@ -642,7 +642,7 @@ func getInternalEndpoint( if err != nil { return "", err } - internalEndpoint, err := cluster.GetInternalEndpoint() + internalEndpoint, err := cluster.GetInternalEndpointRbacFullAccess() if err != nil { return "", err } diff --git a/internal/ovncontroller/configjob.go b/internal/ovncontroller/configjob.go index 268eda5b..b4d4e1c5 100644 --- a/internal/ovncontroller/configjob.go +++ b/internal/ovncontroller/configjob.go @@ -19,6 +19,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" + ovn_common "github.com/openstack-k8s-operators/ovn-operator/internal/common" "sigs.k8s.io/controller-runtime/pkg/client" batchv1 "k8s.io/api/batch/v1" @@ -26,13 +27,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// ConfigJob - prepare job to configure ovn-controller +// ConfigJob - prepare job to configure ovn-controller. +// nodeSystemIDs maps node names to their pre-computed system-id UUIDs. +// When non-empty, the per-node RBAC cert Secret is mounted and SYSTEM_ID is +// passed as an environment variable. func ConfigJob( ctx context.Context, k8sClient client.Client, instance *ovnv1.OVNController, sbCluster *ovnv1.OVNDBCluster, labels map[string]string, + nodeSystemIDs map[string]string, ) ([]*batchv1.Job, error) { var jobs []*batchv1.Job @@ -42,7 +47,7 @@ func ConfigJob( // configuration job automatically right after it will be finished jobTTLAfterFinished := int32(0) - ovnPods, err := getOVNControllerPods( + ovnPods, err := GetOVNControllerPods( ctx, k8sClient, instance, @@ -74,6 +79,40 @@ func ConfigJob( "/usr/local/bin/additional-scripts/configure-ovn.sh", } + volumes := GetOVNControllerVolumes(instance.Name, instance.Namespace, true) + volumeMounts := GetOVNControllerVolumeMounts(true) + + // Per-job env vars start from a copy of the shared ones + jobEnvVars := make(map[string]env.Setter, len(envVars)) + for k, v := range envVars { + jobEnvVars[k] = v + } + + nodeName := ovnPod.Spec.NodeName + if systemID, ok := nodeSystemIDs[nodeName]; ok { + jobEnvVars["SYSTEM_ID"] = env.SetValue(systemID) + + certSecretName := RbacCertSecretName(nodeName) + volumes = append(volumes, corev1.Volume{ + Name: "ovn-rbac-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: certSecretName, + }, + }, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "ovn-rbac-cert", + MountPath: ovn_common.OVNRbacCertMountPath, + ReadOnly: true, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "etc-ovs", + MountPath: "/etc/openvswitch", + ReadOnly: false, + }) + } + jobs = append( jobs, &batchv1.Job{ @@ -100,14 +139,13 @@ func ConfigJob( RunAsUser: &runAsUser, Privileged: &privileged, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: GetOVNControllerVolumeMounts(true), + Env: env.MergeEnvs([]corev1.EnvVar{}, jobEnvVars), + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, }, }, - Volumes: GetOVNControllerVolumes(instance.Name, instance.Namespace, true), - NodeName: ovnPod.Spec.NodeName, - // ^ NodeSelector not required + Volumes: volumes, + NodeName: nodeName, }, }, }, diff --git a/internal/ovncontroller/daemonset.go b/internal/ovncontroller/daemonset.go index d39c6518..5b640781 100644 --- a/internal/ovncontroller/daemonset.go +++ b/internal/ovncontroller/daemonset.go @@ -14,6 +14,7 @@ package ovncontroller import ( "fmt" + "strings" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" @@ -38,7 +39,7 @@ func CreateOVNDaemonSet( volumes := GetOVNControllerVolumes(instance.Name, instance.Namespace, false) mounts := GetOVNControllerVolumeMounts(false) - cmd := []string{ + ovnCmd := []string{ "ovn-controller", "--pidfile", "unix:/run/openvswitch/db.sock", } @@ -59,13 +60,32 @@ func CreateOVNDaemonSet( mounts = append(mounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) } - cmd = append(cmd, []string{ - fmt.Sprintf("--certificate=%s", ovn_common.OVNDbCertPath), - fmt.Sprintf("--private-key=%s", ovn_common.OVNDbKeyPath), + mounts = append(mounts, corev1.VolumeMount{ + Name: "etc-ovs", + MountPath: "/etc/openvswitch", + ReadOnly: true, + }) + ovnCmd = append(ovnCmd, []string{ + fmt.Sprintf("--certificate=%s", ovn_common.OVNControllerCertPath), + fmt.Sprintf("--private-key=%s", ovn_common.OVNControllerKeyPath), fmt.Sprintf("--ca-cert=%s", ovn_common.OVNDbCaCertPath), }...) } + // When RBAC is configured, the config job deploys a per-node client + // certificate to /etc/openvswitch/ after OVS is ready. Wait for the + // cert file before starting ovn-controller so it doesn't connect to + // the SB DB without client authentication. + var cmd []string + if instance.Spec.TLS.Enabled() && instance.Spec.OvnIssuerName != "" { + cmd = []string{ + "/bin/bash", "-c", + "source /usr/local/bin/container-scripts/functions && wait_for_rbac_cert && exec " + strings.Join(ovnCmd, " "), + } + } else { + cmd = ovnCmd + } + ovnControllerLivenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, diff --git a/internal/ovncontroller/utils.go b/internal/ovncontroller/utils.go index c378f350..c4bd51df 100644 --- a/internal/ovncontroller/utils.go +++ b/internal/ovncontroller/utils.go @@ -18,12 +18,29 @@ import ( "sort" "strings" + "github.com/google/uuid" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) +// ComputeSystemID derives a deterministic UUID5 system-id from a node name, +// matching the convention used by OVN for chassis identification. +func ComputeSystemID(nodeName string) string { + return uuid.NewSHA1(uuid.NameSpaceDNS, []byte(nodeName)).String() +} + +// RbacCertName returns the cert-manager Certificate CR name for a given node +func RbacCertName(nodeName string) string { + return fmt.Sprintf("ovn-controller-cert-%s", nodeName) +} + +// RbacCertSecretName returns the Secret name created by cert-manager for a given node's RBAC certificate +func RbacCertSecretName(nodeName string) string { + return "cert-" + RbacCertName(nodeName) +} + func getPhysicalNetworks( instance *ovnv1.OVNController, ) string { @@ -37,7 +54,8 @@ func getPhysicalNetworks( return strings.Join(nicMappings, " ") } -func getOVNControllerPods( +// GetOVNControllerPods returns list of the pods running ovn-controller +func GetOVNControllerPods( ctx context.Context, k8sClient client.Client, instance *ovnv1.OVNController, diff --git a/internal/ovndbcluster/const.go b/internal/ovndbcluster/const.go index 99aaaad7..918cb28a 100644 --- a/internal/ovndbcluster/const.go +++ b/internal/ovndbcluster/const.go @@ -9,6 +9,8 @@ const ( // DbPortSB is the port number for the OVN Southbound database DbPortSB int32 = 6642 + // DbPortSBRBACFullAccess is the port for full-access (non-RBAC) SB connections (used by ovn-northd) + DbPortSBRBACFullAccess int32 = 16642 // RaftPortSB is the port number for the OVN Southbound database Raft protocol RaftPortSB int32 = 6644 ) diff --git a/internal/ovndbcluster/service.go b/internal/ovndbcluster/service.go index c52b73d7..2959ede5 100644 --- a/internal/ovndbcluster/service.go +++ b/internal/ovndbcluster/service.go @@ -38,6 +38,15 @@ func Service( }, } + // Add full-access (non-RBAC) port for SB when TLS is enabled + if instance.Spec.DBType == ovnv1.SBDBType && instance.Spec.TLS.Enabled() { + ports = append(ports, corev1.ServicePort{ + Name: dbPortName + "-full-access", + Port: DbPortSBRBACFullAccess, + Protocol: corev1.ProtocolTCP, + }) + } + // Add metrics port if metrics are enabled and exporter image is specified if instance.Spec.ExporterImage != "" && (instance.Spec.MetricsEnabled == nil || *instance.Spec.MetricsEnabled) { ports = append(ports, corev1.ServicePort{ diff --git a/internal/ovndbcluster/statefulset.go b/internal/ovndbcluster/statefulset.go index bd0c846f..9bcd6437 100644 --- a/internal/ovndbcluster/statefulset.go +++ b/internal/ovndbcluster/statefulset.go @@ -133,6 +133,7 @@ func StatefulSet( } volumes = append(volumes, svc.CreateVolume(serviceName)) volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(serviceName)...) + } // NOTE(ihar) ovndb pods leave the raft cluster on delete; it's important diff --git a/templates/ovncontroller/bin/functions b/templates/ovncontroller/bin/functions index c13eb79a..b01850f0 100755 --- a/templates/ovncontroller/bin/functions +++ b/templates/ovncontroller/bin/functions @@ -50,6 +50,38 @@ function wait_for_ovsdb_server { done } +function wait_for_rbac_cert { + local cert_path="/etc/openvswitch/ovn-controller-cert.pem" + local key_path="/etc/openvswitch/ovn-controller-privkey.pem" + local ca_path="/etc/pki/tls/certs/ovndbca.crt" + + while [ ! -f "$cert_path" ]; do + echo "RBAC certificate not yet available at $cert_path. Waiting..." + sleep 2 + done + + # Wait for ovn-northd to populate RBAC permission tables in the SB DB + # before starting ovn-controller; otherwise all SB operations are denied. + # Check for actual RBAC_Permission entries (not just RBAC_Role) because + # northd may create the role before all permission entries are committed. + local ovn_remote + while true; do + ovn_remote=$(ovs-vsctl --if-exists get open . external-ids:ovn-remote 2>/dev/null | tr -d '"') + if [ -n "$ovn_remote" ] && \ + ovn-sbctl --db="$ovn_remote" \ + --timeout=5 \ + --certificate="$cert_path" \ + --private-key="$key_path" \ + --ca-cert="$ca_path" \ + --no-leader-only \ + find RBAC_Role name=ovn-controller 2>/dev/null | grep -q ovn-controller; then + break + fi + echo "RBAC tables not yet populated by ovn-northd. Waiting..." + sleep 2 + done +} + # configure external-ids in OVS function configure_external_ids { ovs-vsctl set open . external-ids:ovn-bridge=${OVNBridge} diff --git a/templates/ovncontroller/bin/init.sh b/templates/ovncontroller/bin/init.sh index bec5945d..f611ee28 100755 --- a/templates/ovncontroller/bin/init.sh +++ b/templates/ovncontroller/bin/init.sh @@ -16,10 +16,27 @@ source $(dirname $0)/functions +# Install RBAC client certificate if available (mounted from cert-manager Secret). +# This is a pure file copy with no OVS dependency, so do it before +# wait_for_ovsdb_server to unblock ovn-controller's wait_for_rbac_cert +# as early as possible. +if [ -f "/tmp/ovn-rbac-cert/tls.crt" ]; then + mkdir -p /etc/openvswitch + cp /tmp/ovn-rbac-cert/tls.crt /etc/openvswitch/ovn-controller-cert.pem + cp /tmp/ovn-rbac-cert/tls.key /etc/openvswitch/ovn-controller-privkey.pem + chmod 0644 /etc/openvswitch/ovn-controller-cert.pem + chmod 0600 /etc/openvswitch/ovn-controller-privkey.pem +fi + wait_for_ovsdb_server # From now on, we should exit immediatelly when any command exits with non-zero status set -ex +# Set system-id if provided by the controller +if [ -n "${SYSTEM_ID}" ]; then + ovs-vsctl set open . external-ids:system-id=${SYSTEM_ID} +fi + configure_external_ids configure_physical_networks diff --git a/templates/ovncontroller/config/configure-ovn.sh b/templates/ovncontroller/config/configure-ovn.sh index a0806564..d8ddc9e6 100755 --- a/templates/ovncontroller/config/configure-ovn.sh +++ b/templates/ovncontroller/config/configure-ovn.sh @@ -37,7 +37,9 @@ function configure_log_level { done ctl_path=$(find /var/run/ovn/ -maxdepth 1 -name "ovn-controller.*.ctl") - ovn-appctl -t "$ctl_path" vlog/set ${OVNLogLevel} + if [ -n "$ctl_path" ]; then + ovn-appctl -t "$ctl_path" vlog/set ${OVNLogLevel} + fi } diff --git a/templates/ovndbcluster/bin/setup.sh b/templates/ovndbcluster/bin/setup.sh index 364f053b..57dcb7b4 100755 --- a/templates/ovndbcluster/bin/setup.sh +++ b/templates/ovndbcluster/bin/setup.sh @@ -17,8 +17,10 @@ set -ex source $(dirname $0)/functions DB_PORT="{{ .DB_PORT }}" +DB_PORT_FULL_ACCESS="{{ .DB_PORT_FULL_ACCESS }}" {{- if .TLS }} DB_SCHEME="pssl" +SSL_CA_CERT={{.OVNDB_CACERT_PATH}} {{- else }} DB_SCHEME="ptcp" {{- end }} @@ -56,7 +58,7 @@ set "$@" --db-${DB_TYPE}-port=${DB_PORT} {{- if .TLS }} set "$@" --ovn-${DB_TYPE}-db-ssl-key={{.OVNDB_KEY_PATH}} set "$@" --ovn-${DB_TYPE}-db-ssl-cert={{.OVNDB_CERT_PATH}} -set "$@" --ovn-${DB_TYPE}-db-ssl-ca-cert={{.OVNDB_CACERT_PATH}} +set "$@" --ovn-${DB_TYPE}-db-ssl-ca-cert=${SSL_CA_CERT} set "$@" --db-${DB_TYPE}-cluster-local-proto=ssl set "$@" --db-${DB_TYPE}-cluster-remote-proto=ssl set "$@" --db-${DB_TYPE}-create-insecure-remote=no @@ -122,15 +124,26 @@ if [[ "$(hostname)" == "{{ .SERVICE_NAME }}-0" ]]; then export OVN_${DB_TYPE^^}_DAEMON=$(${CTLCMD} --pidfile --detach) {{- if .TLS }} - ${CTLCMD} set-ssl {{.OVNDB_KEY_PATH}} {{.OVNDB_CERT_PATH}} {{.OVNDB_CACERT_PATH}} + ${CTLCMD} set-ssl {{.OVNDB_KEY_PATH}} {{.OVNDB_CERT_PATH}} ${SSL_CA_CERT} + if [[ "${DB_TYPE}" == "sb" ]]; then + # Use RBAC for the connections of the ovn-controller to SB + ${CTLCMD} set-connection role=ovn-controller ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} + # In this case, Northd needs to have full access to the DB so there need + # to be another connection defined for it + # TODO(slaweq): port has to be also set in Northd + ${CTLCMD} -- --id=@conn_uuid create Connection target="${DB_SCHEME}\:${DB_PORT_FULL_ACCESS}" -- add SB_Global . connections @conn_uuid + else + # No RBAC for connecting to the Northbound DB so only one connection + # defined is fine + ${CTLCMD} set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} + fi + {{- else }} ${CTLCMD} del-ssl -{{- end }} - CURRENT_PROBE="$(${CTLCMD} get connection . inactivity_probe || echo [])" - if [ "$CURRENT_PROBE" = "[]" ]; then - CURRENT_PROBE=60000 - fi + # If TLS is disabled, RBAC can't be used, so one connection defined is enough ${CTLCMD} set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} +{{- end }} + # OVN does not support setting inactivity-probe through --remote cli arg so # we have to set it after database is up. # @@ -144,8 +157,14 @@ if [[ "$(hostname)" == "{{ .SERVICE_NAME }}-0" ]]; then # TODO: Consider migrating inactivity probe setting to config files when # we update to ovs 3.3. See --config-file in ovsdb-server(1) for more # details. - while [ "$(${CTLCMD} get connection . inactivity_probe)" != "${CURRENT_PROBE}" ]; do - ${CTLCMD} --inactivity-probe="${CURRENT_PROBE}" set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} + for connection_id in $(${CTLCMD} -f csv --no-headings --columns=_uuid list connection); do + CURRENT_PROBE="$(${CTLCMD} get connection $connection_id inactivity_probe || echo [])" + if [ "$CURRENT_PROBE" = "[]" ]; then + CURRENT_PROBE=60000 + fi + while [ "$(${CTLCMD} get connection $connection_id inactivity_probe)" != "${CURRENT_PROBE}" ]; do + ${CTLCMD} set connection $connection_id inactivity_probe=${CURRENT_PROBE} + done done ${CTLCMD} list connection diff --git a/templates/ovndbcluster/config/runtime-config.sh b/templates/ovndbcluster/config/runtime-config.sh index 3c8726f9..6c974ba1 100755 --- a/templates/ovndbcluster/config/runtime-config.sh +++ b/templates/ovndbcluster/config/runtime-config.sh @@ -74,11 +74,17 @@ if echo "$CONFIG_FLAGS" | grep -q "INACTIVITY_PROBE"; then if [[ "$(hostname)" == "${SERVICE_NAME}-0-config-"* ]]; then echo "Configuring inactivity probe to ${INACTIVITY_PROBE}ms on pod-0..." - # Simple approach - connect directly to running database (OVN_RUNDIR already set) - if ovn-${DB_TYPE}ctl --no-leader-only set connection . inactivity_probe="$INACTIVITY_PROBE"; then - echo "✓ Successfully configured inactivity probe" - else - echo "✗ Failed to configure inactivity probe" + # Set inactivity probe on all connections + FAILED=0 + for CONN_ID in $(ovn-${DB_TYPE}ctl --no-leader-only --format=table --no-headings --columns=_uuid list connection); do + if ovn-${DB_TYPE}ctl --no-leader-only set connection ${CONN_ID} inactivity_probe="$INACTIVITY_PROBE"; then + echo "✓ Successfully configured inactivity probe on connection ${CONN_ID}" + else + echo "✗ Failed to configure inactivity probe on connection ${CONN_ID}" + FAILED=1 + fi + done + if [ "$FAILED" -eq 1 ]; then exit 1 fi else diff --git a/test/functional/base_test.go b/test/functional/base_test.go index 8e7b56b5..9d9014be 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -22,6 +22,7 @@ import ( "strings" "time" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" . "github.com/onsi/gomega" //revive:disable:dot-imports appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -70,6 +71,16 @@ func OVNNorthdConditionGetter(name types.NamespacedName) condition.Conditions { return instance.Status.Conditions } +func CreateReadyOVNNorthd(namespace string, spec ovnv1.OVNNorthdSpec) types.NamespacedName { + name := ovn.CreateOVNNorthd(nil, namespace, spec) + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + th.SimulateStatefulSetReplicaReady(statefulSetName) + return name +} + func GetDefaultOVNDBClusterSpec() ovnv1.OVNDBClusterSpec { return ovnv1.OVNDBClusterSpec{ OVNDBClusterSpecCore: ovnv1.OVNDBClusterSpecCore{ @@ -179,6 +190,73 @@ func DeleteOVNDBClusters(names []types.NamespacedName) { } } +// CreateTLSOVNDBClusters Creates NB and SB OVNDBClusters with TLS enabled. +// Caller must create TLS secrets (CABundleSecretName, OvnDbCertSecretName) first. +func CreateTLSOVNDBClusters(namespace string, nad map[string][]string, replicas int32) []types.NamespacedName { + dbs := []types.NamespacedName{} + for _, db := range []string{ovnv1.NBDBType, ovnv1.SBDBType} { + spec := GetTLSOVNDBClusterSpec() + stringNad := "" + Expect(len(nad)).Should(BeNumerically("<=", 1)) + for k := range nad { + if strings.Contains(k, "/") { + stringNad = strings.Split(k, "/")[1] + } + } + if len(nad) != 0 { + Expect(stringNad).ToNot(Equal("")) + } + spec.DBType = db + spec.NetworkAttachment = stringNad + spec.Replicas = &replicas + instance := CreateOVNDBCluster(namespace, spec) + instanceName := types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} + + dbName := "nb" + if db == ovnv1.SBDBType { + dbName = "sb" + } + statefulSetName := types.NamespacedName{ + Namespace: instance.GetNamespace(), + Name: "ovsdbserver-" + dbName, + } + th.SimulateStatefulSetReplicaReadyWithPods( + statefulSetName, + nad, + ) + Eventually(func(g Gomega) { + ovndbcluster := ovn.GetOVNDBCluster(instanceName) + endpoint := "" + if len(nad) == 0 { + endpoint, _ = ovndbcluster.GetInternalEndpoint() + } else { + endpoint, _ = ovndbcluster.GetExternalEndpoint() + } + g.Expect(endpoint).ToNot(BeEmpty()) + }).Should(Succeed()) + + if db == ovnv1.SBDBType { + Eventually(func(g Gomega) { + ovndbcluster := ovn.GetOVNDBCluster(instanceName) + g.Expect(ovndbcluster.Status.InternalDBAddressRbacFullAccess).ToNot(BeEmpty()) + }).Should(Succeed()) + } + + dbs = append(dbs, instanceName) + } + + logger.Info("TLS OVNDBClusters created", "OVNDBCluster", dbs) + return dbs +} + +func GetCertManagerCert(name types.NamespacedName) *certmgrv1.Certificate { + cert := &certmgrv1.Certificate{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, cert)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + return cert +} + // GetOVNDBCluster Get OVNDBCluster func GetOVNDBCluster(name types.NamespacedName) *ovnv1.OVNDBCluster { return ovn.GetOVNDBCluster(name) @@ -229,6 +307,16 @@ func GetTLSOVNControllerSpec() ovnv1.OVNControllerSpec { return spec } +func GetTLSRbacOVNControllerSpec() ovnv1.OVNControllerSpec { + spec := GetTLSOVNControllerSpec() + spec.OvnIssuerName = OvnIssuerName + return spec +} + +func GetTLSRbacOVNDBClusterSpec() ovnv1.OVNDBClusterSpec { + return GetTLSOVNDBClusterSpec() +} + func CreateOVNController(namespace string, spec ovnv1.OVNControllerSpec) client.Object { name := ovn.CreateOVNController(nil, namespace, spec) diff --git a/test/functional/ovncontroller_controller_test.go b/test/functional/ovncontroller_controller_test.go index 5ff8c667..3d6d882c 100644 --- a/test/functional/ovncontroller_controller_test.go +++ b/test/functional/ovncontroller_controller_test.go @@ -26,15 +26,18 @@ import ( //revive:disable-next-line:dot-imports . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" ovn_common "github.com/openstack-k8s-operators/ovn-operator/internal/common" + "github.com/openstack-k8s-operators/ovn-operator/internal/ovncontroller" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -911,6 +914,8 @@ var _ = Describe("OVNController controller", func() { BeforeEach(func() { dbs := CreateOVNDBClusters(namespace, map[string][]string{}, 1) DeferCleanup(DeleteOVNDBClusters, dbs) + northdName := CreateReadyOVNNorthd(namespace, GetDefaultOVNNorthdSpec()) + DeferCleanup(th.DeleteInstance, GetOVNNorthd(northdName)) instance := CreateOVNController(namespace, GetTLSOVNControllerSpec()) DeferCleanup(th.DeleteInstance, instance) @@ -1045,10 +1050,13 @@ var _ = Describe("OVNController controller", func() { th.AssertVolumeMountExists("ovn-controller-tls-certs", "tls.crt", svcC.VolumeMounts) th.AssertVolumeMountExists("ovn-controller-tls-certs", "ca.crt", svcC.VolumeMounts) - // check cli args + // check etc-ovs volume mount for RBAC certs + th.AssertVolumeMountExists("etc-ovs", "", svcC.VolumeMounts) + + // check cli args use RBAC cert paths Expect(svcC.Command).To(And( - ContainElement(ContainSubstring(fmt.Sprintf("--private-key=%s", ovn_common.OVNDbKeyPath))), - ContainElement(ContainSubstring(fmt.Sprintf("--certificate=%s", ovn_common.OVNDbCertPath))), + ContainElement(ContainSubstring(fmt.Sprintf("--private-key=%s", ovn_common.OVNControllerKeyPath))), + ContainElement(ContainSubstring(fmt.Sprintf("--certificate=%s", ovn_common.OVNControllerCertPath))), ContainElement(ContainSubstring(fmt.Sprintf("--ca-cert=%s", ovn_common.OVNDbCaCertPath))), )) @@ -1165,6 +1173,196 @@ var _ = Describe("OVNController controller", func() { }) }) + When("OVNController is created with TLS and RBAC", func() { + var ovnControllerName types.NamespacedName + var dbs []types.NamespacedName + + BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + issuer := &certmgrv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: OvnIssuerName, + Namespace: namespace, + }, + Spec: certmgrv1.IssuerSpec{ + IssuerConfig: certmgrv1.IssuerConfig{ + SelfSigned: &certmgrv1.SelfSignedIssuer{}, + }, + }, + } + Expect(k8sClient.Create(ctx, issuer)).To(Succeed()) + DeferCleanup(k8sClient.Delete, ctx, issuer) + + dbs = CreateTLSOVNDBClusters(namespace, map[string][]string{}, 1) + DeferCleanup(DeleteOVNDBClusters, dbs) + northdName := CreateReadyOVNNorthd(namespace, GetDefaultOVNNorthdSpec()) + DeferCleanup(th.DeleteInstance, GetOVNNorthd(northdName)) + + instance := CreateOVNController(namespace, GetTLSRbacOVNControllerSpec()) + ovnControllerName = types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} + DeferCleanup(th.DeleteInstance, instance) + }) + + It("creates cert-manager Certificate CRs for each node", func() { + daemonSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetName, + map[string][]string{}, + ) + daemonSetNameOVS := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller-ovs", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetNameOVS, + map[string][]string{}, + ) + + nodeName := daemonSetName.Name + certName := ovncontroller.RbacCertName(nodeName) + certNN := types.NamespacedName{ + Name: certName, + Namespace: namespace, + } + + cert := GetCertManagerCert(certNN) + expectedSystemID := ovncontroller.ComputeSystemID(nodeName) + Expect(cert.Spec.CommonName).To(Equal(expectedSystemID)) + Expect(cert.Spec.IssuerRef.Name).To(Equal(OvnIssuerName)) + Expect(cert.Spec.IssuerRef.Kind).To(Equal("Issuer")) + Expect(cert.Spec.Usages).To(ContainElement(certmgrv1.UsageClientAuth)) + }) + + It("creates config jobs with RBAC cert volumes and SYSTEM_ID env var", func() { + daemonSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetName, + map[string][]string{}, + ) + daemonSetNameOVS := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller-ovs", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetNameOVS, + map[string][]string{}, + ) + + nodeName := daemonSetName.Name + certName := ovncontroller.RbacCertName(nodeName) + certSecretName := ovncontroller.RbacCertSecretName(nodeName) + + // Wait for the Certificate CR to be created, then simulate + // cert-manager by creating the cert Secret + GetCertManagerCert(types.NamespacedName{ + Name: certName, + Namespace: namespace, + }) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: certSecretName, + Namespace: namespace, + })) + + // The config job should now be created with RBAC volumes + configJobName := types.NamespacedName{ + Namespace: ovnControllerName.Namespace, + Name: daemonSetName.Name + "-config", + } + Eventually(func(g Gomega) { + job := th.GetJob(configJobName) + g.Expect(job).ToNot(BeNil()) + + container := job.Spec.Template.Spec.Containers[0] + + // Check SYSTEM_ID env var + systemIDValue := "" + for _, e := range container.Env { + if e.Name == "SYSTEM_ID" { + systemIDValue = e.Value + } + } + expectedSystemID := ovncontroller.ComputeSystemID(nodeName) + g.Expect(systemIDValue).To(Equal(expectedSystemID)) + + // Check RBAC cert volume mount + th.AssertVolumeMountExists("ovn-rbac-cert", "", container.VolumeMounts) + + // Check etc-ovs volume mount (writable for cert install) + hasEtcOvs := false + for _, vm := range container.VolumeMounts { + if vm.Name == "etc-ovs" && vm.MountPath == "/etc/openvswitch" { + hasEtcOvs = true + } + } + g.Expect(hasEtcOvs).To(BeTrue()) + + // Check RBAC cert volume references the cert secret + hasRbacVolume := false + for _, v := range job.Spec.Template.Spec.Volumes { + if v.Name == "ovn-rbac-cert" && v.Secret != nil && v.Secret.SecretName == certSecretName { + hasRbacVolume = true + } + } + g.Expect(hasRbacVolume).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("config job is not created without RBAC cert secret", func() { + daemonSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetName, + map[string][]string{}, + ) + daemonSetNameOVS := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller-ovs", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetNameOVS, + map[string][]string{}, + ) + + nodeName := daemonSetName.Name + certName := ovncontroller.RbacCertName(nodeName) + + // Wait for the Certificate CR to be created, confirming + // the controller reached the RBAC cert step + GetCertManagerCert(types.NamespacedName{ + Name: certName, + Namespace: namespace, + }) + + // Without creating the cert secret, the config job should + // not appear because the controller is waiting for the cert + configJobName := types.NamespacedName{ + Namespace: ovnControllerName.Namespace, + Name: daemonSetName.Name + "-config", + } + Consistently(func(g Gomega) { + job := &batchv1.Job{} + err := k8sClient.Get(ctx, configJobName, job) + g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + }) + When("OVNController is created with nodeSelector", func() { var ovnControllerName types.NamespacedName var daemonSetName types.NamespacedName diff --git a/test/functional/ovndbcluster_controller_test.go b/test/functional/ovndbcluster_controller_test.go index 96049987..a16c15b3 100644 --- a/test/functional/ovndbcluster_controller_test.go +++ b/test/functional/ovndbcluster_controller_test.go @@ -32,6 +32,7 @@ import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" ovn_common "github.com/openstack-k8s-operators/ovn-operator/internal/common" + "github.com/openstack-k8s-operators/ovn-operator/internal/ovndbcluster" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -1256,9 +1257,11 @@ var _ = Describe("OVNDBCluster controller", func() { When("OVNDBCluster is created with TLS", func() { var OVNDBClusterName types.NamespacedName BeforeEach(func() { - spec := GetTLSOVNDBClusterSpec() + spec := GetTLSRbacOVNDBClusterSpec() spec.NetworkAttachment = "internalapi" spec.DBType = ovnv1.SBDBType + metricsEnabled := false + spec.MetricsEnabled = &metricsEnabled instance := CreateOVNDBCluster(namespace, spec) OVNDBClusterName = types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} DeferCleanup(th.DeleteInstance, instance) @@ -1306,7 +1309,7 @@ var _ = Describe("OVNDBCluster controller", func() { ) }) - It("creates a Statefulset with TLS certs attached", func() { + It("creates a Statefulset with TLS and RBAC certs attached", func() { DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ Name: CABundleSecretName, Namespace: namespace, @@ -1463,6 +1466,100 @@ var _ = Describe("OVNDBCluster controller", func() { }, timeout, interval).Should(Succeed()) }) + It("creates services with full-access port for SB", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + Eventually(func(g Gomega) { + serviceListWithClusterType := GetServicesListWithLabel(namespace, map[string]string{"type": "cluster"}) + g.Expect(serviceListWithClusterType.Items).ToNot(BeEmpty()) + svc := serviceListWithClusterType.Items[0] + + hasFullAccessPort := false + for _, port := range svc.Spec.Ports { + if port.Name == "south-full-access" && port.Port == ovndbcluster.DbPortSBRBACFullAccess { + hasFullAccessPort = true + } + } + g.Expect(hasFullAccessPort).To(BeTrue(), + "SB service should have full-access port %d", ovndbcluster.DbPortSBRBACFullAccess) + }, timeout, interval).Should(Succeed()) + }) + + It("sets InternalDBAddressRbacFullAccess in status", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + Eventually(func(g Gomega) { + dbCluster := GetOVNDBCluster(OVNDBClusterName) + g.Expect(dbCluster.Status.InternalDBAddressRbacFullAccess).ToNot(BeEmpty()) + g.Expect(dbCluster.Status.InternalDBAddressRbacFullAccess).To( + ContainSubstring(fmt.Sprintf(":%d", ovndbcluster.DbPortSBRBACFullAccess))) + g.Expect(dbCluster.Status.InternalDBAddressRbacFullAccess).To(HavePrefix("ssl:")) + }, timeout, interval).Should(Succeed()) + }) + + It("creates scripts ConfigMap with RBAC parameters", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + scriptsCM := types.NamespacedName{ + Namespace: OVNDBClusterName.Namespace, + Name: fmt.Sprintf("%s-%s", OVNDBClusterName.Name, "scripts"), + } + Eventually(func() corev1.ConfigMap { + return *th.GetConfigMap(scriptsCM) + }, timeout, interval).ShouldNot(BeNil()) + + Expect(th.GetConfigMap(scriptsCM).Data["setup.sh"]).Should( + ContainSubstring(fmt.Sprintf("DB_PORT_FULL_ACCESS=\"%d\"", ovndbcluster.DbPortSBRBACFullAccess))) + + Expect(th.GetConfigMap(scriptsCM).Data["setup.sh"]).Should( + ContainSubstring("role=ovn-controller")) + }) + }) When("OVNDB is created with topologyref", func() { diff --git a/test/functional/ovnnorthd_controller_test.go b/test/functional/ovnnorthd_controller_test.go index 5474bfc9..c12ecfb8 100644 --- a/test/functional/ovnnorthd_controller_test.go +++ b/test/functional/ovnnorthd_controller_test.go @@ -462,6 +462,61 @@ var _ = Describe("OVNNorthd controller", func() { }) }) + When("OVNNorthd is created with TLS-enabled DBClusters", func() { + var ovnNorthdName types.NamespacedName + + BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + dbs := CreateTLSOVNDBClusters(namespace, map[string][]string{}, 1) + DeferCleanup(DeleteOVNDBClusters, dbs) + + spec := GetTLSOVNNorthdSpec() + ovnNorthdName = ovn.CreateOVNNorthd(nil, namespace, spec) + DeferCleanup(ovn.DeleteOVNNorthd, ovnNorthdName) + }) + + It("uses full-access SB address with port 16642", func() { + stsName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + + Eventually(func(g Gomega) { + sts := th.GetStatefulSet(stsName) + args := sts.Spec.Template.Spec.Containers[0].Args + + // NB should use ssl scheme with standard port 6641 + hasNBArg := false + for _, arg := range args { + if len(arg) > 10 && arg[:10] == "--ovnnb-db" { + g.Expect(arg).To(ContainSubstring("ssl:")) + g.Expect(arg).To(ContainSubstring(":6641")) + hasNBArg = true + } + } + g.Expect(hasNBArg).To(BeTrue(), "should have --ovnnb-db arg") + + // SB should use ssl scheme with full-access port 16642 + hasSBArg := false + for _, arg := range args { + if len(arg) > 10 && arg[:10] == "--ovnsb-db" { + g.Expect(arg).To(ContainSubstring("ssl:")) + g.Expect(arg).To(ContainSubstring(":16642")) + hasSBArg = true + } + } + g.Expect(hasSBArg).To(BeTrue(), "should have --ovnsb-db arg with full-access port") + }, timeout, interval).Should(Succeed()) + }) + }) + When("OVNNorthd is created with default metrics settings", func() { var ovnNorthdName types.NamespacedName diff --git a/test/functional/suite_test.go b/test/functional/suite_test.go index 31127265..df70d831 100644 --- a/test/functional/suite_test.go +++ b/test/functional/suite_test.go @@ -42,6 +42,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" appsv1 "k8s.io/api/apps/v1" @@ -78,6 +79,7 @@ var ( const ( CABundleSecretName = "combined-ca-bundle" //nolint:gosec // G101: Not actual credentials, just secret name constants OvnDbCertSecretName = "ovndb-tls-cert" //nolint:gosec // G101: Not actual credentials, just secret name constants + OvnIssuerName = "ovn-issuer" ) func TestAPIs(t *testing.T) { @@ -105,6 +107,14 @@ var _ = BeforeSuite(func() { "github.com/openstack-k8s-operators/infra-operator/apis", "../../go.mod", "bases/topology.openstack.org_topologies.yaml") Expect(err).ShouldNot(HaveOccurred()) + certManagerCertCRD, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/lib-common/modules/test", "../../go.mod", "openshift_crds/cert-manager/v1/certificates.cert-manager.io-crd.yaml") + Expect(err).ShouldNot(HaveOccurred()) + + certManagerIssuerCRD, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/lib-common/modules/test", "../../go.mod", "openshift_crds/cert-manager/v1/issuers.cert-manager.io-crd.yaml") + Expect(err).ShouldNot(HaveOccurred()) + By("bootstrapping test environment") testEnv = &envtest.Environment{ // Increase this to 60 or 120 seconds for the single-core run @@ -120,6 +130,8 @@ var _ = BeforeSuite(func() { networkv1CRD, infranetworkv1CRD, infratopologyv1CRD, + certManagerCertCRD, + certManagerIssuerCRD, }, }, ErrorIfCRDPathMissing: true, @@ -159,6 +171,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = topologyv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = certmgrv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme logger = ctrl.Log.WithName("---Test---")