From 825808ecfbe615e83a4f773e95278cb9a2c36340 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Fri, 24 Apr 2026 11:23:30 +0200 Subject: [PATCH] Add OVN RBAC support with per-node ovn-controller certificates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable OVN role-based access control (RBAC) on the Southbound database so that ovn-controller nodes can only modify their own chassis rows. When the openstack-operator provides an RBAC issuer name (from a dedicated rootca-ovn-rbac CA, see patch [1]), this patch: * Creates per-node cert-manager Certificate CRs for each ovn-controller pod, with CN set to a deterministic UUID5 system-id derived from the node name (ComputeSystemID). This CN must match the chassis system-id for RBAC to authorize operations. * Copies the RBAC client cert/key into /etc/openvswitch/ on each node via the config job, and switches ovn-controller to use these dedicated paths instead of the shared OVN DB cert. * Mounts the RBAC CA certificate into ovsdbserver-sb pods and builds a combined CA bundle (regular CA + RBAC CA) so the SB database can verify ovn-controller client certificates. * Sets role=ovn-controller on the SB DB connection (port 6642) to enforce RBAC. * Creates a second SB DB listener on port 16642 with full (unrestricted) access, used by ovn-northd. * Updates inactivity probe handling in setup.sh and runtime-config.sh to iterate over all connections, since SB now has two listeners. * ovn-controller POD now waits for the Northd to be ready before start, it is done to avoid race condition when ovn-controller POD could be started before Northd would populate RBAC rules in the SB DB and that could cause issues with connection of the ovn-controller to the SB DB. * remove OVS DaemonSet readiness gate in the ovncontroller controller - it was there to make sure that local ovsdb is up so that config job would be able to store config values in it. But init scripts are already waiting actively for the ovsdb to become active before it anything else will be done. This check is also causing deadlock with deploying ovs and ovn-controller PODs now with RBAC enabled as ovn-controller needs to have certificates ready to start and create br-int brigde. That can't be done if the config job is not started and config job couldn't be started because ovncontroller controller was waiting for the OVS DaemonSet to be ready. * Remove OVS DaemonSet readiness gate from the ovncontroller controller - the gate ensured that the local ovsdb was running before the config job attempted to store configuration values. However, the init scripts already poll for ovsdb availability before doing anything else, making the gate redundant. With RBAC enabled, this gate also causes a deadlock: ovn-controller needs its RBAC certificates to start and create the br-int bridge, but those certificates are deployed by the config job, which cannot run until the OVS DaemonSet is ready — and the OVS DaemonSet cannot become ready without br-int. [1] https://github.com/openstack-k8s-operators/openstack-operator/pull/1906 Related: #OSPRH-1921 Closes: #OSPRH-1922 Assisted-by: claude-opus-4.6 Signed-off-by: Slawek Kaplonski --- .../ovn.openstack.org_ovncontrollers.yaml | 7 + .../ovn.openstack.org_ovndbclusters.yaml | 5 + api/v1beta1/client.go | 21 ++ api/v1beta1/ovncontroller_types.go | 7 + api/v1beta1/ovndbcluster_types.go | 16 ++ cmd/main.go | 2 + .../ovn.openstack.org_ovncontrollers.yaml | 7 + .../ovn.openstack.org_ovndbclusters.yaml | 5 + config/rbac/role.yaml | 20 ++ go.mod | 5 +- go.sum | 18 +- internal/common/const.go | 8 + .../controller/ovncontroller_controller.go | 95 +++++++- .../controller/ovndbcluster_controller.go | 12 ++ internal/controller/ovnnorthd_controller.go | 2 +- internal/ovncontroller/configjob.go | 52 ++++- internal/ovncontroller/daemonset.go | 28 ++- internal/ovncontroller/utils.go | 20 +- internal/ovndbcluster/const.go | 2 + internal/ovndbcluster/service.go | 9 + internal/ovndbcluster/statefulset.go | 1 + templates/ovncontroller/bin/functions | 32 +++ templates/ovncontroller/bin/init.sh | 17 ++ .../ovncontroller/config/configure-ovn.sh | 4 +- templates/ovndbcluster/bin/setup.sh | 37 +++- .../ovndbcluster/config/runtime-config.sh | 16 +- test/functional/base_test.go | 88 ++++++++ .../ovncontroller_controller_test.go | 204 +++++++++++++++++- .../ovndbcluster_controller_test.go | 101 ++++++++- test/functional/ovnnorthd_controller_test.go | 55 +++++ test/functional/suite_test.go | 14 ++ 31 files changed, 867 insertions(+), 43 deletions(-) diff --git a/api/bases/ovn.openstack.org_ovncontrollers.yaml b/api/bases/ovn.openstack.org_ovncontrollers.yaml index 85ccbfe9..6d332395 100644 --- a/api/bases/ovn.openstack.org_ovncontrollers.yaml +++ b/api/bases/ovn.openstack.org_ovncontrollers.yaml @@ -150,6 +150,13 @@ spec: description: Image used for the ovn-controller container (will be set to environmental default if empty) type: string + ovnIssuerName: + description: |- + OvnIssuerName - The name of the cert-manager Issuer used to sign + per-node ovn-controller certificates. When set, the controller + creates cert-manager Certificate resources for each node with + CN matching the chassis system-id for OVN RBAC. + type: string ovnLogLevel: default: info description: OVNLogLevel - Set log level off, emer, err, warn, info diff --git a/api/bases/ovn.openstack.org_ovndbclusters.yaml b/api/bases/ovn.openstack.org_ovndbclusters.yaml index 75c312ad..1402c7c3 100644 --- a/api/bases/ovn.openstack.org_ovndbclusters.yaml +++ b/api/bases/ovn.openstack.org_ovndbclusters.yaml @@ -438,6 +438,11 @@ spec: description: InternalDBAddress - DB IP address used by other Pods in the cluster type: string + internalDbAddressRbacFullAccess: + description: |- + InternalDBAddressRbacFullAccess - DB IP address for full-access (non-RBAC) + connections, used by ovn-northd when RBAC is enabled on SB DB + type: string lastAppliedTopology: description: LastAppliedTopology - the last applied Topology properties: diff --git a/api/v1beta1/client.go b/api/v1beta1/client.go index b42f2c64..9250425c 100644 --- a/api/v1beta1/client.go +++ b/api/v1beta1/client.go @@ -72,6 +72,27 @@ func GetOVNController( return nil, nil } +// GetOVNNorthd - return OVNNorthd instance in the given namespace +func GetOVNNorthd( + ctx context.Context, + h *helper.Helper, + namespace string, +) (*OVNNorthd, error) { + ovnNorthdList := &OVNNorthdList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + err := h.GetClient().List(ctx, ovnNorthdList, listOpts...) + if err != nil { + return nil, err + } + if len(ovnNorthdList.Items) > 0 { + return &ovnNorthdList.Items[0], nil + } + + return nil, nil +} + // GetDBClusterByType - return OVNDBCluster for the given dbType func GetDBClusterByType( ctx context.Context, diff --git a/api/v1beta1/ovncontroller_types.go b/api/v1beta1/ovncontroller_types.go index 35afaf06..c4664cc9 100644 --- a/api/v1beta1/ovncontroller_types.go +++ b/api/v1beta1/ovncontroller_types.go @@ -124,6 +124,13 @@ type OVNControllerSpecCore struct { // +kubebuilder:validation:Optional // MetricsTLS - Parameters related to TLS for metrics daemonset MetricsTLS tls.SimpleService `json:"metricsTLS,omitempty"` + + // +kubebuilder:validation:Optional + // OvnIssuerName - The name of the cert-manager Issuer used to sign + // per-node ovn-controller certificates. When set, the controller + // creates cert-manager Certificate resources for each node with + // CN matching the chassis system-id for OVN RBAC. + OvnIssuerName string `json:"ovnIssuerName,omitempty"` } // OVNControllerStatus defines the observed state of OVNController diff --git a/api/v1beta1/ovndbcluster_types.go b/api/v1beta1/ovndbcluster_types.go index 1ec60e35..b3e3c8ca 100644 --- a/api/v1beta1/ovndbcluster_types.go +++ b/api/v1beta1/ovndbcluster_types.go @@ -171,6 +171,10 @@ type OVNDBClusterStatus struct { // InternalDBAddress - DB IP address used by other Pods in the cluster InternalDBAddress string `json:"internalDbAddress,omitempty"` + // InternalDBAddressRbacFullAccess - DB IP address for full-access (non-RBAC) + // connections, used by ovn-northd when RBAC is enabled on SB DB + InternalDBAddressRbacFullAccess string `json:"internalDbAddressRbacFullAccess,omitempty"` + // NetworkAttachments status of the deployment pods NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` @@ -239,6 +243,18 @@ func (instance OVNDBCluster) GetInternalEndpoint() (string, error) { return instance.Status.InternalDBAddress, nil } +// GetInternalEndpointRbacFullAccess - return the full-access (non-RBAC) internal endpoint for SB DB. +// Falls back to GetInternalEndpoint if the DB is not SB or TLS is not enabled. +func (instance OVNDBCluster) GetInternalEndpointRbacFullAccess() (string, error) { + if instance.Spec.DBType != SBDBType || !instance.Spec.TLS.Enabled() { + return instance.GetInternalEndpoint() + } + if instance.Status.InternalDBAddressRbacFullAccess == "" { + return "", fmt.Errorf("internal RBAC full-access DBEndpoint not ready yet for %s", instance.Spec.DBType) + } + return instance.Status.InternalDBAddressRbacFullAccess, nil +} + // GetExternalEndpoint - return the DNS that openstack dnsmasq can resolve func (instance OVNDBCluster) GetExternalEndpoint() (string, error) { if (instance.Spec.NetworkAttachment != "" || instance.Spec.Override.Service != nil) && instance.Status.DBAddress == "" { diff --git a/cmd/main.go b/cmd/main.go index b17be762..d44c6033 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -46,6 +46,7 @@ import ( // +kubebuilder:scaffold:imports + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" @@ -66,6 +67,7 @@ func init() { utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(infranetworkv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(certmgrv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml b/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml index 85ccbfe9..6d332395 100644 --- a/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml +++ b/config/crd/bases/ovn.openstack.org_ovncontrollers.yaml @@ -150,6 +150,13 @@ spec: description: Image used for the ovn-controller container (will be set to environmental default if empty) type: string + ovnIssuerName: + description: |- + OvnIssuerName - The name of the cert-manager Issuer used to sign + per-node ovn-controller certificates. When set, the controller + creates cert-manager Certificate resources for each node with + CN matching the chassis system-id for OVN RBAC. + type: string ovnLogLevel: default: info description: OVNLogLevel - Set log level off, emer, err, warn, info diff --git a/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml b/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml index 75c312ad..1402c7c3 100644 --- a/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml +++ b/config/crd/bases/ovn.openstack.org_ovndbclusters.yaml @@ -438,6 +438,11 @@ spec: description: InternalDBAddress - DB IP address used by other Pods in the cluster type: string + internalDbAddressRbacFullAccess: + description: |- + InternalDBAddressRbacFullAccess - DB IP address for full-access (non-RBAC) + connections, used by ovn-northd when RBAC is enabled on SB DB + type: string lastAppliedTopology: description: LastAppliedTopology - the last applied Topology properties: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4e6e5293..19df9e46 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -65,6 +65,26 @@ rules: - patch - update - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - issuers + verbs: + - get + - list + - watch - apiGroups: - k8s.cni.cncf.io resources: diff --git a/go.mod b/go.mod index 0879a253..c1056ca5 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/openstack-k8s-operators/ovn-operator go 1.24.4 require ( + github.com/cert-manager/cert-manager v1.16.5 github.com/go-logr/logr v1.4.3 github.com/google/uuid v1.6.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 github.com/onsi/ginkgo/v2 v2.28.2 github.com/onsi/gomega v1.40.0 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260506154724-30a976ba8ef0 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 github.com/openstack-k8s-operators/ovn-operator/api v0.0.0-20230418071801-b5843d9e05fb @@ -65,7 +67,7 @@ require ( github.com/prometheus/procfs v0.16.1 // indirect github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.7 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect @@ -103,6 +105,7 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250902184714-7fc278399c7f // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect + sigs.k8s.io/gateway-api v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect diff --git a/go.sum b/go.sum index 6612f306..f74de97d 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cert-manager/cert-manager v1.16.5 h1:XIhKoS4zQV9RHXAkqQW0NLivvoxAnWzbPsy9BG6cPVc= +github.com/cert-manager/cert-manager v1.16.5/go.mod h1:0DwmIGjMOreiP7/6gAqnjaBRJ+yHCfZ5DP7NNqKV+tY= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -118,6 +120,8 @@ github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e h1:E1OdwSpqWuDPCedyU github.com/openshift/api v0.0.0-20250711200046-c86d80652a9e/go.mod h1:Shkl4HanLwDiiBzakv+con/aMGnVE2MAGvoKp5oyYUo= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31 h1:FWa0vNs175LpV1eSZ60YOGFdbJ3LqxQ1fxfprBRg7T4= github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20260508091801-73f228e6af31/go.mod h1:/S2AN21zV70V1XuL0Of2dCjYWNkKwQSyNI8l/iQVrMs= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260506154724-30a976ba8ef0 h1:Dck0sbRnRRm0JZ9zdbKoJADZqk37pEv7sFUZ8sZwqDA= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:+0PCJgchLUfBcZM3pebsN0pN+d5a22NiC2vy2cpGfRs= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0 h1:vkFvn06Ns9qW4AbzFjFDu8ioosRmhkEZiDrO3DOQhLg= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20260506154724-30a976ba8ef0/go.mod h1:aIuG6lx3aS0vnXweRNdR/Q0SlfOsLIo0OzrqKK7C6xs= github.com/openstack-k8s-operators/lib-common/modules/test v0.6.1-0.20260506154724-30a976ba8ef0 h1:mG3QhS/QWv9Y/AkZZ5OzO6hu6+l5oDXnI/Q5ZUbj6Zs= @@ -143,11 +147,15 @@ github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wx github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -254,9 +262,9 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.31.14 h1:xYn/S/WFJsksI7dk/5uBRd3Umm/D8W5g7sRnd4csotA= @@ -281,6 +289,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsA sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.7 h1:DLABZfMr20A+AwCZOHhcbcu+TqBXnJZaVBri9K3EO48= sigs.k8s.io/controller-runtime v0.19.7/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/gateway-api v1.1.0 h1:DsLDXCi6jR+Xz8/xd0Z1PYl2Pn0TyaFMOPPZIj4inDM= +sigs.k8s.io/gateway-api v1.1.0/go.mod h1:ZH4lHrL2sDi0FHZ9jjneb8kKnGzFWyrTya35sWUTrRs= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/internal/common/const.go b/internal/common/const.go index 1158bac4..273a4601 100644 --- a/internal/common/const.go +++ b/internal/common/const.go @@ -14,6 +14,14 @@ const ( // OVNMetricsKeyPath is the path to the metrics private key file OVNMetricsKeyPath string = "/etc/pki/tls/private/ovnmetrics.key" + // OVNRbacCertMountPath is the mount path for the per-node RBAC certificate in config jobs + OVNRbacCertMountPath string = "/tmp/ovn-rbac-cert" + + // OVNControllerCertPath is the destination path for the RBAC client certificate on the host + OVNControllerCertPath string = "/etc/openvswitch/ovn-controller-cert.pem" + // OVNControllerKeyPath is the destination path for the RBAC client key on the host + OVNControllerKeyPath string = "/etc/openvswitch/ovn-controller-privkey.pem" + // MetricsPort is the port used for metrics MetricsPort int32 = 1981 ) diff --git a/internal/controller/ovncontroller_controller.go b/internal/controller/ovncontroller_controller.go index 5eb75860..4dbb993d 100644 --- a/internal/controller/ovncontroller_controller.go +++ b/internal/controller/ovncontroller_controller.go @@ -37,7 +37,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + certmanager "github.com/openstack-k8s-operators/lib-common/modules/certmanager" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" @@ -96,6 +98,9 @@ func (r *OVNControllerReconciler) GetClient() client.Client { // +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid;privileged,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update +// cert-manager permissions for per-node RBAC certificates +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch // Reconcile reconciles the OVNController CR to deploy OVN controller pods func (r *OVNControllerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { @@ -266,7 +271,9 @@ func (r *OVNControllerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&certmgrv1.Certificate{}). Watches(&ovnv1.OVNDBCluster{}, handler.EnqueueRequestsFromMapFunc(ovnv1.OVNCRNamespaceMapFunc(crs, mgr.GetClient()))). + Watches(&ovnv1.OVNNorthd{}, handler.EnqueueRequestsFromMapFunc(ovnv1.OVNCRNamespaceMapFunc(crs, mgr.GetClient()))). Watches( &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), @@ -641,6 +648,26 @@ func (r *OVNControllerReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) } + // When TLS is enabled, RBAC is enforced on the SB database. The RBAC + // permission tables are populated by ovn-northd, so ovn-controller must + // not start until northd is running; otherwise all SB operations are + // denied with "permission error". + if instance.Spec.TLS.Enabled() { + northd, err := ovnv1.GetOVNNorthd(ctx, helper, instance.Namespace) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to look up OVNNorthd: %w", err) + } + if northd == nil || northd.Status.ReadyCount == 0 { + Log.Info("OVNNorthd is not ready yet, waiting before deploying OVNController...") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + return ctrl.Result{RequeueAfter: time.Duration(5) * time.Second}, nil + } + } + // Define a new DaemonSet object for OVNController dset := daemonset.NewDaemonSet( ovncontroller.CreateOVNDaemonSet(instance, inputHash, ovnServiceLabels, topology), @@ -848,12 +875,70 @@ func (r *OVNControllerReconciler) reconcileNormal(ctx context.Context, instance } // create OVN Config Job - start - // Waits for OVS pods to run the configJob which basically will set config into OVS database - if instance.Status.OVSNumberReady != instance.Status.DesiredNumberScheduled { - Log.Info("OVS DaemonSet not ready yet. Configuration job cannot be started.") - return ctrl.Result{Requeue: true}, nil + + // Create per-node RBAC certificates if configured + nodeSystemIDs := map[string]string{} + if instance.Spec.OvnIssuerName != "" { + ovnPods, podErr := ovncontroller.GetOVNControllerPods(ctx, r.Client, instance) + if podErr != nil { + return ctrl.Result{}, podErr + } + + issuer := &certmgrv1.Issuer{} + if err := helper.GetClient().Get(ctx, types.NamespacedName{ + Name: instance.Spec.OvnIssuerName, + Namespace: instance.Namespace, + }, issuer); err != nil { + return ctrl.Result{}, fmt.Errorf("error getting issuer %s/%s - %w", instance.Spec.OvnIssuerName, instance.Namespace, err) + } + + durationString := certmanager.CertDefaultDuration + if d, ok := issuer.Annotations[certmanager.CertDurationAnnotation]; ok && d != "" { + durationString = d + } + duration, err := time.ParseDuration(durationString) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error parsing certificate duration %s - %w", durationString, err) + } + + var renewBefore *time.Duration + if r, ok := issuer.Annotations[certmanager.CertRenewBeforeAnnotation]; ok && r != "" { + rb, err := time.ParseDuration(r) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error parsing certificate renewBefore %s - %w", r, err) + } + renewBefore = &rb + } + + for _, pod := range ovnPods.Items { + nodeName := pod.Spec.NodeName + if nodeName == "" { + continue + } + systemID := ovncontroller.ComputeSystemID(nodeName) + certName := ovncontroller.RbacCertName(nodeName) + nodeSystemIDs[nodeName] = systemID + + _, certResult, certErr := certmanager.EnsureCert(ctx, helper, certmanager.CertificateRequest{ + IssuerName: instance.Spec.OvnIssuerName, + CertName: certName, + CommonName: &systemID, + Labels: ovnServiceLabels, + Usages: []certmgrv1.KeyUsage{certmgrv1.UsageClientAuth, certmgrv1.UsageDigitalSignature}, + Duration: &duration, + RenewBefore: renewBefore, + }, instance) + if certErr != nil { + return certResult, certErr + } + if (certResult != ctrl.Result{}) { + Log.Info("Waiting for RBAC certificate", "node", nodeName, "cert", certName) + return certResult, nil + } + } } - jobsDef, err := ovncontroller.ConfigJob(ctx, r.Client, instance, sbCluster, ovnServiceLabels) + + jobsDef, err := ovncontroller.ConfigJob(ctx, r.Client, instance, sbCluster, ovnServiceLabels, nodeSystemIDs) if err != nil { Log.Error(err, "Failed to create OVN controller configuration Job") return ctrl.Result{}, err diff --git a/internal/controller/ovndbcluster_controller.go b/internal/controller/ovndbcluster_controller.go index 69f6ce8b..dae9aded 100644 --- a/internal/controller/ovndbcluster_controller.go +++ b/internal/controller/ovndbcluster_controller.go @@ -740,6 +740,7 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance * instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) internalDbAddress := []string{} + internalDbAddressRbacFullAccess := []string{} var svcPort int32 scheme := "tcp" if instance.Spec.TLS.Enabled() { @@ -755,6 +756,9 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance * // TODO: Watch operator.openshift.io resource once cluster domain is customizable clusterDomain := clusterdns.GetDNSClusterDomain() internalDbAddress = append(internalDbAddress, fmt.Sprintf("%s:%s.%s.svc.%s:%d", scheme, svc.Name, svc.Namespace, clusterDomain, svcPort)) + if instance.Spec.DBType == ovnv1.SBDBType && instance.Spec.TLS.Enabled() { + internalDbAddressRbacFullAccess = append(internalDbAddressRbacFullAccess, fmt.Sprintf("%s:%s.%s.svc.%s:%d", scheme, svc.Name, svc.Namespace, clusterDomain, ovndbcluster.DbPortSBRBACFullAccess)) + } } // Note setting this to the singular headless service address (e.g ssl:ovsdbserver-sb...) "works" but will not @@ -764,6 +768,9 @@ func (r *OVNDBClusterReconciler) reconcileNormal(ctx context.Context, instance * // Set DB Address instance.Status.InternalDBAddress = strings.Join(internalDbAddress, ",") + if len(internalDbAddressRbacFullAccess) > 0 { + instance.Status.InternalDBAddressRbacFullAccess = strings.Join(internalDbAddressRbacFullAccess, ",") + } if instance.Spec.DBType == ovnv1.SBDBType && (instance.Spec.NetworkAttachment != "" || instance.Spec.Override.Service != nil) { // This config map will populate the sb db address to edpm, can't use the nb // If there's no networkAttachments the configMap is not needed @@ -1290,6 +1297,11 @@ func (r *OVNDBClusterReconciler) generateServiceConfigMaps( templateParameters["OVN_METRICS_CERT_PATH"] = ovn_common.OVNMetricsCertPath templateParameters["OVN_METRICS_KEY_PATH"] = ovn_common.OVNMetricsKeyPath templateParameters["OVN_RUNDIR"] = "/etc/ovn" + if instance.Spec.DBType == ovnv1.SBDBType && instance.Spec.TLS.Enabled() { + templateParameters["DB_PORT_FULL_ACCESS"] = ovndbcluster.DbPortSBRBACFullAccess + } else { + templateParameters["DB_PORT_FULL_ACCESS"] = templateParameters["DB_PORT"] + } cms := []util.Template{ // ScriptsConfigMap diff --git a/internal/controller/ovnnorthd_controller.go b/internal/controller/ovnnorthd_controller.go index 8651514c..5ef530e5 100644 --- a/internal/controller/ovnnorthd_controller.go +++ b/internal/controller/ovnnorthd_controller.go @@ -642,7 +642,7 @@ func getInternalEndpoint( if err != nil { return "", err } - internalEndpoint, err := cluster.GetInternalEndpoint() + internalEndpoint, err := cluster.GetInternalEndpointRbacFullAccess() if err != nil { return "", err } diff --git a/internal/ovncontroller/configjob.go b/internal/ovncontroller/configjob.go index 268eda5b..b4d4e1c5 100644 --- a/internal/ovncontroller/configjob.go +++ b/internal/ovncontroller/configjob.go @@ -19,6 +19,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/env" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" + ovn_common "github.com/openstack-k8s-operators/ovn-operator/internal/common" "sigs.k8s.io/controller-runtime/pkg/client" batchv1 "k8s.io/api/batch/v1" @@ -26,13 +27,17 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// ConfigJob - prepare job to configure ovn-controller +// ConfigJob - prepare job to configure ovn-controller. +// nodeSystemIDs maps node names to their pre-computed system-id UUIDs. +// When non-empty, the per-node RBAC cert Secret is mounted and SYSTEM_ID is +// passed as an environment variable. func ConfigJob( ctx context.Context, k8sClient client.Client, instance *ovnv1.OVNController, sbCluster *ovnv1.OVNDBCluster, labels map[string]string, + nodeSystemIDs map[string]string, ) ([]*batchv1.Job, error) { var jobs []*batchv1.Job @@ -42,7 +47,7 @@ func ConfigJob( // configuration job automatically right after it will be finished jobTTLAfterFinished := int32(0) - ovnPods, err := getOVNControllerPods( + ovnPods, err := GetOVNControllerPods( ctx, k8sClient, instance, @@ -74,6 +79,40 @@ func ConfigJob( "/usr/local/bin/additional-scripts/configure-ovn.sh", } + volumes := GetOVNControllerVolumes(instance.Name, instance.Namespace, true) + volumeMounts := GetOVNControllerVolumeMounts(true) + + // Per-job env vars start from a copy of the shared ones + jobEnvVars := make(map[string]env.Setter, len(envVars)) + for k, v := range envVars { + jobEnvVars[k] = v + } + + nodeName := ovnPod.Spec.NodeName + if systemID, ok := nodeSystemIDs[nodeName]; ok { + jobEnvVars["SYSTEM_ID"] = env.SetValue(systemID) + + certSecretName := RbacCertSecretName(nodeName) + volumes = append(volumes, corev1.Volume{ + Name: "ovn-rbac-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: certSecretName, + }, + }, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "ovn-rbac-cert", + MountPath: ovn_common.OVNRbacCertMountPath, + ReadOnly: true, + }) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: "etc-ovs", + MountPath: "/etc/openvswitch", + ReadOnly: false, + }) + } + jobs = append( jobs, &batchv1.Job{ @@ -100,14 +139,13 @@ func ConfigJob( RunAsUser: &runAsUser, Privileged: &privileged, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: GetOVNControllerVolumeMounts(true), + Env: env.MergeEnvs([]corev1.EnvVar{}, jobEnvVars), + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, }, }, - Volumes: GetOVNControllerVolumes(instance.Name, instance.Namespace, true), - NodeName: ovnPod.Spec.NodeName, - // ^ NodeSelector not required + Volumes: volumes, + NodeName: nodeName, }, }, }, diff --git a/internal/ovncontroller/daemonset.go b/internal/ovncontroller/daemonset.go index d39c6518..5b640781 100644 --- a/internal/ovncontroller/daemonset.go +++ b/internal/ovncontroller/daemonset.go @@ -14,6 +14,7 @@ package ovncontroller import ( "fmt" + "strings" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/env" @@ -38,7 +39,7 @@ func CreateOVNDaemonSet( volumes := GetOVNControllerVolumes(instance.Name, instance.Namespace, false) mounts := GetOVNControllerVolumeMounts(false) - cmd := []string{ + ovnCmd := []string{ "ovn-controller", "--pidfile", "unix:/run/openvswitch/db.sock", } @@ -59,13 +60,32 @@ func CreateOVNDaemonSet( mounts = append(mounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) } - cmd = append(cmd, []string{ - fmt.Sprintf("--certificate=%s", ovn_common.OVNDbCertPath), - fmt.Sprintf("--private-key=%s", ovn_common.OVNDbKeyPath), + mounts = append(mounts, corev1.VolumeMount{ + Name: "etc-ovs", + MountPath: "/etc/openvswitch", + ReadOnly: true, + }) + ovnCmd = append(ovnCmd, []string{ + fmt.Sprintf("--certificate=%s", ovn_common.OVNControllerCertPath), + fmt.Sprintf("--private-key=%s", ovn_common.OVNControllerKeyPath), fmt.Sprintf("--ca-cert=%s", ovn_common.OVNDbCaCertPath), }...) } + // When RBAC is configured, the config job deploys a per-node client + // certificate to /etc/openvswitch/ after OVS is ready. Wait for the + // cert file before starting ovn-controller so it doesn't connect to + // the SB DB without client authentication. + var cmd []string + if instance.Spec.TLS.Enabled() && instance.Spec.OvnIssuerName != "" { + cmd = []string{ + "/bin/bash", "-c", + "source /usr/local/bin/container-scripts/functions && wait_for_rbac_cert && exec " + strings.Join(ovnCmd, " "), + } + } else { + cmd = ovnCmd + } + ovnControllerLivenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, diff --git a/internal/ovncontroller/utils.go b/internal/ovncontroller/utils.go index c378f350..c4bd51df 100644 --- a/internal/ovncontroller/utils.go +++ b/internal/ovncontroller/utils.go @@ -18,12 +18,29 @@ import ( "sort" "strings" + "github.com/google/uuid" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) +// ComputeSystemID derives a deterministic UUID5 system-id from a node name, +// matching the convention used by OVN for chassis identification. +func ComputeSystemID(nodeName string) string { + return uuid.NewSHA1(uuid.NameSpaceDNS, []byte(nodeName)).String() +} + +// RbacCertName returns the cert-manager Certificate CR name for a given node +func RbacCertName(nodeName string) string { + return fmt.Sprintf("ovn-controller-cert-%s", nodeName) +} + +// RbacCertSecretName returns the Secret name created by cert-manager for a given node's RBAC certificate +func RbacCertSecretName(nodeName string) string { + return "cert-" + RbacCertName(nodeName) +} + func getPhysicalNetworks( instance *ovnv1.OVNController, ) string { @@ -37,7 +54,8 @@ func getPhysicalNetworks( return strings.Join(nicMappings, " ") } -func getOVNControllerPods( +// GetOVNControllerPods returns list of the pods running ovn-controller +func GetOVNControllerPods( ctx context.Context, k8sClient client.Client, instance *ovnv1.OVNController, diff --git a/internal/ovndbcluster/const.go b/internal/ovndbcluster/const.go index 99aaaad7..918cb28a 100644 --- a/internal/ovndbcluster/const.go +++ b/internal/ovndbcluster/const.go @@ -9,6 +9,8 @@ const ( // DbPortSB is the port number for the OVN Southbound database DbPortSB int32 = 6642 + // DbPortSBRBACFullAccess is the port for full-access (non-RBAC) SB connections (used by ovn-northd) + DbPortSBRBACFullAccess int32 = 16642 // RaftPortSB is the port number for the OVN Southbound database Raft protocol RaftPortSB int32 = 6644 ) diff --git a/internal/ovndbcluster/service.go b/internal/ovndbcluster/service.go index c52b73d7..2959ede5 100644 --- a/internal/ovndbcluster/service.go +++ b/internal/ovndbcluster/service.go @@ -38,6 +38,15 @@ func Service( }, } + // Add full-access (non-RBAC) port for SB when TLS is enabled + if instance.Spec.DBType == ovnv1.SBDBType && instance.Spec.TLS.Enabled() { + ports = append(ports, corev1.ServicePort{ + Name: dbPortName + "-full-access", + Port: DbPortSBRBACFullAccess, + Protocol: corev1.ProtocolTCP, + }) + } + // Add metrics port if metrics are enabled and exporter image is specified if instance.Spec.ExporterImage != "" && (instance.Spec.MetricsEnabled == nil || *instance.Spec.MetricsEnabled) { ports = append(ports, corev1.ServicePort{ diff --git a/internal/ovndbcluster/statefulset.go b/internal/ovndbcluster/statefulset.go index bd0c846f..9bcd6437 100644 --- a/internal/ovndbcluster/statefulset.go +++ b/internal/ovndbcluster/statefulset.go @@ -133,6 +133,7 @@ func StatefulSet( } volumes = append(volumes, svc.CreateVolume(serviceName)) volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(serviceName)...) + } // NOTE(ihar) ovndb pods leave the raft cluster on delete; it's important diff --git a/templates/ovncontroller/bin/functions b/templates/ovncontroller/bin/functions index c13eb79a..b01850f0 100755 --- a/templates/ovncontroller/bin/functions +++ b/templates/ovncontroller/bin/functions @@ -50,6 +50,38 @@ function wait_for_ovsdb_server { done } +function wait_for_rbac_cert { + local cert_path="/etc/openvswitch/ovn-controller-cert.pem" + local key_path="/etc/openvswitch/ovn-controller-privkey.pem" + local ca_path="/etc/pki/tls/certs/ovndbca.crt" + + while [ ! -f "$cert_path" ]; do + echo "RBAC certificate not yet available at $cert_path. Waiting..." + sleep 2 + done + + # Wait for ovn-northd to populate RBAC permission tables in the SB DB + # before starting ovn-controller; otherwise all SB operations are denied. + # Check for actual RBAC_Permission entries (not just RBAC_Role) because + # northd may create the role before all permission entries are committed. + local ovn_remote + while true; do + ovn_remote=$(ovs-vsctl --if-exists get open . external-ids:ovn-remote 2>/dev/null | tr -d '"') + if [ -n "$ovn_remote" ] && \ + ovn-sbctl --db="$ovn_remote" \ + --timeout=5 \ + --certificate="$cert_path" \ + --private-key="$key_path" \ + --ca-cert="$ca_path" \ + --no-leader-only \ + find RBAC_Role name=ovn-controller 2>/dev/null | grep -q ovn-controller; then + break + fi + echo "RBAC tables not yet populated by ovn-northd. Waiting..." + sleep 2 + done +} + # configure external-ids in OVS function configure_external_ids { ovs-vsctl set open . external-ids:ovn-bridge=${OVNBridge} diff --git a/templates/ovncontroller/bin/init.sh b/templates/ovncontroller/bin/init.sh index bec5945d..f611ee28 100755 --- a/templates/ovncontroller/bin/init.sh +++ b/templates/ovncontroller/bin/init.sh @@ -16,10 +16,27 @@ source $(dirname $0)/functions +# Install RBAC client certificate if available (mounted from cert-manager Secret). +# This is a pure file copy with no OVS dependency, so do it before +# wait_for_ovsdb_server to unblock ovn-controller's wait_for_rbac_cert +# as early as possible. +if [ -f "/tmp/ovn-rbac-cert/tls.crt" ]; then + mkdir -p /etc/openvswitch + cp /tmp/ovn-rbac-cert/tls.crt /etc/openvswitch/ovn-controller-cert.pem + cp /tmp/ovn-rbac-cert/tls.key /etc/openvswitch/ovn-controller-privkey.pem + chmod 0644 /etc/openvswitch/ovn-controller-cert.pem + chmod 0600 /etc/openvswitch/ovn-controller-privkey.pem +fi + wait_for_ovsdb_server # From now on, we should exit immediatelly when any command exits with non-zero status set -ex +# Set system-id if provided by the controller +if [ -n "${SYSTEM_ID}" ]; then + ovs-vsctl set open . external-ids:system-id=${SYSTEM_ID} +fi + configure_external_ids configure_physical_networks diff --git a/templates/ovncontroller/config/configure-ovn.sh b/templates/ovncontroller/config/configure-ovn.sh index a0806564..d8ddc9e6 100755 --- a/templates/ovncontroller/config/configure-ovn.sh +++ b/templates/ovncontroller/config/configure-ovn.sh @@ -37,7 +37,9 @@ function configure_log_level { done ctl_path=$(find /var/run/ovn/ -maxdepth 1 -name "ovn-controller.*.ctl") - ovn-appctl -t "$ctl_path" vlog/set ${OVNLogLevel} + if [ -n "$ctl_path" ]; then + ovn-appctl -t "$ctl_path" vlog/set ${OVNLogLevel} + fi } diff --git a/templates/ovndbcluster/bin/setup.sh b/templates/ovndbcluster/bin/setup.sh index 364f053b..57dcb7b4 100755 --- a/templates/ovndbcluster/bin/setup.sh +++ b/templates/ovndbcluster/bin/setup.sh @@ -17,8 +17,10 @@ set -ex source $(dirname $0)/functions DB_PORT="{{ .DB_PORT }}" +DB_PORT_FULL_ACCESS="{{ .DB_PORT_FULL_ACCESS }}" {{- if .TLS }} DB_SCHEME="pssl" +SSL_CA_CERT={{.OVNDB_CACERT_PATH}} {{- else }} DB_SCHEME="ptcp" {{- end }} @@ -56,7 +58,7 @@ set "$@" --db-${DB_TYPE}-port=${DB_PORT} {{- if .TLS }} set "$@" --ovn-${DB_TYPE}-db-ssl-key={{.OVNDB_KEY_PATH}} set "$@" --ovn-${DB_TYPE}-db-ssl-cert={{.OVNDB_CERT_PATH}} -set "$@" --ovn-${DB_TYPE}-db-ssl-ca-cert={{.OVNDB_CACERT_PATH}} +set "$@" --ovn-${DB_TYPE}-db-ssl-ca-cert=${SSL_CA_CERT} set "$@" --db-${DB_TYPE}-cluster-local-proto=ssl set "$@" --db-${DB_TYPE}-cluster-remote-proto=ssl set "$@" --db-${DB_TYPE}-create-insecure-remote=no @@ -122,15 +124,26 @@ if [[ "$(hostname)" == "{{ .SERVICE_NAME }}-0" ]]; then export OVN_${DB_TYPE^^}_DAEMON=$(${CTLCMD} --pidfile --detach) {{- if .TLS }} - ${CTLCMD} set-ssl {{.OVNDB_KEY_PATH}} {{.OVNDB_CERT_PATH}} {{.OVNDB_CACERT_PATH}} + ${CTLCMD} set-ssl {{.OVNDB_KEY_PATH}} {{.OVNDB_CERT_PATH}} ${SSL_CA_CERT} + if [[ "${DB_TYPE}" == "sb" ]]; then + # Use RBAC for the connections of the ovn-controller to SB + ${CTLCMD} set-connection role=ovn-controller ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} + # In this case, Northd needs to have full access to the DB so there need + # to be another connection defined for it + # TODO(slaweq): port has to be also set in Northd + ${CTLCMD} -- --id=@conn_uuid create Connection target="${DB_SCHEME}\:${DB_PORT_FULL_ACCESS}" -- add SB_Global . connections @conn_uuid + else + # No RBAC for connecting to the Northbound DB so only one connection + # defined is fine + ${CTLCMD} set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} + fi + {{- else }} ${CTLCMD} del-ssl -{{- end }} - CURRENT_PROBE="$(${CTLCMD} get connection . inactivity_probe || echo [])" - if [ "$CURRENT_PROBE" = "[]" ]; then - CURRENT_PROBE=60000 - fi + # If TLS is disabled, RBAC can't be used, so one connection defined is enough ${CTLCMD} set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} +{{- end }} + # OVN does not support setting inactivity-probe through --remote cli arg so # we have to set it after database is up. # @@ -144,8 +157,14 @@ if [[ "$(hostname)" == "{{ .SERVICE_NAME }}-0" ]]; then # TODO: Consider migrating inactivity probe setting to config files when # we update to ovs 3.3. See --config-file in ovsdb-server(1) for more # details. - while [ "$(${CTLCMD} get connection . inactivity_probe)" != "${CURRENT_PROBE}" ]; do - ${CTLCMD} --inactivity-probe="${CURRENT_PROBE}" set-connection ${DB_SCHEME}:${DB_PORT}:${DB_ADDR} + for connection_id in $(${CTLCMD} -f csv --no-headings --columns=_uuid list connection); do + CURRENT_PROBE="$(${CTLCMD} get connection $connection_id inactivity_probe || echo [])" + if [ "$CURRENT_PROBE" = "[]" ]; then + CURRENT_PROBE=60000 + fi + while [ "$(${CTLCMD} get connection $connection_id inactivity_probe)" != "${CURRENT_PROBE}" ]; do + ${CTLCMD} set connection $connection_id inactivity_probe=${CURRENT_PROBE} + done done ${CTLCMD} list connection diff --git a/templates/ovndbcluster/config/runtime-config.sh b/templates/ovndbcluster/config/runtime-config.sh index 3c8726f9..6c974ba1 100755 --- a/templates/ovndbcluster/config/runtime-config.sh +++ b/templates/ovndbcluster/config/runtime-config.sh @@ -74,11 +74,17 @@ if echo "$CONFIG_FLAGS" | grep -q "INACTIVITY_PROBE"; then if [[ "$(hostname)" == "${SERVICE_NAME}-0-config-"* ]]; then echo "Configuring inactivity probe to ${INACTIVITY_PROBE}ms on pod-0..." - # Simple approach - connect directly to running database (OVN_RUNDIR already set) - if ovn-${DB_TYPE}ctl --no-leader-only set connection . inactivity_probe="$INACTIVITY_PROBE"; then - echo "✓ Successfully configured inactivity probe" - else - echo "✗ Failed to configure inactivity probe" + # Set inactivity probe on all connections + FAILED=0 + for CONN_ID in $(ovn-${DB_TYPE}ctl --no-leader-only --format=table --no-headings --columns=_uuid list connection); do + if ovn-${DB_TYPE}ctl --no-leader-only set connection ${CONN_ID} inactivity_probe="$INACTIVITY_PROBE"; then + echo "✓ Successfully configured inactivity probe on connection ${CONN_ID}" + else + echo "✗ Failed to configure inactivity probe on connection ${CONN_ID}" + FAILED=1 + fi + done + if [ "$FAILED" -eq 1 ]; then exit 1 fi else diff --git a/test/functional/base_test.go b/test/functional/base_test.go index 8e7b56b5..9d9014be 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -22,6 +22,7 @@ import ( "strings" "time" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" . "github.com/onsi/gomega" //revive:disable:dot-imports appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -70,6 +71,16 @@ func OVNNorthdConditionGetter(name types.NamespacedName) condition.Conditions { return instance.Status.Conditions } +func CreateReadyOVNNorthd(namespace string, spec ovnv1.OVNNorthdSpec) types.NamespacedName { + name := ovn.CreateOVNNorthd(nil, namespace, spec) + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + th.SimulateStatefulSetReplicaReady(statefulSetName) + return name +} + func GetDefaultOVNDBClusterSpec() ovnv1.OVNDBClusterSpec { return ovnv1.OVNDBClusterSpec{ OVNDBClusterSpecCore: ovnv1.OVNDBClusterSpecCore{ @@ -179,6 +190,73 @@ func DeleteOVNDBClusters(names []types.NamespacedName) { } } +// CreateTLSOVNDBClusters Creates NB and SB OVNDBClusters with TLS enabled. +// Caller must create TLS secrets (CABundleSecretName, OvnDbCertSecretName) first. +func CreateTLSOVNDBClusters(namespace string, nad map[string][]string, replicas int32) []types.NamespacedName { + dbs := []types.NamespacedName{} + for _, db := range []string{ovnv1.NBDBType, ovnv1.SBDBType} { + spec := GetTLSOVNDBClusterSpec() + stringNad := "" + Expect(len(nad)).Should(BeNumerically("<=", 1)) + for k := range nad { + if strings.Contains(k, "/") { + stringNad = strings.Split(k, "/")[1] + } + } + if len(nad) != 0 { + Expect(stringNad).ToNot(Equal("")) + } + spec.DBType = db + spec.NetworkAttachment = stringNad + spec.Replicas = &replicas + instance := CreateOVNDBCluster(namespace, spec) + instanceName := types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} + + dbName := "nb" + if db == ovnv1.SBDBType { + dbName = "sb" + } + statefulSetName := types.NamespacedName{ + Namespace: instance.GetNamespace(), + Name: "ovsdbserver-" + dbName, + } + th.SimulateStatefulSetReplicaReadyWithPods( + statefulSetName, + nad, + ) + Eventually(func(g Gomega) { + ovndbcluster := ovn.GetOVNDBCluster(instanceName) + endpoint := "" + if len(nad) == 0 { + endpoint, _ = ovndbcluster.GetInternalEndpoint() + } else { + endpoint, _ = ovndbcluster.GetExternalEndpoint() + } + g.Expect(endpoint).ToNot(BeEmpty()) + }).Should(Succeed()) + + if db == ovnv1.SBDBType { + Eventually(func(g Gomega) { + ovndbcluster := ovn.GetOVNDBCluster(instanceName) + g.Expect(ovndbcluster.Status.InternalDBAddressRbacFullAccess).ToNot(BeEmpty()) + }).Should(Succeed()) + } + + dbs = append(dbs, instanceName) + } + + logger.Info("TLS OVNDBClusters created", "OVNDBCluster", dbs) + return dbs +} + +func GetCertManagerCert(name types.NamespacedName) *certmgrv1.Certificate { + cert := &certmgrv1.Certificate{} + Eventually(func(g Gomega) { + g.Expect(k8sClient.Get(ctx, name, cert)).Should(Succeed()) + }, timeout, interval).Should(Succeed()) + return cert +} + // GetOVNDBCluster Get OVNDBCluster func GetOVNDBCluster(name types.NamespacedName) *ovnv1.OVNDBCluster { return ovn.GetOVNDBCluster(name) @@ -229,6 +307,16 @@ func GetTLSOVNControllerSpec() ovnv1.OVNControllerSpec { return spec } +func GetTLSRbacOVNControllerSpec() ovnv1.OVNControllerSpec { + spec := GetTLSOVNControllerSpec() + spec.OvnIssuerName = OvnIssuerName + return spec +} + +func GetTLSRbacOVNDBClusterSpec() ovnv1.OVNDBClusterSpec { + return GetTLSOVNDBClusterSpec() +} + func CreateOVNController(namespace string, spec ovnv1.OVNControllerSpec) client.Object { name := ovn.CreateOVNController(nil, namespace, spec) diff --git a/test/functional/ovncontroller_controller_test.go b/test/functional/ovncontroller_controller_test.go index 5ff8c667..3d6d882c 100644 --- a/test/functional/ovncontroller_controller_test.go +++ b/test/functional/ovncontroller_controller_test.go @@ -26,15 +26,18 @@ import ( //revive:disable-next-line:dot-imports . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" ovn_common "github.com/openstack-k8s-operators/ovn-operator/internal/common" + "github.com/openstack-k8s-operators/ovn-operator/internal/ovncontroller" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -911,6 +914,8 @@ var _ = Describe("OVNController controller", func() { BeforeEach(func() { dbs := CreateOVNDBClusters(namespace, map[string][]string{}, 1) DeferCleanup(DeleteOVNDBClusters, dbs) + northdName := CreateReadyOVNNorthd(namespace, GetDefaultOVNNorthdSpec()) + DeferCleanup(th.DeleteInstance, GetOVNNorthd(northdName)) instance := CreateOVNController(namespace, GetTLSOVNControllerSpec()) DeferCleanup(th.DeleteInstance, instance) @@ -1045,10 +1050,13 @@ var _ = Describe("OVNController controller", func() { th.AssertVolumeMountExists("ovn-controller-tls-certs", "tls.crt", svcC.VolumeMounts) th.AssertVolumeMountExists("ovn-controller-tls-certs", "ca.crt", svcC.VolumeMounts) - // check cli args + // check etc-ovs volume mount for RBAC certs + th.AssertVolumeMountExists("etc-ovs", "", svcC.VolumeMounts) + + // check cli args use RBAC cert paths Expect(svcC.Command).To(And( - ContainElement(ContainSubstring(fmt.Sprintf("--private-key=%s", ovn_common.OVNDbKeyPath))), - ContainElement(ContainSubstring(fmt.Sprintf("--certificate=%s", ovn_common.OVNDbCertPath))), + ContainElement(ContainSubstring(fmt.Sprintf("--private-key=%s", ovn_common.OVNControllerKeyPath))), + ContainElement(ContainSubstring(fmt.Sprintf("--certificate=%s", ovn_common.OVNControllerCertPath))), ContainElement(ContainSubstring(fmt.Sprintf("--ca-cert=%s", ovn_common.OVNDbCaCertPath))), )) @@ -1165,6 +1173,196 @@ var _ = Describe("OVNController controller", func() { }) }) + When("OVNController is created with TLS and RBAC", func() { + var ovnControllerName types.NamespacedName + var dbs []types.NamespacedName + + BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + issuer := &certmgrv1.Issuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: OvnIssuerName, + Namespace: namespace, + }, + Spec: certmgrv1.IssuerSpec{ + IssuerConfig: certmgrv1.IssuerConfig{ + SelfSigned: &certmgrv1.SelfSignedIssuer{}, + }, + }, + } + Expect(k8sClient.Create(ctx, issuer)).To(Succeed()) + DeferCleanup(k8sClient.Delete, ctx, issuer) + + dbs = CreateTLSOVNDBClusters(namespace, map[string][]string{}, 1) + DeferCleanup(DeleteOVNDBClusters, dbs) + northdName := CreateReadyOVNNorthd(namespace, GetDefaultOVNNorthdSpec()) + DeferCleanup(th.DeleteInstance, GetOVNNorthd(northdName)) + + instance := CreateOVNController(namespace, GetTLSRbacOVNControllerSpec()) + ovnControllerName = types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} + DeferCleanup(th.DeleteInstance, instance) + }) + + It("creates cert-manager Certificate CRs for each node", func() { + daemonSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetName, + map[string][]string{}, + ) + daemonSetNameOVS := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller-ovs", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetNameOVS, + map[string][]string{}, + ) + + nodeName := daemonSetName.Name + certName := ovncontroller.RbacCertName(nodeName) + certNN := types.NamespacedName{ + Name: certName, + Namespace: namespace, + } + + cert := GetCertManagerCert(certNN) + expectedSystemID := ovncontroller.ComputeSystemID(nodeName) + Expect(cert.Spec.CommonName).To(Equal(expectedSystemID)) + Expect(cert.Spec.IssuerRef.Name).To(Equal(OvnIssuerName)) + Expect(cert.Spec.IssuerRef.Kind).To(Equal("Issuer")) + Expect(cert.Spec.Usages).To(ContainElement(certmgrv1.UsageClientAuth)) + }) + + It("creates config jobs with RBAC cert volumes and SYSTEM_ID env var", func() { + daemonSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetName, + map[string][]string{}, + ) + daemonSetNameOVS := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller-ovs", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetNameOVS, + map[string][]string{}, + ) + + nodeName := daemonSetName.Name + certName := ovncontroller.RbacCertName(nodeName) + certSecretName := ovncontroller.RbacCertSecretName(nodeName) + + // Wait for the Certificate CR to be created, then simulate + // cert-manager by creating the cert Secret + GetCertManagerCert(types.NamespacedName{ + Name: certName, + Namespace: namespace, + }) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: certSecretName, + Namespace: namespace, + })) + + // The config job should now be created with RBAC volumes + configJobName := types.NamespacedName{ + Namespace: ovnControllerName.Namespace, + Name: daemonSetName.Name + "-config", + } + Eventually(func(g Gomega) { + job := th.GetJob(configJobName) + g.Expect(job).ToNot(BeNil()) + + container := job.Spec.Template.Spec.Containers[0] + + // Check SYSTEM_ID env var + systemIDValue := "" + for _, e := range container.Env { + if e.Name == "SYSTEM_ID" { + systemIDValue = e.Value + } + } + expectedSystemID := ovncontroller.ComputeSystemID(nodeName) + g.Expect(systemIDValue).To(Equal(expectedSystemID)) + + // Check RBAC cert volume mount + th.AssertVolumeMountExists("ovn-rbac-cert", "", container.VolumeMounts) + + // Check etc-ovs volume mount (writable for cert install) + hasEtcOvs := false + for _, vm := range container.VolumeMounts { + if vm.Name == "etc-ovs" && vm.MountPath == "/etc/openvswitch" { + hasEtcOvs = true + } + } + g.Expect(hasEtcOvs).To(BeTrue()) + + // Check RBAC cert volume references the cert secret + hasRbacVolume := false + for _, v := range job.Spec.Template.Spec.Volumes { + if v.Name == "ovn-rbac-cert" && v.Secret != nil && v.Secret.SecretName == certSecretName { + hasRbacVolume = true + } + } + g.Expect(hasRbacVolume).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + + It("config job is not created without RBAC cert secret", func() { + daemonSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetName, + map[string][]string{}, + ) + daemonSetNameOVS := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-controller-ovs", + } + SimulateDaemonsetNumberReadyWithPods( + daemonSetNameOVS, + map[string][]string{}, + ) + + nodeName := daemonSetName.Name + certName := ovncontroller.RbacCertName(nodeName) + + // Wait for the Certificate CR to be created, confirming + // the controller reached the RBAC cert step + GetCertManagerCert(types.NamespacedName{ + Name: certName, + Namespace: namespace, + }) + + // Without creating the cert secret, the config job should + // not appear because the controller is waiting for the cert + configJobName := types.NamespacedName{ + Namespace: ovnControllerName.Namespace, + Name: daemonSetName.Name + "-config", + } + Consistently(func(g Gomega) { + job := &batchv1.Job{} + err := k8sClient.Get(ctx, configJobName, job) + g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue()) + }, timeout, interval).Should(Succeed()) + }) + }) + When("OVNController is created with nodeSelector", func() { var ovnControllerName types.NamespacedName var daemonSetName types.NamespacedName diff --git a/test/functional/ovndbcluster_controller_test.go b/test/functional/ovndbcluster_controller_test.go index 96049987..a16c15b3 100644 --- a/test/functional/ovndbcluster_controller_test.go +++ b/test/functional/ovndbcluster_controller_test.go @@ -32,6 +32,7 @@ import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" ovn_common "github.com/openstack-k8s-operators/ovn-operator/internal/common" + "github.com/openstack-k8s-operators/ovn-operator/internal/ovndbcluster" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -1256,9 +1257,11 @@ var _ = Describe("OVNDBCluster controller", func() { When("OVNDBCluster is created with TLS", func() { var OVNDBClusterName types.NamespacedName BeforeEach(func() { - spec := GetTLSOVNDBClusterSpec() + spec := GetTLSRbacOVNDBClusterSpec() spec.NetworkAttachment = "internalapi" spec.DBType = ovnv1.SBDBType + metricsEnabled := false + spec.MetricsEnabled = &metricsEnabled instance := CreateOVNDBCluster(namespace, spec) OVNDBClusterName = types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} DeferCleanup(th.DeleteInstance, instance) @@ -1306,7 +1309,7 @@ var _ = Describe("OVNDBCluster controller", func() { ) }) - It("creates a Statefulset with TLS certs attached", func() { + It("creates a Statefulset with TLS and RBAC certs attached", func() { DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ Name: CABundleSecretName, Namespace: namespace, @@ -1463,6 +1466,100 @@ var _ = Describe("OVNDBCluster controller", func() { }, timeout, interval).Should(Succeed()) }) + It("creates services with full-access port for SB", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + Eventually(func(g Gomega) { + serviceListWithClusterType := GetServicesListWithLabel(namespace, map[string]string{"type": "cluster"}) + g.Expect(serviceListWithClusterType.Items).ToNot(BeEmpty()) + svc := serviceListWithClusterType.Items[0] + + hasFullAccessPort := false + for _, port := range svc.Spec.Ports { + if port.Name == "south-full-access" && port.Port == ovndbcluster.DbPortSBRBACFullAccess { + hasFullAccessPort = true + } + } + g.Expect(hasFullAccessPort).To(BeTrue(), + "SB service should have full-access port %d", ovndbcluster.DbPortSBRBACFullAccess) + }, timeout, interval).Should(Succeed()) + }) + + It("sets InternalDBAddressRbacFullAccess in status", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + Eventually(func(g Gomega) { + dbCluster := GetOVNDBCluster(OVNDBClusterName) + g.Expect(dbCluster.Status.InternalDBAddressRbacFullAccess).ToNot(BeEmpty()) + g.Expect(dbCluster.Status.InternalDBAddressRbacFullAccess).To( + ContainSubstring(fmt.Sprintf(":%d", ovndbcluster.DbPortSBRBACFullAccess))) + g.Expect(dbCluster.Status.InternalDBAddressRbacFullAccess).To(HavePrefix("ssl:")) + }, timeout, interval).Should(Succeed()) + }) + + It("creates scripts ConfigMap with RBAC parameters", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + scriptsCM := types.NamespacedName{ + Namespace: OVNDBClusterName.Namespace, + Name: fmt.Sprintf("%s-%s", OVNDBClusterName.Name, "scripts"), + } + Eventually(func() corev1.ConfigMap { + return *th.GetConfigMap(scriptsCM) + }, timeout, interval).ShouldNot(BeNil()) + + Expect(th.GetConfigMap(scriptsCM).Data["setup.sh"]).Should( + ContainSubstring(fmt.Sprintf("DB_PORT_FULL_ACCESS=\"%d\"", ovndbcluster.DbPortSBRBACFullAccess))) + + Expect(th.GetConfigMap(scriptsCM).Data["setup.sh"]).Should( + ContainSubstring("role=ovn-controller")) + }) + }) When("OVNDB is created with topologyref", func() { diff --git a/test/functional/ovnnorthd_controller_test.go b/test/functional/ovnnorthd_controller_test.go index 5474bfc9..c12ecfb8 100644 --- a/test/functional/ovnnorthd_controller_test.go +++ b/test/functional/ovnnorthd_controller_test.go @@ -462,6 +462,61 @@ var _ = Describe("OVNNorthd controller", func() { }) }) + When("OVNNorthd is created with TLS-enabled DBClusters", func() { + var ovnNorthdName types.NamespacedName + + BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + dbs := CreateTLSOVNDBClusters(namespace, map[string][]string{}, 1) + DeferCleanup(DeleteOVNDBClusters, dbs) + + spec := GetTLSOVNNorthdSpec() + ovnNorthdName = ovn.CreateOVNNorthd(nil, namespace, spec) + DeferCleanup(ovn.DeleteOVNNorthd, ovnNorthdName) + }) + + It("uses full-access SB address with port 16642", func() { + stsName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + + Eventually(func(g Gomega) { + sts := th.GetStatefulSet(stsName) + args := sts.Spec.Template.Spec.Containers[0].Args + + // NB should use ssl scheme with standard port 6641 + hasNBArg := false + for _, arg := range args { + if len(arg) > 10 && arg[:10] == "--ovnnb-db" { + g.Expect(arg).To(ContainSubstring("ssl:")) + g.Expect(arg).To(ContainSubstring(":6641")) + hasNBArg = true + } + } + g.Expect(hasNBArg).To(BeTrue(), "should have --ovnnb-db arg") + + // SB should use ssl scheme with full-access port 16642 + hasSBArg := false + for _, arg := range args { + if len(arg) > 10 && arg[:10] == "--ovnsb-db" { + g.Expect(arg).To(ContainSubstring("ssl:")) + g.Expect(arg).To(ContainSubstring(":16642")) + hasSBArg = true + } + } + g.Expect(hasSBArg).To(BeTrue(), "should have --ovnsb-db arg with full-access port") + }, timeout, interval).Should(Succeed()) + }) + }) + When("OVNNorthd is created with default metrics settings", func() { var ovnNorthdName types.NamespacedName diff --git a/test/functional/suite_test.go b/test/functional/suite_test.go index 31127265..df70d831 100644 --- a/test/functional/suite_test.go +++ b/test/functional/suite_test.go @@ -42,6 +42,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" appsv1 "k8s.io/api/apps/v1" @@ -78,6 +79,7 @@ var ( const ( CABundleSecretName = "combined-ca-bundle" //nolint:gosec // G101: Not actual credentials, just secret name constants OvnDbCertSecretName = "ovndb-tls-cert" //nolint:gosec // G101: Not actual credentials, just secret name constants + OvnIssuerName = "ovn-issuer" ) func TestAPIs(t *testing.T) { @@ -105,6 +107,14 @@ var _ = BeforeSuite(func() { "github.com/openstack-k8s-operators/infra-operator/apis", "../../go.mod", "bases/topology.openstack.org_topologies.yaml") Expect(err).ShouldNot(HaveOccurred()) + certManagerCertCRD, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/lib-common/modules/test", "../../go.mod", "openshift_crds/cert-manager/v1/certificates.cert-manager.io-crd.yaml") + Expect(err).ShouldNot(HaveOccurred()) + + certManagerIssuerCRD, err := test.GetCRDDirFromModule( + "github.com/openstack-k8s-operators/lib-common/modules/test", "../../go.mod", "openshift_crds/cert-manager/v1/issuers.cert-manager.io-crd.yaml") + Expect(err).ShouldNot(HaveOccurred()) + By("bootstrapping test environment") testEnv = &envtest.Environment{ // Increase this to 60 or 120 seconds for the single-core run @@ -120,6 +130,8 @@ var _ = BeforeSuite(func() { networkv1CRD, infranetworkv1CRD, infratopologyv1CRD, + certManagerCertCRD, + certManagerIssuerCRD, }, }, ErrorIfCRDPathMissing: true, @@ -159,6 +171,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = topologyv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = certmgrv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme logger = ctrl.Log.WithName("---Test---")