Add support for injecting CA certificates into Secrets

README.md

@@ -4,9 +4,9 @@ This operator runs the following OpenShift controllers:
 * **serving cert signer:**
   * Issues a signed serving certificate/key pair to services annotated with 'service.beta.openshift.io/serving-cert-secret-name' via a secret. [See the current OKD documentation for usage.](https://docs.okd.io/latest/security/certificates/service-serving-certificate.html)
 
-* **configmap cabundle injector:**
-  * Watches for configmaps annotated with 'service.beta.openshift.io/inject-cabundle=true' and adds or updates a data item (key "service-ca.crt") containing the PEM-encoded CA signing bundle. Consumers of the configmap can then trust service-ca.crt in their TLS client configuration, allowing connections to services that utilize service-serving certificates.
-  * Note: Explicitly referencing the "service-ca.crt" key in a volumeMount will prevent a pod from starting until the configMap has been injected with the CA bundle (https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#restrictions). This behavior helps ensure that pods start with the CA bundle data available.
+* **configmap/secret cabundle injector:**
+  * Watches for configmaps and secrets annotated with 'service.beta.openshift.io/inject-cabundle=true' and adds or updates a data item (key "service-ca.crt") containing the PEM-encoded CA signing bundle. Consumers of the configmap/secret can then trust service-ca.crt in their TLS client configuration, allowing connections to services that utilize service-serving certificates.
+  * Note: Explicitly referencing the "service-ca.crt" key in a volumeMount will prevent a pod from starting until the configMap/secret has been injected with the CA bundle (https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#restrictions). This behavior helps ensure that pods start with the CA bundle data available.
 
 ```
 $ oc create configmap foobar --from-literal=key1=foo

pkg/controller/cabundleinjector/secret.go

@@ -0,0 +1,81 @@
+package cabundleinjector
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kcoreclient "k8s.io/client-go/kubernetes/typed/core/v1"
+	listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/klog/v2"
+
+	apiannotations "github.com/openshift/api/annotations"
+	"github.com/openshift/library-go/pkg/controller/factory"
+	"github.com/openshift/service-ca-operator/pkg/controller/api"
+)
+
+type secretCABundleInjector struct {
+	client   kcoreclient.SecretsGetter
+	lister   listers.SecretLister
+	caBundle string
+
+	filterFn func(secret *corev1.Secret) bool
+}
+
+func newSecretInjectorConfig(config *caBundleInjectorConfig) controllerConfig {
+	informer := config.kubeInformers.Core().V1().Secrets()
+
+	syncer := &secretCABundleInjector{
+		client:   config.kubeClient.CoreV1(),
+		lister:   informer.Lister(),
+		caBundle: string(config.caBundle),
+	}
+
+	return controllerConfig{
+		name:     "SecretCABundleInjector",
+		sync:     syncer.Sync,
+		informer: informer.Informer(),
+		annotationsChecker: annotationsChecker(
+			api.InjectCABundleAnnotationName,
+		),
+		namespaced: true,
+	}
+}
+
+func (bi *secretCABundleInjector) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
+	namespace, name := namespacedObjectFromQueueKey(syncCtx.QueueKey())
+
+	secret, err := bi.lister.Secrets(namespace).Get(name)
+	if apierrors.IsNotFound(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if bi.filterFn != nil && !bi.filterFn(secret) {
+		return nil
+	}
+
+	// skip updating when the CA bundle is already there
+	if data, ok := secret.Data[api.InjectionDataKey]; ok &&
+		string(data) == bi.caBundle && len(secret.Data) == 1 {
+
+		return nil
+	}
+
+	klog.Infof("updating secret %s/%s with the service signing CA bundle", secret.Namespace, secret.Name)
+
+	// make a copy to avoid mutating cache state
+	secretCopy := secret.DeepCopy()
+	secretCopy.Data = map[string][]byte{api.InjectionDataKey: []byte(bi.caBundle)}
+	// set the owning-component unless someone else has claimed it.
+	if len(secretCopy.Annotations[apiannotations.OpenShiftComponent]) == 0 {
+		secretCopy.Annotations[apiannotations.OpenShiftComponent] = api.OwningJiraComponent
+		secretCopy.Annotations[apiannotations.OpenShiftDescription] = fmt.Sprintf("Secret is added/updated with a data item containing the CA signing bundle that can be used to verify service-serving certificates")
+	}
+
+	_, err = bi.client.Secrets(secretCopy.Namespace).Update(ctx, secretCopy, metav1.UpdateOptions{})
+	return err
+}

pkg/controller/cabundleinjector/starter.go

@@ -89,6 +89,7 @@ func StartCABundleInjector(ctx context.Context, controllerContext *controllercmd
 	configConstructors := []configBuilderFunc{
 		newAPIServiceInjectorConfig,
 		newConfigMapInjectorConfig,
+		newSecretInjectorConfig,
 		newCRDInjectorConfig,
 		newMutatingWebhookInjectorConfig,
 		newValidatingWebhookInjectorConfig,

test/e2e/e2e.go

@@ -106,6 +106,18 @@ var _ = g.Describe("[sig-service-ca] service-ca-operator", func() {
 		})
 	})
 
+	g.Context("ca-bundle-injection-secret", func() {
+		g.It("[Operator][Serial] should inject CA bundle into annotated secrets", func() {
+			testCABundleInjectionSecret(g.GinkgoTB())
+		})
+	})
+
+	g.Context("ca-bundle-injection-secret-update", func() {
+		g.It("[Operator][Serial] should stomp on updated data in CA bundle injection secrets", func() {
+			testCABundleInjectionSecretUpdate(g.GinkgoTB())
+		})
+	})
+
 	g.Context("vulnerable-legacy-ca-bundle-injection-configmap", func() {
 		g.It("[Operator][Serial] should only inject CA bundle for specific configmap names with legacy annotation", func() {
 			testVulnerableLegacyCABundleInjectionConfigMap(g.GinkgoTB())
@@ -756,6 +768,149 @@ func pollForConfigMapChange(t testing.TB, client *kubernetes.Clientset, compareC
 	})
 }
 
+func testCABundleInjectionSecret(t testing.TB) {
+	adminClient, err := getKubeClient()
+	if err != nil {
+		t.Fatalf("error getting kube client: %v", err)
+	}
+
+	ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5))
+	if err != nil {
+		t.Fatalf("could not create test namespace: %v", err)
+	}
+	defer cleanup()
+
+	testSecretName := "test-secret-" + randSeq(5)
+
+	err = createAnnotatedCABundleInjectionSecret(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error creating annotated secret: %v", err)
+	}
+
+	err = pollForCABundleInjectionSecret(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error fetching ca bundle injection secret: %v", err)
+	}
+
+	err = checkSecretCABundleInjectionData(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error when checking ca bundle injection secret: %v", err)
+	}
+}
+
+func testCABundleInjectionSecretUpdate(t testing.TB) {
+	adminClient, err := getKubeClient()
+	if err != nil {
+		t.Fatalf("error getting kube client: %v", err)
+	}
+
+	ns, cleanup, err := createTestNamespace(t, adminClient, "test-"+randSeq(5))
+	if err != nil {
+		t.Fatalf("could not create test namespace: %v", err)
+	}
+	defer cleanup()
+
+	testSecretName := "test-secret-" + randSeq(5)
+
+	err = createAnnotatedCABundleInjectionSecret(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error creating annotated secret: %v", err)
+	}
+
+	err = pollForCABundleInjectionSecret(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error fetching ca bundle injection secret: %v", err)
+	}
+
+	err = checkSecretCABundleInjectionData(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error when checking ca bundle injection secret: %v", err)
+	}
+
+	err = editSecretCABundleInjectionData(t, adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error editing ca bundle injection secret: %v", err)
+	}
+
+	err = checkSecretCABundleInjectionData(adminClient, testSecretName, ns.Name)
+	if err != nil {
+		t.Fatalf("error when checking ca bundle injection secret: %v", err)
+	}
+}
+
+func createAnnotatedCABundleInjectionSecret(client *kubernetes.Clientset, secretName, namespace string) error {
+	obj := &v1.Secret{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: secretName,
+		},
+	}
+	setInjectionAnnotation(&obj.ObjectMeta)
+	_, err := client.CoreV1().Secrets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+	return err
+}
+
+func pollForCABundleInjectionSecret(client *kubernetes.Clientset, secretName, namespace string) error {
+	return wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
+		_, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+		if err != nil && errors.IsNotFound(err) {
+			return false, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	})
+}
+
+func checkSecretCABundleInjectionData(client *kubernetes.Clientset, secretName, namespace string) error {
+	secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if len(secret.Data) != 1 {
+		return fmt.Errorf("unexpected ca bundle injection secret data map length: %v", len(secret.Data))
+	}
+	if _, ok := secret.Data[api.InjectionDataKey]; !ok {
+		return fmt.Errorf("unexpected ca bundle injection secret data: %v", secret.Data)
+	}
+	return nil
+}
+
+func editSecretCABundleInjectionData(t testing.TB, client *kubernetes.Clientset, secretName, namespace string) error {
+	secret, err := client.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	secretCopy := secret.DeepCopy()
+	if len(secretCopy.Data) != 1 {
+		return fmt.Errorf("ca bundle injection secret missing data")
+	}
+	secretCopy.Data["foo"] = []byte("blah")
+	_, err = client.CoreV1().Secrets(namespace).Update(context.TODO(), secretCopy, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	return pollForSecretDataChange(t, client, secretCopy, "foo")
+}
+
+func pollForSecretDataChange(t testing.TB, client *kubernetes.Clientset, compareSecret *v1.Secret, keysToChange ...string) error {
+	return wait.PollImmediate(pollInterval, rotationPollTimeout, func() (bool, error) {
+		s, err := client.CoreV1().Secrets(compareSecret.Namespace).Get(context.TODO(), compareSecret.Name, metav1.GetOptions{})
+		if err != nil {
+			t.Logf("%s: failed to get secret: %v", time.Now().Format(time.RFC1123Z), err)
+			return false, nil
+		}
+		for _, key := range keysToChange {
+			if bytes.Equal(s.Data[key], compareSecret.Data[key]) {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+}
+
 // pollForConfigMapCAInjection polls until the configmap has CA bundle injected.
 // This is different from pollForCABundleInjectionConfigMap which only checks if
 // the configmap exists. This function validates the injection data is present.

test/e2e/e2e_test.go

@@ -479,6 +479,19 @@ func TestE2E(t *testing.T) {
 		testCABundleInjectionConfigMapUpdate(t)
 	})
 
+	// test ca bundle injection secret
+	// NOTE: This test is also available in the OTE framework (test/e2e/e2e.go).
+	// This duplication is temporary until we fully migrate to OTE and validate the new e2e jobs.
+	// Eventually, all tests will run only through the OTE framework.
+	t.Run("ca-bundle-injection-secret", func(t *testing.T) {
+		testCABundleInjectionSecret(t)
+	})
+
+	// test updated data in ca bundle injection secret will be stomped on
+	t.Run("ca-bundle-injection-secret-update", func(t *testing.T) {
+		testCABundleInjectionSecretUpdate(t)
+	})
+
 	// test vulnerable-legacy ca bundle injection configmap
 	// NOTE: This test is also available in the OTE framework (test/e2e/e2e.go).
 	// This duplication is temporary until we fully migrate to OTE and validate the new e2e jobs.