[WIP] CNTRLPLANE-3236: Kms plugin lifecycle

pkg/operator/encryption/controllers/key_controller.go

@@ -223,7 +223,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact
 
 	sort.Sort(sort.StringSlice(reasons))
 	internalReason := strings.Join(reasons, ", ")
-	keySecret, err := c.generateKeySecret(newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason)
+	keySecret, err := c.generateKeySecret(ctx, newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason)
 	if err != nil {
 		return fmt.Errorf("failed to create key: %v", err)
 	}
@@ -260,7 +260,7 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c
 	return nil // we made this key earlier
 }
 
-func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) {
+func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) {
 	bs := crypto.ModeToNewKeyFunc[currentMode]()
 	ks := state.KeyState{
 		Key: apiserverv1.Key{
@@ -281,6 +281,19 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode,
 			},
 			Provider: apiServerEncryption.KMS,
 		}
+		if apiServerEncryption.KMS != nil {
+			secretName := apiServerEncryption.KMS.Vault.Authentication.AppRole.Secret.Name
+			if len(secretName) > 0 {
+				credentialsSecret, err := c.secretClient.Secrets("openshift-config").Get(ctx, secretName, metav1.GetOptions{})
+				if err != nil {
+					return nil, fmt.Errorf("failed to get credentials secret openshift-config/%s: %w", secretName, err)
+				}
+				ks.KMSConfig.Credentials = make(map[string]string, len(credentialsSecret.Data))
+				for k, v := range credentialsSecret.Data {
+					ks.KMSConfig.Credentials[k] = string(v)
+				}
+			}
+		}
 	}
 	return secrets.FromKeyState(c.instanceName, ks)
 }

pkg/operator/encryption/controllers/key_controller_test.go

@@ -46,6 +46,17 @@ func TestKeyController(t *testing.T) {
 	apiServerWithKMS := simpleAPIServer.DeepCopy()
 	apiServerWithKMS.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: encryptiontesting.DefaultKMSProviderConfig}
 
+	vaultCredentialsSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "vault-approle-secret",
+			Namespace: "openshift-config",
+		},
+		Data: map[string][]byte{
+			"VAULT_ROLE_ID":   []byte("test-role-id"),
+			"VAULT_SECRET_ID": []byte("test-secret-id"),
+		},
+	}
+
 	scenarios := []struct {
 		name                     string
 		initialObjects           []runtime.Object
@@ -336,9 +347,10 @@ func TestKeyController(t *testing.T) {
 				{Group: "", Resource: "secrets"},
 			},
 			targetNamespace: "kms",
-			expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"},
+			expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"},
 			initialObjects: []runtime.Object{
 				encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"),
+				vaultCredentialsSecret,
 			},
 			apiServerObjects: []runtime.Object{apiServerWithKMS},
 			validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) {
@@ -418,10 +430,11 @@ func TestKeyController(t *testing.T) {
 			initialObjects: []runtime.Object{
 				encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"),
 				encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("61def964fb967f5d7c44a2af8dab6865"), "aescbc"),
+				vaultCredentialsSecret,
 			},
 			apiServerObjects: []runtime.Object{apiServerWithKMS},
 			targetNamespace:  "kms",
-			expectedActions:  []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"},
+			expectedActions:  []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"},
 			validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) {
 				wasSecretValidated := false
 				for _, action := range actions {
@@ -512,10 +525,11 @@ func TestKeyController(t *testing.T) {
 			initialObjects: []runtime.Object{
 				encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"),
 				encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("identity-key"), "identity"),
+				vaultCredentialsSecret,
 			},
 			apiServerObjects: []runtime.Object{apiServerWithKMS},
 			targetNamespace:  "kms",
-			expectedActions:  []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"},
+			expectedActions:  []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"},
 			validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) {
 				wasSecretValidated := false
 				for _, action := range actions {

pkg/operator/encryption/encryptiondata/config.go

@@ -3,6 +3,7 @@ package encryptiondata
 import (
 	"encoding/base64"
 	"fmt"
+	"reflect"
 	"sort"
 	"strings"
 
@@ -30,6 +31,9 @@ type Config struct {
 	// KMSProviders maps keyID to provider-specific configuration,
 	// carried from Key Secrets into the encryption-config Secret.
 	KMSProviders map[string]*configv1.KMSConfig
+	// KMSCredentials maps keyID to credential key-value pairs,
+	// carried from Key Secrets into the encryption-config Secret.
+	KMSCredentials map[string]map[string]string
 }
 
 func (c *Config) HasEncryptionConfiguration() bool {
@@ -40,6 +44,7 @@ func (c *Config) HasEncryptionConfiguration() bool {
 func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupResourceState) (*Config, error) {
 	resourceConfigs := make([]apiserverconfigv1.ResourceConfiguration, 0, len(encryptionState))
 	var kmsProviders map[string]*configv1.KMSConfig
+	var kmsCredentials map[string]map[string]string
 
 	for gr, grKeys := range encryptionState {
 		resourceConfigs = append(resourceConfigs, apiserverconfigv1.ResourceConfiguration{
@@ -67,6 +72,18 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes
 					kmsProviders[key.Key.Name] = key.KMSConfig.Provider
 				}
 			}
+			if key.HasKMSCredentials() {
+				if kmsCredentials == nil {
+					kmsCredentials = map[string]map[string]string{}
+				}
+				if existing, exists := kmsCredentials[key.Key.Name]; exists {
+					if !reflect.DeepEqual(existing, key.KMSConfig.Credentials) {
+						return nil, fmt.Errorf("KMS credentials mismatch for keyID %s: credentials from different resources must be identical", key.Key.Name)
+					}
+				} else {
+					kmsCredentials[key.Key.Name] = key.KMSConfig.Credentials
+				}
+			}
 		}
 	}
 
@@ -76,8 +93,9 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes
 	})
 
 	return &Config{
-		Encryption:   &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs},
-		KMSProviders: kmsProviders,
+		Encryption:     &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs},
+		KMSProviders:   kmsProviders,
+		KMSCredentials: kmsCredentials,
 	}, nil
 }
 

pkg/operator/encryption/encryptiondata/config_test.go

@@ -886,6 +886,39 @@ func TestSecretRoundtrip(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "KMS with provider config and credentials",
+			cfg: &encryptiondata.Config{
+				Encryption: &apiserverconfigv1.EncryptionConfiguration{
+					TypeMeta: metav1.TypeMeta{
+						Kind:       "EncryptionConfiguration",
+						APIVersion: "apiserver.config.k8s.io/v1",
+					},
+					Resources: []apiserverconfigv1.ResourceConfiguration{{
+						Resources: []string{"secrets"},
+						Providers: []apiserverconfigv1.ProviderConfiguration{{
+							KMS: &apiserverconfigv1.KMSConfiguration{
+								APIVersion: "v2",
+								Name:       "1_secrets",
+								Endpoint:   "unix:///var/run/kmsplugin/kms-1.sock",
+								Timeout:    &metav1.Duration{Duration: 10 * time.Second},
+							},
+						}, {
+							Identity: &apiserverconfigv1.IdentityConfiguration{},
+						}},
+					}},
+				},
+				KMSProviders: map[string]*configv1.KMSConfig{
+					"1": encryptiontesting.DefaultKMSProviderConfig,
+				},
+				KMSCredentials: map[string]map[string]string{
+					"1": {
+						"VAULT_ROLE_ID":   "test-role-id",
+						"VAULT_SECRET_ID": "test-secret-id",
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

pkg/operator/encryption/encryptiondata/secret.go

@@ -1,6 +1,7 @@
 package encryptiondata
 
 import (
+	"encoding/json"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -29,27 +30,41 @@ func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) {
 		return nil, err
 	}
 	var kmsProviders map[string]*configv1.KMSConfig
+	var kmsCredentials map[string]map[string]string
 	for key, value := range encryptionConfigSecret.Data {
-		// Not all data keys are provider configs — the Secret also contains the
-		// encryption-config entry, so skip keys that don't match the pattern.
 		keyID, found, err := kms.KeyIDFromProviderConfigSecretDataKey(key)
 		if err != nil {
 			return nil, fmt.Errorf("failed to extract keyID from data key %s: %w", key, err)
 		}
-		if !found {
+		if found {
+			providerConfig, err := encoding.DecodeKMSConfig(value)
+			if err != nil {
+				return nil, fmt.Errorf("failed to decode KMS provider config for key %s: %w", keyID, err)
+			}
+			if kmsProviders == nil {
+				kmsProviders = map[string]*configv1.KMSConfig{}
+			}
+			kmsProviders[keyID] = providerConfig
 			continue
 		}
-		providerConfig, err := encoding.DecodeKMSConfig(value)
+
+		credKeyID, credFound, err := kms.KeyIDFromCredentialSecretDataKey(key)
 		if err != nil {
-			return nil, fmt.Errorf("failed to decode KMS provider config for key %s: %w", keyID, err)
+			return nil, fmt.Errorf("failed to extract keyID from credential data key %s: %w", key, err)
 		}
-		if kmsProviders == nil {
-			kmsProviders = map[string]*configv1.KMSConfig{}
+		if credFound {
+			credentials := map[string]string{}
+			if err := json.Unmarshal(value, &credentials); err != nil {
+				return nil, fmt.Errorf("failed to decode KMS credentials for key %s: %w", credKeyID, err)
+			}
+			if kmsCredentials == nil {
+				kmsCredentials = map[string]map[string]string{}
+			}
+			kmsCredentials[credKeyID] = credentials
 		}
-		kmsProviders[keyID] = providerConfig
 	}
 
-	return &Config{Encryption: encryptionConfig, KMSProviders: kmsProviders}, nil
+	return &Config{Encryption: encryptionConfig, KMSProviders: kmsProviders, KMSCredentials: kmsCredentials}, nil
 }
 
 func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) {
@@ -93,5 +108,17 @@ func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) {
 		s.Data[dataKey] = encodedProvider
 	}
 
+	for keyID, credentials := range secretData.KMSCredentials {
+		credentialsData, err := json.Marshal(credentials)
+		if err != nil {
+			return nil, fmt.Errorf("failed to encode KMS credentials for key %s: %w", keyID, err)
+		}
+		dataKey, err := kms.ToCredentialSecretDataKeyFor(keyID)
+		if err != nil {
+			return nil, err
+		}
+		s.Data[dataKey] = credentialsData
+	}
+
 	return s, nil
 }

pkg/operator/encryption/kms/helpers.go

@@ -2,15 +2,24 @@ package kms
 
 import (
 	"fmt"
+	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 
+	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/api/features"
 	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
+	"github.com/openshift/library-go/pkg/operator/encryption/encoding"
 	corev1 "k8s.io/api/core/v1"
+	apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 )
 
+var kmsEndpointRegexp = regexp.MustCompile(`^unix:///var/run/kmsplugin/kms-(\d+)\.sock$`)
+
 const providerConfigDataKeyPrefix = "kms-provider-config-"
+const credentialDataKeyPrefix = "kms-secret-data-"
+const credentialsDir = "/etc/kubernetes/static-pod-resources/secrets/encryption-config"
 
 // ToProviderConfigSecretDataKeyFor constructs the data key for storing a KMS provider config in the encryption-config Secret.
 // The keyID must be a valid non-negative integer string.
@@ -34,6 +43,28 @@ func KeyIDFromProviderConfigSecretDataKey(dataKey string) (string, bool, error)
 	return keyID, true, nil
 }
 
+// ToCredentialSecretDataKeyFor constructs the data key for storing KMS credentials in the encryption-config Secret.
+// The keyID must be a valid non-negative integer string.
+func ToCredentialSecretDataKeyFor(keyID string) (string, error) {
+	if _, err := strconv.ParseUint(keyID, 10, 64); err != nil {
+		return "", fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID)
+	}
+	return credentialDataKeyPrefix + keyID, nil
+}
+
+// KeyIDFromCredentialSecretDataKey extracts the keyID from a kms-secret-data data key.
+// Returns the keyID and true if the key matches the "kms-secret-data-<keyID>" pattern.
+func KeyIDFromCredentialSecretDataKey(dataKey string) (string, bool, error) {
+	keyID, found := strings.CutPrefix(dataKey, credentialDataKeyPrefix)
+	if !found || len(keyID) == 0 {
+		return "", false, nil
+	}
+	if _, err := strconv.ParseUint(keyID, 10, 64); err != nil {
+		return "", false, fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID)
+	}
+	return keyID, true, nil
+}
+
 // AddKMSPluginVolumeAndMountToPodSpec conditionally adds the KMS plugin volume mount to the specified container.
 // It assumes the pod spec does not already contain the KMS volume or mount; no deduplication is performed.
 // Deprecated: this is a temporary solution to get KMS TP v1 out. We should come up with a different approach afterwards.
@@ -90,3 +121,50 @@ func AddKMSPluginVolumeAndMountToPodSpec(podSpec *corev1.PodSpec, containerName
 
 	return nil
 }
+
+func findFirstKMSConfiguration(config *apiserverv1.EncryptionConfiguration) *apiserverv1.KMSConfiguration {
+	for _, resource := range config.Resources {
+		for _, provider := range resource.Providers {
+			if provider.KMS != nil {
+				return provider.KMS
+			}
+		}
+	}
+	return nil
+}
+
+func parseKeyIDFromEndpoint(endpoint string) (string, error) {
+	matches := kmsEndpointRegexp.FindStringSubmatch(endpoint)
+	if matches == nil {
+		return "", fmt.Errorf("unexpected KMS endpoint format: %s", endpoint)
+	}
+	return matches[1], nil
+}
+
+func parseProviderConfig(secret *corev1.Secret, kmsConfiguration *apiserverv1.KMSConfiguration) (*configv1.KMSConfig, error) {
+	keyID, err := parseKeyIDFromEndpoint(kmsConfiguration.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse key ID from endpoint: %w", err)
+	}
+	providerConfigKey, err := ToProviderConfigSecretDataKeyFor(keyID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create provider config secret key ID from endpoint: %w", err)
+	}
+	providerConfigData, ok := secret.Data[providerConfigKey]
+	if !ok {
+		return nil, fmt.Errorf("missing provider config key %s in encryption-config secret", providerConfigKey)
+	}
+	kmsConfig, err := encoding.DecodeKMSConfig(providerConfigData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode provider config: %w", err)
+	}
+	return kmsConfig, nil
+}
+
+func parseSecretDataPath(kmsConfiguration *apiserverv1.KMSConfiguration) (string, error) {
+	keyID, err := parseKeyIDFromEndpoint(kmsConfiguration.Endpoint)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse key ID from endpoint: %w", err)
+	}
+	return filepath.Join(credentialsDir, credentialDataKeyPrefix+keyID), nil
+}