From 5221c8280c1ba7448f318cde82784274d4f2604b Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Thu, 7 May 2026 15:00:59 -0400 Subject: [PATCH 1/4] kms: propagate credential data through the encryption-config secret pipeline --- .../encryption/controllers/key_controller.go | 17 ++++++- .../controllers/key_controller_test.go | 20 +++++++-- .../encryption/encryptiondata/config.go | 22 ++++++++- .../encryption/encryptiondata/config_test.go | 33 ++++++++++++++ .../encryption/encryptiondata/secret.go | 45 +++++++++++++++---- pkg/operator/encryption/kms/helpers.go | 23 ++++++++++ pkg/operator/encryption/secrets/secrets.go | 15 +++++++ .../encryption/secrets/secrets_test.go | 25 +++++++++++ pkg/operator/encryption/secrets/types.go | 4 ++ pkg/operator/encryption/state/types.go | 7 +++ pkg/operator/encryption/testing/helpers.go | 11 +++++ 11 files changed, 206 insertions(+), 16 deletions(-) diff --git a/pkg/operator/encryption/controllers/key_controller.go b/pkg/operator/encryption/controllers/key_controller.go index bce23fe788..ff27a125f9 100644 --- a/pkg/operator/encryption/controllers/key_controller.go +++ b/pkg/operator/encryption/controllers/key_controller.go @@ -223,7 +223,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact sort.Sort(sort.StringSlice(reasons)) internalReason := strings.Join(reasons, ", ") - keySecret, err := c.generateKeySecret(newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) + keySecret, err := c.generateKeySecret(ctx, newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) if err != nil { return fmt.Errorf("failed to create key: %v", err) } @@ -260,7 +260,7 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c return nil // we made this key earlier } -func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { +func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { bs := crypto.ModeToNewKeyFunc[currentMode]() ks := state.KeyState{ Key: apiserverv1.Key{ @@ -281,6 +281,19 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, }, Provider: apiServerEncryption.KMS, } + if apiServerEncryption.KMS != nil { + secretName := apiServerEncryption.KMS.Vault.Authentication.AppRole.Secret.Name + if len(secretName) > 0 { + credentialsSecret, err := c.secretClient.Secrets("openshift-config").Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get credentials secret openshift-config/%s: %w", secretName, err) + } + ks.KMSConfig.Credentials = make(map[string]string, len(credentialsSecret.Data)) + for k, v := range credentialsSecret.Data { + ks.KMSConfig.Credentials[k] = string(v) + } + } + } } return secrets.FromKeyState(c.instanceName, ks) } diff --git a/pkg/operator/encryption/controllers/key_controller_test.go b/pkg/operator/encryption/controllers/key_controller_test.go index a5b9d13486..bf77e72182 100644 --- a/pkg/operator/encryption/controllers/key_controller_test.go +++ b/pkg/operator/encryption/controllers/key_controller_test.go @@ -46,6 +46,17 @@ func TestKeyController(t *testing.T) { apiServerWithKMS := simpleAPIServer.DeepCopy() apiServerWithKMS.Spec.Encryption = configv1.APIServerEncryption{Type: "KMS", KMS: encryptiontesting.DefaultKMSProviderConfig} + vaultCredentialsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vault-approle-secret", + Namespace: "openshift-config", + }, + Data: map[string][]byte{ + "VAULT_ROLE_ID": []byte("test-role-id"), + "VAULT_SECRET_ID": []byte("test-secret-id"), + }, + } + scenarios := []struct { name string initialObjects []runtime.Object @@ -336,9 +347,10 @@ func TestKeyController(t *testing.T) { {Group: "", Resource: "secrets"}, }, targetNamespace: "kms", - expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, initialObjects: []runtime.Object{ encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + vaultCredentialsSecret, }, apiServerObjects: []runtime.Object{apiServerWithKMS}, validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { @@ -418,10 +430,11 @@ func TestKeyController(t *testing.T) { initialObjects: []runtime.Object{ encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("61def964fb967f5d7c44a2af8dab6865"), "aescbc"), + vaultCredentialsSecret, }, apiServerObjects: []runtime.Object{apiServerWithKMS}, targetNamespace: "kms", - expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { wasSecretValidated := false for _, action := range actions { @@ -512,10 +525,11 @@ func TestKeyController(t *testing.T) { initialObjects: []runtime.Object{ encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("identity-key"), "identity"), + vaultCredentialsSecret, }, apiServerObjects: []runtime.Object{apiServerWithKMS}, targetNamespace: "kms", - expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { wasSecretValidated := false for _, action := range actions { diff --git a/pkg/operator/encryption/encryptiondata/config.go b/pkg/operator/encryption/encryptiondata/config.go index e51837a5cb..5dbc825cd7 100644 --- a/pkg/operator/encryption/encryptiondata/config.go +++ b/pkg/operator/encryption/encryptiondata/config.go @@ -3,6 +3,7 @@ package encryptiondata import ( "encoding/base64" "fmt" + "reflect" "sort" "strings" @@ -30,6 +31,9 @@ type Config struct { // KMSProviders maps keyID to provider-specific configuration, // carried from Key Secrets into the encryption-config Secret. KMSProviders map[string]*configv1.KMSConfig + // KMSCredentials maps keyID to credential key-value pairs, + // carried from Key Secrets into the encryption-config Secret. + KMSCredentials map[string]map[string]string } func (c *Config) HasEncryptionConfiguration() bool { @@ -40,6 +44,7 @@ func (c *Config) HasEncryptionConfiguration() bool { func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupResourceState) (*Config, error) { resourceConfigs := make([]apiserverconfigv1.ResourceConfiguration, 0, len(encryptionState)) var kmsProviders map[string]*configv1.KMSConfig + var kmsCredentials map[string]map[string]string for gr, grKeys := range encryptionState { resourceConfigs = append(resourceConfigs, apiserverconfigv1.ResourceConfiguration{ @@ -67,6 +72,18 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes kmsProviders[key.Key.Name] = key.KMSConfig.Provider } } + if key.HasKMSCredentials() { + if kmsCredentials == nil { + kmsCredentials = map[string]map[string]string{} + } + if existing, exists := kmsCredentials[key.Key.Name]; exists { + if !reflect.DeepEqual(existing, key.KMSConfig.Credentials) { + return nil, fmt.Errorf("KMS credentials mismatch for keyID %s: credentials from different resources must be identical", key.Key.Name) + } + } else { + kmsCredentials[key.Key.Name] = key.KMSConfig.Credentials + } + } } } @@ -76,8 +93,9 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes }) return &Config{ - Encryption: &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs}, - KMSProviders: kmsProviders, + Encryption: &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs}, + KMSProviders: kmsProviders, + KMSCredentials: kmsCredentials, }, nil } diff --git a/pkg/operator/encryption/encryptiondata/config_test.go b/pkg/operator/encryption/encryptiondata/config_test.go index e5bf4329aa..00ff9ca2d7 100644 --- a/pkg/operator/encryption/encryptiondata/config_test.go +++ b/pkg/operator/encryption/encryptiondata/config_test.go @@ -886,6 +886,39 @@ func TestSecretRoundtrip(t *testing.T) { }, }, }, + { + name: "KMS with provider config and credentials", + cfg: &encryptiondata.Config{ + Encryption: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1_secrets", + Endpoint: "unix:///var/run/kmsplugin/kms-1.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + KMSProviders: map[string]*configv1.KMSConfig{ + "1": encryptiontesting.DefaultKMSProviderConfig, + }, + KMSCredentials: map[string]map[string]string{ + "1": { + "VAULT_ROLE_ID": "test-role-id", + "VAULT_SECRET_ID": "test-secret-id", + }, + }, + }, + }, } for _, tt := range tests { diff --git a/pkg/operator/encryption/encryptiondata/secret.go b/pkg/operator/encryption/encryptiondata/secret.go index b983697026..92c7e76bc1 100644 --- a/pkg/operator/encryption/encryptiondata/secret.go +++ b/pkg/operator/encryption/encryptiondata/secret.go @@ -1,6 +1,7 @@ package encryptiondata import ( + "encoding/json" "fmt" corev1 "k8s.io/api/core/v1" @@ -29,27 +30,41 @@ func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) { return nil, err } var kmsProviders map[string]*configv1.KMSConfig + var kmsCredentials map[string]map[string]string for key, value := range encryptionConfigSecret.Data { - // Not all data keys are provider configs — the Secret also contains the - // encryption-config entry, so skip keys that don't match the pattern. keyID, found, err := kms.KeyIDFromProviderConfigSecretDataKey(key) if err != nil { return nil, fmt.Errorf("failed to extract keyID from data key %s: %w", key, err) } - if !found { + if found { + providerConfig, err := encoding.DecodeKMSConfig(value) + if err != nil { + return nil, fmt.Errorf("failed to decode KMS provider config for key %s: %w", keyID, err) + } + if kmsProviders == nil { + kmsProviders = map[string]*configv1.KMSConfig{} + } + kmsProviders[keyID] = providerConfig continue } - providerConfig, err := encoding.DecodeKMSConfig(value) + + credKeyID, credFound, err := kms.KeyIDFromCredentialSecretDataKey(key) if err != nil { - return nil, fmt.Errorf("failed to decode KMS provider config for key %s: %w", keyID, err) + return nil, fmt.Errorf("failed to extract keyID from credential data key %s: %w", key, err) } - if kmsProviders == nil { - kmsProviders = map[string]*configv1.KMSConfig{} + if credFound { + credentials := map[string]string{} + if err := json.Unmarshal(value, &credentials); err != nil { + return nil, fmt.Errorf("failed to decode KMS credentials for key %s: %w", credKeyID, err) + } + if kmsCredentials == nil { + kmsCredentials = map[string]map[string]string{} + } + kmsCredentials[credKeyID] = credentials } - kmsProviders[keyID] = providerConfig } - return &Config{Encryption: encryptionConfig, KMSProviders: kmsProviders}, nil + return &Config{Encryption: encryptionConfig, KMSProviders: kmsProviders, KMSCredentials: kmsCredentials}, nil } func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) { @@ -93,5 +108,17 @@ func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) { s.Data[dataKey] = encodedProvider } + for keyID, credentials := range secretData.KMSCredentials { + credentialsData, err := json.Marshal(credentials) + if err != nil { + return nil, fmt.Errorf("failed to encode KMS credentials for key %s: %w", keyID, err) + } + dataKey, err := kms.ToCredentialSecretDataKeyFor(keyID) + if err != nil { + return nil, err + } + s.Data[dataKey] = credentialsData + } + return s, nil } diff --git a/pkg/operator/encryption/kms/helpers.go b/pkg/operator/encryption/kms/helpers.go index 5412df517e..8d31fa2f2a 100644 --- a/pkg/operator/encryption/kms/helpers.go +++ b/pkg/operator/encryption/kms/helpers.go @@ -11,6 +11,7 @@ import ( ) const providerConfigDataKeyPrefix = "kms-provider-config-" +const credentialDataKeyPrefix = "kms-secret-data-" // ToProviderConfigSecretDataKeyFor constructs the data key for storing a KMS provider config in the encryption-config Secret. // The keyID must be a valid non-negative integer string. @@ -34,6 +35,28 @@ func KeyIDFromProviderConfigSecretDataKey(dataKey string) (string, bool, error) return keyID, true, nil } +// ToCredentialSecretDataKeyFor constructs the data key for storing KMS credentials in the encryption-config Secret. +// The keyID must be a valid non-negative integer string. +func ToCredentialSecretDataKeyFor(keyID string) (string, error) { + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return credentialDataKeyPrefix + keyID, nil +} + +// KeyIDFromCredentialSecretDataKey extracts the keyID from a kms-secret-data data key. +// Returns the keyID and true if the key matches the "kms-secret-data-" pattern. +func KeyIDFromCredentialSecretDataKey(dataKey string) (string, bool, error) { + keyID, found := strings.CutPrefix(dataKey, credentialDataKeyPrefix) + if !found || len(keyID) == 0 { + return "", false, nil + } + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", false, fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return keyID, true, nil +} + // AddKMSPluginVolumeAndMountToPodSpec conditionally adds the KMS plugin volume mount to the specified container. // It assumes the pod spec does not already contain the KMS volume or mount; no deduplication is performed. // Deprecated: this is a temporary solution to get KMS TP v1 out. We should come up with a different approach afterwards. diff --git a/pkg/operator/encryption/secrets/secrets.go b/pkg/operator/encryption/secrets/secrets.go index 1d6ca40167..c598d5dbc5 100644 --- a/pkg/operator/encryption/secrets/secrets.go +++ b/pkg/operator/encryption/secrets/secrets.go @@ -87,6 +87,13 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { // encryption mode. return state.KeyState{}, fmt.Errorf("%s can not be empty, when mode is KMS", EncryptionSecretKMSProviderConfig) } + if v, ok := s.Data[EncryptionSecretKMSCredentials]; ok && len(v) > 0 { + credentials := map[string]string{} + if err := json.Unmarshal(v, &credentials); err != nil { + return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid %s data: %w", s.Namespace, s.Name, EncryptionSecretKMSCredentials, err) + } + key.KMSConfig.Credentials = credentials + } key.Mode = keyMode default: return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid mode: %s", s.Namespace, s.Name, keyMode) @@ -159,6 +166,14 @@ func FromKeyState(component string, ks state.KeyState) (*corev1.Secret, error) { s.Data[EncryptionSecretKMSProviderConfig] = providerData } + if ks.HasKMSCredentials() { + credentialsData, err := json.Marshal(ks.KMSConfig.Credentials) + if err != nil { + return nil, fmt.Errorf("failed to encode KMS credentials: %w", err) + } + s.Data[EncryptionSecretKMSCredentials] = credentialsData + } + return s, nil } diff --git a/pkg/operator/encryption/secrets/secrets_test.go b/pkg/operator/encryption/secrets/secrets_test.go index 5f0c9f089e..3140d8a1a7 100644 --- a/pkg/operator/encryption/secrets/secrets_test.go +++ b/pkg/operator/encryption/secrets/secrets_test.go @@ -182,6 +182,31 @@ func TestRoundtrip(t *testing.T) { }, }, }, + { + name: "full kms with credentials", + component: "kms", + ks: state.KeyState{ + Key: v1.Key{ + Name: "3", + Secret: base64.StdEncoding.EncodeToString(emptyKey), + }, + Backed: true, + Mode: "KMS", + KMSConfig: &state.KMSConfig{ + Encryption: &v1.KMSConfiguration{ + APIVersion: "v2", + Name: "3", + Endpoint: "unix:///var/run/kmsplugin/kms-3.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + Provider: defaultKMSProviderConfig, + Credentials: map[string]string{ + "VAULT_ROLE_ID": "test-role-id", + "VAULT_SECRET_ID": "test-secret-id", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/operator/encryption/secrets/types.go b/pkg/operator/encryption/secrets/types.go index 17f0ff4311..c19efa8f9d 100644 --- a/pkg/operator/encryption/secrets/types.go +++ b/pkg/operator/encryption/secrets/types.go @@ -57,6 +57,10 @@ const ( // EncryptionSecretKMSProviderConfig is the data field key that stores the serialized provider // configuration for KMS mode in the encryption-key secret. EncryptionSecretKMSProviderConfig = "encryption.apiserver.operator.openshift.io-kms-provider-config" + + // EncryptionSecretKMSCredentials is the data field key for the JSON-encoded KMS + // credential map in the encryption-key secret. + EncryptionSecretKMSCredentials = "encryption.apiserver.operator.openshift.io-kms-secret-data" ) // MigratedGroupResources is the data structured stored in the diff --git a/pkg/operator/encryption/state/types.go b/pkg/operator/encryption/state/types.go index f9adea69a8..298bb6fbe8 100644 --- a/pkg/operator/encryption/state/types.go +++ b/pkg/operator/encryption/state/types.go @@ -53,6 +53,10 @@ func (k *KeyState) HasKMSProvider() bool { return k != nil && k.KMSConfig != nil && k.KMSConfig.Provider != nil } +func (k *KeyState) HasKMSCredentials() bool { + return k != nil && k.KMSConfig != nil && len(k.KMSConfig.Credentials) > 0 +} + // KMSConfig stores all KMS encryption mode related configurations type KMSConfig struct { // Encoded EncryptionConfig that stores the KMS related fields @@ -60,6 +64,9 @@ type KMSConfig struct { // Provider stores KMS provider specific configurations Provider *configv1.KMSConfig + + // Credentials stores credential key-value pairs from the referenced Secret + Credentials map[string]string } type MigrationState struct { diff --git a/pkg/operator/encryption/testing/helpers.go b/pkg/operator/encryption/testing/helpers.go index 96cbaec8ce..a33fa92791 100644 --- a/pkg/operator/encryption/testing/helpers.go +++ b/pkg/operator/encryption/testing/helpers.go @@ -28,6 +28,7 @@ const ( encryptionSecretMigratedResourcesForTest = "encryption.apiserver.operator.openshift.io/migrated-resources" encryptionSecretKMSEncryptionConfigForTest = "encryption.apiserver.operator.openshift.io-kms-encryption-config" encryptionSecretKMSProviderConfigForTest = "encryption.apiserver.operator.openshift.io-kms-provider-config" + encryptionSecretKMSCredentialsForTest = "encryption.apiserver.operator.openshift.io-kms-secret-data" ) func CreateEncryptionKeySecretNoData(targetNS string, grs []schema.GroupResource, keyID uint64) *corev1.Secret { @@ -149,6 +150,16 @@ func CreateEncryptionKeySecretWithCustomKMSConfig(targetNS string, grs []schema. return secret } +func CreateEncryptionKeySecretWithKMSConfigAndCredentials(targetNS string, grs []schema.GroupResource, keyID uint64, credentials map[string]string) *corev1.Secret { + secret := CreateEncryptionKeySecretWithCustomKMSConfig(targetNS, grs, keyID, DefaultKMSProviderConfig) + credentialsData, err := json.Marshal(credentials) + if err != nil { + panic(fmt.Sprintf("failed to encode KMS credentials: %v", err)) + } + secret.Data[encryptionSecretKMSCredentialsForTest] = credentialsData + return secret +} + func CreateDummyKubeAPIPod(name, namespace string, nodeName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ From 407f682fc72e9a69f354f5429fc0df2bfc80730f Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Fri, 8 May 2026 10:20:12 -0400 Subject: [PATCH 2/4] kms: add helper functions for parsing KMS configuration Add utility functions for parsing KMS endpoints, provider configs, and secret data paths from the encryption-config Secret. --- pkg/operator/encryption/kms/helpers.go | 55 ++++++++ pkg/operator/encryption/kms/helpers_test.go | 144 +++++++++++++++++++- 2 files changed, 197 insertions(+), 2 deletions(-) diff --git a/pkg/operator/encryption/kms/helpers.go b/pkg/operator/encryption/kms/helpers.go index 8d31fa2f2a..eeb6a0c24f 100644 --- a/pkg/operator/encryption/kms/helpers.go +++ b/pkg/operator/encryption/kms/helpers.go @@ -2,16 +2,24 @@ package kms import ( "fmt" + "path/filepath" + "regexp" "strconv" "strings" + configv1 "github.com/openshift/api/config/v1" "github.com/openshift/api/features" "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" + "github.com/openshift/library-go/pkg/operator/encryption/encoding" corev1 "k8s.io/api/core/v1" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" ) +var kmsEndpointRegexp = regexp.MustCompile(`^unix:///var/run/kmsplugin/kms-(\d+)\.sock$`) + const providerConfigDataKeyPrefix = "kms-provider-config-" const credentialDataKeyPrefix = "kms-secret-data-" +const credentialsDir = "/etc/kubernetes/static-pod-resources/secrets/encryption-config" // ToProviderConfigSecretDataKeyFor constructs the data key for storing a KMS provider config in the encryption-config Secret. // The keyID must be a valid non-negative integer string. @@ -113,3 +121,50 @@ func AddKMSPluginVolumeAndMountToPodSpec(podSpec *corev1.PodSpec, containerName return nil } + +func findFirstKMSConfiguration(config *apiserverv1.EncryptionConfiguration) *apiserverv1.KMSConfiguration { + for _, resource := range config.Resources { + for _, provider := range resource.Providers { + if provider.KMS != nil { + return provider.KMS + } + } + } + return nil +} + +func parseKeyIDFromEndpoint(endpoint string) (string, error) { + matches := kmsEndpointRegexp.FindStringSubmatch(endpoint) + if matches == nil { + return "", fmt.Errorf("unexpected KMS endpoint format: %s", endpoint) + } + return matches[1], nil +} + +func parseProviderConfig(secret *corev1.Secret, kmsConfiguration *apiserverv1.KMSConfiguration) (*configv1.KMSConfig, error) { + keyID, err := parseKeyIDFromEndpoint(kmsConfiguration.Endpoint) + if err != nil { + return nil, fmt.Errorf("failed to parse key ID from endpoint: %w", err) + } + providerConfigKey, err := ToProviderConfigSecretDataKeyFor(keyID) + if err != nil { + return nil, fmt.Errorf("failed to create provider config secret key ID from endpoint: %w", err) + } + providerConfigData, ok := secret.Data[providerConfigKey] + if !ok { + return nil, fmt.Errorf("missing provider config key %s in encryption-config secret", providerConfigKey) + } + kmsConfig, err := encoding.DecodeKMSConfig(providerConfigData) + if err != nil { + return nil, fmt.Errorf("failed to decode provider config: %w", err) + } + return kmsConfig, nil +} + +func parseSecretDataPath(kmsConfiguration *apiserverv1.KMSConfiguration) (string, error) { + keyID, err := parseKeyIDFromEndpoint(kmsConfiguration.Endpoint) + if err != nil { + return "", fmt.Errorf("failed to parse key ID from endpoint: %w", err) + } + return filepath.Join(credentialsDir, credentialDataKeyPrefix+keyID), nil +} diff --git a/pkg/operator/encryption/kms/helpers_test.go b/pkg/operator/encryption/kms/helpers_test.go index 456d850532..4b51eb4552 100644 --- a/pkg/operator/encryption/kms/helpers_test.go +++ b/pkg/operator/encryption/kms/helpers_test.go @@ -7,9 +7,10 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/api/features" "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" - corev1 "k8s.io/api/core/v1" - + "github.com/openshift/library-go/pkg/operator/encryption/encoding" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" ) func TestKeyIDFromProviderConfigSecretDataKey(t *testing.T) { @@ -114,6 +115,145 @@ func TestToProviderConfigSecretDataKeyFor(t *testing.T) { } } +func TestParseKeyIDFromEndpoint(t *testing.T) { + tests := []struct { + name string + endpoint string + wantKeyID string + wantErr string + }{ + { + name: "standard endpoint", + endpoint: "unix:///var/run/kmsplugin/kms-555.sock", + wantKeyID: "555", + }, + { + name: "single digit key ID", + endpoint: "unix:///var/run/kmsplugin/kms-3.sock", + wantKeyID: "3", + }, + { + name: "missing kms- prefix", + endpoint: "unix:///var/run/kmsplugin/plugin-555.sock", + wantErr: "unexpected KMS endpoint format", + }, + { + name: "missing .sock suffix", + endpoint: "unix:///var/run/kmsplugin/kms-555.socket", + wantErr: "unexpected KMS endpoint format", + }, + { + name: "empty key ID", + endpoint: "unix:///var/run/kmsplugin/kms-.sock", + wantErr: "unexpected KMS endpoint format", + }, + { + name: "no unix prefix", + endpoint: "/var/run/kmsplugin/kms-555.sock", + wantErr: "unexpected KMS endpoint format", + }, + { + name: "no digit key ID", + endpoint: "/var/run/kmsplugin/kms-abc.sock", + wantErr: "unexpected KMS endpoint format", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + keyID, err := parseKeyIDFromEndpoint(tt.endpoint) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.wantKeyID, keyID) + }) + } +} + +func TestParseProviderConfig(t *testing.T) { + vaultConfig := &configv1.KMSConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSConfig{ + KMSPluginImage: "quay.io/test/vault:v1", + VaultAddress: "https://vault.example.com:8200", + VaultNamespace: "my-namespace", + TransitKey: "my-key", + TransitMount: "transit", + }, + } + configBytes, err := encoding.EncodeKMSConfig(vaultConfig) + require.NoError(t, err) + + providerConfigKey, err := ToProviderConfigSecretDataKeyFor("555") + require.NoError(t, err) + + tests := []struct { + name string + secret *corev1.Secret + kmsConfig *apiserverv1.KMSConfiguration + wantErr string + }{ + { + name: "valid provider config", + secret: &corev1.Secret{ + Data: map[string][]byte{ + providerConfigKey: configBytes, + }, + }, + kmsConfig: &apiserverv1.KMSConfiguration{ + Endpoint: "unix:///var/run/kmsplugin/kms-555.sock", + }, + }, + { + name: "missing provider config key", + secret: &corev1.Secret{ + Data: map[string][]byte{}, + }, + kmsConfig: &apiserverv1.KMSConfiguration{ + Endpoint: "unix:///var/run/kmsplugin/kms-555.sock", + }, + wantErr: "missing provider config key", + }, + { + name: "invalid JSON", + secret: &corev1.Secret{ + Data: map[string][]byte{ + providerConfigKey: []byte(`{invalid`), + }, + }, + kmsConfig: &apiserverv1.KMSConfiguration{ + Endpoint: "unix:///var/run/kmsplugin/kms-555.sock", + }, + wantErr: "failed to decode provider config", + }, + { + name: "invalid endpoint", + secret: &corev1.Secret{ + Data: map[string][]byte{}, + }, + kmsConfig: &apiserverv1.KMSConfiguration{ + Endpoint: "invalid-endpoint", + }, + wantErr: "failed to parse key ID from endpoint", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config, err := parseProviderConfig(tt.secret, tt.kmsConfig) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.NotNil(t, config) + require.Equal(t, vaultConfig, config) + }) + } +} + func TestAddKMSPluginVolume(t *testing.T) { directoryOrCreate := corev1.HostPathDirectoryOrCreate From 3b48c83ea2c5c6742cc18422a25986a4e20a5e4e Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Fri, 8 May 2026 10:20:51 -0400 Subject: [PATCH 3/4] kms: add sidecar injection into pod specs Add InjectIntoPodSpec which reads the encryption-config Secret, parses the KMS configuration, and injects a sidecar container with the necessary volume mounts into the API server pod spec. --- pkg/operator/encryption/kms/sidecar.go | 186 +++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 pkg/operator/encryption/kms/sidecar.go diff --git a/pkg/operator/encryption/kms/sidecar.go b/pkg/operator/encryption/kms/sidecar.go new file mode 100644 index 0000000000..983c59e416 --- /dev/null +++ b/pkg/operator/encryption/kms/sidecar.go @@ -0,0 +1,186 @@ +package kms + +import ( + "fmt" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/operator/encryption/encoding" + corev1 "k8s.io/api/core/v1" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" +) + +type OperatorConfig struct { + EncryptionConfigNamespace string + EncryptionConfigSecretName string + APIServerContainerName string +} + +type SidecarProvider interface { + BuildSidecarContainer(name string, kmsConfiguration *apiserverv1.KMSConfiguration) (corev1.Container, error) +} + +func newSidecarProvider(providerConfig *configv1.KMSConfig, secretDataPath string) (SidecarProvider, error) { + return nil, fmt.Errorf("unsupported KMS provider configuration") +} + +func InjectIntoPodSpec(podSpec *corev1.PodSpec, secretLister corev1listers.SecretLister, opConfig OperatorConfig) error { + if podSpec == nil { + return fmt.Errorf("pod spec cannot be nil") + } + + encryptionConfigurationSecret, err := secretLister.Secrets(opConfig.EncryptionConfigNamespace).Get(opConfig.EncryptionConfigSecretName) + if err != nil { + klog.V(4).Infof("KMS is disabled: could not get %s secret: %v", opConfig.EncryptionConfigSecretName, err) + return nil + } + + encryptionConfigurationBytes, ok := encryptionConfigurationSecret.Data["encryption-config"] + if !ok { + klog.V(4).Infof("KMS is disabled: could not get encryption-config key in secret %s", encryptionConfigurationSecret.Name) + return nil + } + + encryptionConfiguration, err := encoding.DecodeEncryptionConfiguration(encryptionConfigurationBytes) + if err != nil { + return fmt.Errorf("failed to decode EncryptionConfiguration from %s/%s secret: %w", opConfig.EncryptionConfigNamespace, opConfig.EncryptionConfigSecretName, err) + } + + // TODO: don't get only the first? Deduplicate and do this in a loop + kmsConfiguration := findFirstKMSConfiguration(encryptionConfiguration) + if kmsConfiguration == nil { + // TODO: should error out instead? + klog.V(4).Infof("KMS is disabled: no KMS provider found in EncryptionConfiguration") + return nil + } + + providerConfig, err := parseProviderConfig(encryptionConfigurationSecret, kmsConfiguration) + if err != nil { + return fmt.Errorf("failed to parse provider config: %w", err) + } + + secretDataPath, err := parseSecretDataPath(kmsConfiguration) + if err != nil { + return fmt.Errorf("failed to parse secret data path: %w", err) + } + + sidecarProvider, err := newSidecarProvider(providerConfig, secretDataPath) + if err != nil { + return fmt.Errorf("failed to create a sidecar provider: %w", err) + } + + klog.V(4).Infof("KMS is enabled: found config, now patching pod spec") + + if err := appendContainer(podSpec, sidecarProvider, "kms-plugin", kmsConfiguration); err != nil { + return err + } + + if err := addSocketVolume(podSpec, "kms-plugin"); err != nil { + return err + } + + if err := addResourceDirMount(podSpec, "kms-plugin"); err != nil { + return err + } + + if err := addSocketVolume(podSpec, opConfig.APIServerContainerName); err != nil { + return err + } + + return nil +} + +func addResourceDirMount(podSpec *corev1.PodSpec, containerName string) error { + containerIndex := -1 + for i, container := range podSpec.Containers { + if container.Name == containerName { + containerIndex = i + break + } + } + if containerIndex < 0 { + return fmt.Errorf("container %s not found", containerName) + } + + container := &podSpec.Containers[containerIndex] + for _, m := range container.VolumeMounts { + if m.Name == "resource-dir" { + return nil + } + } + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{ + Name: "resource-dir", + MountPath: "/etc/kubernetes/static-pod-resources", + ReadOnly: true, + }, + ) + return nil +} + +func appendContainer(podSpec *corev1.PodSpec, provider SidecarProvider, containerName string, kmsConfig *apiserverv1.KMSConfiguration) error { + if podSpec == nil { + return fmt.Errorf("pod spec cannot be nil") + } + + container, err := provider.BuildSidecarContainer(containerName, kmsConfig) + if err != nil { + return fmt.Errorf("failed to build sidecar container: %w", err) + } + + podSpec.Containers = append(podSpec.Containers, container) + return nil +} + +func addSocketVolume(podSpec *corev1.PodSpec, containerName string) error { + containerIndex := -1 + for i, container := range podSpec.Containers { + if container.Name == containerName { + containerIndex = i + break + } + } + + if containerIndex < 0 { + return fmt.Errorf("container %s not found", containerName) + } + + container := &podSpec.Containers[containerIndex] + foundMount := false + for _, m := range container.VolumeMounts { + if m.Name == "kms-plugin-socket" { + foundMount = true + break + } + } + if !foundMount { + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{ + Name: "kms-plugin-socket", + MountPath: "/var/run/kmsplugin", + }, + ) + } + + foundVolume := false + for _, volume := range podSpec.Volumes { + if volume.Name == "kms-plugin-socket" { + foundVolume = true + break + } + } + + if !foundVolume { + podSpec.Volumes = append(podSpec.Volumes, + corev1.Volume{ + Name: "kms-plugin-socket", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) + } + + return nil +} From 793d171edacd37451b6f6e52d0f87c95f09c9177 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Fri, 8 May 2026 10:21:11 -0400 Subject: [PATCH 4/4] kms: add Vault sidecar provider plugin Implement the VaultSidecarProvider which builds a sidecar container for the vault-kube-kms binary with AppRole authentication. Wire it into the newSidecarProvider factory function. --- pkg/operator/encryption/kms/plugins/vault.go | 63 +++ .../encryption/kms/plugins/vault_test.go | 93 +++++ pkg/operator/encryption/kms/sidecar.go | 8 +- pkg/operator/encryption/kms/sidecar_test.go | 372 ++++++++++++++++++ 4 files changed, 535 insertions(+), 1 deletion(-) create mode 100644 pkg/operator/encryption/kms/plugins/vault.go create mode 100644 pkg/operator/encryption/kms/plugins/vault_test.go create mode 100644 pkg/operator/encryption/kms/sidecar_test.go diff --git a/pkg/operator/encryption/kms/plugins/vault.go b/pkg/operator/encryption/kms/plugins/vault.go new file mode 100644 index 0000000000..66cdc4ae54 --- /dev/null +++ b/pkg/operator/encryption/kms/plugins/vault.go @@ -0,0 +1,63 @@ +package plugins + +import ( + "fmt" + + configv1 "github.com/openshift/api/config/v1" + corev1 "k8s.io/api/core/v1" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + "k8s.io/utils/ptr" +) + +type VaultSidecarProvider struct { + Config *configv1.VaultKMSConfig + SecretDataPath string +} + +func NewVaultSidecarProvider(providerConfig *configv1.KMSConfig, secretDataPath string) (*VaultSidecarProvider, error) { + return &VaultSidecarProvider{ + Config: &providerConfig.Vault, + SecretDataPath: secretDataPath, + }, nil +} + +func (v *VaultSidecarProvider) BuildSidecarContainer(name string, kmsConfig *apiserverv1.KMSConfiguration) (corev1.Container, error) { + if v.Config == nil { + return corev1.Container{}, fmt.Errorf("vault config cannot be nil") + } + + // FIXME: this is fragile. TBD + args := fmt.Sprintf(`set -e + CREDS=$(cat %s) + SECRET_ID=${CREDS#*\"VAULT_SECRET_ID\":\"} + SECRET_ID=${SECRET_ID%%%%\"*} + ROLE_ID=${CREDS#*\"VAULT_ROLE_ID\":\"} + ROLE_ID=${ROLE_ID%%%%\"*} + printf '%%s' "$SECRET_ID" > /tmp/secret-id + exec /vault-kube-kms \ + -listen-address=%s \ + -vault-address=%s \ + -vault-namespace=%s \ + -transit-mount=%s \ + -transit-key=%s \ + -approle-role-id=$ROLE_ID \ + -approle-secret-id-path=/tmp/secret-id`, + v.SecretDataPath, + kmsConfig.Endpoint, + v.Config.VaultAddress, + v.Config.VaultNamespace, + v.Config.TransitMount, + v.Config.TransitKey, + ) + + return corev1.Container{ + Name: name, + Image: v.Config.KMSPluginImage, + Command: []string{"/bin/sh", "-c"}, + Args: []string{args}, + // This is necessary to read the secret data in /etc/kubernetes/static-pod-resources/secrets/encryption-config + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(0)), + }, + }, nil +} diff --git a/pkg/operator/encryption/kms/plugins/vault_test.go b/pkg/operator/encryption/kms/plugins/vault_test.go new file mode 100644 index 0000000000..61a9269340 --- /dev/null +++ b/pkg/operator/encryption/kms/plugins/vault_test.go @@ -0,0 +1,93 @@ +package plugins + +import ( + "fmt" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/require" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + "k8s.io/utils/ptr" +) + +func TestVaultSidecarProvider_BuildSidecarContainer(t *testing.T) { + tests := []struct { + name string + vaultConfig *configv1.VaultKMSConfig + secretDataPath string + containerName string + kmsConfig *apiserverv1.KMSConfiguration + wantErr string + }{ + { + name: "builds container with correct args", + secretDataPath: "/etc/kubernetes/static-pod-resources/secrets/encryption-config/kms-secret-data-555", + vaultConfig: &configv1.VaultKMSConfig{ + KMSPluginImage: "quay.io/test/vault:v2", + VaultAddress: "https://vault.example.com:8200", + VaultNamespace: "my-namespace", + TransitKey: "my-key", + TransitMount: "transit", + }, + containerName: "kms-plugin", + kmsConfig: &apiserverv1.KMSConfiguration{ + APIVersion: "v2", + Name: "555_secrets", + Endpoint: "unix:///var/run/kmsplugin/kms-555.sock", + }, + }, + { + name: "nil vault config", + secretDataPath: "/etc/kubernetes/static-pod-resources/secrets/encryption-config/kms-secret-data-555", + vaultConfig: nil, + containerName: "kms-plugin", + kmsConfig: &apiserverv1.KMSConfiguration{}, + wantErr: "vault config cannot be nil", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := &VaultSidecarProvider{ + Config: tt.vaultConfig, + SecretDataPath: tt.secretDataPath, + } + + container, err := provider.BuildSidecarContainer(tt.containerName, tt.kmsConfig) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.containerName, container.Name) + require.Equal(t, tt.vaultConfig.KMSPluginImage, container.Image) + require.Equal(t, []string{"/bin/sh", "-c"}, container.Command) + + expectedArgs := fmt.Sprintf(`set -e + CREDS=$(cat %s) + SECRET_ID=${CREDS#*\"VAULT_SECRET_ID\":\"} + SECRET_ID=${SECRET_ID%%%%\"*} + ROLE_ID=${CREDS#*\"VAULT_ROLE_ID\":\"} + ROLE_ID=${ROLE_ID%%%%\"*} + printf '%%s' "$SECRET_ID" > /tmp/secret-id + exec /vault-kube-kms \ + -listen-address=%s \ + -vault-address=%s \ + -vault-namespace=%s \ + -transit-mount=%s \ + -transit-key=%s \ + -approle-role-id=$ROLE_ID \ + -approle-secret-id-path=/tmp/secret-id`, + tt.secretDataPath, + tt.kmsConfig.Endpoint, + tt.vaultConfig.VaultAddress, + tt.vaultConfig.VaultNamespace, + tt.vaultConfig.TransitMount, + tt.vaultConfig.TransitKey, + ) + require.Len(t, container.Args, 1) + require.Equal(t, expectedArgs, container.Args[0]) + require.Equal(t, ptr.To(int64(0)), container.SecurityContext.RunAsUser) + }) + } +} diff --git a/pkg/operator/encryption/kms/sidecar.go b/pkg/operator/encryption/kms/sidecar.go index 983c59e416..76d99c30a3 100644 --- a/pkg/operator/encryption/kms/sidecar.go +++ b/pkg/operator/encryption/kms/sidecar.go @@ -5,6 +5,7 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/library-go/pkg/operator/encryption/encoding" + "github.com/openshift/library-go/pkg/operator/encryption/kms/plugins" corev1 "k8s.io/api/core/v1" apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" corev1listers "k8s.io/client-go/listers/core/v1" @@ -22,7 +23,12 @@ type SidecarProvider interface { } func newSidecarProvider(providerConfig *configv1.KMSConfig, secretDataPath string) (SidecarProvider, error) { - return nil, fmt.Errorf("unsupported KMS provider configuration") + switch providerConfig.Type { + case configv1.VaultKMSProvider: + return plugins.NewVaultSidecarProvider(providerConfig, secretDataPath) + default: + return nil, fmt.Errorf("unsupported KMS provider configuration") + } } func InjectIntoPodSpec(podSpec *corev1.PodSpec, secretLister corev1listers.SecretLister, opConfig OperatorConfig) error { diff --git a/pkg/operator/encryption/kms/sidecar_test.go b/pkg/operator/encryption/kms/sidecar_test.go new file mode 100644 index 0000000000..6f4c4b754e --- /dev/null +++ b/pkg/operator/encryption/kms/sidecar_test.go @@ -0,0 +1,372 @@ +package kms + +import ( + "encoding/json" + "fmt" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/operator/encryption/encoding" + "github.com/openshift/library-go/pkg/operator/encryption/kms/plugins" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiserverv1 "k8s.io/apiserver/pkg/apis/apiserver/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" +) + +func TestNewSidecarProvider(t *testing.T) { + tests := []struct { + name string + config *configv1.KMSConfig + secretDataPath string + wantErr string + }{ + { + name: "vault provider", + secretDataPath: "/etc/kubernetes/static-pod-resources/secrets/encryption-config/kms-secret-data-1", + config: &configv1.KMSConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSConfig{ + KMSPluginImage: "quay.io/test/vault:v1", + VaultAddress: "https://vault.example.com:8200", + TransitKey: "my-key", + }, + }, + }, + { + name: "unsupported provider", + secretDataPath: "/etc/kubernetes/static-pod-resources/secrets/encryption-config/kms-secret-data-1", + config: &configv1.KMSConfig{}, + wantErr: "unsupported KMS provider configuration", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider, err := newSidecarProvider(tt.config, tt.secretDataPath) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.NotNil(t, provider) + }) + } +} + +func TestAppendContainer(t *testing.T) { + kmsConfig := &apiserverv1.KMSConfiguration{ + APIVersion: "v2", + Name: "test", + Endpoint: "unix:///var/run/kmsplugin/kms-1.sock", + } + + tests := []struct { + name string + podSpec *corev1.PodSpec + wantErr string + }{ + { + name: "sidecar container added", + podSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + }, + { + name: "nil pod spec", + podSpec: nil, + wantErr: "pod spec cannot be nil", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := &plugins.VaultSidecarProvider{ + Config: &configv1.VaultKMSConfig{ + KMSPluginImage: "quay.io/test/vault:v1", + VaultAddress: "https://vault.example.com:8200", + VaultNamespace: "ns", + TransitKey: "key", + TransitMount: "transit", + }, + SecretDataPath: "/etc/kubernetes/static-pod-resources/secrets/encryption-config/kms-secret-data-1", + } + + err := appendContainer(tt.podSpec, provider, "kms-plugin", kmsConfig) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Len(t, tt.podSpec.Containers, 2) + require.Equal(t, "kms-plugin", tt.podSpec.Containers[1].Name) + require.Equal(t, "quay.io/test/vault:v1", tt.podSpec.Containers[1].Image) + }) + } +} + +func TestInjectIntoPodSpec(t *testing.T) { + secretLister := func(secrets ...*corev1.Secret) corev1listers.SecretLister { + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, s := range secrets { + indexer.Add(s) + } + return corev1listers.NewSecretLister(indexer) + } + + vaultConfig := &configv1.KMSConfig{ + Type: configv1.VaultKMSProvider, + Vault: configv1.VaultKMSConfig{ + KMSPluginImage: "quay.io/test/vault:v1", + VaultAddress: "https://vault.example.com:8200", + VaultNamespace: "my-namespace", + TransitKey: "my-key", + TransitMount: "transit", + Authentication: configv1.VaultAuthentication{ + Type: configv1.VaultAuthenticationTypeAppRole, + AppRole: configv1.VaultAppRoleAuthentication{ + Secret: configv1.VaultSecretReference{Name: "vault-kms-credentials"}, + }, + }, + }, + } + providerConfigBytes, err := encoding.EncodeKMSConfig(vaultConfig) + require.NoError(t, err) + + providerConfigKey, err := ToProviderConfigSecretDataKeyFor("555") + require.NoError(t, err) + + credentialsKey, err := ToCredentialSecretDataKeyFor("555") + require.NoError(t, err) + + credentials := map[string]string{ + "VAULT_ROLE_ID": "role-id", + "VAULT_SECRET_ID": "secret-id", + } + credentialsBytes, err := json.Marshal(credentials) + require.NoError(t, err) + + encryptionConfig := &apiserverv1.EncryptionConfiguration{ + Resources: []apiserverv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: []apiserverv1.ProviderConfiguration{ + { + KMS: &apiserverv1.KMSConfiguration{ + APIVersion: "v2", + Name: "555_secrets", + Endpoint: "unix:///var/run/kmsplugin/kms-555.sock", + }, + }, + }, + }, + }, + } + encryptionConfigBytes, err := encoding.EncodeEncryptionConfiguration(encryptionConfig) + require.NoError(t, err) + + encryptionConfigSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "encryption-config", + Namespace: "openshift-kube-apiserver", + }, + Data: map[string][]byte{ + "encryption-config": encryptionConfigBytes, + providerConfigKey: providerConfigBytes, + credentialsKey: credentialsBytes, + }, + } + + credentialsFile := "/etc/kubernetes/static-pod-resources/secrets/encryption-config/kms-secret-data-555" + sidecarArgs := fmt.Sprintf(`set -e + CREDS=$(cat %s) + SECRET_ID=${CREDS#*\"VAULT_SECRET_ID\":\"} + SECRET_ID=${SECRET_ID%%%%\"*} + ROLE_ID=${CREDS#*\"VAULT_ROLE_ID\":\"} + ROLE_ID=${ROLE_ID%%%%\"*} + printf '%%s' "$SECRET_ID" > /tmp/secret-id + exec /vault-kube-kms \ + -listen-address=%s \ + -vault-address=%s \ + -vault-namespace=%s \ + -transit-mount=%s \ + -transit-key=%s \ + -approle-role-id=$ROLE_ID \ + -approle-secret-id-path=/tmp/secret-id`, + credentialsFile, + "unix:///var/run/kmsplugin/kms-555.sock", + "https://vault.example.com:8200", + "my-namespace", + "transit", + "my-key", + ) + + socketMount := corev1.VolumeMount{ + Name: "kms-plugin-socket", + MountPath: "/var/run/kmsplugin", + } + socketVolume := corev1.Volume{ + Name: "kms-plugin-socket", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + resourceDirMount := corev1.VolumeMount{ + Name: "resource-dir", + MountPath: "/etc/kubernetes/static-pod-resources", + ReadOnly: true, + } + resourceDirVolume := corev1.Volume{ + Name: "resource-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kubernetes/static-pod-resources", + }, + }, + } + + opConfig := OperatorConfig{ + EncryptionConfigNamespace: "openshift-kube-apiserver", + EncryptionConfigSecretName: "encryption-config", + APIServerContainerName: "kube-apiserver", + } + + tests := []struct { + name string + actualPodSpec *corev1.PodSpec + expectedPodSpec *corev1.PodSpec + lister corev1listers.SecretLister + wantErr string + }{ + { + name: "sidecar and volumes injected", + actualPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + Volumes: []corev1.Volume{resourceDirVolume}, + }, + expectedPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-apiserver", + VolumeMounts: []corev1.VolumeMount{socketMount}, + }, + { + Name: "kms-plugin", + Image: "quay.io/test/vault:v1", + Command: []string{"/bin/sh", "-c"}, + Args: []string{sidecarArgs}, + VolumeMounts: []corev1.VolumeMount{socketMount, resourceDirMount}, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(int64(0)), + }, + }, + }, + Volumes: []corev1.Volume{resourceDirVolume, socketVolume}, + }, + lister: secretLister(encryptionConfigSecret), + }, + { + name: "nil pod spec", + actualPodSpec: nil, + lister: secretLister(encryptionConfigSecret), + wantErr: "pod spec cannot be nil", + }, + { + name: "missing encryption-config secret: pod spec unchanged", + actualPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + expectedPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + lister: secretLister(), + }, + { + name: "missing encryption-config key in secret: pod spec unchanged", + actualPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + expectedPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + lister: secretLister(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "encryption-config", + Namespace: "openshift-kube-apiserver", + }, + Data: map[string][]byte{"other-key": []byte("data")}, + }), + }, + { + name: "no KMS provider in EncryptionConfiguration: pod spec unchanged", + actualPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + expectedPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "kube-apiserver"}, + }, + }, + lister: func() corev1listers.SecretLister { + noKMSConfig := &apiserverv1.EncryptionConfiguration{ + Resources: []apiserverv1.ResourceConfiguration{ + { + Resources: []string{"secrets"}, + Providers: []apiserverv1.ProviderConfiguration{ + {AESCBC: &apiserverv1.AESConfiguration{}}, + }, + }, + }, + } + noKMSBytes, err := encoding.EncodeEncryptionConfiguration(noKMSConfig) + require.NoError(t, err) + return secretLister(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "encryption-config", + Namespace: "openshift-kube-apiserver", + }, + Data: map[string][]byte{"encryption-config": noKMSBytes}, + }) + }(), + }, + { + name: "missing API server container", + actualPodSpec: &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "other-container"}, + }, + }, + lister: secretLister(encryptionConfigSecret), + wantErr: "container kube-apiserver not found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := InjectIntoPodSpec(tt.actualPodSpec, tt.lister, opConfig) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedPodSpec, tt.actualPodSpec) + }) + } +}