diff --git a/pkg/operator/encryption/controllers/key_controller.go b/pkg/operator/encryption/controllers/key_controller.go index 3c930469d5..835efa7603 100644 --- a/pkg/operator/encryption/controllers/key_controller.go +++ b/pkg/operator/encryption/controllers/key_controller.go @@ -116,7 +116,9 @@ func NewKeyController( WithInformers( apiServerInformer.Informer(), operatorClient.Informer(), + kubeInformersForNamespaces.InformersFor("openshift-config-managed").Core().V1().Secrets().Informer(), + // TODO: add informer for openshift-config namespace to watch referenced Secrets for KMS plugin secret data changes deployer, ).ToController( c.controllerInstanceName, @@ -223,7 +225,7 @@ func (c *keyController) checkAndCreateKeys(ctx context.Context, syncContext fact sort.Sort(sort.StringSlice(reasons)) internalReason := strings.Join(reasons, ", ") - keySecret, err := c.generateKeySecret(newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) + keySecret, err := c.generateKeySecret(ctx, newKeyID, currentMode, apiEncryptionConfiguration, internalReason, externalReason) if err != nil { return fmt.Errorf("failed to create key: %v", err) } @@ -260,7 +262,7 @@ func (c *keyController) validateExistingSecret(ctx context.Context, keySecret *c return nil // we made this key earlier } -func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { +func (c *keyController) generateKeySecret(ctx context.Context, keyID uint64, currentMode state.Mode, apiServerEncryption configv1.APIServerEncryption, internalReason, externalReason string) (*corev1.Secret, error) { bs := crypto.ModeToNewKeyFunc[currentMode]() ks := state.KeyState{ Key: apiserverv1.Key{ @@ -281,6 +283,22 @@ func (c *keyController) generateKeySecret(keyID uint64, currentMode state.Mode, }, Plugin: apiServerEncryption.KMS, } + + if secretName, expectedKeys := referencedSecretName(apiServerEncryption.KMS); len(secretName) > 0 { + refSecret, err := c.secretClient.Secrets("openshift-config").Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s in openshift-config: %w", secretName, err) + } + sd := make(map[string][]byte, len(expectedKeys)) + for _, key := range expectedKeys { + v, ok := refSecret.Data[key] + if !ok { + return nil, fmt.Errorf("secret %s in openshift-config is missing required key %q", secretName, key) + } + sd[secretName+"-"+key] = v + } + ks.KMS.SecretData = sd + } } return secrets.FromKeyState(c.instanceName, ks) } @@ -383,6 +401,15 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern return latestKeyID, "rotation-interval-has-passed", time.Since(latestKey.Migrated.Timestamp) > encryptionSecretMigrationInterval } +func referencedSecretName(plugin configv1.KMSPluginConfig) (string, []string) { + switch plugin.Type { + case configv1.VaultKMSProvider: + return plugin.Vault.Authentication.AppRole.Secret.Name, []string{"role-id", "secret-id"} + default: + return "", nil + } +} + // TODO make this un-settable once set // ex: we could require the tech preview no upgrade flag to be set before we will honor this field type unsupportedEncryptionConfig struct { diff --git a/pkg/operator/encryption/controllers/key_controller_test.go b/pkg/operator/encryption/controllers/key_controller_test.go index 8d636325ee..ed807184c5 100644 --- a/pkg/operator/encryption/controllers/key_controller_test.go +++ b/pkg/operator/encryption/controllers/key_controller_test.go @@ -336,9 +336,10 @@ func TestKeyController(t *testing.T) { {Group: "", Resource: "secrets"}, }, targetNamespace: "kms", - expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, initialObjects: []runtime.Object{ encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), + encryptiontesting.CreateVaultAppRoleSecret("vault-approle-secret", "test-role-id", "test-secret-id"), }, apiServerObjects: []runtime.Object{apiServerWithKMS}, validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { @@ -381,6 +382,14 @@ func TestKeyController(t *testing.T) { ts.Errorf("unexpected kms-plugin-config: %s", kmsPluginConfigData) } + // Verify secret data is carried + if roleID := string(actualSecret.Data["encryption.apiserver.operator.openshift.io-kms-plugin-secret-vault-approle-secret-role-id"]); roleID != "test-role-id" { + ts.Errorf("expected role-id secret data to be 'test-role-id', got %q", roleID) + } + if secretID := string(actualSecret.Data["encryption.apiserver.operator.openshift.io-kms-plugin-secret-vault-approle-secret-secret-id"]); secretID != "test-secret-id" { + ts.Errorf("expected secret-id secret data to be 'test-secret-id', got %q", secretID) + } + // Verify internal reason if actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"] != "secrets-key-does-not-exist" { ts.Errorf("unexpected internal reason: %s", actualSecret.Annotations["encryption.apiserver.operator.openshift.io/internal-reason"]) @@ -418,10 +427,11 @@ func TestKeyController(t *testing.T) { initialObjects: []runtime.Object{ encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("61def964fb967f5d7c44a2af8dab6865"), "aescbc"), + encryptiontesting.CreateVaultAppRoleSecret("vault-approle-secret", "test-role-id", "test-secret-id"), }, apiServerObjects: []runtime.Object{apiServerWithKMS}, targetNamespace: "kms", - expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { wasSecretValidated := false for _, action := range actions { @@ -512,10 +522,11 @@ func TestKeyController(t *testing.T) { initialObjects: []runtime.Object{ encryptiontesting.CreateDummyKubeAPIPod("kube-apiserver-1", "kms", "node-1"), encryptiontesting.CreateEncryptionKeySecretWithRawKeyWithMode("kms", []schema.GroupResource{{Group: "", Resource: "secrets"}}, 5, []byte("identity-key"), "identity"), + encryptiontesting.CreateVaultAppRoleSecret("vault-approle-secret", "test-role-id", "test-secret-id"), }, apiServerObjects: []runtime.Object{apiServerWithKMS}, targetNamespace: "kms", - expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "create:secrets:openshift-config-managed", "create:events:kms"}, + expectedActions: []string{"list:pods:kms", "get:secrets:kms", "list:secrets:openshift-config-managed", "get:secrets:openshift-config", "create:secrets:openshift-config-managed", "create:events:kms"}, validateFunc: func(ts *testing.T, actions []clientgotesting.Action, targetNamespace string, targetGRs []schema.GroupResource) { wasSecretValidated := false for _, action := range actions { @@ -667,7 +678,7 @@ func TestKeyController(t *testing.T) { // - target namespace: pods and secrets // - openshift-config-managed: secrets // note that the informer factory is not used in the test - it's only needed to create the controller - kubeInformers := v1helpers.NewKubeInformersForNamespaces(fakeKubeClient, "openshift-config-managed", scenario.targetNamespace) + kubeInformers := v1helpers.NewKubeInformersForNamespaces(fakeKubeClient, "openshift-config-managed", "openshift-config", scenario.targetNamespace) fakeSecretClient := fakeKubeClient.CoreV1() fakePodClient := fakeKubeClient.CoreV1() fakeConfigClient := configv1clientfake.NewSimpleClientset(scenario.apiServerObjects...) diff --git a/pkg/operator/encryption/encryptiondata/config.go b/pkg/operator/encryption/encryptiondata/config.go index 40a855a81f..bed09e7c07 100644 --- a/pkg/operator/encryption/encryptiondata/config.go +++ b/pkg/operator/encryption/encryptiondata/config.go @@ -30,6 +30,10 @@ type Config struct { // KMSPlugins maps keyID to plugin-specific configuration, // carried from Key Secrets into the encryption-config Secret. KMSPlugins map[string]configv1.KMSPluginConfig + // KMSSecretData maps keyID to secret data carried from + // Key Secrets into the encryption-config Secret. + // Inner map keys use the short form (e.g. "app-role-role-id"). + KMSSecretData map[string]map[string][]byte } func (c *Config) HasEncryptionConfiguration() bool { @@ -40,6 +44,7 @@ func (c *Config) HasEncryptionConfiguration() bool { func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupResourceState) (*Config, error) { resourceConfigs := make([]apiserverconfigv1.ResourceConfiguration, 0, len(encryptionState)) var kmsPlugins map[string]configv1.KMSPluginConfig + var kmsSecretData map[string]map[string][]byte for gr, grKeys := range encryptionState { resourceConfigs = append(resourceConfigs, apiserverconfigv1.ResourceConfiguration{ @@ -47,11 +52,11 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes Providers: stateToProviders(gr.Resource, grKeys), }) - // Collect KMS plugin configs from read keys (which already include the write key). - // We iterate over encryptionState which is keyed by GroupResource, so the same - // keyID is seen once per resource (e.g. key "1" for secrets and key "1" for configmaps). - // Since all resources share the same Key Secret, the plugin config is identical - // across duplicates and we only need to keep the first occurrence. + // Collect KMS plugin configs and secret data from read keys (which already + // include the write key). We iterate over encryptionState which is keyed by + // GroupResource, so the same keyID is seen once per resource. Since all + // resources share the same Key Secret, the plugin config and secret data are + // identical across duplicates and we only need to keep the first occurrence. for _, key := range grKeys.ReadKeys { if key.HasKMSPlugin() { if kmsPlugins == nil { @@ -67,6 +72,18 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes kmsPlugins[key.Key.Name] = key.KMS.Plugin } } + if key.HasKMSSecretData() { + if kmsSecretData == nil { + kmsSecretData = map[string]map[string][]byte{} + } + if existing, exists := kmsSecretData[key.Key.Name]; exists { + if !equality.Semantic.DeepEqual(existing, key.KMS.SecretData) { + return nil, fmt.Errorf("KMS secret data mismatch for keyID %s: secret data from different resources must be identical", key.Key.Name) + } + } else { + kmsSecretData[key.Key.Name] = key.KMS.SecretData + } + } } } @@ -76,8 +93,9 @@ func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupRes }) return &Config{ - Encryption: &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs}, - KMSPlugins: kmsPlugins, + Encryption: &apiserverconfigv1.EncryptionConfiguration{Resources: resourceConfigs}, + KMSPlugins: kmsPlugins, + KMSSecretData: kmsSecretData, }, nil } diff --git a/pkg/operator/encryption/encryptiondata/config_test.go b/pkg/operator/encryption/encryptiondata/config_test.go index 181fbb0730..bbf0af16f4 100644 --- a/pkg/operator/encryption/encryptiondata/config_test.go +++ b/pkg/operator/encryption/encryptiondata/config_test.go @@ -851,6 +851,39 @@ func TestSecretRoundtrip(t *testing.T) { }, }, }, + { + name: "KMS with provider config and secret data", + cfg: &encryptiondata.Config{ + Encryption: &apiserverconfigv1.EncryptionConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "EncryptionConfiguration", + APIVersion: "apiserver.config.k8s.io/v1", + }, + Resources: []apiserverconfigv1.ResourceConfiguration{{ + Resources: []string{"secrets"}, + Providers: []apiserverconfigv1.ProviderConfiguration{{ + KMS: &apiserverconfigv1.KMSConfiguration{ + APIVersion: "v2", + Name: "1_secrets", + Endpoint: "unix:///var/run/kmsplugin/kms-1.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + }, { + Identity: &apiserverconfigv1.IdentityConfiguration{}, + }}, + }}, + }, + KMSPlugins: map[string]configv1.KMSPluginConfig{ + "1": encryptiontesting.DefaultKMSPluginConfig, + }, + KMSSecretData: map[string]map[string][]byte{ + "1": { + "vault-approle-secret-role-id": []byte("test-role-id"), + "vault-approle-secret-secret-id": []byte("test-secret-id"), + }, + }, + }, + }, { name: "KMS with multiple provider configs", cfg: &encryptiondata.Config{ diff --git a/pkg/operator/encryption/encryptiondata/secret.go b/pkg/operator/encryption/encryptiondata/secret.go index 0df884c62c..4d363eda0f 100644 --- a/pkg/operator/encryption/encryptiondata/secret.go +++ b/pkg/operator/encryption/encryptiondata/secret.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strconv" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -17,11 +18,15 @@ import ( "github.com/openshift/library-go/pkg/operator/encryption/state" ) -// EncryptionConfSecretName is the name of the final encryption config secret that is revisioned per apiserver rollout. -const EncryptionConfSecretName = "encryption-config" - -// EncryptionConfSecretKey is the map data key used to store the raw bytes of the final encryption config. -const EncryptionConfSecretKey = "encryption-config" +const ( + // EncryptionConfSecretName is the name of the final encryption config secret that is revisioned per apiserver rollout. + EncryptionConfSecretName = "encryption-config" + // EncryptionConfSecretKey is the map data key used to store the raw bytes of the final encryption config. + EncryptionConfSecretKey = "encryption-config" + // encryptionConfigSecretDataPrefix is the data key prefix for KMS plugin secret + // data entries in the encryption-config Secret. Full key: "kms-plugin-secret-{secretDataKey}-{keyID}". + encryptionConfigSecretDataPrefix = "kms-plugin-secret-" +) func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) { data, ok := encryptionConfigSecret.Data[EncryptionConfSecretKey] @@ -56,7 +61,29 @@ func FromSecret(encryptionConfigSecret *corev1.Secret) (*Config, error) { kmsPlugins[keyID] = pluginConfig } - return &Config{Encryption: encryptionConfig, KMSPlugins: kmsPlugins}, nil + // Extract secret data entries from the encryption-config Secret. + // Data keys follow the format "kms-plugin-secret-{secretDataKey}-{keyID}" + // (e.g. "kms-plugin-secret-app-role-role-id-1"). KeyIDFromSecretDataKey + // returns the keyID (e.g. "1") and the secretDataKey (e.g. "app-role-role-id"). + var kmsSecretData map[string]map[string][]byte + for key, value := range encryptionConfigSecret.Data { + keyID, secretDataKey, found, err := keyIDFromSecretDataKey(key) + if err != nil { + return nil, fmt.Errorf("failed to extract keyID from secret data key %s: %w", key, err) + } + if !found { + continue + } + if kmsSecretData == nil { + kmsSecretData = map[string]map[string][]byte{} + } + if kmsSecretData[keyID] == nil { + kmsSecretData[keyID] = map[string][]byte{} + } + kmsSecretData[keyID][secretDataKey] = value + } + + return &Config{Encryption: encryptionConfig, KMSPlugins: kmsPlugins, KMSSecretData: kmsSecretData}, nil } func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) { @@ -100,6 +127,19 @@ func ToSecret(ns, name string, secretData *Config) (*corev1.Secret, error) { s.Data[dataKey] = encodedPlugin } + // Write secret data entries to the encryption-config Secret. + // Each secret dataKey (e.g. "app-role-role-id") is combined with the keyID + // (e.g. "1") to produce the data key "kms-plugin-secret-app-role-role-id-1". + for keyID, keySecretData := range secretData.KMSSecretData { + for secretDataKey, value := range keySecretData { + dataKey, err := toSecretDataKeyFor(secretDataKey, keyID) + if err != nil { + return nil, err + } + s.Data[dataKey] = value + } + } + return s, nil } @@ -145,3 +185,26 @@ func ExtractUniqueAndSortedKMSConfigurations(secretData *Config) ([]*apiserverco }) return result, nil } + +func toSecretDataKeyFor(secretDataKey, keyID string) (string, error) { + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return encryptionConfigSecretDataPrefix + secretDataKey + "-" + keyID, nil +} + +func keyIDFromSecretDataKey(dataKey string) (string, string, bool, error) { + rest, found := strings.CutPrefix(dataKey, encryptionConfigSecretDataPrefix) + if !found { + return "", "", false, nil + } + i := strings.LastIndex(rest, "-") + if i < 1 { + return "", "", false, nil + } + keyID := rest[i+1:] + if _, err := strconv.ParseUint(keyID, 10, 64); err != nil { + return "", "", false, fmt.Errorf("invalid keyID %q: must be a non-negative integer", keyID) + } + return keyID, rest[:i], true, nil +} diff --git a/pkg/operator/encryption/secrets/secrets.go b/pkg/operator/encryption/secrets/secrets.go index e2e59efd6d..30e30256c3 100644 --- a/pkg/operator/encryption/secrets/secrets.go +++ b/pkg/operator/encryption/secrets/secrets.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -87,6 +88,14 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { // encryption mode. return state.KeyState{}, fmt.Errorf("%s can not be empty, when mode is KMS", EncryptionSecretKMSPluginConfig) } + for dataKey, value := range s.Data { + if rawKey, found := strings.CutPrefix(dataKey, EncryptionSecretKMSSecretDataPrefix); found && len(rawKey) > 0 { + if key.KMS.SecretData == nil { + key.KMS.SecretData = map[string][]byte{} + } + key.KMS.SecretData[rawKey] = value + } + } key.Mode = keyMode default: return state.KeyState{}, fmt.Errorf("secret %s/%s has invalid mode: %s", s.Namespace, s.Name, keyMode) @@ -159,6 +168,12 @@ func FromKeyState(component string, ks state.KeyState) (*corev1.Secret, error) { s.Data[EncryptionSecretKMSPluginConfig] = pluginData } + if ks.HasKMSSecretData() { + for dataKey, value := range ks.KMS.SecretData { + s.Data[EncryptionSecretKMSSecretDataPrefix+dataKey] = value + } + } + return s, nil } diff --git a/pkg/operator/encryption/secrets/secrets_test.go b/pkg/operator/encryption/secrets/secrets_test.go index 1d73ea9cf1..bc6a2659f0 100644 --- a/pkg/operator/encryption/secrets/secrets_test.go +++ b/pkg/operator/encryption/secrets/secrets_test.go @@ -182,6 +182,39 @@ func TestRoundtrip(t *testing.T) { }, }, }, + { + name: "full kms with secret data", + component: "kms", + ks: state.KeyState{ + Key: v1.Key{ + Name: "1", + Secret: base64.StdEncoding.EncodeToString(emptyKey), + }, + Backed: true, + Mode: "KMS", + KMS: &state.KMSState{ + Encryption: &v1.KMSConfiguration{ + APIVersion: "v2", + Name: "1", + Endpoint: "unix:///var/run/kmsplugin/kms-1.sock", + Timeout: &metav1.Duration{Duration: 10 * time.Second}, + }, + Plugin: defaultKMSPluginConfig, + SecretData: map[string][]byte{ + "vault-approle-secret-role-id": []byte("test-role-id"), + "vault-approle-secret-secret-id": []byte("test-secret-id"), + }, + }, + Migrated: state.MigrationState{ + Timestamp: now, + Resources: []schema.GroupResource{ + {Resource: "secrets"}, + }, + }, + InternalReason: "internal", + ExternalReason: "external", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/operator/encryption/secrets/types.go b/pkg/operator/encryption/secrets/types.go index 2795afbc0d..e08c46670d 100644 --- a/pkg/operator/encryption/secrets/types.go +++ b/pkg/operator/encryption/secrets/types.go @@ -57,6 +57,11 @@ const ( // EncryptionSecretKMSPluginConfig is the data field key that stores the serialized KMS plugin // configuration for KMS mode in the encryption-key secret. EncryptionSecretKMSPluginConfig = "encryption.apiserver.operator.openshift.io-kms-plugin-config" + + // EncryptionSecretKMSSecretDataPrefix is the data field key prefix for secret data values + // fetched from the referenced secret in openshift-config. The full data key is + // constructed by appending "-{secretName}-{dataKey}" to this prefix. + EncryptionSecretKMSSecretDataPrefix = "encryption.apiserver.operator.openshift.io-kms-plugin-secret-" ) // MigratedGroupResources is the data structured stored in the diff --git a/pkg/operator/encryption/state/types.go b/pkg/operator/encryption/state/types.go index acaf493377..83c6d4088e 100644 --- a/pkg/operator/encryption/state/types.go +++ b/pkg/operator/encryption/state/types.go @@ -53,6 +53,10 @@ func (k *KeyState) HasKMSPlugin() bool { return k != nil && k.KMS != nil && k.KMS.Plugin != (configv1.KMSPluginConfig{}) } +func (k *KeyState) HasKMSSecretData() bool { + return k != nil && k.KMS != nil && len(k.KMS.SecretData) > 0 +} + // KMSState stores all KMS encryption mode related configurations type KMSState struct { // Encoded EncryptionConfig that stores the KMS related fields @@ -60,6 +64,10 @@ type KMSState struct { // Plugin stores KMS plugin specific configurations Plugin configv1.KMSPluginConfig + + // SecretData stores fetched values from the referenced secret + // in openshift-config. Keys use the format "{secretName}-{keyName}". + SecretData map[string][]byte } type MigrationState struct { diff --git a/pkg/operator/encryption/testing/helpers.go b/pkg/operator/encryption/testing/helpers.go index df4865d937..4faa7342e7 100644 --- a/pkg/operator/encryption/testing/helpers.go +++ b/pkg/operator/encryption/testing/helpers.go @@ -149,6 +149,20 @@ func CreateEncryptionKeySecretWithCustomKMSPluginConfig(targetNS string, grs []s return secret } +func CreateVaultAppRoleSecret(name, roleID, secretID string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "openshift-config", + }, + Data: map[string][]byte{ + "role-id": []byte(roleID), + "secret-id": []byte(secretID), + }, + Type: corev1.SecretTypeOpaque, + } +} + func CreateDummyKubeAPIPod(name, namespace string, nodeName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index 7f56615569..f1c9a581db 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -67,7 +67,7 @@ func TestEncryptionIntegration(tt *testing.T) { // kube clients kubeClient, err := kubernetes.NewForConfig(kubeConfig) require.NoError(t, err) - kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "openshift-config-managed") + kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "openshift-config-managed", "openshift-config") apiextensionsClient, err := v1.NewForConfig(kubeConfig) require.NoError(t, err) @@ -305,6 +305,41 @@ func TestEncryptionIntegration(tt *testing.T) { } } + verifyKMSSecretData := func() { + t.Helper() + encryptionConfigSecret, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-config-%s", component), metav1.GetOptions{}) + require.NoError(t, err) + cfg, err := encryptiondata.FromSecret(encryptionConfigSecret) + require.NoError(t, err) + + expectedKeyIDs := map[string]bool{} + for _, rc := range cfg.Encryption.Resources { + for _, p := range rc.Providers { + if p.KMS != nil { + parts := strings.SplitN(p.KMS.Name, "_", 2) + require.Len(t, parts, 2, "unexpected KMS provider name format: %s", p.KMS.Name) + expectedKeyIDs[parts[0]] = true + } + } + } + + for keyID := range expectedKeyIDs { + secretData, ok := cfg.KMSSecretData[keyID] + require.True(t, ok, "expected secret data for keyID %s in encryption-config secret", keyID) + require.NotEmpty(t, secretData, "expected non-empty secret data for keyID %s", keyID) + + // Verify actual values match the source secret + require.Equal(t, "test-role-id", secretData["vault-approle-secret-role-id"], "role-id secret data mismatch for keyID %s", keyID) + require.Equal(t, "test-secret-id", secretData["vault-approle-secret-secret-id"], "secret-id secret data mismatch for keyID %s", keyID) + + // Verify Key Secret also carries the secret data + keySecret, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-%s", component, keyID), metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, "test-role-id", string(keySecret.Data[secrets.EncryptionSecretKMSSecretDataPrefix+"vault-approle-secret-role-id"]), "key secret %s role-id secret data mismatch", keyID) + require.Equal(t, "test-secret-id", string(keySecret.Data[secrets.EncryptionSecretKMSSecretDataPrefix+"vault-approle-secret-secret-id"]), "key secret %s secret-id secret data mismatch", keyID) + } + } + t.Logf("Wait for initial Encrypted condition") waitForConditionStatus("Encrypted", operatorv1.ConditionFalse) @@ -452,6 +487,18 @@ func TestEncryptionIntegration(tt *testing.T) { ) waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) + t.Logf("Create vault AppRole vault AppRole secret") + _, err = kubeClient.CoreV1().Secrets("openshift-config").Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "vault-approle-secret", Namespace: "openshift-config"}, + Data: map[string][]byte{ + "role-id": []byte("test-role-id"), + "secret-id": []byte("test-secret-id"), + }, + Type: corev1.SecretTypeOpaque, + }, metav1.CreateOptions{}) + require.NoError(t, err) + defer kubeClient.CoreV1().Secrets("openshift-config").Delete(ctx, "vault-approle-secret", metav1.DeleteOptions{}) + t.Logf("Switch to KMS") _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS","kms":{"type":"Vault","vault":{"kmsPluginImage":"registry.example.com/kms-plugin@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890","vaultAddress":"https://vault.example.com","authentication":{"type":"AppRole","appRole":{"secret":{"name":"vault-approle-secret"}}},"transitKey":"test-transit-key"}}}}}`), metav1.PatchOptions{}) require.NoError(t, err) @@ -466,6 +513,7 @@ func TestEncryptionIntegration(tt *testing.T) { waitForMigration("8") waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) verifyKMSPlugins() + verifyKMSSecretData() t.Logf("Verify KMS key secret contains provider config") kmsKeySecret, err := kubeClient.CoreV1().Secrets("openshift-config-managed").Get(ctx, fmt.Sprintf("encryption-key-%s-8", component), metav1.GetOptions{}) @@ -489,6 +537,7 @@ func TestEncryptionIntegration(tt *testing.T) { ) waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) verifyKMSPlugins() + verifyKMSSecretData() t.Logf("Switch back to KMS") _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS","kms":{"type":"Vault","vault":{"kmsPluginImage":"registry.example.com/kms-plugin@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890","vaultAddress":"https://vault.example.com","authentication":{"type":"AppRole","appRole":{"secret":{"name":"vault-approle-secret"}}},"transitKey":"test-transit-key"}}}}}`), metav1.PatchOptions{}) @@ -504,6 +553,7 @@ func TestEncryptionIntegration(tt *testing.T) { waitForMigration("10") waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) verifyKMSPlugins() + verifyKMSSecretData() t.Logf("Rotate KMS key via aescbc (KMS->AESCBC->KMS)") _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"aescbc","kms":null}}}`), metav1.PatchOptions{}) @@ -516,6 +566,7 @@ func TestEncryptionIntegration(tt *testing.T) { ) waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) verifyKMSPlugins() + verifyKMSSecretData() t.Logf("Switch back to KMS after rotation") _, err = fakeApiServerClient.Patch(ctx, "cluster", types.MergePatchType, []byte(`{"spec":{"encryption":{"type":"KMS","kms":{"type":"Vault","vault":{"kmsPluginImage":"registry.example.com/kms-plugin@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890","vaultAddress":"https://vault.example.com","authentication":{"type":"AppRole","appRole":{"secret":{"name":"vault-approle-secret"}}},"transitKey":"test-transit-key"}}}}}`), metav1.PatchOptions{}) @@ -531,6 +582,7 @@ func TestEncryptionIntegration(tt *testing.T) { waitForMigration("12") waitForConditionStatus("Encrypted", operatorv1.ConditionTrue) verifyKMSPlugins() + verifyKMSSecretData() t.Logf("Delete the encryption-config while in KMS mode") _, err = kubeClient.CoreV1().Secrets("openshift-config-managed").Patch(ctx, fmt.Sprintf("encryption-config-%s", component), types.JSONPatchType, []byte(`[{"op":"remove","path":"/metadata/finalizers"}]`), metav1.PatchOptions{})