From db7c0171ef1505dc383c71cd7bfa8a535139102f Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Mon, 23 Mar 2026 10:56:49 +0800 Subject: [PATCH 01/11] feat(encryption): add encryption primitives and wire format Signed-off-by: tenfyzhong --- pkg/encryption/cipher.go | 107 ++++++++++++++++ pkg/encryption/cipher_test.go | 35 +++++ pkg/encryption/data_key_id.go | 34 +++++ pkg/encryption/data_key_id_24be.go | 32 +++++ pkg/encryption/format.go | 157 +++++++++++++++++++++++ pkg/encryption/format_test.go | 198 +++++++++++++++++++++++++++++ pkg/encryption/types.go | 44 +++++++ pkg/errors/error.go | 31 +++++ 8 files changed, 638 insertions(+) create mode 100644 pkg/encryption/cipher.go create mode 100644 pkg/encryption/cipher_test.go create mode 100644 pkg/encryption/data_key_id.go create mode 100644 pkg/encryption/data_key_id_24be.go create mode 100644 pkg/encryption/format.go create mode 100644 pkg/encryption/format_test.go create mode 100644 pkg/encryption/types.go diff --git a/pkg/encryption/cipher.go b/pkg/encryption/cipher.go new file mode 100644 index 0000000000..483752cc92 --- /dev/null +++ b/pkg/encryption/cipher.go @@ -0,0 +1,107 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + + cerrors "github.com/pingcap/ticdc/pkg/errors" +) + +// Cipher is the interface for encryption/decryption operations +type Cipher interface { + // Encrypt encrypts data using the provided key and IV + Encrypt(data, key, iv []byte) ([]byte, error) + + // Decrypt decrypts data using the provided key and IV + Decrypt(data, key, iv []byte) ([]byte, error) + + // IVSize returns the required IV size in bytes + IVSize() int +} + +// AES256CTRCipher implements AES-CTR encryption for AES key sizes. +type AES256CTRCipher struct{} + +// NewAES256CTRCipher creates a new AES-CTR cipher. +func NewAES256CTRCipher() *AES256CTRCipher { + return &AES256CTRCipher{} +} + +// IVSize returns the IV size for AES-CTR (16 bytes) +func (c *AES256CTRCipher) IVSize() int { + return aes.BlockSize +} + +func isValidAESKeySize(key []byte) bool { + switch len(key) { + case 16, 24, 32: + return true + default: + return false + } +} + +// Encrypt encrypts data using AES-CTR. +func (c *AES256CTRCipher) Encrypt(data, key, iv []byte) ([]byte, error) { + if !isValidAESKeySize(key) { + return nil, cerrors.ErrEncryptionFailed.GenWithStackByArgs("key must be 16, 24, or 32 bytes for AES-CTR") + } + if len(iv) != c.IVSize() { + return nil, cerrors.ErrEncryptionFailed.GenWithStackByArgs("IV must be 16 bytes") + } + + block, err := aes.NewCipher(key) + if err != nil { + return nil, cerrors.ErrEncryptionFailed.Wrap(err) + } + + stream := cipher.NewCTR(block, iv) + ciphertext := make([]byte, len(data)) + stream.XORKeyStream(ciphertext, data) + + return ciphertext, nil +} + +// Decrypt decrypts data using AES-CTR. +func (c *AES256CTRCipher) Decrypt(data, key, iv []byte) ([]byte, error) { + if !isValidAESKeySize(key) { + return nil, cerrors.ErrDecryptionFailed.GenWithStackByArgs("key must be 16, 24, or 32 bytes for AES-CTR") + } + if len(iv) != c.IVSize() { + return nil, cerrors.ErrDecryptionFailed.GenWithStackByArgs("IV must be 16 bytes") + } + + block, err := aes.NewCipher(key) + if err != nil { + return nil, cerrors.ErrDecryptionFailed.Wrap(err) + } + + stream := cipher.NewCTR(block, iv) + plaintext := make([]byte, len(data)) + stream.XORKeyStream(plaintext, data) + + return plaintext, nil +} + +// GenerateIV generates a random IV of the specified size +func GenerateIV(size int) ([]byte, error) { + iv := make([]byte, size) + if _, err := rand.Read(iv); err != nil { + return nil, cerrors.ErrEncryptionFailed.Wrap(err) + } + return iv, nil +} diff --git a/pkg/encryption/cipher_test.go b/pkg/encryption/cipher_test.go new file mode 100644 index 0000000000..e980428d77 --- /dev/null +++ b/pkg/encryption/cipher_test.go @@ -0,0 +1,35 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAES256CTREncryptDecrypt(t *testing.T) { + key := []byte("0123456789abcdef0123456789abcdef") // 32 bytes + iv := []byte("1234567890abcdef") // 16 bytes + plain := []byte("hello world") + + cipherImpl := NewAES256CTRCipher() + encrypted, err := cipherImpl.Encrypt(plain, key, iv) + require.NoError(t, err) + require.NotEqual(t, plain, encrypted) + + decrypted, err := cipherImpl.Decrypt(encrypted, key, iv) + require.NoError(t, err) + require.Equal(t, plain, decrypted) +} diff --git a/pkg/encryption/data_key_id.go b/pkg/encryption/data_key_id.go new file mode 100644 index 0000000000..c5e1003969 --- /dev/null +++ b/pkg/encryption/data_key_id.go @@ -0,0 +1,34 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import "github.com/pingcap/errors" + +// DataKeyID represents a 3-byte data key identifier in the encryption header. +type DataKeyID [3]byte + +// ToString converts DataKeyID to string. +func (id DataKeyID) ToString() string { + return string(id[:]) +} + +// DataKeyIDFromString creates DataKeyID from string (must be 3 bytes). +func DataKeyIDFromString(s string) (DataKeyID, error) { + if len(s) != 3 { + return DataKeyID{}, errors.New("data key ID must be exactly 3 bytes") + } + var id DataKeyID + copy(id[:], s) + return id, nil +} diff --git a/pkg/encryption/data_key_id_24be.go b/pkg/encryption/data_key_id_24be.go new file mode 100644 index 0000000000..18a9cad6e6 --- /dev/null +++ b/pkg/encryption/data_key_id_24be.go @@ -0,0 +1,32 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import cerrors "github.com/pingcap/ticdc/pkg/errors" + +func encodeDataKeyID24BE(id uint32) (string, error) { + if id > 0xFFFFFF { + return "", cerrors.ErrInvalidDataKeyID.GenWithStackByArgs("data key ID exceeds 24-bit range") + } + b := [3]byte{byte(id >> 16), byte(id >> 8), byte(id)} + return string(b[:]), nil +} + +func decodeDataKeyID24BE(id string) (uint32, error) { + if len(id) != 3 { + return 0, cerrors.ErrInvalidDataKeyID.GenWithStackByArgs("data key ID must be 3 bytes") + } + b := []byte(id) + return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]), nil +} diff --git a/pkg/encryption/format.go b/pkg/encryption/format.go new file mode 100644 index 0000000000..db0e4d5392 --- /dev/null +++ b/pkg/encryption/format.go @@ -0,0 +1,157 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + cerrors "github.com/pingcap/ticdc/pkg/errors" +) + +const ( + // EncryptionHeaderSize is the size of encryption header (4 bytes) + // Format: [version(1 byte)][dataKeyID(3 bytes)] + EncryptionHeaderSize = 4 + + // VersionUnencrypted indicates data is not encrypted + VersionUnencrypted byte = 0x00 +) + +// EncryptionHeader represents the 4-byte encryption header +// Format: [version(1 byte)][dataKeyID(3 bytes)] +type EncryptionHeader struct { + Version byte + DataKeyID [3]byte +} + +// EncodeEncryptedData encodes data with encryption header +// Format: [version(1)][dataKeyID(3)][encryptedData] +// The version byte comes from the encryption metadata obtained from TiKV +func EncodeEncryptedData(data []byte, version byte, dataKeyID string) ([]byte, error) { + if len(dataKeyID) != 3 { + return nil, cerrors.ErrInvalidDataKeyID.GenWithStackByArgs("data key ID must be 3 bytes") + } + + if version == VersionUnencrypted { + return nil, cerrors.ErrEncryptionFailed.GenWithStackByArgs("version cannot be 0 for encrypted data") + } + + result := make([]byte, EncryptionHeaderSize+len(data)) + result[0] = version + copy(result[1:4], dataKeyID) + copy(result[4:], data) + + return result, nil +} + +// DecodeEncryptedData decodes data and extracts encryption header +// Returns: (version, dataKeyID, encryptedData, error) +func DecodeEncryptedData(data []byte) (byte, string, []byte, error) { + if len(data) < EncryptionHeaderSize { + return 0, "", nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("data too short for encryption header") + } + + version := data[0] + var dataKeyID [3]byte + copy(dataKeyID[:], data[1:4]) + encryptedData := data[4:] + + return version, string(dataKeyID[:]), encryptedData, nil +} + +// IsEncrypted checks if data is encrypted by examining the version byte +// Data is considered encrypted if version != 0 (VersionUnencrypted) +// The caller should validate that the version matches expected versions from TiKV metadata +func IsEncrypted(data []byte) bool { + if len(data) < EncryptionHeaderSize { + return false + } + return data[0] != VersionUnencrypted +} + +// IsEncryptedWithVersion checks if data is encrypted with a specific version +// This is useful when you know the expected version from TiKV metadata +func IsEncryptedWithVersion(data []byte, expectedVersion byte) bool { + if len(data) < EncryptionHeaderSize { + return false + } + return data[0] == expectedVersion +} + +// GetVersion extracts the version byte from data +// Returns 0 if data is too short +func GetVersion(data []byte) byte { + if len(data) < EncryptionHeaderSize { + return 0 + } + return data[0] +} + +// EncodeUnencryptedData encodes unencrypted data with version=0 header +// This creates a unified format where all new data has the 4-byte header +func EncodeUnencryptedData(data []byte) []byte { + result := make([]byte, EncryptionHeaderSize+len(data)) + result[0] = VersionUnencrypted + // DataKeyID is zero for unencrypted data (3 bytes) + result[1] = 0 + result[2] = 0 + result[3] = 0 + copy(result[4:], data) + return result +} + +// DecodeUnencryptedData decodes unencrypted data (removes header if present) +func DecodeUnencryptedData(data []byte) ([]byte, error) { + if len(data) < EncryptionHeaderSize { + // No header, return as-is (backward compatibility) + return data, nil + } + + version := data[0] + dataKeyID1, dataKeyID2, dataKeyID3 := data[1], data[2], data[3] + dataKeyIDIsZero := dataKeyID1 == 0 && dataKeyID2 == 0 && dataKeyID3 == 0 + + if version == VersionUnencrypted && dataKeyIDIsZero { + // New-format unencrypted data with header, remove header + return data[4:], nil + } + + // For backward compatibility, treat any other format as legacy unencrypted data + // This includes: + // - Legacy data without header (any pattern) + // - Data that might look like encrypted but is actually legacy + // The caller is responsible for ensuring data is not actually encrypted + return data, nil +} + +// ExtractDataKeyID extracts the data key ID from encrypted data +func ExtractDataKeyID(data []byte) (string, error) { + if len(data) < EncryptionHeaderSize { + return "", cerrors.ErrDecodeFailed.GenWithStackByArgs("data too short") + } + + version := data[0] + dataKeyID1, dataKeyID2, dataKeyID3 := data[1], data[2], data[3] + dataKeyIDIsZero := dataKeyID1 == 0 && dataKeyID2 == 0 && dataKeyID3 == 0 + + // Only extract key ID from data that definitively looks like new-format encrypted: + // - version != 0 (encrypted data has non-zero version) + // - DataKeyID is non-zero (encrypted data always has a valid key ID) + if version != VersionUnencrypted && !dataKeyIDIsZero { + var keyID [3]byte + copy(keyID[:], data[1:4]) + return string(keyID[:]), nil + } + + // Otherwise, this is not encrypted data (legacy data or new-format unencrypted) + return "", cerrors.ErrDecodeFailed.GenWithStackByArgs("data is not encrypted") +} diff --git a/pkg/encryption/format_test.go b/pkg/encryption/format_test.go new file mode 100644 index 0000000000..f37a310b82 --- /dev/null +++ b/pkg/encryption/format_test.go @@ -0,0 +1,198 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "testing" + + cerrors "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestEncodeEncryptedDataInvalidKey(t *testing.T) { + // Key ID must be exactly 3 bytes + _, err := EncodeEncryptedData([]byte("payload"), 0x01, "ab") + require.Error(t, err) + require.True(t, cerrors.ErrInvalidDataKeyID.Equal(err)) + + _, err = EncodeEncryptedData([]byte("payload"), 0x01, "abcd") + require.Error(t, err) + require.True(t, cerrors.ErrInvalidDataKeyID.Equal(err)) +} + +func TestEncodeEncryptedDataInvalidVersion(t *testing.T) { + // Version cannot be 0 for encrypted data + _, err := EncodeEncryptedData([]byte("payload"), VersionUnencrypted, "abc") + require.Error(t, err) +} + +func TestEncodeDecodeEncryptedData(t *testing.T) { + data := []byte("payload") + keyID := "abc" // 3 bytes + version := byte(0x01) + + encoded, err := EncodeEncryptedData(data, version, keyID) + require.NoError(t, err) + require.True(t, IsEncrypted(encoded)) + + // Verify version byte is set correctly + require.Equal(t, version, encoded[0]) + + decodedVersion, decodedKeyID, body, err := DecodeEncryptedData(encoded) + require.NoError(t, err) + require.Equal(t, version, decodedVersion) + require.Equal(t, keyID, decodedKeyID) + require.Equal(t, data, body) +} + +func TestEncodeDecodeWithDifferentVersions(t *testing.T) { + data := []byte("payload") + keyID := "xyz" + + // Test with different version values that might come from TiKV + versions := []byte{0x01, 0x02, 0x10, 0xFF} + + for _, version := range versions { + encoded, err := EncodeEncryptedData(data, version, keyID) + require.NoError(t, err) + require.True(t, IsEncrypted(encoded)) + require.Equal(t, version, GetVersion(encoded)) + + decodedVersion, decodedKeyID, body, err := DecodeEncryptedData(encoded) + require.NoError(t, err) + require.Equal(t, version, decodedVersion) + require.Equal(t, keyID, decodedKeyID) + require.Equal(t, data, body) + } +} + +func TestEncodeUnencryptedData(t *testing.T) { + raw := []byte("plain") + encoded := EncodeUnencryptedData(raw) + + // Unencrypted data with header should NOT be detected as encrypted + // because the version byte is VersionUnencrypted (0x00) + require.False(t, IsEncrypted(encoded)) + require.Equal(t, VersionUnencrypted, encoded[0]) + + decoded, err := DecodeUnencryptedData(encoded) + require.NoError(t, err) + require.Equal(t, raw, decoded) +} + +func TestIsEncryptedWithLegacyData(t *testing.T) { + // Legacy unencrypted data (no header) should not be detected as encrypted + // because it's too short for the header + shortData := []byte("abc") + require.False(t, IsEncrypted(shortData)) + + // Data with version=0 (first byte is 0x00) is not encrypted + unencryptedWithHeader := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05} + require.False(t, IsEncrypted(unencryptedWithHeader)) +} + +func TestIsEncryptedWithVersionByte(t *testing.T) { + // Data with non-zero version byte should be detected as encrypted + encryptedData := []byte{0x01, 'a', 'b', 'c', 'd', 'a', 't', 'a'} + require.True(t, IsEncrypted(encryptedData)) + + // Data with different version values + for _, v := range []byte{0x01, 0x02, 0x10, 0xFF} { + data := []byte{v, 'a', 'b', 'c', 'd', 'a', 't', 'a'} + require.True(t, IsEncrypted(data)) + } + + // Data too short should not be detected as encrypted + shortData := []byte{0x01, 'a', 'b'} + require.False(t, IsEncrypted(shortData)) +} + +func TestIsEncryptedWithVersion(t *testing.T) { + data := []byte{0x05, 'a', 'b', 'c', 'd', 'a', 't', 'a'} + + // Should match when version matches + require.True(t, IsEncryptedWithVersion(data, 0x05)) + + // Should not match when version doesn't match + require.False(t, IsEncryptedWithVersion(data, 0x01)) + require.False(t, IsEncryptedWithVersion(data, 0x00)) +} + +func TestGetVersion(t *testing.T) { + // Normal data + data := []byte{0x05, 'a', 'b', 'c', 'd', 'a', 't', 'a'} + require.Equal(t, byte(0x05), GetVersion(data)) + + // Short data returns 0 + shortData := []byte{0x05, 'a', 'b'} + require.Equal(t, byte(0x00), GetVersion(shortData)) +} + +func TestDecodeUnencryptedDataBackwardCompatibility(t *testing.T) { + // Legacy data without header should be returned as-is + // Use data that is too short to have a header (length < 4) + legacyData := []byte("legacy") + decoded, err := DecodeUnencryptedData(legacyData) + require.NoError(t, err) + require.Equal(t, legacyData, decoded) + + // Also test with data that has non-zero DataKeyID pattern + // This can't be confused with new-format encrypted data (which would have non-zero key ID) + // and can't be confused with new-format unencrypted (which has zero key ID) + legacyData2 := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05} + decoded2, err := DecodeUnencryptedData(legacyData2) + require.NoError(t, err) + require.Equal(t, legacyData2, decoded2) +} + +func TestDecodeUnencryptedDataWithEncryptedData(t *testing.T) { + // For backward compatibility, DecodeUnencryptedData treats any format as legacy unencrypted data + // and returns the data as-is. It does not return an error even for encrypted-looking data. + // The caller is responsible for ensuring data is not actually encrypted. + encryptedData := []byte{0x01, 'a', 'b', 'c', 'd', 'a', 't', 'a'} + decoded, err := DecodeUnencryptedData(encryptedData) + require.NoError(t, err) + require.Equal(t, encryptedData, decoded) +} + +func TestExtractDataKeyID(t *testing.T) { + data := []byte("payload") + keyID := "xyz" + version := byte(0x01) + + encoded, err := EncodeEncryptedData(data, version, keyID) + require.NoError(t, err) + + extractedKeyID, err := ExtractDataKeyID(encoded) + require.NoError(t, err) + require.Equal(t, keyID, extractedKeyID) +} + +func TestExtractDataKeyIDFromUnencryptedData(t *testing.T) { + // Trying to extract key ID from unencrypted data should return error + unencryptedData := EncodeUnencryptedData([]byte("plain")) + _, err := ExtractDataKeyID(unencryptedData) + require.Error(t, err) + + // Trying to extract key ID from legacy data that is too short should return error + legacyData := []byte("abc") // 3 bytes < 4 + _, err = ExtractDataKeyID(legacyData) + require.Error(t, err) + + // Legacy data with non-zero bytes in positions 1-3 should also return error + // (This can't be confused with new-format encrypted data which has non-zero key ID) + legacyData2 := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05} + _, err = ExtractDataKeyID(legacyData2) + require.Error(t, err) +} diff --git a/pkg/encryption/types.go b/pkg/encryption/types.go new file mode 100644 index 0000000000..d3b88640f3 --- /dev/null +++ b/pkg/encryption/types.go @@ -0,0 +1,44 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +// EncryptionMeta is aligned with kvproto `keyspace_encryptionpb.EncryptionMeta`. +type EncryptionMeta struct { + KeyspaceId uint32 `json:"keyspace_id,omitempty"` + Current *EncryptionEpoch `json:"current,omitempty"` + MasterKey *MasterKey `json:"master_key,omitempty"` + DataKeys map[uint32]*DataKey `json:"data_keys,omitempty"` + History []*EncryptionEpoch `json:"history,omitempty"` +} + +// EncryptionEpoch is aligned with kvproto `keyspace_encryptionpb.EncryptionEpoch`. +type EncryptionEpoch struct { + FileId uint64 `json:"file_id,omitempty"` + DataKeyId uint32 `json:"data_key_id,omitempty"` + CreatedAt uint64 `json:"created_at,omitempty"` +} + +// MasterKey is aligned with kvproto `keyspace_encryptionpb.MasterKey`. +type MasterKey struct { + Vendor string `json:"vendor,omitempty"` + CmekId string `json:"cmek_id,omitempty"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Ciphertext []byte `json:"ciphertext,omitempty"` +} + +// DataKey is aligned with kvproto `keyspace_encryptionpb.DataKey`. +type DataKey struct { + Ciphertext []byte `json:"ciphertext,omitempty"` +} diff --git a/pkg/errors/error.go b/pkg/errors/error.go index 3ad854eee1..3d0ce1c0cb 100644 --- a/pkg/errors/error.go +++ b/pkg/errors/error.go @@ -821,6 +821,37 @@ var ( "unimplemented IOType: %d", errors.RFCCodeText("CDC:ErrUnimplementedIOType"), ) + + // encryption related errors + ErrEncryptionMetaNotFound = errors.Normalize( + "encryption meta not found", + errors.RFCCodeText("CDC:ErrEncryptionMetaNotFound"), + ) + + ErrUnsupportedEncryptionAlgorithm = errors.Normalize( + "unsupported encryption algorithm: %s", + errors.RFCCodeText("CDC:ErrUnsupportedEncryptionAlgorithm"), + ) + + ErrEncryptionFailed = errors.Normalize( + "encryption failed: %s", + errors.RFCCodeText("CDC:ErrEncryptionFailed"), + ) + + ErrDecryptionFailed = errors.Normalize( + "decryption failed: %s", + errors.RFCCodeText("CDC:ErrDecryptionFailed"), + ) + + ErrInvalidDataKeyID = errors.Normalize( + "invalid data key ID: %s", + errors.RFCCodeText("CDC:ErrInvalidDataKeyID"), + ) + + ErrDataKeyNotFound = errors.Normalize( + "data key not found: %s", + errors.RFCCodeText("CDC:ErrDataKeyNotFound"), + ) ) // ErrorType defines the type of application errors From 16993f5a99704fb05bedb4eb46dd7b8f8b78d42e Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Mon, 23 Mar 2026 10:56:58 +0800 Subject: [PATCH 02/11] feat(encryption): add tikv encryption metadata http client Signed-off-by: tenfyzhong --- pkg/encryption/json_types.go | 62 ++++ pkg/encryption/tikv_http_client.go | 422 ++++++++++++++++++++++++ pkg/encryption/tikv_http_client_test.go | 312 ++++++++++++++++++ 3 files changed, 796 insertions(+) create mode 100644 pkg/encryption/json_types.go create mode 100644 pkg/encryption/tikv_http_client.go create mode 100644 pkg/encryption/tikv_http_client_test.go diff --git a/pkg/encryption/json_types.go b/pkg/encryption/json_types.go new file mode 100644 index 0000000000..b468846921 --- /dev/null +++ b/pkg/encryption/json_types.go @@ -0,0 +1,62 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "encoding/base64" + "encoding/json" + "fmt" +) + +// ByteArray supports decoding either: +// - a JSON string (base64-encoded bytes), or +// - a JSON array of uint8 values (TiKV status API style). +type ByteArray []byte + +func (b *ByteArray) UnmarshalJSON(data []byte) error { + if len(data) == 0 || string(data) == "null" { + *b = nil + return nil + } + + switch data[0] { + case '"': + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return err + } + *b = decoded + return nil + case '[': + var ints []int + if err := json.Unmarshal(data, &ints); err != nil { + return err + } + out := make([]byte, len(ints)) + for i, v := range ints { + if v < 0 || v > 255 { + return fmt.Errorf("byte value out of range: %d", v) + } + out[i] = byte(v) + } + *b = out + return nil + default: + return fmt.Errorf("unsupported JSON type for bytes: %s", string(data)) + } +} diff --git a/pkg/encryption/tikv_http_client.go b/pkg/encryption/tikv_http_client.go new file mode 100644 index 0000000000..96421dbc35 --- /dev/null +++ b/pkg/encryption/tikv_http_client.go @@ -0,0 +1,422 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + oldproto "github.com/gogo/protobuf/proto" + "github.com/pingcap/errors" + "github.com/pingcap/log" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/httputil" + "github.com/pingcap/ticdc/pkg/security" + "github.com/pingcap/tidb/pkg/util/engine" + pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/opt" + "go.uber.org/zap" +) + +type tikvEncryptionHTTPClient struct { + pdClient pd.Client + httpClient *httputil.Client + httpScheme string + httpTimeout time.Duration +} + +func NewTiKVEncryptionHTTPClient(pdClient pd.Client, credential *security.Credential) (TiKVEncryptionClient, error) { + httpClient, err := httputil.NewClient(credential) + if err != nil { + return nil, err + } + + httpScheme := "http" + if credential != nil && credential.IsTLSEnabled() { + httpScheme = "https" + } + + return &tikvEncryptionHTTPClient{ + pdClient: pdClient, + httpClient: httpClient, + httpScheme: httpScheme, + httpTimeout: 5 * time.Second, + }, nil +} + +func (c *tikvEncryptionHTTPClient) GetKeyspaceEncryptionMeta(ctx context.Context, keyspaceID uint32) (*EncryptionMeta, error) { + stores, err := c.pdClient.GetAllStores(ctx, opt.WithExcludeTombstone()) + if err != nil { + log.Warn("failed to list TiKV stores", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return nil, errors.Trace(err) + } + + var lastErr error + for _, store := range stores { + if engine.IsTiFlash(store) { + continue + } + + statusAddr := store.GetStatusAddress() + if statusAddr == "" { + continue + } + + meta, err := c.getEncryptionMetaFromStore(ctx, store.GetId(), statusAddr, keyspaceID) + if err == nil { + return meta, nil + } + if cerrors.ErrEncryptionMetaNotFound.Equal(err) { + lastErr = err + continue + } + lastErr = err + } + + if lastErr == nil { + lastErr = cerrors.ErrEncryptionMetaNotFound + } + return nil, lastErr +} + +func (c *tikvEncryptionHTTPClient) getEncryptionMetaFromStore(ctx context.Context, storeID uint64, statusAddr string, keyspaceID uint32) (*EncryptionMeta, error) { + storeURL := c.buildStatusURL(statusAddr, keyspaceID) + contentType := "" + + reqCtx, cancel := context.WithTimeout(ctx, c.httpTimeout) + defer cancel() + + resp, err := c.httpClient.Get(reqCtx, storeURL) + if err != nil { + log.Warn("failed to fetch encryption meta from TiKV store", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL), + zap.Error(err)) + return nil, errors.Trace(err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Warn("failed to read encryption meta response body", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL), + zap.Error(err)) + return nil, errors.Trace(err) + } + contentType = resp.Header.Get("Content-Type") + + if resp.StatusCode == http.StatusNotFound { + log.Debug("encryption meta not found on TiKV store", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL)) + return nil, cerrors.ErrEncryptionMetaNotFound + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + log.Warn("unexpected encryption meta response status", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL), + zap.Int("statusCode", resp.StatusCode), + zap.Int("bodySize", len(body)), + zap.String("body", truncateBytesForLog(body, 256))) + return nil, errors.Errorf("[%d] %s", resp.StatusCode, body) + } + + metaResp := &encryptionMetaResponse{} + responseFormat := "json" + if jsonErr := json.Unmarshal(body, metaResp); jsonErr != nil { + log.Debug("failed to decode encryption meta response as json, fallback to protobuf", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL), + zap.String("contentType", contentType), + zap.Int("bodySize", len(body)), + zap.Error(jsonErr)) + + metaResp, err = decodeEncryptionMetaResponseFromProtobuf(body) + if err != nil { + decodeErr := errors.Annotatef(err, "json decode failed: %v", jsonErr) + log.Warn("failed to decode encryption meta response", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL), + zap.String("contentType", contentType), + zap.Int("bodySize", len(body)), + zap.String("body", truncateBytesForLog(body, 256)), + zap.Error(decodeErr)) + return nil, errors.Trace(decodeErr) + } + responseFormat = "protobuf" + } + + meta, err := metaResp.toEncryptionMeta() + if err != nil { + log.Warn("failed to convert encryption meta response", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("keyspaceID", keyspaceID), + zap.String("url", storeURL), + zap.Error(err)) + return nil, err + } + + if meta.KeyspaceId != 0 && meta.KeyspaceId != keyspaceID { + log.Warn("encryption meta keyspace ID mismatch", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.Uint32("requestedKeyspaceID", keyspaceID), + zap.Uint32("metaKeyspaceID", meta.KeyspaceId), + zap.String("url", storeURL)) + } + + masterKeyCiphertextLen := 0 + if meta.MasterKey != nil { + masterKeyCiphertextLen = len(meta.MasterKey.Ciphertext) + } + + log.Info("fetched valid encryption meta from TiKV store", + zap.Uint64("storeID", storeID), + zap.String("statusAddr", statusAddr), + zap.String("responseFormat", responseFormat), + zap.String("contentType", contentType), + zap.Uint32("requestedKeyspaceID", keyspaceID), + zap.Uint32("metaKeyspaceID", meta.KeyspaceId), + zap.Uint32("currentDataKeyID", meta.Current.DataKeyId), + zap.Uint8("version", byte(meta.Current.DataKeyId&0xFF)), + zap.Int("dataKeyCount", len(meta.DataKeys)), + zap.Int("historyCount", len(meta.History)), + zap.String("kmsVendor", safeKMSVendor(meta.MasterKey)), + zap.String("cmekID", safeCMEKID(meta.MasterKey)), + zap.Int("masterKeyCiphertextLen", masterKeyCiphertextLen)) + + return meta, nil +} + +func (c *tikvEncryptionHTTPClient) buildStatusURL(statusAddr string, keyspaceID uint32) string { + if strings.Contains(statusAddr, "://") { + return fmt.Sprintf("%s/encryption/get-meta?keyspace_id=%d", strings.TrimRight(statusAddr, "/"), keyspaceID) + } + return fmt.Sprintf("%s://%s/encryption/get-meta?keyspace_id=%d", c.httpScheme, statusAddr, keyspaceID) +} + +type encryptionMetaResponse struct { + KeyspaceId uint32 `json:"keyspace_id"` + Current encryptionEpochResponse `json:"current"` + MasterKey masterKeyResponse `json:"master_key"` + DataKeys map[uint32]dataKeyResponse `json:"data_keys"` + History []encryptionEpochResponse `json:"history"` +} + +type encryptionEpochResponse struct { + FileId uint64 `json:"file_id"` + DataKeyId uint32 `json:"data_key_id"` + CreatedAt uint64 `json:"created_at"` +} + +type masterKeyResponse struct { + Vendor string `json:"vendor"` + CmekId string `json:"cmek_id"` + Region string `json:"region"` + Endpoint string `json:"endpoint"` + Ciphertext ByteArray `json:"ciphertext"` +} + +type dataKeyResponse struct { + Ciphertext ByteArray `json:"ciphertext"` +} + +type keyspaceEncryptionMetaPB struct { + KeyspaceId uint32 `protobuf:"varint,1,opt,name=keyspace_id,json=keyspaceId,proto3"` + Current *keyspaceEncryptionEpochPB `protobuf:"bytes,2,opt,name=current,proto3"` + MasterKey *keyspaceMasterKeyPB `protobuf:"bytes,3,opt,name=master_key,json=masterKey,proto3"` + DataKeys map[uint32]*keyspaceDataKeyPB `protobuf:"bytes,4,rep,name=data_keys,json=dataKeys,proto3" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + History []*keyspaceEncryptionEpochPB `protobuf:"bytes,5,rep,name=history,proto3"` +} + +func (m *keyspaceEncryptionMetaPB) Reset() { *m = keyspaceEncryptionMetaPB{} } +func (m *keyspaceEncryptionMetaPB) String() string { return "" } +func (*keyspaceEncryptionMetaPB) ProtoMessage() {} + +type keyspaceEncryptionEpochPB struct { + FileId uint64 `protobuf:"varint,1,opt,name=file_id,json=fileId,proto3"` + DataKeyId uint32 `protobuf:"varint,2,opt,name=data_key_id,json=dataKeyId,proto3"` + CreatedAt uint64 `protobuf:"varint,3,opt,name=created_at,json=createdAt,proto3"` +} + +func (m *keyspaceEncryptionEpochPB) Reset() { *m = keyspaceEncryptionEpochPB{} } +func (m *keyspaceEncryptionEpochPB) String() string { return "" } +func (*keyspaceEncryptionEpochPB) ProtoMessage() {} + +type keyspaceMasterKeyPB struct { + Vendor string `protobuf:"bytes,1,opt,name=vendor,proto3"` + CmekId string `protobuf:"bytes,2,opt,name=cmek_id,json=cmekId,proto3"` + Region string `protobuf:"bytes,3,opt,name=region,proto3"` + Endpoint string `protobuf:"bytes,4,opt,name=endpoint,proto3"` + Ciphertext []byte `protobuf:"bytes,5,opt,name=ciphertext,proto3"` +} + +func (m *keyspaceMasterKeyPB) Reset() { *m = keyspaceMasterKeyPB{} } +func (m *keyspaceMasterKeyPB) String() string { return "" } +func (*keyspaceMasterKeyPB) ProtoMessage() {} + +type keyspaceDataKeyPB struct { + Ciphertext []byte `protobuf:"bytes,1,opt,name=ciphertext,proto3"` +} + +func (m *keyspaceDataKeyPB) Reset() { *m = keyspaceDataKeyPB{} } +func (m *keyspaceDataKeyPB) String() string { return "" } +func (*keyspaceDataKeyPB) ProtoMessage() {} + +func decodeEncryptionMetaResponseFromProtobuf(body []byte) (*encryptionMetaResponse, error) { + metaPB := &keyspaceEncryptionMetaPB{} + if err := oldproto.Unmarshal(body, metaPB); err != nil { + return nil, errors.Trace(err) + } + if metaPB.Current == nil && metaPB.MasterKey == nil && len(metaPB.DataKeys) == 0 && len(metaPB.History) == 0 && metaPB.KeyspaceId == 0 { + return nil, errors.New("protobuf payload does not contain encryption meta fields") + } + return metaPB.toEncryptionMetaResponse(), nil +} + +func (m *keyspaceEncryptionMetaPB) toEncryptionMetaResponse() *encryptionMetaResponse { + resp := &encryptionMetaResponse{ + KeyspaceId: m.KeyspaceId, + DataKeys: make(map[uint32]dataKeyResponse, len(m.DataKeys)), + History: make([]encryptionEpochResponse, 0, len(m.History)), + } + + if m.Current != nil { + resp.Current = encryptionEpochResponse{ + FileId: m.Current.FileId, + DataKeyId: m.Current.DataKeyId, + CreatedAt: m.Current.CreatedAt, + } + } + + if m.MasterKey != nil { + resp.MasterKey = masterKeyResponse{ + Vendor: m.MasterKey.Vendor, + CmekId: m.MasterKey.CmekId, + Region: m.MasterKey.Region, + Endpoint: m.MasterKey.Endpoint, + Ciphertext: ByteArray(m.MasterKey.Ciphertext), + } + } + + for id, dataKey := range m.DataKeys { + if dataKey == nil { + continue + } + resp.DataKeys[id] = dataKeyResponse{ + Ciphertext: ByteArray(dataKey.Ciphertext), + } + } + + for _, epoch := range m.History { + if epoch == nil { + continue + } + resp.History = append(resp.History, encryptionEpochResponse{ + FileId: epoch.FileId, + DataKeyId: epoch.DataKeyId, + CreatedAt: epoch.CreatedAt, + }) + } + + return resp +} + +func (r *encryptionMetaResponse) toEncryptionMeta() (*EncryptionMeta, error) { + if r.Current.DataKeyId == 0 { + log.Warn("invalid encryption meta from TiKV: current data key ID is empty", + zap.Uint32("metaKeyspaceID", r.KeyspaceId)) + return nil, cerrors.ErrEncryptionMetaNotFound + } + + version := byte(r.Current.DataKeyId & 0xFF) + if version == VersionUnencrypted { + log.Warn("invalid encryption meta from TiKV: version must be non-zero", + zap.Uint32("metaKeyspaceID", r.KeyspaceId), + zap.Uint32("currentDataKeyID", r.Current.DataKeyId)) + return nil, cerrors.ErrEncryptionFailed.GenWithStackByArgs("version must be non-zero") + } + + dataKeys := make(map[uint32]*DataKey, len(r.DataKeys)) + for id, dk := range r.DataKeys { + dataKeys[id] = &DataKey{Ciphertext: []byte(dk.Ciphertext)} + } + + if _, ok := dataKeys[r.Current.DataKeyId]; !ok { + log.Warn("invalid encryption meta from TiKV: current data key missing", + zap.Uint32("metaKeyspaceID", r.KeyspaceId), + zap.Uint32("currentDataKeyID", r.Current.DataKeyId), + zap.Int("dataKeyCount", len(dataKeys))) + return nil, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("current data key not found") + } + + history := make([]*EncryptionEpoch, 0, len(r.History)) + for _, epoch := range r.History { + history = append(history, &EncryptionEpoch{ + FileId: epoch.FileId, + DataKeyId: epoch.DataKeyId, + CreatedAt: epoch.CreatedAt, + }) + } + + return &EncryptionMeta{ + KeyspaceId: r.KeyspaceId, + Current: &EncryptionEpoch{ + FileId: r.Current.FileId, + DataKeyId: r.Current.DataKeyId, + CreatedAt: r.Current.CreatedAt, + }, + MasterKey: r.MasterKey.toMasterKey(), + DataKeys: dataKeys, + History: history, + }, nil +} + +func (r *masterKeyResponse) toMasterKey() *MasterKey { + return &MasterKey{ + Vendor: r.Vendor, + CmekId: r.CmekId, + Region: r.Region, + Endpoint: r.Endpoint, + Ciphertext: []byte(r.Ciphertext), + } +} + +func truncateBytesForLog(b []byte, max int) string { + if len(b) <= max { + return string(b) + } + return fmt.Sprintf("%s...(truncated, %d bytes total)", string(b[:max]), len(b)) +} diff --git a/pkg/encryption/tikv_http_client_test.go b/pkg/encryption/tikv_http_client_test.go new file mode 100644 index 0000000000..bbd79a46e0 --- /dev/null +++ b/pkg/encryption/tikv_http_client_test.go @@ -0,0 +1,312 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + oldproto "github.com/gogo/protobuf/proto" + "github.com/pingcap/kvproto/pkg/metapb" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" + pd "github.com/tikv/pd/client" + pdopt "github.com/tikv/pd/client/opt" +) + +type mockTiKVMetaPDClient struct { + pd.Client + stores []*metapb.Store +} + +func (m *mockTiKVMetaPDClient) GetAllStores(ctx context.Context, opts ...pdopt.GetStoreOption) ([]*metapb.Store, error) { + return m.stores, nil +} + +func TestTiKVEncryptionHTTPClientGetKeyspaceEncryptionMeta(t *testing.T) { + t.Parallel() + + const keyspaceID = uint32(1) + const dataKeyID = uint32(0x010203) // 24-bit big-endian -> [0x01 0x02 0x03] + + handler := http.NewServeMux() + handler.HandleFunc("/encryption/get-meta", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + qs := r.URL.Query() + if qs.Get("keyspace_id") != "1" { + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "keyspace_id": 1, + "current": {"file_id": 1, "data_key_id": 66051, "created_at": 0}, + "master_key": {"vendor": "aws-kms", "cmek_id": "cmek-1", "region": "us-west-1", "endpoint": "", "ciphertext": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]}, + "data_keys": { + "66051": {"ciphertext": [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]} + }, + "history": [] +}`)) + }) + + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + srvURL, err := url.Parse(srv.URL) + require.NoError(t, err) + statusAddr := srvURL.Host + + pdCli := &mockTiKVMetaPDClient{ + stores: []*metapb.Store{ + {Id: 1, StatusAddress: statusAddr, Address: "unused"}, + }, + } + + client, err := NewTiKVEncryptionHTTPClient(pdCli, nil) + require.NoError(t, err) + + meta, err := client.GetKeyspaceEncryptionMeta(context.Background(), keyspaceID) + require.NoError(t, err) + require.NotNil(t, meta) + require.Equal(t, keyspaceID, meta.KeyspaceId) + require.NotNil(t, meta.Current) + + expectedKeyID := string([]byte{0x01, 0x02, 0x03}) + currentKeyID, err := encodeDataKeyID24BE(meta.Current.DataKeyId) + require.NoError(t, err) + require.Equal(t, expectedKeyID, currentKeyID) + require.Equal(t, dataKeyID, meta.Current.DataKeyId) + + dk, ok := meta.DataKeys[dataKeyID] + require.True(t, ok) + require.Len(t, dk.Ciphertext, 32) +} + +func TestTiKVEncryptionHTTPClientGetKeyspaceEncryptionMetaFromProtobuf(t *testing.T) { + t.Parallel() + + const keyspaceID = uint32(2) + const dataKeyID = uint32(0x010203) + + metaPB := &testKeyspaceEncryptionMetaPB{ + KeyspaceId: keyspaceID, + Current: &testEncryptionEpochPB{ + FileId: 1, + DataKeyId: dataKeyID, + CreatedAt: 123, + }, + MasterKey: &testMasterKeyPB{ + Vendor: "aws", + CmekId: "cmek-2", + Region: "eu-west-2", + Endpoint: "http://0.0.0.0:8080", + Ciphertext: []byte{1, 2, 3, 4}, + }, + DataKeys: map[uint32]*testDataKeyPB{ + dataKeyID: {Ciphertext: []byte{9, 8, 7, 6}}, + }, + History: []*testEncryptionEpochPB{ + { + FileId: 2, + DataKeyId: dataKeyID, + CreatedAt: 124, + }, + }, + } + + payload, err := oldproto.Marshal(metaPB) + require.NoError(t, err) + + handler := http.NewServeMux() + handler.HandleFunc("/encryption/get-meta", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/protobuf") + _, _ = w.Write(payload) + }) + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + srvURL, err := url.Parse(srv.URL) + require.NoError(t, err) + + pdCli := &mockTiKVMetaPDClient{ + stores: []*metapb.Store{ + {Id: 1, StatusAddress: srvURL.Host}, + }, + } + + client, err := NewTiKVEncryptionHTTPClient(pdCli, nil) + require.NoError(t, err) + + meta, err := client.GetKeyspaceEncryptionMeta(context.Background(), keyspaceID) + require.NoError(t, err) + require.Equal(t, keyspaceID, meta.KeyspaceId) + require.Equal(t, dataKeyID, meta.Current.DataKeyId) + require.Equal(t, "aws", meta.MasterKey.Vendor) + require.Equal(t, []byte{9, 8, 7, 6}, meta.DataKeys[dataKeyID].Ciphertext) +} + +func TestTiKVEncryptionHTTPClientNotFoundReturnsErrEncryptionMetaNotFound(t *testing.T) { + t.Parallel() + + handler := http.NewServeMux() + handler.HandleFunc("/encryption/get-meta", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) + }) + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + srvURL, err := url.Parse(srv.URL) + require.NoError(t, err) + statusAddr := srvURL.Host + + pdCli := &mockTiKVMetaPDClient{ + stores: []*metapb.Store{ + {Id: 1, StatusAddress: statusAddr}, + }, + } + + client, err := NewTiKVEncryptionHTTPClient(pdCli, nil) + require.NoError(t, err) + + _, err = client.GetKeyspaceEncryptionMeta(context.Background(), 1) + require.True(t, cerrors.ErrEncryptionMetaNotFound.Equal(err), "err=%v", err) +} + +func TestTiKVEncryptionHTTPClientRejectsVersionZeroMeta(t *testing.T) { + t.Parallel() + + handler := http.NewServeMux() + handler.HandleFunc("/encryption/get-meta", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "keyspace_id": 1, + "current": {"file_id": 1, "data_key_id": 66048, "created_at": 0}, + "master_key": {"vendor": "aws-kms", "cmek_id": "cmek-1", "region": "us-west-1", "endpoint": "", "ciphertext": [0,1,2]}, + "data_keys": {"66048": {"ciphertext": [1,2,3]}}, + "history": [] +}`)) + }) + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + srvURL, err := url.Parse(srv.URL) + require.NoError(t, err) + pdCli := &mockTiKVMetaPDClient{ + stores: []*metapb.Store{ + {Id: 1, StatusAddress: srvURL.Host}, + }, + } + + client, err := NewTiKVEncryptionHTTPClient(pdCli, nil) + require.NoError(t, err) + + _, err = client.GetKeyspaceEncryptionMeta(context.Background(), 1) + require.True(t, cerrors.ErrEncryptionFailed.Equal(err), "err=%v", err) +} + +func TestTiKVEncryptionHTTPClientRejectsMetaMissingCurrentDataKey(t *testing.T) { + t.Parallel() + + handler := http.NewServeMux() + handler.HandleFunc("/encryption/get-meta", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "keyspace_id": 1, + "current": {"file_id": 1, "data_key_id": 66051, "created_at": 0}, + "master_key": {"vendor": "aws-kms", "cmek_id": "cmek-1", "region": "us-west-1", "endpoint": "", "ciphertext": [0,1,2]}, + "data_keys": {"66052": {"ciphertext": [1,2,3]}}, + "history": [] +}`)) + }) + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + + srvURL, err := url.Parse(srv.URL) + require.NoError(t, err) + pdCli := &mockTiKVMetaPDClient{ + stores: []*metapb.Store{ + {Id: 1, StatusAddress: srvURL.Host}, + }, + } + + client, err := NewTiKVEncryptionHTTPClient(pdCli, nil) + require.NoError(t, err) + + _, err = client.GetKeyspaceEncryptionMeta(context.Background(), 1) + require.True(t, cerrors.ErrDataKeyNotFound.Equal(err), "err=%v", err) +} + +func TestByteArrayUnmarshalSupportsUint8Array(t *testing.T) { + t.Parallel() + + var b ByteArray + err := b.UnmarshalJSON([]byte(`[0, 1, 2, 255]`)) + require.NoError(t, err) + require.Equal(t, []byte{0, 1, 2, 255}, []byte(b)) + + var bad ByteArray + err = bad.UnmarshalJSON([]byte(`[256]`)) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "out of range")) +} + +type testKeyspaceEncryptionMetaPB struct { + KeyspaceId uint32 `protobuf:"varint,1,opt,name=keyspace_id,json=keyspaceId,proto3"` + Current *testEncryptionEpochPB `protobuf:"bytes,2,opt,name=current,proto3"` + MasterKey *testMasterKeyPB `protobuf:"bytes,3,opt,name=master_key,json=masterKey,proto3"` + DataKeys map[uint32]*testDataKeyPB `protobuf:"bytes,4,rep,name=data_keys,json=dataKeys,proto3" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + History []*testEncryptionEpochPB `protobuf:"bytes,5,rep,name=history,proto3"` +} + +func (m *testKeyspaceEncryptionMetaPB) Reset() { *m = testKeyspaceEncryptionMetaPB{} } +func (m *testKeyspaceEncryptionMetaPB) String() string { return "" } +func (*testKeyspaceEncryptionMetaPB) ProtoMessage() {} + +type testEncryptionEpochPB struct { + FileId uint64 `protobuf:"varint,1,opt,name=file_id,json=fileId,proto3"` + DataKeyId uint32 `protobuf:"varint,2,opt,name=data_key_id,json=dataKeyId,proto3"` + CreatedAt uint64 `protobuf:"varint,3,opt,name=created_at,json=createdAt,proto3"` +} + +func (m *testEncryptionEpochPB) Reset() { *m = testEncryptionEpochPB{} } +func (m *testEncryptionEpochPB) String() string { return "" } +func (*testEncryptionEpochPB) ProtoMessage() {} + +type testMasterKeyPB struct { + Vendor string `protobuf:"bytes,1,opt,name=vendor,proto3"` + CmekId string `protobuf:"bytes,2,opt,name=cmek_id,json=cmekId,proto3"` + Region string `protobuf:"bytes,3,opt,name=region,proto3"` + Endpoint string `protobuf:"bytes,4,opt,name=endpoint,proto3"` + Ciphertext []byte `protobuf:"bytes,5,opt,name=ciphertext,proto3"` +} + +func (m *testMasterKeyPB) Reset() { *m = testMasterKeyPB{} } +func (m *testMasterKeyPB) String() string { return "" } +func (*testMasterKeyPB) ProtoMessage() {} + +type testDataKeyPB struct { + Ciphertext []byte `protobuf:"bytes,1,opt,name=ciphertext,proto3"` +} + +func (m *testDataKeyPB) Reset() { *m = testDataKeyPB{} } +func (m *testDataKeyPB) String() string { return "" } +func (*testDataKeyPB) ProtoMessage() {} From 64aff72e730b1dad84b5ddbb7a0ede7c0f33f3e3 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Mon, 23 Mar 2026 10:57:14 +0800 Subject: [PATCH 03/11] feat(encryption): add aws and gcp kms clients Signed-off-by: tenfyzhong --- go.mod | 17 +- go.sum | 22 +- pkg/config/debug.go | 67 ++++++ pkg/config/server.go | 1 + pkg/encryption/kms/aws_kms.go | 88 ++++++++ pkg/encryption/kms/client.go | 335 ++++++++++++++++++++++++++++++ pkg/encryption/kms/client_test.go | 206 ++++++++++++++++++ pkg/encryption/kms/gcp_kms.go | 76 +++++++ pkg/encryption/kms/mock_client.go | 112 ++++++++++ 9 files changed, 906 insertions(+), 18 deletions(-) create mode 100644 pkg/encryption/kms/aws_kms.go create mode 100644 pkg/encryption/kms/client.go create mode 100644 pkg/encryption/kms/client_test.go create mode 100644 pkg/encryption/kms/gcp_kms.go create mode 100644 pkg/encryption/kms/mock_client.go diff --git a/go.mod b/go.mod index 543aa60447..dfd788d952 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module github.com/pingcap/ticdc -go 1.25.8 +go 1.25.5 require ( + cloud.google.com/go/kms v1.15.8 cloud.google.com/go/storage v1.39.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 github.com/BurntSushi/toml v1.5.0 @@ -12,10 +13,11 @@ require ( github.com/agiledragon/gomonkey/v2 v2.11.0 github.com/apache/pulsar-client-go v0.13.0 github.com/aws/aws-sdk-go v1.55.5 - github.com/aws/aws-sdk-go-v2 v1.40.0 + github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.2 github.com/aws/aws-sdk-go-v2/credentials v1.19.2 github.com/aws/aws-sdk-go-v2/service/glue v1.134.1 + github.com/aws/aws-sdk-go-v2/service/kms v1.49.5 github.com/benbjohnson/clock v1.3.5 github.com/bradleyjkemp/grpc-tools v0.2.5 github.com/cenkalti/backoff/v4 v4.2.1 @@ -91,6 +93,7 @@ require ( golang.org/x/term v0.34.0 golang.org/x/text v0.29.0 golang.org/x/time v0.12.0 + google.golang.org/api v0.170.0 google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.36.6 ) @@ -109,8 +112,7 @@ require ( cloud.google.com/go v0.112.2 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.7 // indirect - cloud.google.com/go/kms v1.15.8 // indirect - filippo.io/edwards25519 v1.1.1 // indirect + filippo.io/edwards25519 v1.1.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/AthenZ/athenz v1.10.39 // indirect @@ -142,8 +144,8 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect @@ -155,7 +157,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 // indirect - github.com/aws/smithy-go v1.23.2 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.14.3 // indirect github.com/blacktear23/go-proxyprotocol v1.0.6 // indirect @@ -363,7 +365,6 @@ require ( golang.org/x/mod v0.27.0 // indirect golang.org/x/tools v0.36.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/api v0.170.0 // indirect google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250425173222-7b384671a197 // indirect diff --git a/go.sum b/go.sum index 4bc5eb9cb6..272f2aae8c 100644 --- a/go.sum +++ b/go.sum @@ -1243,8 +1243,8 @@ cloud.google.com/go/workflows v1.12.2/go.mod h1:+OmBIgNqYJPVggnMo9nqmizW0qEXHhmn cloud.google.com/go/workflows v1.12.3/go.mod h1:fmOUeeqEwPzIU81foMjTRQIdwQHADi/vEr1cx9R1m5g= cloud.google.com/go/workflows v1.12.4/go.mod h1:yQ7HUqOkdJK4duVtMeBCAOPiN1ZF1E9pAMX51vpwB/w= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= -filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= @@ -1392,8 +1392,8 @@ github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc= -github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= github.com/aws/aws-sdk-go-v2/config v1.32.2 h1:4liUsdEpUUPZs5WVapsJLx5NPmQhQdez7nYFcovrytk= @@ -1402,10 +1402,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.2 h1:qZry8VUyTK4VIo5aEdUcBjPZHL2v github.com/aws/aws-sdk-go-v2/credentials v1.19.2/go.mod h1:YUqm5a1/kBnoK+/NY5WEiMocZihKSo15/tJdmdXnM5g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 h1:WZVR5DbDgxzA0BJeudId89Kmgy6DIU4ORpxwsVHz0qA= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14/go.mod h1:Dadl9QO0kHgbrH1GRqGiZdYtW5w+IXXaBNCHTIaheM4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 h1:PZHqQACxYb8mYgms4RZbhZG0a7dPW06xOjmaH0EJC/I= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14/go.mod h1:VymhrMJUWs69D8u0/lZ7jSB6WgaG/NqHi3gX0aYf6U0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 h1:bOS19y6zlJwagBfHxs0ESzr1XCOU2KXJCWcq3E2vfjY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14/go.mod h1:1ipeGBMAxZ0xcTm6y6paC2C/J6f6OO7LBODV9afuAyM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.14 h1:ITi7qiDSv/mSGDSWNpZ4k4Ve0DQR6Ug2SJQ8zEHoDXg= @@ -1420,6 +1420,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 h1:FIouAnCE github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14/go.mod h1:UTwDc5COa5+guonQU8qBikJo1ZJ4ln2r1MkF7Dqag1E= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14 h1:FzQE21lNtUor0Fb7QNgnEyiRCBlolLTX/Z1j65S7teM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.14/go.mod h1:s1ydyWG9pm3ZwmmYN21HKyG9WzAZhYVW85wMHs5FV6w= +github.com/aws/aws-sdk-go-v2/service/kms v1.49.5 h1:DKibav4XF66XSeaXcrn9GlWGHos6D/vJ4r7jsK7z5CE= +github.com/aws/aws-sdk-go-v2/service/kms v1.49.5/go.mod h1:1SdcmEGUEQE1mrU2sIgeHtcMSxHuybhPvuEPANzIDfI= github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1 h1:OgQy/+0+Kc3khtqiEOk23xQAglXi3Tj0y5doOxbi5tg= github.com/aws/aws-sdk-go-v2/service/s3 v1.92.1/go.mod h1:wYNqY3L02Z3IgRYxOBPH9I1zD9Cjh9hI5QOy/eOjQvw= github.com/aws/aws-sdk-go-v2/service/signin v1.0.2 h1:MxMBdKTYBjPQChlJhi4qlEueqB1p1KcbTEa7tD5aqPs= @@ -1430,8 +1432,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10 h1:GtsxyiF3Nd3JahRBJbxLCCd github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.10/go.mod h1:/j67Z5XBVDx8nZVp9EuFM9/BS5dvBznbqILGuu73hug= github.com/aws/aws-sdk-go-v2/service/sts v1.41.2 h1:a5UTtD4mHBU3t0o6aHQZFJTNKVfxFWfPX7J0Lr7G+uY= github.com/aws/aws-sdk-go-v2/service/sts v1.41.2/go.mod h1:6TxbXoDSgBQ225Qd8Q+MbxUxUh6TtNKwbRt/EPS9xso= -github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= -github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 148e927430..b544f8fd0e 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -36,6 +36,9 @@ type DebugConfig struct { SchemaStore *SchemaStoreConfig `toml:"schema-store" json:"schema_store"` EventService *EventServiceConfig `toml:"event-service" json:"event_service"` + + // Encryption is the configuration for CMEK encryption at rest + Encryption *EncryptionConfig `toml:"encryption" json:"encryption"` } // ValidateAndAdjust validates and adjusts the debug configuration @@ -140,3 +143,67 @@ func NewDefaultEventServiceConfig() *EventServiceConfig { EnableRemoteEventService: true, } } + +// EncryptionConfig represents config for CMEK encryption at rest +type EncryptionConfig struct { + // EnableEncryption enables encryption for data at rest + EnableEncryption bool `toml:"enable-encryption" json:"enable_encryption"` + + // MetaRefreshInterval is the interval for refreshing encryption metadata (default: 1 hour) + MetaRefreshInterval TomlDuration `toml:"meta-refresh-interval" json:"meta_refresh_interval"` + + // MetaCacheTTL is the TTL for caching encryption metadata (default: 1 hour) + MetaCacheTTL TomlDuration `toml:"meta-cache-ttl" json:"meta_cache_ttl"` + + // AllowDegradeOnError allows graceful degradation to unencrypted mode on encryption errors + AllowDegradeOnError bool `toml:"allow-degrade-on-error" json:"allow_degrade_on_error"` + + // KMS contains optional KMS client overrides. If unset, TiCDC will use the + // default credential chain of the corresponding cloud provider. + KMS *KMSConfig `toml:"kms" json:"kms"` +} + +// KMSConfig contains KMS configuration for different cloud providers. +type KMSConfig struct { + AWS *AWSKMSConfig `toml:"aws" json:"aws"` + GCP *GCPKMSConfig `toml:"gcp" json:"gcp"` +} + +type AWSKMSConfig struct { + // Region overrides the region from TiKV encryption metadata. + Region string `toml:"region" json:"region"` + // Endpoint overrides the endpoint from TiKV encryption metadata. + Endpoint string `toml:"endpoint" json:"endpoint"` + + // Profile configures the AWS shared config profile to use. + Profile string `toml:"profile" json:"profile"` + + // Static credentials. If AccessKey is set, SecretAccessKey must also be set. + AccessKey string `toml:"access-key" json:"access_key"` + SecretAccessKey string `toml:"secret-access-key" json:"secret_access_key"` + SessionToken string `toml:"session-token" json:"session_token"` +} + +type GCPKMSConfig struct { + // Endpoint overrides the endpoint from TiKV encryption metadata. + Endpoint string `toml:"endpoint" json:"endpoint"` + + // CredentialsFile specifies a service account JSON file path. + CredentialsFile string `toml:"credentials-file" json:"credentials_file"` + // CredentialsJSON specifies a service account JSON content. + CredentialsJSON string `toml:"credentials-json" json:"credentials_json"` +} + +// NewDefaultEncryptionConfig returns the default encryption configuration +func NewDefaultEncryptionConfig() *EncryptionConfig { + return &EncryptionConfig{ + EnableEncryption: false, + MetaRefreshInterval: TomlDuration(1 * time.Hour), + MetaCacheTTL: TomlDuration(1 * time.Hour), + AllowDegradeOnError: true, + KMS: &KMSConfig{ + AWS: &AWSKMSConfig{}, + GCP: &GCPKMSConfig{}, + }, + } +} diff --git a/pkg/config/server.go b/pkg/config/server.go index 099a1deabd..51a69537ad 100644 --- a/pkg/config/server.go +++ b/pkg/config/server.go @@ -126,6 +126,7 @@ var defaultServerConfig = &ServerConfig{ EventStore: NewDefaultEventStoreConfig(), SchemaStore: NewDefaultSchemaStoreConfig(), EventService: NewDefaultEventServiceConfig(), + Encryption: NewDefaultEncryptionConfig(), }, ClusterID: "default", GcTunerMemoryThreshold: DisableMemoryLimit, diff --git a/pkg/encryption/kms/aws_kms.go b/pkg/encryption/kms/aws_kms.go new file mode 100644 index 0000000000..66ee547a61 --- /dev/null +++ b/pkg/encryption/kms/aws_kms.go @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kms + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + awskms "github.com/aws/aws-sdk-go-v2/service/kms" +) + +type awsKMSDecryptor struct { + client *awskms.Client +} + +func (d *awsKMSDecryptor) Decrypt(ctx context.Context, keyID string, ciphertext []byte) ([]byte, error) { + input := &awskms.DecryptInput{ + CiphertextBlob: ciphertext, + } + if keyID != "" { + input.KeyId = aws.String(keyID) + } + out, err := d.client.Decrypt(ctx, input) + if err != nil { + return nil, err + } + return out.Plaintext, nil +} + +func newAWSDecryptor(ctx context.Context, cfg awsClientConfig) (awsDecryptor, error) { + opts := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(cfg.Region), + } + if cfg.Profile != "" { + opts = append(opts, awsconfig.WithSharedConfigProfile(cfg.Profile)) + } + if cfg.AccessKey != "" { + opts = append(opts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(cfg.AccessKey, cfg.SecretAccessKey, cfg.SessionToken), + )) + } + if cfg.Endpoint != "" { + endpointURL := normalizeAWSEndpoint(cfg.Endpoint) + resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...any) (aws.Endpoint, error) { + if service != awskms.ServiceID { + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + } + return aws.Endpoint{ + URL: endpointURL, + SigningRegion: cfg.Region, + HostnameImmutable: true, + }, nil + }) + opts = append(opts, awsconfig.WithEndpointResolverWithOptions(resolver)) + } + + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, opts...) + if err != nil { + return nil, err + } + + return &awsKMSDecryptor{client: awskms.NewFromConfig(awsCfg)}, nil +} + +func normalizeAWSEndpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "" + } + if strings.Contains(endpoint, "://") { + return endpoint + } + return "https://" + endpoint +} diff --git a/pkg/encryption/kms/client.go b/pkg/encryption/kms/client.go new file mode 100644 index 0000000000..df497ddc1b --- /dev/null +++ b/pkg/encryption/kms/client.go @@ -0,0 +1,335 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kms + +import ( + "context" + "strings" + "sync" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/config" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +type awsClientConfig struct { + Region string + Endpoint string + Profile string + AccessKey string + SecretAccessKey string + SessionToken string +} + +type gcpClientConfig struct { + Endpoint string + CredentialsFile string + CredentialsJSON string +} + +type awsDecryptor interface { + Decrypt(ctx context.Context, keyID string, ciphertext []byte) ([]byte, error) +} + +type gcpDecryptor interface { + Decrypt(ctx context.Context, keyID string, ciphertext []byte) ([]byte, error) + Close() error +} + +type ( + awsDecryptorFactory func(ctx context.Context, cfg awsClientConfig) (awsDecryptor, error) + gcpDecryptorFactory func(ctx context.Context, cfg gcpClientConfig) (gcpDecryptor, error) +) + +type ClientOption func(*client) + +func WithAWSDecryptorFactory(factory awsDecryptorFactory) ClientOption { + return func(c *client) { + c.awsFactory = factory + } +} + +func WithGCPDecryptorFactory(factory gcpDecryptorFactory) ClientOption { + return func(c *client) { + c.gcpFactory = factory + } +} + +type awsClientKey struct { + region string + endpoint string +} + +type gcpClientKey struct { + endpoint string +} + +type client struct { + cfg *config.EncryptionConfig + + awsFactory awsDecryptorFactory + gcpFactory gcpDecryptorFactory + + awsMu sync.Mutex + awsClients map[awsClientKey]awsDecryptor + + gcpMu sync.Mutex + gcpClients map[gcpClientKey]gcpDecryptor +} + +func NewClient(cfg *config.EncryptionConfig, opts ...ClientOption) (KMSClient, error) { + c := &client{ + cfg: cfg, + awsClients: make(map[awsClientKey]awsDecryptor), + gcpClients: make(map[gcpClientKey]gcpDecryptor), + awsFactory: newAWSDecryptor, + gcpFactory: newGCPDecryptor, + } + for _, opt := range opts { + opt(c) + } + if err := c.validateConfig(); err != nil { + return nil, err + } + return c, nil +} + +func (c *client) DecryptMasterKey(ctx context.Context, ciphertext []byte, keyID string, vendor string, region string, endpoint string) ([]byte, error) { + normalizedVendor := normalizeVendor(vendor) + switch normalizedVendor { + case vendorAWS: + awsCfg := c.resolveAWS(region, endpoint) + if awsCfg.Region == "" { + log.Warn("aws kms region is empty", + zap.String("vendor", vendor), + zap.String("keyID", keyID), + zap.String("endpoint", awsCfg.Endpoint)) + return nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("aws kms region is empty") + } + decryptor, err := c.getAWSDecryptor(ctx, awsCfg) + if err != nil { + log.Warn("failed to create aws kms client", + zap.String("vendor", vendor), + zap.String("keyID", keyID), + zap.String("region", awsCfg.Region), + zap.String("endpoint", awsCfg.Endpoint), + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + plaintext, err := decryptor.Decrypt(ctx, keyID, ciphertext) + if err != nil { + log.Warn("failed to decrypt master key via aws kms", + zap.String("vendor", vendor), + zap.String("keyID", keyID), + zap.String("region", awsCfg.Region), + zap.String("endpoint", awsCfg.Endpoint), + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + return plaintext, nil + case vendorGCP: + gcpCfg := c.resolveGCP(endpoint) + decryptor, err := c.getGCPDecryptor(ctx, gcpCfg) + if err != nil { + log.Warn("failed to create gcp kms client", + zap.String("vendor", vendor), + zap.String("keyID", keyID), + zap.String("endpoint", gcpCfg.Endpoint), + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + plaintext, err := decryptor.Decrypt(ctx, keyID, ciphertext) + if err != nil { + log.Warn("failed to decrypt master key via gcp kms", + zap.String("vendor", vendor), + zap.String("keyID", keyID), + zap.String("endpoint", gcpCfg.Endpoint), + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + return plaintext, nil + default: + log.Warn("unsupported KMS vendor", + zap.String("vendor", vendor), + zap.String("normalizedVendor", normalizedVendor), + zap.String("keyID", keyID)) + return nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("unsupported KMS vendor: " + vendor) + } +} + +func (c *client) Close() { + c.gcpMu.Lock() + defer c.gcpMu.Unlock() + + for _, cli := range c.gcpClients { + _ = cli.Close() + } + clear(c.gcpClients) +} + +const ( + vendorAWS = "aws" + vendorGCP = "gcp" +) + +func normalizeVendor(v string) string { + v = strings.TrimSpace(v) + v = strings.ToLower(v) + switch v { + case "aws-kms", "aws": + return vendorAWS + case "gcp-kms", "gcp": + return vendorGCP + default: + return v + } +} + +func (c *client) validateConfig() error { + if c.cfg == nil || c.cfg.KMS == nil { + return nil + } + + if c.cfg.KMS.AWS != nil { + awsCfg := c.cfg.KMS.AWS + if awsCfg.AccessKey != "" || awsCfg.SecretAccessKey != "" || awsCfg.SessionToken != "" { + if awsCfg.AccessKey == "" || awsCfg.SecretAccessKey == "" { + return cerrors.ErrEncryptionFailed.GenWithStackByArgs("aws kms access-key and secret-access-key must be set together") + } + if awsCfg.Profile != "" { + return cerrors.ErrEncryptionFailed.GenWithStackByArgs("aws kms profile and static credentials are mutually exclusive") + } + } + } + + if c.cfg.KMS.GCP != nil { + gcpCfg := c.cfg.KMS.GCP + if gcpCfg.CredentialsFile != "" && gcpCfg.CredentialsJSON != "" { + return cerrors.ErrEncryptionFailed.GenWithStackByArgs("gcp kms credentials-file and credentials-json are mutually exclusive") + } + } + + return nil +} + +func (c *client) resolveAWS(metaRegion, metaEndpoint string) awsClientConfig { + resolved := awsClientConfig{ + Region: metaRegion, + Endpoint: metaEndpoint, + } + + if c.cfg == nil || c.cfg.KMS == nil || c.cfg.KMS.AWS == nil { + return resolved + } + awsCfg := c.cfg.KMS.AWS + if awsCfg.Region != "" { + resolved.Region = awsCfg.Region + } + if awsCfg.Endpoint != "" { + resolved.Endpoint = awsCfg.Endpoint + } + resolved.Profile = awsCfg.Profile + resolved.AccessKey = awsCfg.AccessKey + resolved.SecretAccessKey = awsCfg.SecretAccessKey + resolved.SessionToken = awsCfg.SessionToken + return resolved +} + +func (c *client) resolveGCP(metaEndpoint string) gcpClientConfig { + resolved := gcpClientConfig{ + Endpoint: metaEndpoint, + } + + if c.cfg == nil || c.cfg.KMS == nil || c.cfg.KMS.GCP == nil { + return resolved + } + gcpCfg := c.cfg.KMS.GCP + if gcpCfg.Endpoint != "" { + resolved.Endpoint = gcpCfg.Endpoint + } + resolved.CredentialsFile = gcpCfg.CredentialsFile + resolved.CredentialsJSON = gcpCfg.CredentialsJSON + return resolved +} + +func (c *client) getAWSDecryptor(ctx context.Context, cfg awsClientConfig) (awsDecryptor, error) { + if c.awsFactory == nil { + log.Error("aws kms decryptor factory is nil") + return nil, cerrors.ErrEncryptionFailed.GenWithStackByArgs("aws kms decryptor factory is nil") + } + + key := awsClientKey{region: cfg.Region, endpoint: cfg.Endpoint} + + c.awsMu.Lock() + if existing, ok := c.awsClients[key]; ok { + c.awsMu.Unlock() + return existing, nil + } + c.awsMu.Unlock() + + cli, err := c.awsFactory(ctx, cfg) + if err != nil { + log.Warn("failed to initialize aws kms decryptor", + zap.String("region", cfg.Region), + zap.String("endpoint", cfg.Endpoint), + zap.Bool("hasProfile", cfg.Profile != ""), + zap.Bool("hasStaticCredentials", cfg.AccessKey != ""), + zap.Error(err)) + return nil, err + } + + c.awsMu.Lock() + defer c.awsMu.Unlock() + if existing, ok := c.awsClients[key]; ok { + return existing, nil + } + c.awsClients[key] = cli + return cli, nil +} + +func (c *client) getGCPDecryptor(ctx context.Context, cfg gcpClientConfig) (gcpDecryptor, error) { + if c.gcpFactory == nil { + log.Error("gcp kms decryptor factory is nil") + return nil, cerrors.ErrEncryptionFailed.GenWithStackByArgs("gcp kms decryptor factory is nil") + } + + key := gcpClientKey{endpoint: cfg.Endpoint} + + c.gcpMu.Lock() + if existing, ok := c.gcpClients[key]; ok { + c.gcpMu.Unlock() + return existing, nil + } + c.gcpMu.Unlock() + + cli, err := c.gcpFactory(ctx, cfg) + if err != nil { + log.Warn("failed to initialize gcp kms decryptor", + zap.String("endpoint", cfg.Endpoint), + zap.Bool("hasCredentialsFile", cfg.CredentialsFile != ""), + zap.Bool("hasCredentialsJSON", cfg.CredentialsJSON != ""), + zap.Error(err)) + return nil, err + } + + c.gcpMu.Lock() + defer c.gcpMu.Unlock() + if existing, ok := c.gcpClients[key]; ok { + _ = cli.Close() + return existing, nil + } + c.gcpClients[key] = cli + return cli, nil +} diff --git a/pkg/encryption/kms/client_test.go b/pkg/encryption/kms/client_test.go new file mode 100644 index 0000000000..af0ca83a82 --- /dev/null +++ b/pkg/encryption/kms/client_test.go @@ -0,0 +1,206 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kms + +import ( + "context" + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +type fakeDecryptor struct { + lastKeyID string + lastCipher []byte + plaintextResp []byte +} + +func (d *fakeDecryptor) Decrypt(ctx context.Context, keyID string, ciphertext []byte) ([]byte, error) { + d.lastKeyID = keyID + d.lastCipher = append(d.lastCipher[:0], ciphertext...) + return append([]byte(nil), d.plaintextResp...), nil +} + +type fakeClosableDecryptor struct { + fakeDecryptor + closed bool +} + +func (d *fakeClosableDecryptor) Close() error { + d.closed = true + return nil +} + +func TestKMSClientDecryptMasterKeyAWSUsesResolvedRegionEndpointAndCachesClient(t *testing.T) { + t.Parallel() + + const ( + region = "us-west-2" + endpoint = "https://kms.us-west-2.amazonaws.com" + keyID = "arn:aws:kms:us-west-2:123456789012:key/abcd" + vendor = "aws-kms" + ) + ciphertext := []byte{1, 2, 3} + plaintext := []byte{9, 8, 7} + + var calls int + decryptor := &fakeDecryptor{plaintextResp: plaintext} + awsFactory := func(ctx context.Context, cfg awsClientConfig) (awsDecryptor, error) { + calls++ + require.Equal(t, region, cfg.Region) + require.Equal(t, endpoint, cfg.Endpoint) + return decryptor, nil + } + + cli, err := NewClient(&config.EncryptionConfig{}, WithAWSDecryptorFactory(awsFactory)) + require.NoError(t, err) + + out1, err := cli.DecryptMasterKey(context.Background(), ciphertext, keyID, vendor, region, endpoint) + require.NoError(t, err) + require.Equal(t, plaintext, out1) + require.Equal(t, keyID, decryptor.lastKeyID) + require.Equal(t, ciphertext, decryptor.lastCipher) + + out2, err := cli.DecryptMasterKey(context.Background(), ciphertext, keyID, vendor, region, endpoint) + require.NoError(t, err) + require.Equal(t, plaintext, out2) + require.Equal(t, 1, calls, "expected client to be cached") +} + +func TestKMSClientDecryptMasterKeyGCPUsesResolvedEndpointAndCachesClient(t *testing.T) { + t.Parallel() + + const ( + endpoint = "cloudkms.googleapis.com:443" + keyID = "projects/p/locations/l/keyRings/r/cryptoKeys/k" + vendor = "gcp-kms" + ) + ciphertext := []byte{4, 5, 6} + plaintext := []byte{6, 5, 4} + + var calls int + decryptor := &fakeClosableDecryptor{fakeDecryptor: fakeDecryptor{plaintextResp: plaintext}} + gcpFactory := func(ctx context.Context, cfg gcpClientConfig) (gcpDecryptor, error) { + calls++ + require.Equal(t, endpoint, cfg.Endpoint) + return decryptor, nil + } + + cli, err := NewClient(&config.EncryptionConfig{}, WithGCPDecryptorFactory(gcpFactory)) + require.NoError(t, err) + + out1, err := cli.DecryptMasterKey(context.Background(), ciphertext, keyID, vendor, "", endpoint) + require.NoError(t, err) + require.Equal(t, plaintext, out1) + require.Equal(t, keyID, decryptor.lastKeyID) + require.Equal(t, ciphertext, decryptor.lastCipher) + + out2, err := cli.DecryptMasterKey(context.Background(), ciphertext, keyID, vendor, "", endpoint) + require.NoError(t, err) + require.Equal(t, plaintext, out2) + require.Equal(t, 1, calls, "expected client to be cached") +} + +func TestKMSClientDecryptMasterKeyUnknownVendorReturnsError(t *testing.T) { + t.Parallel() + + cli, err := NewClient(&config.EncryptionConfig{}) + require.NoError(t, err) + + _, err = cli.DecryptMasterKey(context.Background(), []byte{1}, "kid", "unknown", "r", "e") + require.Error(t, err) +} + +func TestKMSClientAWSConfigOverridesRegionAndEndpoint(t *testing.T) { + t.Parallel() + + cfg := &config.EncryptionConfig{ + KMS: &config.KMSConfig{ + AWS: &config.AWSKMSConfig{ + Region: "override-region", + Endpoint: "https://override.endpoint", + Profile: "test-profile", + }, + GCP: &config.GCPKMSConfig{}, + }, + } + + awsFactory := func(ctx context.Context, cfg awsClientConfig) (awsDecryptor, error) { + require.Equal(t, "override-region", cfg.Region) + require.Equal(t, "https://override.endpoint", cfg.Endpoint) + require.Equal(t, "test-profile", cfg.Profile) + return &fakeDecryptor{plaintextResp: []byte{1}}, nil + } + + cli, err := NewClient(cfg, WithAWSDecryptorFactory(awsFactory)) + require.NoError(t, err) + _, err = cli.DecryptMasterKey(context.Background(), []byte{1}, "kid", "aws-kms", "meta-region", "meta-endpoint") + require.NoError(t, err) +} + +func TestKMSClientGCPConfigOverridesEndpointAndCredentials(t *testing.T) { + t.Parallel() + + cfg := &config.EncryptionConfig{ + KMS: &config.KMSConfig{ + AWS: &config.AWSKMSConfig{}, + GCP: &config.GCPKMSConfig{ + Endpoint: "override.gcp.endpoint:443", + CredentialsFile: "/tmp/creds.json", + }, + }, + } + + gcpFactory := func(ctx context.Context, cfg gcpClientConfig) (gcpDecryptor, error) { + require.Equal(t, "override.gcp.endpoint:443", cfg.Endpoint) + require.Equal(t, "/tmp/creds.json", cfg.CredentialsFile) + require.Empty(t, cfg.CredentialsJSON) + return &fakeClosableDecryptor{fakeDecryptor: fakeDecryptor{plaintextResp: []byte{1}}}, nil + } + + cli, err := NewClient(cfg, WithGCPDecryptorFactory(gcpFactory)) + require.NoError(t, err) + _, err = cli.DecryptMasterKey(context.Background(), []byte{1}, "kid", "gcp-kms", "", "meta-endpoint") + require.NoError(t, err) +} + +func TestNewClientValidatesAWSStaticCredentials(t *testing.T) { + t.Parallel() + + _, err := NewClient(&config.EncryptionConfig{ + KMS: &config.KMSConfig{ + AWS: &config.AWSKMSConfig{ + AccessKey: "ak", + }, + GCP: &config.GCPKMSConfig{}, + }, + }) + require.Error(t, err) +} + +func TestNewClientValidatesGCPCredentialsExclusive(t *testing.T) { + t.Parallel() + + _, err := NewClient(&config.EncryptionConfig{ + KMS: &config.KMSConfig{ + AWS: &config.AWSKMSConfig{}, + GCP: &config.GCPKMSConfig{ + CredentialsFile: "/tmp/a.json", + CredentialsJSON: "{}", + }, + }, + }) + require.Error(t, err) +} diff --git a/pkg/encryption/kms/gcp_kms.go b/pkg/encryption/kms/gcp_kms.go new file mode 100644 index 0000000000..0ecc09d2f6 --- /dev/null +++ b/pkg/encryption/kms/gcp_kms.go @@ -0,0 +1,76 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kms + +import ( + "context" + "net/url" + "strings" + + cloudkms "cloud.google.com/go/kms/apiv1" + "cloud.google.com/go/kms/apiv1/kmspb" + "google.golang.org/api/option" +) + +type gcpKMSDecryptor struct { + client *cloudkms.KeyManagementClient +} + +func (d *gcpKMSDecryptor) Decrypt(ctx context.Context, keyID string, ciphertext []byte) ([]byte, error) { + resp, err := d.client.Decrypt(ctx, &kmspb.DecryptRequest{ + Name: keyID, + Ciphertext: ciphertext, + }) + if err != nil { + return nil, err + } + return resp.Plaintext, nil +} + +func (d *gcpKMSDecryptor) Close() error { + return d.client.Close() +} + +func newGCPDecryptor(ctx context.Context, cfg gcpClientConfig) (gcpDecryptor, error) { + var opts []option.ClientOption + if cfg.Endpoint != "" { + opts = append(opts, option.WithEndpoint(normalizeGCPEndpoint(cfg.Endpoint))) + } + if cfg.CredentialsFile != "" { + opts = append(opts, option.WithCredentialsFile(cfg.CredentialsFile)) + } + if cfg.CredentialsJSON != "" { + opts = append(opts, option.WithCredentialsJSON([]byte(cfg.CredentialsJSON))) + } + + client, err := cloudkms.NewKeyManagementClient(ctx, opts...) + if err != nil { + return nil, err + } + return &gcpKMSDecryptor{client: client}, nil +} + +func normalizeGCPEndpoint(endpoint string) string { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "" + } + if strings.Contains(endpoint, "://") { + parsed, err := url.Parse(endpoint) + if err == nil && parsed.Host != "" { + return parsed.Host + } + } + return endpoint +} diff --git a/pkg/encryption/kms/mock_client.go b/pkg/encryption/kms/mock_client.go new file mode 100644 index 0000000000..fcbb39f32c --- /dev/null +++ b/pkg/encryption/kms/mock_client.go @@ -0,0 +1,112 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kms + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + + "github.com/pingcap/log" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +// KMSClient is the interface for Key Management Service operations +type KMSClient interface { + // DecryptMasterKey decrypts the master key using the specified KMS parameters + DecryptMasterKey(ctx context.Context, ciphertext []byte, keyID string, vendor string, region string, endpoint string) ([]byte, error) +} + +// MockKMSClient is a mock implementation of KMSClient for development and testing +// It uses an in-memory mock key for decryption +type MockKMSClient struct { + // mockKey is a fixed key used for mock decryption (32 bytes for AES-256) + mockKey []byte +} + +// NewMockKMSClient creates a new mock KMS client +func NewMockKMSClient() *MockKMSClient { + // Generate a fixed mock key for testing + // In a real implementation, this would be retrieved from KMS + mockKey := make([]byte, 32) + // Use a deterministic key for testing (in production, this would come from KMS) + for i := range mockKey { + mockKey[i] = byte(i % 256) + } + + return &MockKMSClient{ + mockKey: mockKey, + } +} + +// DecryptMasterKey decrypts the master key using mock KMS +// In a real implementation, this would call the actual KMS service +func (c *MockKMSClient) DecryptMasterKey(ctx context.Context, ciphertext []byte, keyID string, vendor string, region string, endpoint string) ([]byte, error) { + if len(ciphertext) < aes.BlockSize { + return nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("ciphertext too short") + } + + log.Debug("mock KMS client: decrypting master key", + zap.String("keyID", keyID), + zap.String("vendor", vendor), + zap.String("region", region), + zap.Int("ciphertextLen", len(ciphertext))) + + // In mock implementation, we use AES-256-CTR to decrypt + // The ciphertext is encrypted with the mock key + block, err := aes.NewCipher(c.mockKey) + if err != nil { + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + + // Extract IV from the beginning of ciphertext + iv := ciphertext[:aes.BlockSize] + encryptedData := ciphertext[aes.BlockSize:] + + // Decrypt using CTR mode + stream := cipher.NewCTR(block, iv) + plaintext := make([]byte, len(encryptedData)) + stream.XORKeyStream(plaintext, encryptedData) + + return plaintext, nil +} + +// EncryptMasterKey encrypts a master key using mock KMS (for testing) +// This is used to generate mock ciphertext +func (c *MockKMSClient) EncryptMasterKey(plaintext []byte) ([]byte, error) { + block, err := aes.NewCipher(c.mockKey) + if err != nil { + return nil, cerrors.ErrEncodeFailed.Wrap(err) + } + + // Generate random IV + iv := make([]byte, aes.BlockSize) + if _, err := rand.Read(iv); err != nil { + return nil, cerrors.ErrEncodeFailed.Wrap(err) + } + + // Encrypt using CTR mode + stream := cipher.NewCTR(block, iv) + ciphertext := make([]byte, len(plaintext)) + stream.XORKeyStream(ciphertext, plaintext) + + // Prepend IV to ciphertext + result := make([]byte, aes.BlockSize+len(ciphertext)) + copy(result[:aes.BlockSize], iv) + copy(result[aes.BlockSize:], ciphertext) + + return result, nil +} From 73311f16f5df57a69b288c763cb2f94c39c6d313 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Mon, 23 Mar 2026 10:57:31 +0800 Subject: [PATCH 04/11] feat(encryption): add metadata cache and data encryption manager Signed-off-by: tenfyzhong --- pkg/encryption/encryption_manager.go | 199 ++++++++ pkg/encryption/encryption_manager_test.go | 154 +++++++ pkg/encryption/manager.go | 532 ++++++++++++++++++++++ pkg/encryption/manager_test.go | 148 ++++++ pkg/encryption/mock_tikv_client.go | 171 +++++++ pkg/encryption/mock_tikv_client_test.go | 40 ++ 6 files changed, 1244 insertions(+) create mode 100644 pkg/encryption/encryption_manager.go create mode 100644 pkg/encryption/encryption_manager_test.go create mode 100644 pkg/encryption/manager.go create mode 100644 pkg/encryption/manager_test.go create mode 100644 pkg/encryption/mock_tikv_client.go create mode 100644 pkg/encryption/mock_tikv_client_test.go diff --git a/pkg/encryption/encryption_manager.go b/pkg/encryption/encryption_manager.go new file mode 100644 index 0000000000..ad7493c6cb --- /dev/null +++ b/pkg/encryption/encryption_manager.go @@ -0,0 +1,199 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/config" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +// EncryptionManager is the main interface for encryption/decryption operations +type EncryptionManager interface { + // EncryptData encrypts data for a keyspace + // Returns encrypted data with header, or original data if encryption is not enabled + EncryptData(ctx context.Context, keyspaceID uint32, data []byte) ([]byte, error) + + // DecryptData decrypts data for a keyspace + // Automatically detects if data is encrypted and handles accordingly + DecryptData(ctx context.Context, keyspaceID uint32, encryptedData []byte) ([]byte, error) +} + +type encryptionManager struct { + metaManager EncryptionMetaManager +} + +// NewEncryptionManager creates a new encryption manager +func NewEncryptionManager(metaManager EncryptionMetaManager) EncryptionManager { + return &encryptionManager{ + metaManager: metaManager, + } +} + +// EncryptData encrypts data for a keyspace +func (m *encryptionManager) EncryptData(ctx context.Context, keyspaceID uint32, data []byte) ([]byte, error) { + allowDegrade := true + serverCfg := config.GetGlobalServerConfig() + if serverCfg != nil && serverCfg.Debug != nil && serverCfg.Debug.Encryption != nil { + allowDegrade = serverCfg.Debug.Encryption.AllowDegradeOnError + } + + // Get current data key, key ID and version together to avoid mismatch when keys rotate. + dataKey, currentDataKeyID, version, err := m.metaManager.GetCurrentDataKey(ctx, keyspaceID) + if err != nil { + if allowDegrade { + log.Warn("failed to get current data key, degrade to plaintext", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return data, nil + } + log.Error("failed to get current data key", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return nil, cerrors.ErrEncryptionFailed.Wrap(err) + } + + if len(dataKey) == 0 { + log.Debug("encryption not enabled for keyspace", + zap.Uint32("keyspaceID", keyspaceID)) + return data, nil + } + + cipherImpl := NewAES256CTRCipher() + + // Generate IV + iv, err := GenerateIV(cipherImpl.IVSize()) + if err != nil { + log.Error("failed to generate IV", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return nil, cerrors.ErrEncryptionFailed.Wrap(err) + } + + // Encrypt data + encryptedData, err := cipherImpl.Encrypt(data, dataKey, iv) + if err != nil { + log.Error("failed to encrypt data", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return nil, cerrors.ErrEncryptionFailed.Wrap(err) + } + + // Prepend IV to encrypted data + encryptedWithIV := make([]byte, len(iv)+len(encryptedData)) + copy(encryptedWithIV, iv) + copy(encryptedWithIV[len(iv):], encryptedData) + + // Encode with encryption header + result, err := EncodeEncryptedData(encryptedWithIV, version, currentDataKeyID) + if err != nil { + log.Error("failed to encode encrypted data", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint8("version", version), + zap.Binary("dataKeyID", []byte(currentDataKeyID)), + zap.Error(err)) + return nil, cerrors.ErrEncryptionFailed.Wrap(err) + } + + log.Debug("data encrypted successfully", + zap.Uint32("keyspaceID", keyspaceID), + zap.String("dataKeyID", currentDataKeyID), + zap.Int("originalSize", len(data)), + zap.Int("encryptedSize", len(result))) + + return result, nil +} + +// DecryptData decrypts data for a keyspace +func (m *encryptionManager) DecryptData(ctx context.Context, keyspaceID uint32, encryptedData []byte) ([]byte, error) { + // Check if data is encrypted + if !IsEncrypted(encryptedData) { + // Data is not encrypted, return as-is (backward compatibility) + log.Debug("data is not encrypted", + zap.Uint32("keyspaceID", keyspaceID)) + return encryptedData, nil + } + + // Decode encryption header + version, dataKeyID, dataWithIV, err := DecodeEncryptedData(encryptedData) + if err != nil { + log.Warn("failed to decode encrypted data header", + zap.Uint32("keyspaceID", keyspaceID), + zap.Int("encryptedSize", len(encryptedData)), + zap.Error(err)) + return nil, cerrors.ErrDecryptionFailed.Wrap(err) + } + + if version == VersionUnencrypted { + // Should not happen if IsEncrypted returned true, but handle it anyway + return dataWithIV, nil + } + + dataKey, err := m.metaManager.GetDataKey(ctx, keyspaceID, dataKeyID) + if err != nil { + log.Warn("failed to get data key for decryption", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint8("version", version), + zap.Binary("dataKeyID", []byte(dataKeyID)), + zap.Error(err)) + return nil, cerrors.ErrDecryptionFailed.Wrap(err) + } + + if len(dataKey) == 0 { + log.Warn("data key is empty for decryption", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint8("version", version), + zap.Binary("dataKeyID", []byte(dataKeyID))) + return nil, cerrors.ErrDecryptionFailed.GenWithStackByArgs("data key is empty") + } + + cipherImpl := NewAES256CTRCipher() + + // Extract IV from the beginning of data + ivSize := cipherImpl.IVSize() + if len(dataWithIV) < ivSize { + log.Warn("encrypted data too short for IV", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint8("version", version), + zap.Binary("dataKeyID", []byte(dataKeyID)), + zap.Int("dataWithIVSize", len(dataWithIV)), + zap.Int("expectedIVSize", ivSize)) + return nil, cerrors.ErrDecryptionFailed.GenWithStackByArgs("data too short for IV") + } + + iv := dataWithIV[:ivSize] + encryptedDataOnly := dataWithIV[ivSize:] + + // Decrypt data + plaintext, err := cipherImpl.Decrypt(encryptedDataOnly, dataKey, iv) + if err != nil { + log.Warn("failed to decrypt data", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint8("version", version), + zap.Binary("dataKeyID", []byte(dataKeyID)), + zap.Error(err)) + return nil, cerrors.ErrDecryptionFailed.Wrap(err) + } + + log.Debug("data decrypted successfully", + zap.Uint32("keyspaceID", keyspaceID), + zap.String("dataKeyID", dataKeyID), + zap.Int("encryptedSize", len(encryptedData)), + zap.Int("plaintextSize", len(plaintext))) + + return plaintext, nil +} diff --git a/pkg/encryption/encryption_manager_test.go b/pkg/encryption/encryption_manager_test.go new file mode 100644 index 0000000000..8a8959a220 --- /dev/null +++ b/pkg/encryption/encryption_manager_test.go @@ -0,0 +1,154 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "bytes" + "context" + "errors" + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +type mockMetaManager struct { + currentKey []byte + currentKeyID string + version byte + currentKeyErr error + dataKeys map[string][]byte +} + +func (m *mockMetaManager) IsEncryptionEnabled(ctx context.Context, keyspaceID uint32) bool { + return true +} + +func (m *mockMetaManager) GetCurrentDataKey(ctx context.Context, keyspaceID uint32) ([]byte, string, byte, error) { + return m.currentKey, m.currentKeyID, m.version, m.currentKeyErr +} + +func (m *mockMetaManager) GetDataKey(ctx context.Context, keyspaceID uint32, dataKeyID string) ([]byte, error) { + if m.dataKeys == nil { + if m.currentKeyID == dataKeyID && len(m.currentKey) > 0 { + return m.currentKey, nil + } + return nil, errors.New("data key not found") + } + key, ok := m.dataKeys[dataKeyID] + if !ok { + return nil, errors.New("data key not found") + } + return key, nil +} + +func (m *mockMetaManager) Start(ctx context.Context) error { return nil } +func (m *mockMetaManager) Stop() {} + +func setAllowDegradeOnError(t *testing.T, allow bool) func() { + t.Helper() + original := config.GetGlobalServerConfig().Clone() + updated := original.Clone() + updated.Debug.Encryption.AllowDegradeOnError = allow + config.StoreGlobalServerConfig(updated) + return func() { + config.StoreGlobalServerConfig(original) + } +} + +func TestEncryptDataAllowDegradeOnError(t *testing.T) { + restore := setAllowDegradeOnError(t, true) + defer restore() + + meta := &mockMetaManager{ + currentKeyErr: errors.New("boom"), + } + manager := NewEncryptionManager(meta) + input := []byte("payload") + + output, err := manager.EncryptData(context.Background(), 1, input) + require.NoError(t, err) + require.Equal(t, input, output) +} + +func TestEncryptDataDisallowDegradeOnError(t *testing.T) { + restore := setAllowDegradeOnError(t, false) + defer restore() + + meta := &mockMetaManager{ + currentKeyErr: errors.New("boom"), + } + manager := NewEncryptionManager(meta) + _, err := manager.EncryptData(context.Background(), 1, []byte("payload")) + require.Error(t, err) +} + +func TestEncryptDataDisabledSkipsEncryption(t *testing.T) { + restore := setAllowDegradeOnError(t, false) + defer restore() + + meta := &mockMetaManager{} + manager := NewEncryptionManager(meta) + input := []byte("payload") + + output, err := manager.EncryptData(context.Background(), 1, input) + require.NoError(t, err) + require.Equal(t, input, output) +} + +func TestEncryptDecryptRoundTrip(t *testing.T) { + restore := setAllowDegradeOnError(t, false) + defer restore() + + key := bytes.Repeat([]byte{0x11}, 32) + meta := &mockMetaManager{ + currentKey: key, + currentKeyID: "K01", + version: 0x01, + } + manager := NewEncryptionManager(meta) + + input := []byte("round-trip-payload") + encrypted, err := manager.EncryptData(context.Background(), 1, input) + require.NoError(t, err) + require.NotEqual(t, input, encrypted) + require.True(t, IsEncrypted(encrypted)) + + decrypted, err := manager.DecryptData(context.Background(), 1, encrypted) + require.NoError(t, err) + require.Equal(t, input, decrypted) +} + +func TestEncryptDecryptRoundTripWithAES128Key(t *testing.T) { + restore := setAllowDegradeOnError(t, false) + defer restore() + + key := bytes.Repeat([]byte{0x22}, 16) + meta := &mockMetaManager{ + currentKey: key, + currentKeyID: "K02", + version: 0x01, + } + manager := NewEncryptionManager(meta) + + input := []byte("round-trip-with-16-byte-key") + encrypted, err := manager.EncryptData(context.Background(), 1, input) + require.NoError(t, err) + require.NotEqual(t, input, encrypted) + require.True(t, IsEncrypted(encrypted)) + + decrypted, err := manager.DecryptData(context.Background(), 1, encrypted) + require.NoError(t, err) + require.Equal(t, input, decrypted) +} diff --git a/pkg/encryption/manager.go b/pkg/encryption/manager.go new file mode 100644 index 0000000000..5392896206 --- /dev/null +++ b/pkg/encryption/manager.go @@ -0,0 +1,532 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "sync" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/encryption/kms" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +// EncryptionMetaManager manages encryption metadata for keyspaces +type EncryptionMetaManager interface { + // IsEncryptionEnabled checks if encryption is enabled for a keyspace + IsEncryptionEnabled(ctx context.Context, keyspaceID uint32) bool + + // GetCurrentDataKey gets the current data key for a keyspace. + // It returns the plaintext data key, the 3-byte data key ID used in the encryption header, + // and the encryption format version (derived from current.data_key_id & 0xFF). + // + // Returning key and key ID together avoids mismatches if TiKV rotates keys between calls. + GetCurrentDataKey(ctx context.Context, keyspaceID uint32) (dataKey []byte, dataKeyID string, version byte, err error) + + // GetDataKey gets a data key by ID. + GetDataKey(ctx context.Context, keyspaceID uint32, dataKeyID string) ([]byte, error) + + // Start starts the background refresh goroutine + Start(ctx context.Context) error + + // Stop stops the background refresh goroutine + Stop() +} + +type encryptionMetaManager struct { + tikvClient TiKVEncryptionClient + kmsClient kms.KMSClient + + metaCache map[uint32]*cachedMeta + metaMu sync.RWMutex + dataKeyCache map[uint32]map[string]*cachedDataKey + dataKeyMu sync.RWMutex + + ttl time.Duration + refreshInterval time.Duration + stopCh chan struct{} + stopOnce sync.Once + wg sync.WaitGroup +} + +type cachedMeta struct { + meta *EncryptionMeta + timestamp time.Time +} + +type cachedDataKey struct { + key []byte + timestamp time.Time +} + +// NewEncryptionMetaManager creates a new encryption meta manager +func NewEncryptionMetaManager(tikvClient TiKVEncryptionClient, kmsClient kms.KMSClient) EncryptionMetaManager { + return &encryptionMetaManager{ + tikvClient: tikvClient, + kmsClient: kmsClient, + metaCache: make(map[uint32]*cachedMeta), + dataKeyCache: make(map[uint32]map[string]*cachedDataKey), + ttl: 1 * time.Hour, // Default TTL: 1 hour + refreshInterval: 1 * time.Hour, // Default refresh interval: 1 hour + stopCh: make(chan struct{}), + } +} + +// IsEncryptionEnabled checks if encryption is enabled for a keyspace +func (m *encryptionMetaManager) IsEncryptionEnabled(ctx context.Context, keyspaceID uint32) bool { + meta, err := m.getMeta(ctx, keyspaceID) + if err != nil { + log.Warn("failed to get encryption meta", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + // If we can't get meta, encryption is not enabled + return false + } + return meta != nil +} + +// GetCurrentDataKey gets the current data key for a keyspace +func (m *encryptionMetaManager) GetCurrentDataKey(ctx context.Context, keyspaceID uint32) ([]byte, string, byte, error) { + meta, err := m.getMeta(ctx, keyspaceID) + if err != nil { + log.Warn("failed to get encryption meta for current data key", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return nil, "", 0, err + } + + if meta == nil { + return nil, "", 0, nil + } + + if meta.Current == nil || meta.Current.DataKeyId == 0 { + log.Warn("encryption meta current data key ID is empty", + zap.Uint32("keyspaceID", keyspaceID)) + return nil, "", 0, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("current data key ID is empty") + } + + currentKeyID, err := encodeDataKeyID24BE(meta.Current.DataKeyId) + if err != nil { + log.Warn("failed to encode current data key ID", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("dataKeyID", meta.Current.DataKeyId), + zap.Error(err)) + return nil, "", 0, err + } + + version := byte(meta.Current.DataKeyId & 0xFF) + if version == VersionUnencrypted { + log.Warn("invalid encryption meta version derived from current data key ID", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("dataKeyID", meta.Current.DataKeyId)) + return nil, "", 0, cerrors.ErrEncryptionFailed.GenWithStackByArgs("version must be non-zero") + } + + // Check cache first. + m.dataKeyMu.RLock() + if keyspaceCache, ok := m.dataKeyCache[keyspaceID]; ok { + if cached, ok := keyspaceCache[currentKeyID]; ok { + if time.Since(cached.timestamp) < m.ttl { + key := make([]byte, len(cached.key)) + copy(key, cached.key) + m.dataKeyMu.RUnlock() + return key, currentKeyID, version, nil + } + } + } + m.dataKeyMu.RUnlock() + + dataKey, ok := meta.DataKeys[meta.Current.DataKeyId] + if !ok { + log.Warn("current data key not found in encryption meta", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("dataKeyID", meta.Current.DataKeyId)) + return nil, "", 0, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("current data key not found") + } + + plaintextKey, err := m.decryptDataKey(ctx, meta.MasterKey, dataKey.Ciphertext) + if err != nil { + log.Warn("failed to decrypt current data key", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("dataKeyID", meta.Current.DataKeyId), + zap.Binary("dataKeyIDBytes", []byte(currentKeyID)), + zap.String("kmsVendor", safeKMSVendor(meta.MasterKey)), + zap.String("cmekID", safeCMEKID(meta.MasterKey)), + zap.Error(err)) + return nil, "", 0, err + } + + m.dataKeyMu.Lock() + if m.dataKeyCache[keyspaceID] == nil { + m.dataKeyCache[keyspaceID] = make(map[string]*cachedDataKey) + } + m.dataKeyCache[keyspaceID][currentKeyID] = &cachedDataKey{ + key: plaintextKey, + timestamp: time.Now(), + } + m.dataKeyMu.Unlock() + + return plaintextKey, currentKeyID, version, nil +} + +// GetDataKey gets a data key by ID. +func (m *encryptionMetaManager) GetDataKey(ctx context.Context, keyspaceID uint32, dataKeyID string) ([]byte, error) { + // Check cache first + m.dataKeyMu.RLock() + if keyspaceCache, ok := m.dataKeyCache[keyspaceID]; ok { + if cached, ok := keyspaceCache[dataKeyID]; ok { + // Check if cache is still valid + if time.Since(cached.timestamp) < m.ttl { + key := make([]byte, len(cached.key)) + copy(key, cached.key) + m.dataKeyMu.RUnlock() + return key, nil + } + } + } + m.dataKeyMu.RUnlock() + + // Get meta to find the data key + meta, err := m.getMeta(ctx, keyspaceID) + if err != nil { + log.Warn("failed to get encryption meta for data key", + zap.Uint32("keyspaceID", keyspaceID), + zap.Binary("dataKeyID", []byte(dataKeyID)), + zap.Error(err)) + return nil, err + } + + if meta == nil { + log.Warn("encryption not enabled when looking up data key", + zap.Uint32("keyspaceID", keyspaceID), + zap.Binary("dataKeyID", []byte(dataKeyID))) + return nil, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("encryption not enabled") + } + + id, err := decodeDataKeyID24BE(dataKeyID) + if err != nil { + log.Warn("failed to decode data key ID", + zap.Uint32("keyspaceID", keyspaceID), + zap.Binary("dataKeyID", []byte(dataKeyID)), + zap.Error(err)) + return nil, err + } + + dataKey, ok := meta.DataKeys[id] + if !ok { + log.Warn("data key not found in encryption meta", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("dataKeyID", id), + zap.Binary("dataKeyIDBytes", []byte(dataKeyID))) + return nil, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("data key not found: " + dataKeyID) + } + + // Decrypt the data key using master key + plaintextKey, err := m.decryptDataKey(ctx, meta.MasterKey, dataKey.Ciphertext) + if err != nil { + log.Warn("failed to decrypt data key", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("dataKeyID", id), + zap.Binary("dataKeyIDBytes", []byte(dataKeyID)), + zap.String("kmsVendor", safeKMSVendor(meta.MasterKey)), + zap.String("cmekID", safeCMEKID(meta.MasterKey)), + zap.Error(err)) + return nil, err + } + + // Cache the decrypted key + m.dataKeyMu.Lock() + if m.dataKeyCache[keyspaceID] == nil { + m.dataKeyCache[keyspaceID] = make(map[string]*cachedDataKey) + } + m.dataKeyCache[keyspaceID][dataKeyID] = &cachedDataKey{ + key: plaintextKey, + timestamp: time.Now(), + } + m.dataKeyMu.Unlock() + + return plaintextKey, nil +} + +// getMeta gets encryption metadata, with caching +func (m *encryptionMetaManager) getMeta(ctx context.Context, keyspaceID uint32) (*EncryptionMeta, error) { + // Check cache first + m.metaMu.RLock() + if cached, ok := m.metaCache[keyspaceID]; ok { + // Check if cache is still valid + if time.Since(cached.timestamp) < m.ttl { + meta := cached.meta + m.metaMu.RUnlock() + if meta == nil { + log.Debug("using cached empty encryption meta", + zap.Uint32("keyspaceID", keyspaceID)) + } else { + log.Debug("using cached encryption meta", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("metaKeyspaceID", meta.KeyspaceId), + zap.Uint32("currentDataKeyID", meta.Current.DataKeyId), + zap.Uint8("version", byte(meta.Current.DataKeyId&0xFF)), + zap.Int("dataKeyCount", len(meta.DataKeys))) + } + return meta, nil + } + } + m.metaMu.RUnlock() + + // Fetch from TiKV + meta, err := m.tikvClient.GetKeyspaceEncryptionMeta(ctx, keyspaceID) + if err != nil { + // If we get ErrEncryptionMetaNotFound, cache nil to avoid repeated lookups + if cerrors.ErrEncryptionMetaNotFound.Equal(err) { + log.Info("encryption meta not found for keyspace", + zap.Uint32("keyspaceID", keyspaceID)) + m.metaMu.Lock() + m.metaCache[keyspaceID] = &cachedMeta{ + meta: nil, + timestamp: time.Now(), + } + m.metaMu.Unlock() + return nil, nil + } + log.Warn("failed to fetch encryption meta from TiKV", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + return nil, err + } + + // Cache the result (including nil if enabled=false) + m.metaMu.Lock() + m.metaCache[keyspaceID] = &cachedMeta{ + meta: meta, + timestamp: time.Now(), + } + m.metaMu.Unlock() + + if meta == nil { + log.Info("encryption meta loaded as empty", + zap.Uint32("keyspaceID", keyspaceID)) + return nil, nil + } + + log.Info("encryption meta loaded", + zap.Uint32("keyspaceID", keyspaceID), + zap.Uint32("metaKeyspaceID", meta.KeyspaceId), + zap.Uint32("currentDataKeyID", meta.Current.DataKeyId), + zap.Uint8("version", byte(meta.Current.DataKeyId&0xFF)), + zap.Int("dataKeyCount", len(meta.DataKeys)), + zap.Int("historyCount", len(meta.History)), + zap.String("kmsVendor", safeKMSVendor(meta.MasterKey)), + zap.String("cmekID", safeCMEKID(meta.MasterKey))) + + return meta, nil +} + +// decryptDataKey decrypts a data key using the master key +func (m *encryptionMetaManager) decryptDataKey(ctx context.Context, masterKey *MasterKey, dataKeyCiphertext []byte) ([]byte, error) { + if masterKey == nil { + log.Warn("failed to decrypt data key: master key is nil") + return nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("master key is nil") + } + + // Decrypt master key from KMS + masterKeyPlaintext, err := m.kmsClient.DecryptMasterKey( + ctx, + masterKey.Ciphertext, + masterKey.CmekId, + masterKey.Vendor, + masterKey.Region, + masterKey.Endpoint, + ) + if err != nil { + log.Warn("failed to decrypt master key via KMS", + zap.String("kmsVendor", masterKey.Vendor), + zap.String("cmekID", masterKey.CmekId), + zap.String("region", masterKey.Region), + zap.String("endpoint", masterKey.Endpoint), + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + + if len(masterKeyPlaintext) != 32 { + log.Warn("invalid master key plaintext length", + zap.Int("length", len(masterKeyPlaintext))) + return nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("master key plaintext must be 32 bytes") + } + + // Decrypt data key using master key (AES-CTR). + block, err := aes.NewCipher(masterKeyPlaintext) + if err != nil { + log.Warn("failed to initialize AES cipher for data key decryption", + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + + // New format: [iv(16)][ciphertext(payload)]. + if len(dataKeyCiphertext) > aes.BlockSize { + key, method, hasMethod, err := decryptDataKeyPayload( + block, dataKeyCiphertext[:aes.BlockSize], dataKeyCiphertext[aes.BlockSize:], true, + ) + if err == nil { + fields := []zap.Field{ + zap.String("format", "iv_prefixed"), + zap.Int("ciphertextLen", len(dataKeyCiphertext)), + zap.Int("keyLen", len(key)), + } + if hasMethod { + fields = append(fields, zap.Uint8("method", method)) + } + log.Debug("decoded data key ciphertext", fields...) + return key, nil + } + log.Debug("failed to decode iv prefixed data key ciphertext, fallback to legacy format", + zap.Int("ciphertextLen", len(dataKeyCiphertext)), + zap.Error(err)) + } + + // Legacy format: ciphertext only, decrypted with zero IV. + key, method, hasMethod, err := decryptDataKeyPayload( + block, make([]byte, aes.BlockSize), dataKeyCiphertext, false, + ) + if err != nil { + log.Warn("invalid data key ciphertext", + zap.Int("length", len(dataKeyCiphertext)), + zap.Error(err)) + return nil, cerrors.ErrDecodeFailed.Wrap(err) + } + fields := []zap.Field{ + zap.String("format", "legacy_zero_iv"), + zap.Int("ciphertextLen", len(dataKeyCiphertext)), + zap.Int("keyLen", len(key)), + } + if hasMethod { + fields = append(fields, zap.Uint8("method", method)) + } + log.Debug("decoded data key ciphertext", fields...) + return key, nil +} + +func decryptDataKeyPayload(block cipher.Block, iv []byte, ciphertext []byte, requireMethodPrefix bool) ([]byte, byte, bool, error) { + if len(iv) != aes.BlockSize { + return nil, 0, false, cerrors.ErrDecodeFailed.GenWithStackByArgs("iv must be 16 bytes") + } + stream := cipher.NewCTR(block, iv) + plaintext := make([]byte, len(ciphertext)) + stream.XORKeyStream(plaintext, ciphertext) + + key, method, hasMethod, err := parsePlaintextDataKey(plaintext, requireMethodPrefix) + if err != nil { + return nil, 0, false, err + } + return key, method, hasMethod, nil +} + +func parsePlaintextDataKey(plaintext []byte, requireMethodPrefix bool) ([]byte, byte, bool, error) { + if requireMethodPrefix { + switch len(plaintext) { + case 17, 25, 33: + method := plaintext[0] + key := make([]byte, len(plaintext)-1) + copy(key, plaintext[1:]) + return key, method, true, nil + default: + return nil, 0, false, cerrors.ErrDecodeFailed.GenWithStackByArgs("invalid data key plaintext length") + } + } + + switch len(plaintext) { + case 16, 24, 32: + key := make([]byte, len(plaintext)) + copy(key, plaintext) + return key, 0, false, nil + default: + return nil, 0, false, cerrors.ErrDecodeFailed.GenWithStackByArgs("invalid data key plaintext length") + } +} + +func safeKMSVendor(masterKey *MasterKey) string { + if masterKey == nil { + return "" + } + return masterKey.Vendor +} + +func safeCMEKID(masterKey *MasterKey) string { + if masterKey == nil { + return "" + } + return masterKey.CmekId +} + +// Start starts the background refresh goroutine +func (m *encryptionMetaManager) Start(ctx context.Context) error { + m.wg.Add(1) + go m.refreshLoop(ctx) + return nil +} + +func (m *encryptionMetaManager) Stop() { + m.stopOnce.Do(func() { + close(m.stopCh) + }) + m.wg.Wait() +} + +func (m *encryptionMetaManager) Close() { + m.Stop() +} + +// refreshLoop periodically refreshes encryption metadata +func (m *encryptionMetaManager) refreshLoop(ctx context.Context) { + defer m.wg.Done() + + ticker := time.NewTicker(m.refreshInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-m.stopCh: + return + case <-ticker.C: + m.refreshAll(ctx) + } + } +} + +func (m *encryptionMetaManager) refreshAll(ctx context.Context) { + m.metaMu.RLock() + keyspaceIDs := make([]uint32, 0, len(m.metaCache)) + for keyspaceID := range m.metaCache { + keyspaceIDs = append(keyspaceIDs, keyspaceID) + } + m.metaMu.RUnlock() + + for _, keyspaceID := range keyspaceIDs { + m.metaMu.Lock() + delete(m.metaCache, keyspaceID) + m.metaMu.Unlock() + + _, err := m.getMeta(ctx, keyspaceID) + if err != nil { + log.Warn("failed to refresh encryption meta", + zap.Uint32("keyspaceID", keyspaceID), + zap.Error(err)) + } + } +} diff --git a/pkg/encryption/manager_test.go b/pkg/encryption/manager_test.go new file mode 100644 index 0000000000..53c9e5bedf --- /dev/null +++ b/pkg/encryption/manager_test.go @@ -0,0 +1,148 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "testing" + + "github.com/pingcap/ticdc/pkg/encryption/kms" + "github.com/stretchr/testify/require" +) + +type staticKMSClient struct { + plaintext []byte +} + +func (c *staticKMSClient) DecryptMasterKey(ctx context.Context, ciphertext []byte, keyID string, vendor string, region string, endpoint string) ([]byte, error) { + return c.plaintext, nil +} + +var _ kms.KMSClient = (*staticKMSClient)(nil) + +type staticTiKVEncryptionClient struct { + meta *EncryptionMeta + err error +} + +func (c *staticTiKVEncryptionClient) GetKeyspaceEncryptionMeta(ctx context.Context, keyspaceID uint32) (*EncryptionMeta, error) { + return c.meta, c.err +} + +func TestEncryptionMetaManagerDecryptDataKeyUsesZeroIV(t *testing.T) { + t.Parallel() + + masterKeyPlaintext := make([]byte, 32) + for i := range masterKeyPlaintext { + masterKeyPlaintext[i] = byte(i + 1) + } + + dataKeyPlaintext := make([]byte, 32) + for i := range dataKeyPlaintext { + dataKeyPlaintext[i] = byte(0xA0 + i) + } + + block, err := aes.NewCipher(masterKeyPlaintext) + require.NoError(t, err) + iv := make([]byte, aes.BlockSize) + stream := cipher.NewCTR(block, iv) + dataKeyCiphertext := make([]byte, len(dataKeyPlaintext)) + stream.XORKeyStream(dataKeyCiphertext, dataKeyPlaintext) + + dataKeyID := uint32(0x4b3031) // "K01" + + meta := &EncryptionMeta{ + KeyspaceId: 1, + Current: &EncryptionEpoch{ + FileId: 1, + DataKeyId: dataKeyID, + CreatedAt: 0, + }, + MasterKey: &MasterKey{ + Vendor: "aws-kms", + CmekId: "cmek-1", + Region: "us-west-1", + Ciphertext: []byte{1, 2, 3}, + }, + DataKeys: map[uint32]*DataKey{ + dataKeyID: {Ciphertext: dataKeyCiphertext}, + }, + } + + tikvClient := &staticTiKVEncryptionClient{meta: meta} + kmsClient := &staticKMSClient{plaintext: masterKeyPlaintext} + mgr := NewEncryptionMetaManager(tikvClient, kmsClient) + + gotKey, err := mgr.GetDataKey(context.Background(), 1, "K01") + require.NoError(t, err) + require.Equal(t, dataKeyPlaintext, gotKey) +} + +func TestEncryptionMetaManagerDecryptDataKeySupportsIVPrefixedCiphertext(t *testing.T) { + t.Parallel() + + masterKeyPlaintext := make([]byte, 32) + for i := range masterKeyPlaintext { + masterKeyPlaintext[i] = byte(i + 1) + } + + // TiKV may store data key payload as: [method(1)][key(16/24/32)]. + // method byte is not guaranteed to be within a small enum range. + method := byte(0xF5) + dataKeyPlaintext := make([]byte, 16) + for i := range dataKeyPlaintext { + dataKeyPlaintext[i] = byte(0xB0 + i) + } + payload := append([]byte{method}, dataKeyPlaintext...) + + block, err := aes.NewCipher(masterKeyPlaintext) + require.NoError(t, err) + iv := []byte("1234567890abcdef") + stream := cipher.NewCTR(block, iv) + payloadCiphertext := make([]byte, len(payload)) + stream.XORKeyStream(payloadCiphertext, payload) + + // New format: [iv(16)][ciphertext(payload)]. + dataKeyCiphertext := append(append([]byte{}, iv...), payloadCiphertext...) + + dataKeyID := uint32(0x4b3131) // "K11" + + meta := &EncryptionMeta{ + KeyspaceId: 1, + Current: &EncryptionEpoch{ + FileId: 1, + DataKeyId: dataKeyID, + CreatedAt: 0, + }, + MasterKey: &MasterKey{ + Vendor: "aws-kms", + CmekId: "cmek-1", + Region: "us-west-1", + Ciphertext: []byte{1, 2, 3}, + }, + DataKeys: map[uint32]*DataKey{ + dataKeyID: {Ciphertext: dataKeyCiphertext}, + }, + } + + tikvClient := &staticTiKVEncryptionClient{meta: meta} + kmsClient := &staticKMSClient{plaintext: masterKeyPlaintext} + mgr := NewEncryptionMetaManager(tikvClient, kmsClient) + + gotKey, err := mgr.GetDataKey(context.Background(), 1, "K11") + require.NoError(t, err) + require.Equal(t, dataKeyPlaintext, gotKey) +} diff --git a/pkg/encryption/mock_tikv_client.go b/pkg/encryption/mock_tikv_client.go new file mode 100644 index 0000000000..a57a3354a6 --- /dev/null +++ b/pkg/encryption/mock_tikv_client.go @@ -0,0 +1,171 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/encryption/kms" + cerrors "github.com/pingcap/ticdc/pkg/errors" + "go.uber.org/zap" +) + +// TiKVEncryptionClient is the interface for getting encryption metadata from TiKV +type TiKVEncryptionClient interface { + // GetKeyspaceEncryptionMeta gets the encryption metadata for a keyspace + GetKeyspaceEncryptionMeta(ctx context.Context, keyspaceID uint32) (*EncryptionMeta, error) +} + +// MockTiKVClient is a mock implementation of TiKVEncryptionClient for development and testing +type MockTiKVClient struct { + // metaMap stores mock encryption metadata by keyspace ID + metaMap map[uint32]*EncryptionMeta + // notFoundKeyspaces stores keyspace IDs that should return ErrEncryptionMetaNotFound + notFoundKeyspaces map[uint32]bool +} + +// NewMockTiKVClient creates a new mock TiKV client +func NewMockTiKVClient() *MockTiKVClient { + client := &MockTiKVClient{ + metaMap: make(map[uint32]*EncryptionMeta), + notFoundKeyspaces: make(map[uint32]bool), + } + + // Initialize with some default mock data for testing + client.initDefaultMockData() + + return client +} + +// initDefaultMockData initializes default mock encryption metadata +func (c *MockTiKVClient) initDefaultMockData() { + // Create a mock keyspace with encryption enabled + // Data key IDs carry the encryption format version in their low 8 bits. + mockDataKeyID1 := uint32(0x010001) + mockDataKeyID2 := uint32(0x020001) + + // Generate mock master key plaintext (32 bytes for AES-256) + masterKeyPlaintext := make([]byte, 32) + if _, err := rand.Read(masterKeyPlaintext); err != nil { + log.Panic("failed to generate random master key plaintext", zap.Error(err)) + } + + // Generate mock data key plaintext (32 bytes for AES-256) + dataKey1Plaintext := make([]byte, 32) + if _, err := rand.Read(dataKey1Plaintext); err != nil { + log.Panic("failed to generate random data key plaintext", zap.Error(err)) + } + dataKey2Plaintext := make([]byte, 32) + if _, err := rand.Read(dataKey2Plaintext); err != nil { + log.Panic("failed to generate random data key plaintext", zap.Error(err)) + } + + // Encrypt data keys using master key (AES-256-CTR with zero IV) + block, err := aes.NewCipher(masterKeyPlaintext) + if err != nil { + log.Panic("failed to create AES cipher for data key encryption", zap.Error(err)) + } + iv := make([]byte, aes.BlockSize) + stream := cipher.NewCTR(block, iv) + + dataKey1Ciphertext := make([]byte, len(dataKey1Plaintext)) + stream.XORKeyStream(dataKey1Ciphertext, dataKey1Plaintext) + + // Reset stream by creating a new CTR stream to ensure deterministic encryption per key. + stream = cipher.NewCTR(block, iv) + dataKey2Ciphertext := make([]byte, len(dataKey2Plaintext)) + stream.XORKeyStream(dataKey2Ciphertext, dataKey2Plaintext) + + // Encrypt master key plaintext via mock KMS to generate a realistic ciphertext. + kmsClient := kms.NewMockKMSClient() + masterKeyCiphertext, err := kmsClient.EncryptMasterKey(masterKeyPlaintext) + if err != nil { + log.Panic("failed to encrypt master key via mock KMS", zap.Error(err)) + } + + meta := &EncryptionMeta{ + KeyspaceId: 1, + Current: &EncryptionEpoch{ + FileId: 1, + DataKeyId: mockDataKeyID2, + CreatedAt: 0, + }, + MasterKey: &MasterKey{ + Vendor: "aws-kms", + CmekId: "foobar1", + Region: "us-west-1", + Ciphertext: masterKeyCiphertext, + }, + DataKeys: map[uint32]*DataKey{ + mockDataKeyID1: {Ciphertext: dataKey1Ciphertext}, + mockDataKeyID2: {Ciphertext: dataKey2Ciphertext}, + }, + History: nil, + } + + // Use keyspace ID 1 as default enabled keyspace + c.metaMap[1] = meta + + // Create a mock keyspace with encryption disabled. + c.notFoundKeyspaces[2] = true +} + +// GetKeyspaceEncryptionMeta gets the encryption metadata for a keyspace +func (c *MockTiKVClient) GetKeyspaceEncryptionMeta(ctx context.Context, keyspaceID uint32) (*EncryptionMeta, error) { + // Check if this keyspace should return not found error + if c.notFoundKeyspaces[keyspaceID] { + log.Debug("mock TiKV client: encryption meta not found", + zap.Uint32("keyspaceID", keyspaceID)) + return nil, cerrors.ErrEncryptionMetaNotFound + } + + // Return mock metadata if available + if meta, ok := c.metaMap[keyspaceID]; ok { + log.Debug("mock TiKV client: returning encryption meta", + zap.Uint32("keyspaceID", keyspaceID), + zap.Bool("enabled", meta != nil)) + return meta, nil + } + + // Default behavior: return not found for unknown keyspaces + // This simulates classic architecture or unconfigured encryption + log.Debug("mock TiKV client: encryption meta not found (unknown keyspace)", + zap.Uint32("keyspaceID", keyspaceID)) + return nil, cerrors.ErrEncryptionMetaNotFound +} + +// SetKeyspaceMeta sets mock encryption metadata for a keyspace (for testing) +func (c *MockTiKVClient) SetKeyspaceMeta(keyspaceID uint32, meta *EncryptionMeta) { + c.metaMap[keyspaceID] = meta +} + +// SetKeyspaceNotFound sets a keyspace to return cerrors.ErrEncryptionMetaNotFound (for testing) +func (c *MockTiKVClient) SetKeyspaceNotFound(keyspaceID uint32) { + c.notFoundKeyspaces[keyspaceID] = true +} + +// ClearKeyspaceNotFound clears the not found flag for a keyspace (for testing) +func (c *MockTiKVClient) ClearKeyspaceNotFound(keyspaceID uint32) { + delete(c.notFoundKeyspaces, keyspaceID) +} + +// GetKeyspaceMeta gets the stored mock metadata (for testing) +func (c *MockTiKVClient) GetKeyspaceMeta(keyspaceID uint32) (*EncryptionMeta, bool) { + meta, ok := c.metaMap[keyspaceID] + return meta, ok +} diff --git a/pkg/encryption/mock_tikv_client_test.go b/pkg/encryption/mock_tikv_client_test.go new file mode 100644 index 0000000000..d64deb7609 --- /dev/null +++ b/pkg/encryption/mock_tikv_client_test.go @@ -0,0 +1,40 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package encryption + +import ( + "context" + "testing" + + cerrors "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestMockTiKVClientNotFound(t *testing.T) { + cli := NewMockTiKVClient() + meta, err := cli.GetKeyspaceEncryptionMeta(context.Background(), 999) + require.Nil(t, meta) + require.True(t, cerrors.ErrEncryptionMetaNotFound.Equal(err)) +} + +func TestMockTiKVClientEnabledKeyspace(t *testing.T) { + cli := NewMockTiKVClient() + meta, err := cli.GetKeyspaceEncryptionMeta(context.Background(), 1) + require.NoError(t, err) + require.NotNil(t, meta) + require.Equal(t, uint32(1), meta.KeyspaceId) + require.NotNil(t, meta.Current) + require.NotZero(t, meta.Current.DataKeyId) + require.NotEmpty(t, meta.DataKeys) +} From af36df9f1104b16eb972302a17ae3ffa632184b2 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 14:02:34 +0800 Subject: [PATCH 05/11] build: align root Go version with tools module Signed-off-by: tenfyzhong --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index dfd788d952..446cab3f12 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/pingcap/ticdc -go 1.25.5 +go 1.25.8 require ( cloud.google.com/go/kms v1.15.8 From 00fc2559d8f15a97e08552e8a36cd382794bb323 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 14:16:16 +0800 Subject: [PATCH 06/11] encryption,config: read encryption config from server root Signed-off-by: tenfyzhong --- pkg/config/debug.go | 3 --- pkg/config/server.go | 1 - pkg/encryption/encryption_manager.go | 4 ++-- pkg/encryption/encryption_manager_test.go | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/config/debug.go b/pkg/config/debug.go index 827e7f941b..148e927430 100644 --- a/pkg/config/debug.go +++ b/pkg/config/debug.go @@ -36,9 +36,6 @@ type DebugConfig struct { SchemaStore *SchemaStoreConfig `toml:"schema-store" json:"schema_store"` EventService *EventServiceConfig `toml:"event-service" json:"event_service"` - - // Encryption is the configuration for CMEK encryption at rest - Encryption *EncryptionConfig `toml:"encryption" json:"encryption"` } // ValidateAndAdjust validates and adjusts the debug configuration diff --git a/pkg/config/server.go b/pkg/config/server.go index e56a56c135..887051eb8a 100644 --- a/pkg/config/server.go +++ b/pkg/config/server.go @@ -127,7 +127,6 @@ var defaultServerConfig = &ServerConfig{ EventStore: NewDefaultEventStoreConfig(), SchemaStore: NewDefaultSchemaStoreConfig(), EventService: NewDefaultEventServiceConfig(), - Encryption: NewDefaultEncryptionConfig(), }, ClusterID: "default", GcTunerMemoryThreshold: DisableMemoryLimit, diff --git a/pkg/encryption/encryption_manager.go b/pkg/encryption/encryption_manager.go index ad7493c6cb..063285100d 100644 --- a/pkg/encryption/encryption_manager.go +++ b/pkg/encryption/encryption_manager.go @@ -48,8 +48,8 @@ func NewEncryptionManager(metaManager EncryptionMetaManager) EncryptionManager { func (m *encryptionManager) EncryptData(ctx context.Context, keyspaceID uint32, data []byte) ([]byte, error) { allowDegrade := true serverCfg := config.GetGlobalServerConfig() - if serverCfg != nil && serverCfg.Debug != nil && serverCfg.Debug.Encryption != nil { - allowDegrade = serverCfg.Debug.Encryption.AllowDegradeOnError + if serverCfg != nil && serverCfg.Encryption != nil { + allowDegrade = serverCfg.Encryption.AllowDegradeOnError } // Get current data key, key ID and version together to avoid mismatch when keys rotate. diff --git a/pkg/encryption/encryption_manager_test.go b/pkg/encryption/encryption_manager_test.go index 8a8959a220..1d60674b20 100644 --- a/pkg/encryption/encryption_manager_test.go +++ b/pkg/encryption/encryption_manager_test.go @@ -60,7 +60,7 @@ func setAllowDegradeOnError(t *testing.T, allow bool) func() { t.Helper() original := config.GetGlobalServerConfig().Clone() updated := original.Clone() - updated.Debug.Encryption.AllowDegradeOnError = allow + updated.Encryption.AllowDegradeOnError = allow config.StoreGlobalServerConfig(updated) return func() { config.StoreGlobalServerConfig(original) From e9c1fcfdbc4f7ab0f0d90e6085fa3f795b69ca96 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 14:26:53 +0800 Subject: [PATCH 07/11] encryption: replace errors.New with cerrors Signed-off-by: tenfyzhong --- pkg/encryption/encryption_manager_test.go | 10 +++++----- pkg/encryption/tikv_http_client.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/encryption/encryption_manager_test.go b/pkg/encryption/encryption_manager_test.go index 1d60674b20..7995d07839 100644 --- a/pkg/encryption/encryption_manager_test.go +++ b/pkg/encryption/encryption_manager_test.go @@ -16,10 +16,10 @@ package encryption import ( "bytes" "context" - "errors" "testing" "github.com/pingcap/ticdc/pkg/config" + cerrors "github.com/pingcap/ticdc/pkg/errors" "github.com/stretchr/testify/require" ) @@ -44,11 +44,11 @@ func (m *mockMetaManager) GetDataKey(ctx context.Context, keyspaceID uint32, dat if m.currentKeyID == dataKeyID && len(m.currentKey) > 0 { return m.currentKey, nil } - return nil, errors.New("data key not found") + return nil, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("data key not found") } key, ok := m.dataKeys[dataKeyID] if !ok { - return nil, errors.New("data key not found") + return nil, cerrors.ErrDataKeyNotFound.GenWithStackByArgs("data key not found") } return key, nil } @@ -72,7 +72,7 @@ func TestEncryptDataAllowDegradeOnError(t *testing.T) { defer restore() meta := &mockMetaManager{ - currentKeyErr: errors.New("boom"), + currentKeyErr: cerrors.ErrEncryptionFailed.GenWithStackByArgs("boom"), } manager := NewEncryptionManager(meta) input := []byte("payload") @@ -87,7 +87,7 @@ func TestEncryptDataDisallowDegradeOnError(t *testing.T) { defer restore() meta := &mockMetaManager{ - currentKeyErr: errors.New("boom"), + currentKeyErr: cerrors.ErrEncryptionFailed.GenWithStackByArgs("boom"), } manager := NewEncryptionManager(meta) _, err := manager.EncryptData(context.Background(), 1, []byte("payload")) diff --git a/pkg/encryption/tikv_http_client.go b/pkg/encryption/tikv_http_client.go index 5abefee73d..16b74d92fb 100644 --- a/pkg/encryption/tikv_http_client.go +++ b/pkg/encryption/tikv_http_client.go @@ -285,7 +285,7 @@ func decodeEncryptionMetaResponseFromProtobuf(body []byte) (*encryptionMetaRespo return nil, errors.Trace(err) } if metaPB.Current == nil && metaPB.MasterKey == nil && len(metaPB.DataKeys) == 0 && len(metaPB.History) == 0 && metaPB.KeyspaceId == 0 { - return nil, errors.New("protobuf payload does not contain encryption meta fields") + return nil, cerrors.ErrDecodeFailed.GenWithStackByArgs("protobuf payload does not contain encryption meta fields") } return metaPB.toEncryptionMetaResponse(), nil } From 89b3686a54facc564c9316f3991d7ed1108b1e40 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 14:30:36 +0800 Subject: [PATCH 08/11] docs: clarify cerrors usage in AGENTS Signed-off-by: tenfyzhong --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index d4c1eb49fe..3cd8e56942 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ - Functions: use camel case and **do not** include `_` (e.g. `getPartitionNum`, not `get_partition_num`). - Variables: use lowerCamelCase (e.g. `flushInterval`, not `flush_interval`). - Logging: structured logs via `github.com/pingcap/log` + `zap` fields; message strings should **not** include function names and should avoid `-` (use spaces instead). -- Errors: when an error comes from a third party/library call, wrap it immediately with `errors.Trace(err)` or `errors.WrapError(...)` to attach a stack trace; upstream callers should propagate wrapped errors without wrapping again. +- Errors: when an error comes from a third party/library call, wrap it immediately with `errors.Trace(err)` or `errors.WrapError(...)` to attach a stack trace; upstream callers should propagate wrapped errors without wrapping again. Avoid using errors.New to create error objects; instead, utilize the predefined objects available in the cerrors package. ## Testing Guidelines From d5aca71c21bdfa3a2788ed9489e7e6a46fe82f2d Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 14:39:56 +0800 Subject: [PATCH 09/11] encryption: update file headers to 2026 Signed-off-by: tenfyzhong --- pkg/encryption/encryption_manager.go | 2 +- pkg/encryption/encryption_manager_test.go | 2 +- pkg/encryption/manager.go | 2 +- pkg/encryption/mock_tikv_client.go | 2 +- pkg/encryption/mock_tikv_client_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/encryption/encryption_manager.go b/pkg/encryption/encryption_manager.go index 063285100d..4fea198626 100644 --- a/pkg/encryption/encryption_manager.go +++ b/pkg/encryption/encryption_manager.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/encryption/encryption_manager_test.go b/pkg/encryption/encryption_manager_test.go index 7995d07839..9d87895c89 100644 --- a/pkg/encryption/encryption_manager_test.go +++ b/pkg/encryption/encryption_manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/encryption/manager.go b/pkg/encryption/manager.go index 5392896206..d5c8cd2d66 100644 --- a/pkg/encryption/manager.go +++ b/pkg/encryption/manager.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/encryption/mock_tikv_client.go b/pkg/encryption/mock_tikv_client.go index a57a3354a6..7dd005176e 100644 --- a/pkg/encryption/mock_tikv_client.go +++ b/pkg/encryption/mock_tikv_client.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/encryption/mock_tikv_client_test.go b/pkg/encryption/mock_tikv_client_test.go index d64deb7609..8333c27598 100644 --- a/pkg/encryption/mock_tikv_client_test.go +++ b/pkg/encryption/mock_tikv_client_test.go @@ -1,4 +1,4 @@ -// Copyright 2025 PingCAP, Inc. +// Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 51d98d3978277298e1f672a29b5c8d60beae76a7 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 14:42:33 +0800 Subject: [PATCH 10/11] docs(agents): Fix error handling formatting in guidelines - Correct formatting of `errors.New` and `cerrors` package references - Use backticks for consistency with other code references in document Signed-off-by: tenfyzhong --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 3cd8e56942..470f8569e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ - Functions: use camel case and **do not** include `_` (e.g. `getPartitionNum`, not `get_partition_num`). - Variables: use lowerCamelCase (e.g. `flushInterval`, not `flush_interval`). - Logging: structured logs via `github.com/pingcap/log` + `zap` fields; message strings should **not** include function names and should avoid `-` (use spaces instead). -- Errors: when an error comes from a third party/library call, wrap it immediately with `errors.Trace(err)` or `errors.WrapError(...)` to attach a stack trace; upstream callers should propagate wrapped errors without wrapping again. Avoid using errors.New to create error objects; instead, utilize the predefined objects available in the cerrors package. +- Errors: when an error comes from a third party/library call, wrap it immediately with `errors.Trace(err)` or `errors.WrapError(...)` to attach a stack trace; upstream callers should propagate wrapped errors without wrapping again. Avoid using `errors.New` to create error objects; instead, utilize the predefined objects available in the `cerrors` package. ## Testing Guidelines From 8ec7f5dbbe755083af7d5c0b2d163637ca934786 Mon Sep 17 00:00:00 2001 From: tenfyzhong Date: Thu, 16 Apr 2026 15:29:35 +0800 Subject: [PATCH 11/11] *: create empty commit Signed-off-by: tenfyzhong