From 54cb1279fc7d0495d446812697f2633feb51222b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 15:46:04 -0800 Subject: [PATCH 01/32] first pass at KC GCM enc/dec --- src/s3_encryption/__init__.py | 11 +- src/s3_encryption/key_derivation.py | 114 +++++ src/s3_encryption/materials/__init__.py | 4 +- src/s3_encryption/materials/materials.py | 20 +- src/s3_encryption/metadata.py | 7 +- src/s3_encryption/pipelines.py | 178 ++++++- test/integration/test_i_s3_encryption.py | 479 +++++------------- .../test_i_s3_encryption_instruction_file.py | 13 +- .../test_i_s3_encryption_multithreaded.py | 19 +- test/test_pipelines.py | 7 +- 10 files changed, 454 insertions(+), 398 deletions(-) create mode 100644 src/s3_encryption/key_derivation.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index a3558195..25473050 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -15,6 +15,7 @@ DefaultCryptoMaterialsManager, ) from .materials.keyring import AbstractKeyring +from .materials.materials import AlgorithmSuite, CommitmentPolicy from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline S3_METADATA_PREFIX = "x-amz-meta-" @@ -25,6 +26,12 @@ class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" keyring: AbstractKeyring + algorithm_suite: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + commitment_policy: CommitmentPolicy = field( + default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) cmm: AbstractCryptoMaterialsManager = field() ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file ##= type=implementation @@ -92,7 +99,9 @@ def on_put_object_before_call(self, params, **kwargs): pipeline = PutEncryptedObjectPipeline(self.config.cmm) encrypted_data, encryption_metadata = pipeline.encrypt( - body_bytes, encryption_context=encryption_context + body_bytes, + encryption_context=encryption_context, + algorithm_suite=self.config.algorithm_suite, ) params["body"] = encrypted_data diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py new file mode 100644 index 00000000..0d3514c0 --- /dev/null +++ b/src/s3_encryption/key_derivation.py @@ -0,0 +1,114 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Key derivation for S3 Encryption Client key-committing algorithm suites. + +Implements HKDF-based key derivation as specified in: + specification/s3-encryption/key-derivation.md + +For ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + - Extract: HKDF-SHA512, salt = Message ID (28 bytes), IKM = plaintext data key + - Expand (DEK): info = suite_id_bytes + b"DERIVEKEY", output = 32 bytes + - Expand (Commit Key): info = suite_id_bytes + b"COMMITKEY", output = 28 bytes +""" + +import hmac + +from cryptography.hazmat.primitives.hashes import SHA512 +from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +# Algorithm suite ID for S3EC.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +SUITE_ID_BYTES = b"\x00\x73" + +# Output lengths +ENCRYPTION_KEY_LENGTH = 32 # 256 bits +COMMIT_KEY_LENGTH = 28 # 224 bits +MESSAGE_ID_LENGTH = 28 # 224 bits (used as HKDF salt) + +# Fixed IV for KC GCM: 12 bytes of 0x01 +KC_GCM_IV = b"\x01" * 12 + + +def _hkdf_extract(salt: bytes, ikm: bytes) -> bytes: + """HKDF extract step using HMAC-SHA512. + + Args: + salt: The salt value (Message ID). + ikm: Input keying material (plaintext data key). + + Returns: + The pseudorandom key (PRK). + """ + return hmac.new(salt, ikm, "sha512").digest() + + +def _hkdf_expand(prk: bytes, info: bytes, length: int) -> bytes: + """HKDF expand step using SHA-512. + + Args: + prk: Pseudorandom key from extract step. + info: Context/application-specific info string. + length: Desired output length in bytes. + + Returns: + Output keying material of the requested length. + """ + hkdf = HKDFExpand(algorithm=SHA512(), length=length, info=info) + return hkdf.derive(prk) + + +def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, bytes]: + """Derive the encryption key and commitment key from a plaintext data key. + + Uses HKDF with SHA-512 as specified in the S3EC key derivation spec. + + Args: + plaintext_data_key: The plaintext data key from the keyring (32 bytes). + message_id: The generated Message ID used as the HKDF salt (28 bytes). + + Returns: + A tuple of (derived_encryption_key, commit_key). + """ + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The hash function MUST be specified by the algorithm suite commitment settings. + ##% The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + ##% The salt MUST be the Message ID with the length defined in the algorithm suite. + prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The input info MUST be a concatenation of the algorithm suite ID as bytes + ##% followed by the string DERIVEKEY as UTF8 encoded bytes. + derived_encryption_key = _hkdf_expand( + prk, info=SUITE_ID_BYTES + b"DERIVEKEY", length=ENCRYPTION_KEY_LENGTH + ) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The input info MUST be a concatenation of the algorithm suite ID as bytes + ##% followed by the string COMMITKEY as UTF8 encoded bytes. + commit_key = _hkdf_expand( + prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH + ) + + return derived_encryption_key, commit_key + + +def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> None: + """Verify key commitment in constant time. + + Args: + stored_commitment: The commitment value from the object metadata. + derived_commitment: The commitment value derived from the data key. + + Raises: + S3EncryptionClientSecurityError: If the commitment values do not match. + """ + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##% the verification of the derived key commitment value MUST be done in constant time. + ##% the client MUST throw an exception when the derived key commitment value + ##% and stored key commitment value do not match. + from .exceptions import S3EncryptionClientSecurityError + + if not hmac.compare_digest(stored_commitment, derived_commitment): + raise S3EncryptionClientSecurityError( + "Key commitment verification failed: stored commitment does not match derived commitment." + ) diff --git a/src/s3_encryption/materials/__init__.py b/src/s3_encryption/materials/__init__.py index c67d5802..c5cc7d6d 100644 --- a/src/s3_encryption/materials/__init__.py +++ b/src/s3_encryption/materials/__init__.py @@ -10,7 +10,7 @@ from .encrypted_data_key import EncryptedDataKey from .keyring import AbstractKeyring from .kms_keyring import KmsKeyring -from .materials import EncryptionMaterials +from .materials import AlgorithmSuite, CommitmentPolicy, EncryptionMaterials __all__ = [ "AbstractKeyring", @@ -18,5 +18,7 @@ "AbstractCryptoMaterialsManager", "DefaultCryptoMaterialsManager", "EncryptedDataKey", + "AlgorithmSuite", + "CommitmentPolicy", "EncryptionMaterials", ] diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 3966e17c..0bca41a4 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -8,11 +8,25 @@ """ from typing import Any - +from enum import Enum from attrs import define, field from .encrypted_data_key import EncryptedDataKey +class AlgorithmSuite(Enum): + """Algorithm suites supported by the S3 Encryption Client.""" + + ALG_AES_256_GCM_IV12_TAG16_NO_KDF = "AES/GCM/NoPadding" + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = "AES/GCM/HKDF/CommitKey" + + +class CommitmentPolicy(Enum): + """Commitment policies controlling key-commitment behavior.""" + + FORBID_ENCRYPT_ALLOW_DECRYPT = "ForbidEncryptAllowDecrypt" + REQUIRE_ENCRYPT_ALLOW_DECRYPT = "RequireEncryptAllowDecrypt" + REQUIRE_ENCRYPT_REQUIRE_DECRYPT = "RequireEncryptRequireDecrypt" + @define class EncryptionMaterials: @@ -27,6 +41,9 @@ class EncryptionMaterials: plaintext_data_key (Optional[bytes]): The plaintext data key """ + algorithm_suite: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ) encryption_context: dict[str, str] = field(factory=dict) encrypted_data_key: EncryptedDataKey | None = field(default=None) plaintext_data_key: bytes | None = field(default=None) @@ -87,6 +104,7 @@ class DecryptionMaterials: encryption_context_stored: dict[str, str] = field(factory=dict) encryption_context_from_request: dict[str, str] = field(factory=dict) plaintext_data_key: bytes | None = field(default=None) + algorithm_suite: AlgorithmSuite | None = field(default=None) @classmethod def from_dict(cls, materials_dict: dict[str, Any]) -> "DecryptionMaterials": diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 5c8bbda3..f1e4ed25 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -51,7 +51,7 @@ class ObjectMetadata: # V3 format fields (compressed) content_cipher_v3: str | None = field(default=None) encrypted_data_key_v3: str | None = field(default=None) - mat_desc_v3: str | None = field(default=None) + mat_desc_v3: str | dict | None = field(default=None) encryption_context_v3: str | None = field(default=None) encrypted_data_key_algorithm_v3: str | None = field(default=None) key_commitment_v3: str | None = field(default=None) @@ -150,7 +150,10 @@ def to_dict(self) -> dict[str, str]: result[self.ENCRYPTED_DATA_KEY_V3] = self.encrypted_data_key_v3 if self.mat_desc_v3 is not None: - result[self.MAT_DESC_V3] = self.mat_desc_v3 + if isinstance(self.mat_desc_v3, dict): + result[self.MAT_DESC_V3] = json.dumps(self.mat_desc_v3) + else: + result[self.MAT_DESC_V3] = self.mat_desc_v3 if self.encryption_context_v3 is not None: result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 02a5a9c9..a5f0330a 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -14,9 +14,16 @@ from .exceptions import S3EncryptionClientError from .instruction_file import fetch_instruction_file +from .key_derivation import ( + KC_GCM_IV, + MESSAGE_ID_LENGTH, + SUITE_ID_BYTES, + derive_keys, + verify_commitment, +) from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import DecryptionMaterials, EncryptionMaterials +from .materials.materials import AlgorithmSuite, DecryptionMaterials, EncryptionMaterials from .metadata import ObjectMetadata @@ -30,45 +37,56 @@ class PutEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() - def encrypt(self, plaintext, encryption_context=None): + def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): """Encrypt the data before it is stored in S3. Args: plaintext (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption + algorithm_suite (AlgorithmSuite, optional): Algorithm suite to use Returns: bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ + if algorithm_suite is None: + algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + # Create encryption materials request with encryption context copy enc_mats_request = EncryptionMaterials( - encryption_context={} if encryption_context is None else encryption_context.copy() + algorithm_suite=algorithm_suite, + encryption_context={} if encryption_context is None else encryption_context.copy(), ) # Get encryption materials from the crypto materials manager enc_mats = self.cmm.get_encryption_materials(enc_mats_request) - # Generate initialization vector - iv = os.urandom(12) - - # Encrypt the data if enc_mats.plaintext_data_key is None: raise RuntimeError("No plaintext data key found!") - - aesgcm = AESGCM(enc_mats.plaintext_data_key) - ciphertext = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) - encrypted_data = ciphertext - b64_iv = base64.b64encode(iv).decode("utf-8") - - # Get the encrypted data key if enc_mats.encrypted_data_key is None: raise RuntimeError("No encrypted data key found!") edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key + + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + return self._encrypt_kc_gcm(plaintext, enc_mats, edk_bytes) + else: + return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) + + def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): + """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" + ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, + ##% with the plaintext data key, the generated IV, and the tag length defined + ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + iv = os.urandom(12) + aesgcm = AESGCM(enc_mats.plaintext_data_key) + encrypted_data = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) + + b64_iv = base64.b64encode(iv).decode("utf-8") b64_edk = base64.b64encode(edk_bytes).decode("utf-8") - # Create metadata using the ObjectMetadata class metadata = ObjectMetadata( encrypted_data_key_v2=b64_edk, encrypted_data_key_algorithm="kms+context", @@ -77,10 +95,50 @@ def encrypt(self, plaintext, encryption_context=None): encrypted_data_key_context=enc_mats.encryption_context, ) - # Convert to dictionary for storage in S3 metadata - encryption_metadata = metadata.to_dict() + return encrypted_data, metadata.to_dict() + + def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): + """Encrypt using ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (V3 format).""" + ##= specification/s3-encryption/encryption.md#content-encryption + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + message_id = os.urandom(MESSAGE_ID_LENGTH) + + ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_hkdf_sha512_commit_key + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key as described in Key Derivation. + derived_encryption_key, commit_key = derive_keys(enc_mats.plaintext_data_key, message_id) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% the IV used in the AES-GCM content encryption/decryption MUST consist + ##% entirely of bytes with the value 0x01. + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + aesgcm = AESGCM(derived_encryption_key) + encrypted_data = aesgcm.encrypt( + nonce=KC_GCM_IV, data=plaintext, associated_data=SUITE_ID_BYTES + ) + + b64_edk = base64.b64encode(edk_bytes).decode("utf-8") + b64_message_id = base64.b64encode(message_id).decode("utf-8") + b64_commit_key = base64.b64encode(commit_key).decode("utf-8") + + # V3 metadata format + # x-amz-c: content cipher identifier (compressed algorithm suite) + # x-amz-w: wrapping algorithm identifier + # x-amz-3: encrypted data key + # x-amz-i: message ID + # x-amz-d: key commitment + # x-amz-m: material description (encryption context as JSON) + metadata = ObjectMetadata( + content_cipher_v3="115", + encrypted_data_key_algorithm_v3="02", + encrypted_data_key_v3=b64_edk, + message_id_v3=b64_message_id, + key_commitment_v3=b64_commit_key, + mat_desc_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, + ) - return encrypted_data, encryption_metadata + return encrypted_data, metadata.to_dict() @define @@ -156,9 +214,12 @@ def decrypt( if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) elif metadata.is_v2_format(): + # TODO: this is not how this works dec_materials = self._decrypt_v2(metadata, encryption_context) + dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF elif metadata.is_v3_format(): dec_materials = self._decrypt_v3(metadata, encryption_context) + dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY else: raise S3EncryptionClientError( "Unable to determine S3 Encryption Client message format." @@ -172,8 +233,20 @@ def decrypt( ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. # Perform decryption - aesgcm = AESGCM(dec_materials.plaintext_data_key) - return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) + # TODO: include CBC here too + match dec_materials.algorithm_suite: + case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##% The client MUST NOT provide any AAD when encrypting with + ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + aesgcm = AESGCM(dec_materials.plaintext_data_key) + return aesgcm.decrypt( + nonce=dec_materials.iv, data=encrypted_data, associated_data=None + ) + case AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + return self._decrypt_kc_gcm_content(dec_materials, encrypted_data, metadata) + case _: + raise S3EncryptionClientError("Unknown algorithm suite!") def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" @@ -215,7 +288,68 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) + # V3 compressed wrapping algorithm identifiers → canonical names + _V3_WRAP_ALG_MAP = { + "02": "kms+context", + } + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V3 decryption materials.""" - # TODO: Implement V3 decryption - raise NotImplementedError("V3 decryption not yet implemented") + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v3) + + # Map V3 compressed wrapping algorithm to canonical key_provider_info + raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "02" + wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) + + encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info=wrap_alg, + encrypted_data_key=edk_bytes, + ) + + # V3 stores encryption context in mat_desc_v3 (as dict or JSON string) + stored_context = {} + if metadata.mat_desc_v3 is not None: + if isinstance(metadata.mat_desc_v3, dict): + stored_context = metadata.mat_desc_v3 + elif isinstance(metadata.mat_desc_v3, str): + import json + + stored_context = json.loads(metadata.mat_desc_v3) + + dec_materials = DecryptionMaterials( + encrypted_data_keys=[encrypted_data_key], + encryption_context_stored=stored_context, + encryption_context_from_request=encryption_context, + ) + + return self.cmm.decrypt_materials(dec_materials) + + def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): + """Decrypt content encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + + Performs HKDF key derivation, key commitment verification, and AES-GCM decryption. + """ + message_id = base64.b64decode(metadata.message_id_v3) + stored_commitment = base64.b64decode(metadata.key_commitment_v3) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key. + derived_encryption_key, derived_commitment = derive_keys( + dec_materials.plaintext_data_key, message_id + ) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##% the client MUST verify the key commitment values match before deriving + ##% the derived encryption key. + verify_commitment(stored_commitment, derived_commitment) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% the IV used in the AES-GCM content encryption/decryption MUST consist + ##% entirely of bytes with the value 0x01. + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + aesgcm = AESGCM(derived_encryption_key) + return aesgcm.decrypt( + nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES + ) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 616f8da4..d4d5bcaa 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -9,6 +9,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -16,457 +17,207 @@ "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" ) - -def test_simple_roundtrip_ascii_string(): - key = "simple-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - data = "test input for simple v3 round trip" - +# Parameterized algorithm suite configurations. +# Each entry is (algorithm_suite, commitment_policy, id_label). +# "default" uses the client defaults (KC GCM + Require/Require). +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +def _make_client(algorithm_suite, commitment_policy): + """Create an S3EncryptionClient with the given algorithm config.""" kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(input) - print("Output:") - print(output) - raise RuntimeError - print("Success!") - + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) -def test_empty_string_roundtrip(): - key = "empty-string-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - data = "" # Empty string as test data +def _unique_key(prefix): + """Generate a unique S3 key with a timestamp suffix.""" + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_simple_roundtrip_ascii_string(algorithm_suite, commitment_policy): + key = _unique_key("simple-rt-") + data = "test input for simple v3 round trip" - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) # Using repr to clearly show it's an empty string - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Empty string encrypted and decrypted correctly.") + assert output == data -def test_no_body_roundtrip(): - key = "no-body-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_empty_string_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("empty-string-rt-") + data = "" - # Expected data when no Body is provided (empty bytes) - expected_data = b"" - - kms_client = boto3.client("kms", region_name=region) + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + assert output == data - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_no_body_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("no-body-rt-") + expected_data = b"" - # Call put_object without providing a Body parameter + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read() + assert output == expected_data - if output != expected_data: - print("Uh oh! Output doesn't match expected empty bytes!") - print("Expected:") - print(repr(expected_data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print( - "Success! Object with no Body parameter encrypted and decrypted correctly as empty bytes." - ) - - -def test_unicode_string_roundtrip(): - key = "unicode-string-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with unusual Unicode characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_unicode_string_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("unicode-string-rt-") data = "Unicode test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!, ½⅓¼⅕⅙⅐⅛⅑⅒⅔⅖⅗⅘⅙⅚⅜⅝⅞" - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Boto3 encodes to utf-8 in put_object but does not - # decode in get_object; do so manually to complete the - # round trip + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Unicode string encrypted and decrypted correctly.") - + assert output == data -def test_specific_encoding_utf8_roundtrip(): - key = "utf8-encoding-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with mixed characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_specific_encoding_utf8_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("utf8-encoding-rt-") data = "UTF-8 encoding test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!" - - # Explicitly encode as UTF-8 before sending encoded_data = data.encode("utf-8") - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the pre-encoded bytes to put_object + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes and decode with the same encoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! UTF-8 encoded string encrypted and decrypted correctly.") + assert output == data -def test_specific_encoding_latin1_roundtrip(): - key = "latin1-encoding-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - # String with Latin-1 compatible characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_specific_encoding_latin1_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("latin1-encoding-rt-") data = "Latin-1 encoding test: éèêë àâäãåá çñ ¿¡ øæå ØÆÅÉÈÊËÀÂÄÃÅÁ" - - # Explicitly encode as Latin-1 before sending encoded_data = data.encode("latin-1") - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the pre-encoded bytes to put_object + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes and decode with the same encoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("latin-1") - - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Latin-1 encoded string encrypted and decrypted correctly.") + assert output == data -def test_binary_data_roundtrip(): - key = "binary-data-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - # Create some binary data (not valid in any particular encoding) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_binary_data_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("binary-data-rt-") data = bytes(range(256)) - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the binary data directly + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes without decoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read() + assert output == data - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Binary data encrypted and decrypted correctly.") - -def test_invalid_body_types(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_invalid_body_types(algorithm_suite, commitment_policy): """Test that put_object raises an exception when given invalid body types.""" - key = "invalid-body-type" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + key = _unique_key("invalid-body-type-") - # Test with integer - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=42) - assert "Invalid type for parameter Body" in str(excinfo.value) + s3ec = _make_client(algorithm_suite, commitment_policy) - # Test with float - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=3.14) - assert "Invalid type for parameter Body" in str(excinfo.value) + for body in [42, 3.14, [1, 2, 3], {"key": "value"}, True, None]: + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=body) + assert "Invalid type for parameter Body" in str(excinfo.value) - # Test with list - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=[1, 2, 3]) - assert "Invalid type for parameter Body" in str(excinfo.value) - # Test with dictionary - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body={"key": "value"}) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with boolean - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=True) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with None (also raises an exception) - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=None) - assert "Invalid type for parameter Body" in str(excinfo.value) - - print("Success! All invalid body types correctly raised exceptions.") - - -def test_user_metadata_preservation(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_user_metadata_preservation(algorithm_suite, commitment_policy): """Test that user-provided metadata is preserved during encryption.""" - key = "metadata-preservation-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - + key = _unique_key("metadata-preservation-rt-") data = "Test data with user metadata" - - # User metadata to include user_metadata = { "author": "test-user", "version": "1.0", "description": "Test object with custom metadata", } - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with user metadata + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) + response = s3ec.get_object(Bucket=bucket, Key=key) - # Get the object back - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Verify the data decrypts correctly output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - - # Verify user metadata is preserved - returned_metadata = response.get("Metadata", {}) + assert output == data + returned_metadata = response.get("Metadata", {}) for key_name, expected_value in user_metadata.items(): - if key_name not in returned_metadata: - print(f"Uh oh! User metadata key '{key_name}' is missing!") - print("Expected metadata:") - print(user_metadata) - print("Returned metadata:") - print(returned_metadata) - raise RuntimeError - - if returned_metadata[key_name] != expected_value: - print(f"Uh oh! User metadata value for '{key_name}' doesn't match!") - print(f"Expected: {expected_value}") - print(f"Got: {returned_metadata[key_name]}") - raise RuntimeError - - print("Success! User metadata preserved correctly during encryption/decryption.") - print(f"User metadata: {user_metadata}") - print(f"Returned metadata keys: {list(returned_metadata.keys())}") - - -def test_encryption_context_roundtrip(): - """Test that EncryptionContext is properly used during encryption and required for decryption.""" - key = "encryption-context-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + assert key_name in returned_metadata, f"User metadata key '{key_name}' is missing" + assert returned_metadata[key_name] == expected_value - data = "Test data with encryption context" - # Encryption context to use for additional authenticated data +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_roundtrip(algorithm_suite, commitment_policy): + """Test that EncryptionContext is properly used during encryption and required for decryption.""" + key = _unique_key("encryption-context-rt-") + data = "Test data with encryption context" encryption_context = { "department": "engineering", "project": "s3-encryption", "environment": "test", } - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) - # Get the object back WITH the same encryption context - get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": encryption_context} - response = s3ec.get_object(**get_req) - - # Verify the data decrypts correctly output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError + assert output == data - print("Success! Encryption context used correctly during encryption/decryption.") - print(f"Encryption context: {encryption_context}") - -def test_encryption_context_mismatch(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_mismatch(algorithm_suite, commitment_policy): """Test that decryption fails when EncryptionContext doesn't match.""" - key = "encryption-context-mismatch" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - + key = _unique_key("encryption-context-mismatch-") data = "Test data with encryption context" - - # Original encryption context encryption_context = {"department": "engineering", "project": "s3-encryption"} - - # Wrong encryption context for decryption wrong_encryption_context = {"department": "marketing", "project": "s3-encryption"} - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) - # Try to get the object back with WRONG encryption context - should fail - get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": wrong_encryption_context} - - try: - s3ec.get_object(**get_req) - # If we get here, the test failed - decryption should have failed - print("Uh oh! Decryption succeeded with wrong encryption context!") - print(f"Original context: {encryption_context}") - print(f"Wrong context used: {wrong_encryption_context}") - raise RuntimeError("Expected decryption to fail with mismatched encryption context") - except S3EncryptionClientError as e: - # This is expected - decryption should fail - print("Success! Decryption correctly failed with mismatched encryption context.") - print(f"Error message: {str(e)}") - except Exception as e: - # Some other error occurred - print(f"Unexpected error type: {type(e).__name__}") - print(f"Error message: {str(e)}") - raise - - -def test_encryption_context_missing_on_decrypt(): - """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" - key = "encryption-context-missing" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=wrong_encryption_context) - data = "Test data with encryption context" - # Encryption context used during encryption +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_missing_on_decrypt(algorithm_suite, commitment_policy): + """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" + key = _unique_key("encryption-context-missing-") + data = "Test data with encryption context" encryption_context = {"department": "engineering", "project": "s3-encryption"} - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) - # Try to get the object back WITHOUT encryption context - should fail - get_req = {"Bucket": bucket, "Key": key} - - try: - s3ec.get_object(**get_req) - # If we get here, the test failed - decryption should have failed - print("Uh oh! Decryption succeeded without providing required encryption context!") - print(f"Original context: {encryption_context}") - raise RuntimeError("Expected decryption to fail when encryption context not provided") - except S3EncryptionClientError as e: - # This is expected - decryption should fail - print("Success! Decryption correctly failed when encryption context was not provided.") - print(f"Error message: {str(e)}") - except Exception as e: - # Some other error occurred - print(f"Unexpected error type: {type(e).__name__}") - print(f"Error message: {str(e)}") - raise + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 467ddc7a..cc10b6f6 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -7,6 +7,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy # Static test objects bucket bucket = os.environ.get("CI_S3_STATIC_TEST_BUCKET", "s3ec-static-test-objects") @@ -60,7 +61,11 @@ def test_decrypt_v2_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -83,7 +88,11 @@ def test_decrypt_v3_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 419ca7ea..f04d7fe5 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -16,6 +16,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -38,7 +39,11 @@ def test_multithreaded_encryption_context_isolation(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) # Number of threads to test with @@ -150,7 +155,11 @@ def test_multithreaded_rapid_context_switching(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) num_iterations = 20 @@ -228,7 +237,11 @@ def test_multithreaded_mixed_with_and_without_context(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) errors = [] diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 9f40cd5c..3f87be1e 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -182,8 +182,11 @@ def test_decrypt_v3_from_instruction_file(self): mock_keyring.on_decrypt.return_value = mock_dec_materials - # This should fail with NotImplementedError since V3 decryption isn't implemented yet - with pytest.raises(NotImplementedError, match="V3 decryption not yet implemented"): + # V3 decryption is now implemented; with fake commitment data, + # key commitment verification will fail. + from s3_encryption.exceptions import S3EncryptionClientSecurityError + + with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") # Verify instruction file was fetched From 281531e7c05c4fd216e10fcc64cad1cf3b0ff21b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 15:52:27 -0800 Subject: [PATCH 02/32] first pass test server --- .../amazon/encryption/s3/TestUtils.java | 2 +- test-server/python-v3-server/src/main.py | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 2b9cd062..c3c804aa 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -155,7 +155,7 @@ public class TestUtils { public static final Set IMPROVED_VERSIONS = Set.of( JAVA_V4, - // PYTHON_V3, + PYTHON_V3, GO_V4, NET_V4, CPP_V3, diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v3-server/src/main.py index 6c57c6bd..eb3855b7 100755 --- a/test-server/python-v3-server/src/main.py +++ b/test-server/python-v3-server/src/main.py @@ -7,6 +7,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy import boto3 import uvicorn import json @@ -67,6 +68,19 @@ def create_s3_encryption_client_error( ) +# Maps from Smithy model enum strings to Python AlgorithmSuite/CommitmentPolicy enums +_ALGORITHM_SUITE_MAP = { + "ALG_AES_256_GCM_IV12_TAG16_NO_KDF": AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY": AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, +} + +_COMMITMENT_POLICY_MAP = { + "FORBID_ENCRYPT_ALLOW_DECRYPT": CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + "REQUIRE_ENCRYPT_ALLOW_DECRYPT": CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + "REQUIRE_ENCRYPT_REQUIRE_DECRYPT": CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, +} + + @app.put("/object/{bucket}/{key}") async def put_object(bucket: str, key: str, request: Request): """ @@ -185,7 +199,23 @@ async def client_endpoint(request: Request): enable_legacy_wrapping_algorithms=enable_legacy_wrapping_algorithms, ) wrapped_client = boto3.client("s3") - client_config = S3EncryptionClientConfig(keyring) + + # Build config kwargs, only including algorithm_suite and commitment_policy if provided + config_kwargs = {"keyring": keyring} + + encryption_algorithm = config_data.get("encryptionAlgorithm") + if encryption_algorithm is not None: + if encryption_algorithm not in _ALGORITHM_SUITE_MAP: + raise ValueError(f"Unknown encryption algorithm: {encryption_algorithm}") + config_kwargs["algorithm_suite"] = _ALGORITHM_SUITE_MAP[encryption_algorithm] + + commitment_policy = config_data.get("commitmentPolicy") + if commitment_policy is not None: + if commitment_policy not in _COMMITMENT_POLICY_MAP: + raise ValueError(f"Unknown commitment policy: {commitment_policy}") + config_kwargs["commitment_policy"] = _COMMITMENT_POLICY_MAP[commitment_policy] + + client_config = S3EncryptionClientConfig(**config_kwargs) # Create S3EncryptionClient client = S3EncryptionClient(wrapped_client, client_config) From abb0e216383c2a13252a741f999246bd3be12fc4 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 16:17:04 -0800 Subject: [PATCH 03/32] fix duvet --- src/s3_encryption/key_derivation.py | 12 +++++++----- src/s3_encryption/pipelines.py | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index 0d3514c0..0b0e8b6b 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -70,9 +70,11 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by A tuple of (derived_encryption_key, commit_key). """ ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% The hash function MUST be specified by the algorithm suite commitment settings. - ##% The input keying material MUST be the plaintext data key (PDK) generated by the key provider. - ##% The salt MUST be the Message ID with the length defined in the algorithm suite. + ##% - The hash function MUST be specified by the algorithm suite commitment settings. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key) ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -103,8 +105,8 @@ def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> No S3EncryptionClientSecurityError: If the commitment values do not match. """ ##= specification/s3-encryption/decryption.md#decrypting-with-commitment - ##% the verification of the derived key commitment value MUST be done in constant time. - ##% the client MUST throw an exception when the derived key commitment value + ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value ##% and stored key commitment value do not match. from .exceptions import S3EncryptionClientSecurityError diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index a5f0330a..be891dd8 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -75,7 +75,7 @@ def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" - ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, ##% with the plaintext data key, the generated IV, and the tag length defined ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. @@ -104,14 +104,15 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##% or Message ID defined in the algorithm suite. message_id = os.urandom(MESSAGE_ID_LENGTH) - ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_hkdf_sha512_commit_key + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key ##% The client MUST use HKDF to derive the key commitment value and the derived - ##% encrypting key as described in Key Derivation. + ##% encrypting key as described in [Key Derivation](key-derivation.md). derived_encryption_key, commit_key = derive_keys(enc_mats.plaintext_data_key, message_id) ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% the IV used in the AES-GCM content encryption/decryption MUST consist - ##% entirely of bytes with the value 0x01. + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) encrypted_data = aesgcm.encrypt( @@ -236,7 +237,7 @@ def decrypt( # TODO: include CBC here too match dec_materials.algorithm_suite: case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##% The client MUST NOT provide any AAD when encrypting with ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. aesgcm = AESGCM(dec_materials.plaintext_data_key) @@ -333,21 +334,21 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): message_id = base64.b64decode(metadata.message_id_v3) stored_commitment = base64.b64decode(metadata.key_commitment_v3) - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% The client MUST use HKDF to derive the key commitment value and the derived - ##% encrypting key. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##% The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). derived_encryption_key, derived_commitment = derive_keys( dec_materials.plaintext_data_key, message_id ) ##= specification/s3-encryption/decryption.md#decrypting-with-commitment - ##% the client MUST verify the key commitment values match before deriving - ##% the derived encryption key. + ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving + ##% the [derived encryption key](./key-derivation.md#hkdf-operation). verify_commitment(stored_commitment, derived_commitment) ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% the IV used in the AES-GCM content encryption/decryption MUST consist - ##% entirely of bytes with the value 0x01. + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) return aesgcm.decrypt( From 5e0626b6613f7e28e3a447601ddc68c59f92f290 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 16:29:33 -0800 Subject: [PATCH 04/32] fix wrap alg --- src/s3_encryption/metadata.py | 7 +++-- src/s3_encryption/pipelines.py | 52 +++++++++++++++++++++++++--------- test/test_pipelines.py | 34 +++++++++++++++++++--- 3 files changed, 74 insertions(+), 19 deletions(-) diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index f1e4ed25..de57b953 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -52,7 +52,7 @@ class ObjectMetadata: content_cipher_v3: str | None = field(default=None) encrypted_data_key_v3: str | None = field(default=None) mat_desc_v3: str | dict | None = field(default=None) - encryption_context_v3: str | None = field(default=None) + encryption_context_v3: str | dict | None = field(default=None) encrypted_data_key_algorithm_v3: str | None = field(default=None) key_commitment_v3: str | None = field(default=None) message_id_v3: str | None = field(default=None) @@ -156,7 +156,10 @@ def to_dict(self) -> dict[str, str]: result[self.MAT_DESC_V3] = self.mat_desc_v3 if self.encryption_context_v3 is not None: - result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 + if isinstance(self.encryption_context_v3, dict): + result[self.ENCRYPTION_CONTEXT_V3] = json.dumps(self.encryption_context_v3) + else: + result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 if self.encrypted_data_key_algorithm_v3 is not None: result[self.ENCRYPTED_DATA_KEY_ALGORITHM_V3] = self.encrypted_data_key_algorithm_v3 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index be891dd8..f66702b8 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -125,18 +125,18 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): # V3 metadata format # x-amz-c: content cipher identifier (compressed algorithm suite) - # x-amz-w: wrapping algorithm identifier + # x-amz-w: wrapping algorithm identifier (12 = kms+context) # x-amz-3: encrypted data key # x-amz-i: message ID # x-amz-d: key commitment - # x-amz-m: material description (encryption context as JSON) + # x-amz-t: encryption context (for kms+context wrapping) metadata = ObjectMetadata( content_cipher_v3="115", - encrypted_data_key_algorithm_v3="02", + encrypted_data_key_algorithm_v3="12", encrypted_data_key_v3=b64_edk, message_id_v3=b64_message_id, key_commitment_v3=b64_commit_key, - mat_desc_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, + encryption_context_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, ) return encrypted_data, metadata.to_dict() @@ -289,34 +289,60 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) - # V3 compressed wrapping algorithm identifiers → canonical names + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The V3 format uses compression here such that each wrapping algorithm + ##% is represented by a two digit string. + ##% The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + ##% The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + ##% The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. _V3_WRAP_ALG_MAP = { - "02": "kms+context", + "02": "AES/GCM", + "12": "kms+context", + "22": "RSA-OAEP-SHA1", } + # Wrapping algorithms that this client currently supports for decryption + _SUPPORTED_WRAP_ALGS = {"kms+context", "kms"} + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V3 decryption materials.""" edk_bytes = base64.b64decode(metadata.encrypted_data_key_v3) # Map V3 compressed wrapping algorithm to canonical key_provider_info - raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "02" + raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "12" wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) + if wrap_alg not in self._SUPPORTED_WRAP_ALGS: + raise S3EncryptionClientError( + f"Unsupported wrapping algorithm: {wrap_alg}. " + f"The S3 Encryption Client for Python currently supports: " + f"{', '.join(sorted(self._SUPPORTED_WRAP_ALGS))}." + ) + encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", key_provider_info=wrap_alg, encrypted_data_key=edk_bytes, ) - # V3 stores encryption context in mat_desc_v3 (as dict or JSON string) + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. + ##% The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). + # For kms+context, the stored context comes from x-amz-t (encryption_context_v3). + # For AES/GCM and RSA-OAEP-SHA1, it comes from x-amz-m (mat_desc_v3). stored_context = {} - if metadata.mat_desc_v3 is not None: - if isinstance(metadata.mat_desc_v3, dict): - stored_context = metadata.mat_desc_v3 - elif isinstance(metadata.mat_desc_v3, str): + if wrap_alg == "kms+context": + raw_ctx = metadata.encryption_context_v3 + else: + raw_ctx = metadata.mat_desc_v3 + + if raw_ctx is not None: + if isinstance(raw_ctx, dict): + stored_context = raw_ctx + elif isinstance(raw_ctx, str): import json - stored_context = json.loads(metadata.mat_desc_v3) + stored_context = json.loads(raw_ctx) dec_materials = DecryptionMaterials( encrypted_data_keys=[encrypted_data_key], diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 3f87be1e..a4637c9d 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -127,7 +127,7 @@ def test_decrypt_v2_from_instruction_file(self): ##% In the V3 message format, only the content metadata related to ##% the encrypted data is stored in the Instruction File. def test_decrypt_v3_from_instruction_file(self): - """Test decrypting V3 format with instruction file.""" + """Test decrypting V3 format with instruction file (kms+context wrapping).""" # Object metadata contains V3 content keys only object_metadata = { "x-amz-c": "115", # Compressed algorithm suite @@ -136,11 +136,11 @@ def test_decrypt_v3_from_instruction_file(self): } # Instruction file contains encrypted data key and wrapping algorithm + # Uses "12" (kms+context) with "x-amz-t" for encryption context instruction_file_metadata = { "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), - "x-amz-w": "02", # AES/GCM - "x-amz-m": json.dumps({"test-instruction": "material-desc-instruction"}), - "x-amz-crypto-instr-file": "", + "x-amz-w": "12", # kms+context + "x-amz-t": json.dumps({"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}), } # Create mock S3 client @@ -242,3 +242,29 @@ def test_decrypt_with_custom_instruction_file_suffix(self): mock_s3_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test-key.custom-suffix" ) + + def test_decrypt_v3_unsupported_wrap_alg(self): + """Test that V3 decryption with unsupported wrapping algorithm gives a useful error.""" + from s3_encryption.exceptions import S3EncryptionClientError + + # V3 metadata with AES/GCM wrapping (02) — not supported by this client + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-w": "02", # AES/GCM — unsupported + "x-amz-m": json.dumps({"some": "material-desc"}), + "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), + "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline(cmm) + + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": metadata, + } + + with pytest.raises(S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM"): + pipeline.decrypt(mock_response) From ec78e5694cc15b42b3ce04e12e745d952afae967 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 16:39:51 -0800 Subject: [PATCH 05/32] fix duvet --- src/s3_encryption/pipelines.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index f66702b8..f94370f7 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -290,11 +290,13 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) ##= specification/s3-encryption/data-format/content-metadata.md#v3-only - ##% The V3 format uses compression here such that each wrapping algorithm - ##% is represented by a two digit string. - ##% The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. - ##% The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. - ##% The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. + ##% The V3 format uses compression here such that each wrapping algorithm is represented by a two digit string. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. _V3_WRAP_ALG_MAP = { "02": "AES/GCM", "12": "kms+context", @@ -327,6 +329,7 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). # For kms+context, the stored context comes from x-amz-t (encryption_context_v3). # For AES/GCM and RSA-OAEP-SHA1, it comes from x-amz-m (mat_desc_v3). From ad89776ef3535d1d588213081c26965c9b5f7bcc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 17:01:55 -0800 Subject: [PATCH 06/32] fixup --- src/s3_encryption/materials/kms_keyring.py | 9 ++++++++- src/s3_encryption/metadata.py | 2 +- test/test_metadata.py | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index a32493fc..da4fd8ed 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -10,6 +10,7 @@ from botocore import client from ..exceptions import S3EncryptionClientError +from ..materials.materials import AlgorithmSuite from .encrypted_data_key import EncryptedDataKey from .keyring import S3Keyring @@ -76,7 +77,13 @@ def on_encrypt(self, enc_materials): ##= specification/s3-encryption/materials/s3-kms-keyring.md#supported-wrapping-algorithm-modes ##= type=implication ##% The KmsKeyring MUST NOT support encryption using KmsV1 mode. - encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" + # For committing algorithm suites (V3), the encryption context algorithm + # value is the algorithm suite ID as a string ("115"), not the cipher name. + # For non-committing suites (V2), use the cipher name ("AES/GCM/NoPadding"). + if enc_materials.algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + encryption_context["aws:x-amz-cek-alg"] = "115" + else: + encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" # Python implementation uses KMS GenerateDataKey instead of the spec's # EncryptDataKey pattern diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index de57b953..0b0fbce6 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -137,7 +137,7 @@ def to_dict(self) -> dict[str, str]: if self.content_cipher is not None: result[self.CONTENT_CIPHER] = self.content_cipher - if self.content_cipher_tag_length is not None: + if self.content_cipher_tag_length is not None and not self.is_v3_format(): result[self.CONTENT_CIPHER_TAG_LENGTH] = self.content_cipher_tag_length if self.instruction_file is not None: diff --git a/test/test_metadata.py b/test/test_metadata.py index ba783bf5..55c1f0b2 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -56,7 +56,7 @@ def test_to_dict(self): # Verify that fields that are None are not included in the dictionary assert "x-amz-key" not in metadata_dict assert "x-amz-matdesc" not in metadata_dict - # Note: content_cipher_tag_length has a default value of "128" + # content_cipher_tag_length defaults to "128" for V1/V2 assert metadata_dict.get("x-amz-tag-len") == "128" assert "x-amz-crypto-instr-file" not in metadata_dict @@ -124,6 +124,9 @@ def test_to_dict_v3_fields(self): assert metadata_dict["x-amz-m"] == "mat-desc" assert metadata_dict["x-amz-t"] == "encryption-context" + # V3 metadata must NOT include V1/V2-only keys like x-amz-tag-len + assert "x-amz-tag-len" not in metadata_dict + def test_is_v1_format(self): metadata = ObjectMetadata( content_iv="iv", From 2757ea62d122a304f0ada91d8349e37f8393c049 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 19:02:39 -0800 Subject: [PATCH 07/32] implement CBC --- src/s3_encryption/__init__.py | 19 +++ src/s3_encryption/materials/materials.py | 6 + src/s3_encryption/pipelines.py | 144 +++++++++++++++++- .../test_i_s3_encryption_instruction_file.py | 12 +- 4 files changed, 165 insertions(+), 16 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 25473050..93996945 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -32,6 +32,11 @@ class S3EncryptionClientConfig: commitment_policy: CommitmentPolicy = field( default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT ) + ##= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + ##% The S3EC MUST support the option to enable or disable legacy unauthenticated modes (content encryption algorithms). + ##= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + ##% The option to enable legacy unauthenticated modes MUST be set to false by default. + enable_legacy_unauthenticated_modes: bool = field(default=False) cmm: AbstractCryptoMaterialsManager = field() ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file ##= type=implementation @@ -48,6 +53,18 @@ class S3EncryptionClientConfig: def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) + ##= specification/s3-encryption/client.md#encryption-algorithm + ##% The S3EC MUST validate that the configured encryption algorithm is not legacy. + ##= specification/s3-encryption/client.md#encryption-algorithm + ##% If the configured encryption algorithm is legacy, then the S3EC MUST throw an exception. + def __attrs_post_init__(self): + if self.algorithm_suite.is_legacy: + raise S3EncryptionClientError( + f"Cannot configure S3 Encryption Client with legacy algorithm suite " + f"{self.algorithm_suite.name}. Legacy algorithm suites are only " + f"supported for decryption (and enable_legacy_unauthenticated_modes is True)." + ) + class S3EncryptionClientPlugin: """Plugin that adds encryption/decryption capabilities to a boto3 S3 client. @@ -147,6 +164,8 @@ def on_get_object_after_call(self, parsed, **kwargs): pipeline = GetEncryptedObjectPipeline( self.config.cmm, s3_client=getattr(self._context, "s3_client", None), + commitment_policy=self.config.commitment_policy, + enable_legacy_unauthenticated_modes=self.config.enable_legacy_unauthenticated_modes, ) decrypted_data = pipeline.decrypt( response, diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 0bca41a4..ecfcdb13 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -16,9 +16,15 @@ class AlgorithmSuite(Enum): """Algorithm suites supported by the S3 Encryption Client.""" + ALG_AES_256_CBC_IV16_NO_KDF = "AES/CBC/PKCS5Padding" ALG_AES_256_GCM_IV12_TAG16_NO_KDF = "AES/GCM/NoPadding" ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = "AES/GCM/HKDF/CommitKey" + @property + def is_legacy(self) -> bool: + """Return True if this algorithm suite is a legacy unauthenticated mode.""" + return self == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + class CommitmentPolicy(Enum): """Commitment policies controlling key-commitment behavior.""" diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index f94370f7..b30847af 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -12,7 +12,7 @@ from attrs import define, field from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from .exceptions import S3EncryptionClientError +from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from .instruction_file import fetch_instruction_file from .key_derivation import ( KC_GCM_IV, @@ -23,7 +23,7 @@ ) from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import AlgorithmSuite, DecryptionMaterials, EncryptionMaterials +from .materials.materials import AlgorithmSuite, CommitmentPolicy, DecryptionMaterials, EncryptionMaterials from .metadata import ObjectMetadata @@ -152,6 +152,59 @@ class GetEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() s3_client: object = field(default=None) + commitment_policy: CommitmentPolicy = field( + default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + enable_legacy_unauthenticated_modes: bool = field(default=False) + + # Map content cipher metadata values to AlgorithmSuite + _CONTENT_CIPHER_TO_ALGORITHM_SUITE = { + "AES/CBC/PKCS5Padding": AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + "AES/GCM/NoPadding": AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "115": AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + } + + def _determine_algorithm_suite(self, metadata) -> AlgorithmSuite: + """Determine the algorithm suite from object metadata. + + V1 objects are always CBC. + V2/V3 objects check x-amz-cek-alg / x-amz-c to determine the content algorithm. + """ + if metadata.is_v1_format(): + ##= specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility + ##= type=citation + ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version. + return AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + + if metadata.is_v2_format(): + cek_alg = metadata.content_cipher + if cek_alg is None: + raise S3EncryptionClientError( + "V2 format object missing required x-amz-cek-alg metadata." + ) + suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) + if suite is None: + raise S3EncryptionClientError( + f"Unknown content encryption algorithm: {cek_alg}" + ) + return suite + + if metadata.is_v3_format(): + cek_alg = metadata.content_cipher_v3 + if cek_alg is None: + raise S3EncryptionClientError( + "V3 format object missing required x-amz-c metadata." + ) + suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) + if suite is None: + raise S3EncryptionClientError( + f"Unknown content encryption algorithm: {cek_alg}" + ) + return suite + + raise S3EncryptionClientError( + "Unable to determine S3 Encryption Client message format." + ) def decrypt( self, @@ -211,31 +264,68 @@ def decrypt( "BUT Instruction File is being used. This is an illegal combination. " f"bucket: {bucket}\n key:{key}\n instruction_file:{instruction_key}" ) + + # Determine the algorithm suite from the metadata + algorithm_suite = self._determine_algorithm_suite(metadata) + # Determine which format we're dealing with and get decryption materials if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) elif metadata.is_v2_format(): - # TODO: this is not how this works dec_materials = self._decrypt_v2(metadata, encryption_context) - dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF elif metadata.is_v3_format(): dec_materials = self._decrypt_v3(metadata, encryption_context) - dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY else: raise S3EncryptionClientError( "Unable to determine S3 Encryption Client message format." ) + dec_materials.algorithm_suite = algorithm_suite + ##= specification/s3-encryption/decryption.md#cbc-decryption - ##= type=TODO ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + if algorithm_suite == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites + ##% unless specifically configured to do so. + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, + ##% the client MUST throw an exception when attempting to decrypt an object encrypted + ##% with a legacy unauthenticated algorithm suite. + if not self.enable_legacy_unauthenticated_modes: + raise S3EncryptionClientError( + "Cannot decrypt object encrypted with ALG_AES_256_CBC_IV16_NO_KDF. " + "The S3 Encryption Client is not configured to decrypt objects using " + "legacy unauthenticated algorithm suites. " + "Set enable_legacy_unauthenticated_modes=True to allow decryption " + "of objects encrypted with CBC." + ) - # Perform decryption - # TODO: include CBC here too + ##= specification/s3-encryption/decryption.md#key-commitment + ##% The S3EC MUST validate the algorithm suite used for decryption against the + ##% key commitment policy before attempting to decrypt the content ciphertext. + ##= specification/s3-encryption/decryption.md#key-commitment + ##% If the commitment policy requires decryption using a committing algorithm suite, + ##% and the algorithm suite associated with the object does not support key commitment, + ##% then the S3EC MUST throw an exception. + if ( + self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + and dec_materials.algorithm_suite != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ): + raise S3EncryptionClientError( + "Configuration conflict: cannot decrypt non-key-committing object " + "when commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT. " + "Use REQUIRE_ENCRYPT_ALLOW_DECRYPT or FORBID_ENCRYPT_ALLOW_DECRYPT " + "to allow decryption of non-committing objects." + ) + + # Perform decryption based on algorithm suite match dec_materials.algorithm_suite: + case AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: + return self._decrypt_cbc_content(dec_materials, encrypted_data) case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##% The client MUST NOT provide any AAD when encrypting with @@ -289,6 +379,44 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) + def _decrypt_cbc_content(self, dec_materials, encrypted_data): + """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF. + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, + ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or + ##% PKCS7Padding compatible padding for a 16-byte block cipher + ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). + """ + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##% If the cipher object cannot be created as described above, + ##% Decryption MUST fail. + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##% The error SHOULD detail why the cipher could not be initialized + ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). + try: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.padding import PKCS7 + + cipher = Cipher( + algorithms.AES(dec_materials.plaintext_data_key), + modes.CBC(dec_materials.iv), + ) + decryptor = cipher.decryptor() + padded_plaintext = decryptor.update(encrypted_data) + decryptor.finalize() + + # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) + unpadder = PKCS7(128).unpadder() + plaintext = unpadder.update(padded_plaintext) + unpadder.finalize() + + return plaintext + except Exception as e: + raise S3EncryptionClientSecurityError( + f"Failed to decrypt CBC content: {e}. " + "Ensure the underlying crypto provider supports AES/CBC/PKCS7Padding." + ) from e + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The V3 format uses compression here such that each wrapping algorithm is represented by a two digit string. ##= specification/s3-encryption/data-format/content-metadata.md#v3-only diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index cc10b6f6..bf00807c 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -27,8 +27,6 @@ } -# TODO(cbc): enable once CBC decryption is implemented -@pytest.mark.skip(reason="V1 CBC decryption not yet implemented") def test_decrypt_v1_instruction_file(): """Test decrypting V1 object with instruction file. @@ -40,7 +38,8 @@ def test_decrypt_v1_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig(keyring, enable_legacy_unauthenticated_modes=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -75,8 +74,6 @@ def test_decrypt_v2_instruction_file(): print("Success! V2 instruction file decryption completed.") -# TODO(v3): enable once v3 is implemented -@pytest.mark.skip(reason="V3 decryption not yet implemented") def test_decrypt_v3_instruction_file(): """Test decrypting V3 object with instruction file. @@ -90,15 +87,14 @@ def test_decrypt_v3_instruction_file(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - assert output != "static-v3-instruction-file-from-java-v4" + assert output == "static-v3-instruction-file-from-java-v4" print("Success! V3 instruction file decryption completed.") From 711edee087cc90100a5870606741c317a01973b5 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 19:04:57 -0800 Subject: [PATCH 08/32] format --- src/s3_encryption/key_derivation.py | 5 +-- src/s3_encryption/materials/kms_keyring.py | 5 ++- src/s3_encryption/materials/materials.py | 4 +- src/s3_encryption/pipelines.py | 41 ++++++++----------- .../test_i_s3_encryption_instruction_file.py | 7 +++- test/test_pipelines.py | 8 +++- 6 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index 0b0e8b6b..58335a4b 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -15,7 +15,6 @@ from cryptography.hazmat.primitives.hashes import SHA512 from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand -from cryptography.hazmat.primitives.kdf.hkdf import HKDF # Algorithm suite ID for S3EC.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY SUITE_ID_BYTES = b"\x00\x73" @@ -87,9 +86,7 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string COMMITKEY as UTF8 encoded bytes. - commit_key = _hkdf_expand( - prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH - ) + commit_key = _hkdf_expand(prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH) return derived_encryption_key, commit_key diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index da4fd8ed..821a0012 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -80,7 +80,10 @@ def on_encrypt(self, enc_materials): # For committing algorithm suites (V3), the encryption context algorithm # value is the algorithm suite ID as a string ("115"), not the cipher name. # For non-committing suites (V2), use the cipher name ("AES/GCM/NoPadding"). - if enc_materials.algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + if ( + enc_materials.algorithm_suite + == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ): encryption_context["aws:x-amz-cek-alg"] = "115" else: encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index ecfcdb13..39c6add6 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -7,12 +7,14 @@ and decryption operations. """ -from typing import Any from enum import Enum +from typing import Any + from attrs import define, field from .encrypted_data_key import EncryptedDataKey + class AlgorithmSuite(Enum): """Algorithm suites supported by the S3 Encryption Client.""" diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index b30847af..25efc0de 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -23,7 +23,12 @@ ) from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import AlgorithmSuite, CommitmentPolicy, DecryptionMaterials, EncryptionMaterials +from .materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, + EncryptionMaterials, +) from .metadata import ObjectMetadata @@ -70,8 +75,7 @@ def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: return self._encrypt_kc_gcm(plaintext, enc_mats, edk_bytes) - else: - return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) + return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" @@ -136,7 +140,9 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): encrypted_data_key_v3=b64_edk, message_id_v3=b64_message_id, key_commitment_v3=b64_commit_key, - encryption_context_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, + encryption_context_v3=( + enc_mats.encryption_context if enc_mats.encryption_context else None + ), ) return encrypted_data, metadata.to_dict() @@ -184,27 +190,19 @@ def _determine_algorithm_suite(self, metadata) -> AlgorithmSuite: ) suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) if suite is None: - raise S3EncryptionClientError( - f"Unknown content encryption algorithm: {cek_alg}" - ) + raise S3EncryptionClientError(f"Unknown content encryption algorithm: {cek_alg}") return suite if metadata.is_v3_format(): cek_alg = metadata.content_cipher_v3 if cek_alg is None: - raise S3EncryptionClientError( - "V3 format object missing required x-amz-c metadata." - ) + raise S3EncryptionClientError("V3 format object missing required x-amz-c metadata.") suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) if suite is None: - raise S3EncryptionClientError( - f"Unknown content encryption algorithm: {cek_alg}" - ) + raise S3EncryptionClientError(f"Unknown content encryption algorithm: {cek_alg}") return suite - raise S3EncryptionClientError( - "Unable to determine S3 Encryption Client message format." - ) + raise S3EncryptionClientError("Unable to determine S3 Encryption Client message format.") def decrypt( self, @@ -313,7 +311,8 @@ def decrypt( ##% then the S3EC MUST throw an exception. if ( self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - and dec_materials.algorithm_suite != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + and dec_materials.algorithm_suite + != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ): raise S3EncryptionClientError( "Configuration conflict: cannot decrypt non-key-committing object " @@ -408,9 +407,7 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) unpadder = PKCS7(128).unpadder() - plaintext = unpadder.update(padded_plaintext) + unpadder.finalize() - - return plaintext + return unpadder.update(padded_plaintext) + unpadder.finalize() except Exception as e: raise S3EncryptionClientSecurityError( f"Failed to decrypt CBC content: {e}. " @@ -508,6 +505,4 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) - return aesgcm.decrypt( - nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES - ) + return aesgcm.decrypt(nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index bf00807c..81ff0366 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -38,8 +38,11 @@ def test_decrypt_v1_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring, enable_legacy_unauthenticated_modes=True, - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + config = S3EncryptionClientConfig( + keyring, + enable_legacy_unauthenticated_modes=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/test_pipelines.py b/test/test_pipelines.py index a4637c9d..f4e3d250 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -186,7 +186,9 @@ def test_decrypt_v3_from_instruction_file(self): # key commitment verification will fail. from s3_encryption.exceptions import S3EncryptionClientSecurityError - with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") # Verify instruction file was fetched @@ -266,5 +268,7 @@ def test_decrypt_v3_unsupported_wrap_alg(self): "Metadata": metadata, } - with pytest.raises(S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM"): + with pytest.raises( + S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM" + ): pipeline.decrypt(mock_response) From a620bcb56a84009da6e987975c12eb9be3bc6a27 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 2 Mar 2026 10:06:37 -0800 Subject: [PATCH 09/32] test-server legacy unauth --- test-server/python-v3-server/src/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v3-server/src/main.py index eb3855b7..3f32f389 100755 --- a/test-server/python-v3-server/src/main.py +++ b/test-server/python-v3-server/src/main.py @@ -189,6 +189,7 @@ async def client_endpoint(request: Request): key_material = config_data.get("keyMaterial", {}) enable_legacy_wrapping_algorithms = config_data.get("enableLegacyWrappingAlgorithms", False) + enable_legacy_unauthenticated_modes = config_data.get("enableLegacyUnauthenticatedModes", False) # TODO pull region from ARN kms_client = boto3.client("kms", region_name="us-west-2") @@ -201,7 +202,10 @@ async def client_endpoint(request: Request): wrapped_client = boto3.client("s3") # Build config kwargs, only including algorithm_suite and commitment_policy if provided - config_kwargs = {"keyring": keyring} + config_kwargs = { + "keyring": keyring, + "enable_legacy_unauthenticated_modes": enable_legacy_unauthenticated_modes, + } encryption_algorithm = config_data.get("encryptionAlgorithm") if encryption_algorithm is not None: From 27e8fc891f718be36c831759e63ac7342dafd287 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 4 Mar 2026 17:03:51 -0800 Subject: [PATCH 10/32] fix custom suffix test --- test/integration/test_i_s3_encryption_instruction_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 81ff0366..7a7505b5 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -149,7 +149,11 @@ def test_decrypt_v2_instruction_file_custom_suffix(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring, instruction_file_suffix=".custom-suffix-instruction") + config = S3EncryptionClientConfig( + keyring, + instruction_file_suffix=".custom-suffix-instruction", + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) From 05d376b27819bb352c739238d38bb39c60c67f5e Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 4 Mar 2026 17:31:38 -0800 Subject: [PATCH 11/32] tighter validation --- src/s3_encryption/pipelines.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 25efc0de..5d82ad6f 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -244,6 +244,25 @@ def decrypt( instruction_key = key + instruction_suffix instruction_metadata = fetch_instruction_file(self.s3_client, bucket, instruction_key) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation + ##% - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. + ##% - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. + ##% - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. + v3_object_metadata_exclusive_keys = { + ObjectMetadata.CONTENT_CIPHER_V3, + ObjectMetadata.KEY_COMMITMENT_V3, + ObjectMetadata.MESSAGE_ID_V3, + } + forbidden_keys_in_instruction = set(instruction_metadata.keys()) & v3_object_metadata_exclusive_keys + if forbidden_keys_in_instruction: + raise S3EncryptionClientError( + "Instruction file is tampered, instruction file contains object metadata " + f"exclusive mapkeys: {forbidden_keys_in_instruction}. " + f"bucket: {bucket}\n key:{key}\n instruction_file:{instruction_key}" + ) + instruction_metadata.update(encryption_metadata) metadata = ObjectMetadata.from_dict(instruction_metadata) ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files From 155caa11758b67a89f96ab986b14a0352ae5933d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 15:02:27 -0800 Subject: [PATCH 12/32] fix duvet --- src/s3_encryption/pipelines.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 5d82ad6f..30650a44 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -248,7 +248,11 @@ def decrypt( ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files ##= type=implementation ##% - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation ##% - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=implementation ##% - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. v3_object_metadata_exclusive_keys = { ObjectMetadata.CONTENT_CIPHER_V3, From ffa2a78b2b71c87b911fe0e4aa447e8f0c2ea338 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 15:23:38 -0800 Subject: [PATCH 13/32] improve code --- src/s3_encryption/key_derivation.py | 4 +- src/s3_encryption/pipelines.py | 50 +++++++++---------- .../test_i_s3_encryption_instruction_file.py | 8 +-- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index 58335a4b..d4bc28b1 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -16,6 +16,8 @@ from cryptography.hazmat.primitives.hashes import SHA512 from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand +from .exceptions import S3EncryptionClientSecurityError + # Algorithm suite ID for S3EC.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY SUITE_ID_BYTES = b"\x00\x73" @@ -105,8 +107,6 @@ def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> No ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value ##% and stored key commitment value do not match. - from .exceptions import S3EncryptionClientSecurityError - if not hmac.compare_digest(stored_commitment, derived_commitment): raise S3EncryptionClientSecurityError( "Key commitment verification failed: stored commitment does not match derived commitment." diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 30650a44..90d9f049 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -7,10 +7,13 @@ """ import base64 +import json import os from attrs import define, field +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.padding import PKCS7 from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from .instruction_file import fetch_instruction_file @@ -363,39 +366,41 @@ def decrypt( def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" - iv_bytes = base64.b64decode(metadata.content_iv) - edk_bytes = base64.b64decode(metadata.encrypted_data_key_v2) - - encrypted_data_key = EncryptedDataKey( - key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=edk_bytes, - ) - - dec_materials = DecryptionMaterials( - iv=iv_bytes, - encrypted_data_keys=[encrypted_data_key], - encryption_context_stored=metadata.encrypted_data_key_context or {}, - encryption_context_from_request=encryption_context, + return self._decrypt_v1_v2( + iv_b64=metadata.content_iv, + edk_b64=metadata.encrypted_data_key_v2, + wrap_alg=metadata.encrypted_data_key_algorithm, + stored_context=metadata.encrypted_data_key_context or {}, + encryption_context=encryption_context, ) - return self.cmm.decrypt_materials(dec_materials) - def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V1 decryption materials.""" - iv_bytes = base64.b64decode(metadata.content_iv) - edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) + return self._decrypt_v1_v2( + iv_b64=metadata.content_iv, + edk_b64=metadata.encrypted_data_key_v1, + wrap_alg=metadata.encrypted_data_key_algorithm, + stored_context=metadata.encrypted_data_key_context or {}, + encryption_context=encryption_context, + ) + + def _decrypt_v1_v2( + self, iv_b64, edk_b64, wrap_alg, stored_context, encryption_context + ) -> DecryptionMaterials: + """Shared logic for preparing V1/V2 decryption materials.""" + iv_bytes = base64.b64decode(iv_b64) + edk_bytes = base64.b64decode(edk_b64) encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, + key_provider_info=wrap_alg, encrypted_data_key=edk_bytes, ) dec_materials = DecryptionMaterials( iv=iv_bytes, encrypted_data_keys=[encrypted_data_key], - encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_stored=stored_context, encryption_context_from_request=encryption_context, ) @@ -418,9 +423,6 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): ##% The error SHOULD detail why the cipher could not be initialized ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). try: - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - from cryptography.hazmat.primitives.padding import PKCS7 - cipher = Cipher( algorithms.AES(dec_materials.plaintext_data_key), modes.CBC(dec_materials.iv), @@ -491,8 +493,6 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: if isinstance(raw_ctx, dict): stored_context = raw_ctx elif isinstance(raw_ctx, str): - import json - stored_context = json.loads(raw_ctx) dec_materials = DecryptionMaterials( diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 7a7505b5..2f14d9dd 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -123,8 +123,6 @@ def test_decrypt_invalid_instruction_file(): print(f"Error message: {exc_info.value}") -# TODO(v3): enable once v3 is implemented -@pytest.mark.skip(reason="V3 decryption not yet implemented") def test_decrypt_v3_instruction_file_custom_suffix(): """Test decrypting V3 object with a custom instruction file suffix.""" key = TEST_OBJECTS["v3_instruction_file"] @@ -132,7 +130,11 @@ def test_decrypt_v3_instruction_file_custom_suffix(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring, instruction_file_suffix=".custom-suffix-instruction") + config = S3EncryptionClientConfig( + keyring, + instruction_file_suffix=".custom-suffix-instruction", + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) From 8dd7eb49640493e5869b3ec878338e8a038ceb4f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 15:39:11 -0800 Subject: [PATCH 14/32] more better code --- src/s3_encryption/__init__.py | 42 +++++++++++++++++++++------------- src/s3_encryption/pipelines.py | 10 -------- test/test_pipelines.py | 21 ++++++++--------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 93996945..0604a209 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -20,6 +20,17 @@ S3_METADATA_PREFIX = "x-amz-meta-" +# Thread-local context attribute names +_CTX_ENCRYPTION_CONTEXT = "encryption_context" +_CTX_BUCKET = "bucket" +_CTX_KEY = "key" +_CTX_S3_CLIENT = "s3_client" +_CTX_INSTRUCTION_FILE_MODE = "instruction_file_mode" + +# Attributes to clean up after get_object completes +# (s3_client is intentionally excluded — it is not request-scoped) +_GET_OBJECT_CLEANUP_ATTRS = (_CTX_ENCRYPTION_CONTEXT, _CTX_BUCKET, _CTX_KEY) + @define class S3EncryptionClientConfig: @@ -91,7 +102,7 @@ def on_put_object_before_call(self, params, **kwargs): params: Dictionary of parameters for the PutObject call (after serialization) **kwargs: Additional event arguments """ - if getattr(self._context, "instruction_file_mode", False): + if getattr(self._context, _CTX_INSTRUCTION_FILE_MODE, False): raise S3EncryptionClientError( "Instruction file mode is exclusively for reading instruction files " "and not supported in put_object!" @@ -112,7 +123,7 @@ def on_put_object_before_call(self, params, **kwargs): # Unexpected body type - should not happen as boto3 validates before this point raise S3EncryptionClientError("Unexpected type of body parameter!") - encryption_context = getattr(self._context, "encryption_context", None) + encryption_context = getattr(self._context, _CTX_ENCRYPTION_CONTEXT, None) pipeline = PutEncryptedObjectPipeline(self.config.cmm) encrypted_data, encryption_metadata = pipeline.encrypt( @@ -144,12 +155,12 @@ def on_get_object_after_call(self, parsed, **kwargs): **kwargs: Additional event arguments (includes 'params' with request parameters) """ # Check if plaintext mode is enabled via thread-local flag - if getattr(self._context, "instruction_file_mode", False): + if getattr(self._context, _CTX_INSTRUCTION_FILE_MODE, False): self.process_instruction_file(parsed) return # Get encryption context from thread-local storage (set by get_object wrapper) - encryption_context = getattr(self._context, "encryption_context", None) + encryption_context = getattr(self._context, _CTX_ENCRYPTION_CONTEXT, None) # The parsed response already has the Body as a StreamingBody # We need to read it, decrypt it, and replace it @@ -163,15 +174,15 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( self.config.cmm, - s3_client=getattr(self._context, "s3_client", None), + s3_client=getattr(self._context, _CTX_S3_CLIENT, None), commitment_policy=self.config.commitment_policy, enable_legacy_unauthenticated_modes=self.config.enable_legacy_unauthenticated_modes, ) decrypted_data = pipeline.decrypt( response, encryption_context, - bucket=getattr(self._context, "bucket", None), - key=getattr(self._context, "key", None), + bucket=getattr(self._context, _CTX_BUCKET, None), + key=getattr(self._context, _CTX_KEY, None), instruction_suffix=self.config.instruction_file_suffix, ) @@ -191,7 +202,7 @@ def process_instruction_file(self, parsed): Args: parsed: Dictionary containing the parsed response """ - instruction_key = getattr(self._context, "key", None) + instruction_key = getattr(self._context, _CTX_KEY, None) # In plaintext mode, parse instruction file and append to metadata existing_metadata = parsed.get("Metadata", {}) @@ -270,8 +281,8 @@ def put_object(self, **kwargs): raise S3EncryptionClientError(f"Failed to encrypt object: {str(e)}") from e finally: # Clean up thread-local storage - if hasattr(self._plugin._context, "encryption_context"): - delattr(self._plugin._context, "encryption_context") + if hasattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT): + delattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT) def get_object(self, **kwargs): """Download and decrypt an object from S3. @@ -294,12 +305,12 @@ def get_object(self, **kwargs): encryption_context = kwargs.pop("EncryptionContext", None) # Store encryption context in thread-local storage for the event handler - self._plugin._context.encryption_context = encryption_context + setattr(self._plugin._context, _CTX_ENCRYPTION_CONTEXT, encryption_context) # Store wrapped client in thread-local storage for # the event handler to fetch instruction files - self._plugin._context.s3_client = self.wrapped_s3_client - self._plugin._context.bucket = kwargs.get("Bucket") - self._plugin._context.key = kwargs.get("Key") + setattr(self._plugin._context, _CTX_S3_CLIENT, self.wrapped_s3_client) + setattr(self._plugin._context, _CTX_BUCKET, kwargs.get("Bucket")) + setattr(self._plugin._context, _CTX_KEY, kwargs.get("Key")) try: return self.wrapped_s3_client.get_object(**kwargs) @@ -312,7 +323,6 @@ def get_object(self, **kwargs): finally: # Clean up thread-local storage; # do not clean up the client as it is not thread local only - attrs = ["encryption_context", "Bucket", "Key"] - for attr in attrs: + for attr in _GET_OBJECT_CLEANUP_ATTRS: if hasattr(self._plugin._context, attr): delattr(self._plugin._context, attr) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 90d9f049..f589afa2 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -453,9 +453,6 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): "22": "RSA-OAEP-SHA1", } - # Wrapping algorithms that this client currently supports for decryption - _SUPPORTED_WRAP_ALGS = {"kms+context", "kms"} - def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V3 decryption materials.""" edk_bytes = base64.b64decode(metadata.encrypted_data_key_v3) @@ -464,13 +461,6 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "12" wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) - if wrap_alg not in self._SUPPORTED_WRAP_ALGS: - raise S3EncryptionClientError( - f"Unsupported wrapping algorithm: {wrap_alg}. " - f"The S3 Encryption Client for Python currently supports: " - f"{', '.join(sorted(self._SUPPORTED_WRAP_ALGS))}." - ) - encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", key_provider_info=wrap_alg, diff --git a/test/test_pipelines.py b/test/test_pipelines.py index f4e3d250..251ce26e 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -8,8 +8,10 @@ import pytest +from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import DecryptionMaterials from s3_encryption.pipelines import GetEncryptedObjectPipeline @@ -168,8 +170,6 @@ def test_decrypt_v3_from_instruction_file(self): } # Mock the keyring to return decryption materials - from s3_encryption.materials.materials import DecryptionMaterials - plaintext_data_key = os.urandom(32) mock_dec_materials = DecryptionMaterials( @@ -184,8 +184,6 @@ def test_decrypt_v3_from_instruction_file(self): # V3 decryption is now implemented; with fake commitment data, # key commitment verification will fail. - from s3_encryption.exceptions import S3EncryptionClientSecurityError - with pytest.raises( S3EncryptionClientSecurityError, match="Key commitment verification failed" ): @@ -244,22 +242,23 @@ def test_decrypt_with_custom_instruction_file_suffix(self): mock_s3_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test-key.custom-suffix" ) - def test_decrypt_v3_unsupported_wrap_alg(self): - """Test that V3 decryption with unsupported wrapping algorithm gives a useful error.""" - from s3_encryption.exceptions import S3EncryptionClientError - - # V3 metadata with AES/GCM wrapping (02) — not supported by this client + """Test that V3 decryption with unsupported wrapping algorithm is rejected by the keyring.""" + # V3 metadata with AES/GCM wrapping (02) — not supported by the KMS keyring metadata = { "x-amz-c": "115", "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), - "x-amz-w": "02", # AES/GCM — unsupported + "x-amz-w": "02", # AES/GCM — unsupported by KMS keyring "x-amz-m": json.dumps({"some": "material-desc"}), "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), } mock_keyring = Mock(spec=S3Keyring) + # The keyring rejects wrapping algorithms it doesn't support + mock_keyring.on_decrypt.side_effect = S3EncryptionClientError( + "AES/GCM is not a valid key wrapping algorithm!" + ) cmm = DefaultCryptoMaterialsManager(mock_keyring) pipeline = GetEncryptedObjectPipeline(cmm) @@ -269,6 +268,6 @@ def test_decrypt_v3_unsupported_wrap_alg(self): } with pytest.raises( - S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM" + S3EncryptionClientError, match="AES/GCM is not a valid key wrapping algorithm" ): pipeline.decrypt(mock_response) From 99ef305227bfcce73610ccceedf943983bc15a8f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 20:40:16 -0800 Subject: [PATCH 15/32] decrypt annotations --- .../decryption_exceptions.md | 49 +++ src/s3_encryption/key_derivation.py | 3 + src/s3_encryption/pipelines.py | 14 + test/test_decryption.py | 380 ++++++++++++++++++ 4 files changed, 446 insertions(+) create mode 100644 compliance_exceptions/decryption_exceptions.md create mode 100644 test/test_decryption.py diff --git a/compliance_exceptions/decryption_exceptions.md b/compliance_exceptions/decryption_exceptions.md new file mode 100644 index 00000000..4919f2ce --- /dev/null +++ b/compliance_exceptions/decryption_exceptions.md @@ -0,0 +1,49 @@ +# Compliance Exceptions for Decryption Implementation + +## Summary + +The Python S3 Encryption Client does not currently support Ranged Gets. +Ranged Gets allow downloading and decrypting a subset of bytes from an encrypted S3 object. +This is an optional feature per the specification ("MAY support") and is planned for a future release. + +## Ranged Gets + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% The S3EC MAY support the "range" parameter on GetObject which specifies a subset of bytes to download and decrypt. + +Justification: Ranged Gets are not yet implemented in the Python S3 Encryption Client. The specification uses MAY, making this an optional feature. This is planned for a future release. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the S3EC supports Ranged Gets, the S3EC MUST adjust the customer-provided range to include the beginning and end of the cipher blocks for the given range. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, this requirement will be fulfilled. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the object was encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF, then ALG_AES_256_CTR_IV16_TAG16_NO_KDF MUST be used to decrypt the range of the object. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, the correct CTR-mode algorithm suite will be used for GCM-encrypted objects. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the object was encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, then ALG_AES_256_CTR_HKDF_SHA512_COMMIT_KEY MUST be used to decrypt the range of the object. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, the correct CTR-mode algorithm suite will be used for key-committing objects. + +--- + +##= specification/s3-encryption/decryption.md#ranged-gets +##= type=exception +##% If the GetObject response contains a range, but the GetObject request does not contain a range, the S3EC MUST throw an exception. + +Justification: Not applicable since Ranged Gets are not yet supported. When Ranged Gets are implemented, this validation will be added to detect unexpected range responses. + +--- diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index d4bc28b1..cbd4b7db 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -104,7 +104,10 @@ def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> No S3EncryptionClientSecurityError: If the commitment values do not match. """ ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value ##% and stored key commitment value do not match. if not hmac.compare_digest(stored_commitment, derived_commitment): diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index f589afa2..d1821bfc 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -307,15 +307,18 @@ def decrypt( dec_materials.algorithm_suite = algorithm_suite ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. if algorithm_suite == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=implementation ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites ##% unless specifically configured to do so. ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=implementation ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, ##% the client MUST throw an exception when attempting to decrypt an object encrypted ##% with a legacy unauthenticated algorithm suite. @@ -329,9 +332,11 @@ def decrypt( ) ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=implementation ##% The S3EC MUST validate the algorithm suite used for decryption against the ##% key commitment policy before attempting to decrypt the content ciphertext. ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=implementation ##% If the commitment policy requires decryption using a committing algorithm suite, ##% and the algorithm suite associated with the object does not support key commitment, ##% then the S3EC MUST throw an exception. @@ -410,6 +415,7 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF. ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or @@ -417,9 +423,11 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). """ ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation ##% If the cipher object cannot be created as described above, ##% Decryption MUST fail. ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=implementation ##% The error SHOULD detail why the cipher could not be initialized ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). try: @@ -508,6 +516,12 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): ) ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation + ##% When using an algorithm suite which supports key commitment, the client MUST verify + ##% that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the + ##% same bytes as the stored key commitment retrieved from the stored object's metadata. + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=implementation ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving ##% the [derived encryption key](./key-derivation.md#hkdf-operation). verify_commitment(stored_commitment, derived_commitment) diff --git a/test/test_decryption.py b/test/test_decryption.py new file mode 100644 index 00000000..211adea2 --- /dev/null +++ b/test/test_decryption.py @@ -0,0 +1,380 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for decryption specification compliance annotations. + +Each test in this module corresponds to a MUST/SHOULD requirement from +specification/s3-encryption/decryption.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import base64 +import hmac +import os +from io import BytesIO +from unittest.mock import MagicMock, Mock + +import pytest + +from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError +from s3_encryption.key_derivation import derive_keys, verify_commitment +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, +) +from s3_encryption.pipelines import GetEncryptedObjectPipeline + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + enable_legacy=False, + s3_client=None, + keyring_side_effect=None, + keyring_return=None, +): + """Create a GetEncryptedObjectPipeline with a mocked CMM/keyring.""" + mock_keyring = Mock(spec=S3Keyring) + if keyring_side_effect is not None: + mock_keyring.on_decrypt.side_effect = keyring_side_effect + elif keyring_return is not None: + mock_keyring.on_decrypt.return_value = keyring_return + cmm = DefaultCryptoMaterialsManager(mock_keyring) + return GetEncryptedObjectPipeline( + cmm, + s3_client=s3_client, + commitment_policy=commitment_policy, + enable_legacy_unauthenticated_modes=enable_legacy, + ) + + +def _v1_cbc_metadata(): + """Return V1 (CBC) object metadata dict.""" + return { + "x-amz-iv": base64.b64encode(os.urandom(16)).decode(), + "x-amz-key": base64.b64encode(b"encrypted-key").decode(), + "x-amz-matdesc": '{"kms_cmk_id": "key-id"}', + } + + +def _v2_gcm_metadata(): + """Return V2 (GCM, no KDF) object metadata dict.""" + return { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode(), + "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": '{}', + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + + +def _response(metadata, body=b"ciphertext"): + return {"Body": BytesIO(body), "Metadata": metadata} + + +# --------------------------------------------------------------------------- +# CBC Decryption +# --------------------------------------------------------------------------- + +class TestCBCDecryption: + """Tests for specification/s3-encryption/decryption.md#cbc-decryption.""" + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, + ##% the S3EC MUST throw an error which details that client was + ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + def test_cbc_object_rejected_when_legacy_disabled(self): + """CBC-encrypted objects MUST be rejected when legacy modes are disabled.""" + plaintext_key = os.urandom(32) + dec_mats = DecryptionMaterials( + iv=os.urandom(16), + plaintext_data_key=plaintext_key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=False, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientError, match="ALG_AES_256_CBC_IV16_NO_KDF"): + pipeline.decrypt(_response(_v1_cbc_metadata())) + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, + ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or + ##% PKCS7Padding compatible padding for a 16-byte block cipher + ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). + def test_cbc_decryption_succeeds_when_legacy_enabled(self): + """CBC decryption MUST work with PKCS7-compatible padding when legacy is enabled.""" + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.padding import PKCS7 + + plaintext = b"hello world, this is a CBC test!!" + key = os.urandom(32) + iv = os.urandom(16) + + # Encrypt with AES-CBC + PKCS7 padding + padder = PKCS7(128).padder() + padded = padder.update(plaintext) + padder.finalize() + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + encryptor = cipher.encryptor() + ciphertext = encryptor.update(padded) + encryptor.finalize() + + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key": base64.b64encode(b"encrypted-key").decode(), + "x-amz-matdesc": '{"kms_cmk_id": "key-id"}', + } + + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + result = pipeline.decrypt(_response(metadata, ciphertext)) + assert result == plaintext + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% If the cipher object cannot be created as described above, + ##% Decryption MUST fail. + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##= type=test + ##% The error SHOULD detail why the cipher could not be initialized + ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). + def test_cbc_decryption_fails_with_wrong_key(self): + """CBC decryption MUST fail (with detail) when the cipher cannot decrypt.""" + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.padding import PKCS7 + + plaintext = b"hello world, this is a CBC test!!" + real_key = os.urandom(32) + wrong_key = os.urandom(32) + iv = os.urandom(16) + + padder = PKCS7(128).padder() + padded = padder.update(plaintext) + padder.finalize() + cipher = Cipher(algorithms.AES(real_key), modes.CBC(iv)) + encryptor = cipher.encryptor() + ciphertext = encryptor.update(padded) + encryptor.finalize() + + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key": base64.b64encode(b"encrypted-key").decode(), + "x-amz-matdesc": '{"kms_cmk_id": "key-id"}', + } + + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=wrong_key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientSecurityError, match="Failed to decrypt CBC content"): + pipeline.decrypt(_response(metadata, ciphertext)) + + +# --------------------------------------------------------------------------- +# Decrypting with Commitment +# --------------------------------------------------------------------------- + +class TestDecryptingWithCommitment: + """Tests for specification/s3-encryption/decryption.md#decrypting-with-commitment.""" + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the client MUST verify + ##% that the [derived key commitment](./key-derivation.md#hkdf-operation) contains the + ##% same bytes as the stored key commitment retrieved from the stored object's metadata. + def test_commitment_verified_against_stored_metadata(self): + """The derived commitment MUST match the stored commitment from metadata.""" + key = os.urandom(32) + message_id = os.urandom(28) + _, correct_commitment = derive_keys(key, message_id) + + # Should not raise + verify_commitment(correct_commitment, correct_commitment) + + # Tampered commitment must fail + tampered = bytearray(correct_commitment) + tampered[0] ^= 0xFF + with pytest.raises(S3EncryptionClientSecurityError): + verify_commitment(bytes(tampered), correct_commitment) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + def test_commitment_verification_uses_constant_time_compare(self): + """Verification MUST use constant-time comparison (hmac.compare_digest).""" + stored = os.urandom(28) + derived = os.urandom(28) + + # verify_commitment delegates to hmac.compare_digest; confirm it raises + # on mismatch (the constant-time property is guaranteed by hmac.compare_digest). + with pytest.raises(S3EncryptionClientSecurityError): + verify_commitment(stored, derived) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value + ##% and stored key commitment value do not match. + def test_commitment_mismatch_throws_exception(self): + """Mismatched commitment values MUST raise an exception.""" + stored = os.urandom(28) + derived = os.urandom(28) + + with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): + verify_commitment(stored, derived) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##= type=test + ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving + ##% the [derived encryption key](./key-derivation.md#hkdf-operation). + def test_commitment_verified_before_content_decryption(self): + """Commitment verification MUST happen before content decryption is attempted.""" + key = os.urandom(32) + message_id = os.urandom(28) + _, real_commitment = derive_keys(key, message_id) + + # Build V3 metadata with a wrong commitment + wrong_commitment = os.urandom(28) + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key").decode(), + "x-amz-w": "12", + "x-amz-t": "{}", + "x-amz-d": base64.b64encode(wrong_commitment).decode(), + "x-amz-i": base64.b64encode(message_id).decode(), + } + + dec_mats = DecryptionMaterials( + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + + # Must fail at commitment check, not at AES-GCM decryption + with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): + pipeline.decrypt(_response(metadata, b"fake-ciphertext")) + + +# --------------------------------------------------------------------------- +# Key Commitment Policy +# --------------------------------------------------------------------------- + +class TestKeyCommitmentPolicy: + """Tests for specification/s3-encryption/decryption.md#key-commitment.""" + + ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=test + ##% The S3EC MUST validate the algorithm suite used for decryption against the + ##% key commitment policy before attempting to decrypt the content ciphertext. + ##= specification/s3-encryption/decryption.md#key-commitment + ##= type=test + ##% If the commitment policy requires decryption using a committing algorithm suite, + ##% and the algorithm suite associated with the object does not support key commitment, + ##% then the S3EC MUST throw an exception. + def test_require_decrypt_rejects_non_committing_suite(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST reject non-committing algorithm suites.""" + dec_mats = DecryptionMaterials( + iv=os.urandom(12), + plaintext_data_key=os.urandom(32), + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): + pipeline.decrypt(_response(_v2_gcm_metadata())) + + def test_allow_decrypt_accepts_non_committing_suite(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST allow non-committing algorithm suites.""" + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + key = os.urandom(32) + iv = os.urandom(12) + plaintext = b"test data for allow-decrypt policy" + ciphertext = AESGCM(key).encrypt(iv, plaintext, None) + + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": "{}", + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + result = pipeline.decrypt(_response(metadata, ciphertext)) + assert result == plaintext + + +# --------------------------------------------------------------------------- +# Legacy Decryption +# --------------------------------------------------------------------------- + +class TestLegacyDecryption: + """Tests for specification/s3-encryption/decryption.md#legacy-decryption.""" + + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=test + ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites + ##% unless specifically configured to do so. + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##= type=test + ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, + ##% the client MUST throw an exception when attempting to decrypt an object encrypted + ##% with a legacy unauthenticated algorithm suite. + def test_legacy_cbc_rejected_by_default(self): + """Legacy CBC objects MUST be rejected unless enable_legacy_unauthenticated_modes is True.""" + dec_mats = DecryptionMaterials( + iv=os.urandom(16), + plaintext_data_key=os.urandom(32), + algorithm_suite=AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + ) + pipeline = _make_pipeline( + enable_legacy=False, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + + with pytest.raises(S3EncryptionClientError, match="not configured to decrypt"): + pipeline.decrypt(_response(_v1_cbc_metadata())) From 98a42c37759e9d77616c87170d6953b991fbb8d5 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 21:08:20 -0800 Subject: [PATCH 16/32] all the spec, all the tests! --- .../encryption_exceptions.md | 63 +++++ .../key_commitment_exceptions.md | 39 +++ src/s3_encryption/key_derivation.py | 24 +- src/s3_encryption/pipelines.py | 58 ++++ test/test_encryption.py | 250 +++++++++++++++++ test/test_key_commitment.py | 149 ++++++++++ test/test_key_derivation.py | 258 ++++++++++++++++++ 7 files changed, 839 insertions(+), 2 deletions(-) create mode 100644 compliance_exceptions/encryption_exceptions.md create mode 100644 compliance_exceptions/key_commitment_exceptions.md create mode 100644 test/test_encryption.py create mode 100644 test/test_key_commitment.py create mode 100644 test/test_key_derivation.py diff --git a/compliance_exceptions/encryption_exceptions.md b/compliance_exceptions/encryption_exceptions.md new file mode 100644 index 00000000..bf7d9f62 --- /dev/null +++ b/compliance_exceptions/encryption_exceptions.md @@ -0,0 +1,63 @@ +# Compliance Exceptions for Encryption Implementation + +## Summary + +The Python S3 Encryption Client does not implement AES-CTR algorithm suites (used only for ranged-get decryption), +does not yet validate IV/Message ID for zero values, does not validate maximum plaintext length, +and relies on Python's `cryptography` library to automatically append GCM auth tags. + +## AES-CTR Algorithm Suites + +##= specification/s3-encryption/encryption.md#alg-aes-256-ctr-hkdf-sha512-commit-key +##= type=exception +##% Attempts to encrypt using key committing AES-CTR MUST fail. + +Justification: The AES-CTR algorithm suites are only used for ranged-get decryption. Since ranged gets are not yet implemented, these algorithm suites are not defined in the `AlgorithmSuite` enum and cannot be selected for encryption. The constraint is satisfied structurally. + +--- + +##= specification/s3-encryption/encryption.md#alg-aes-256-ctr-iv16-tag16-no-kdf +##= type=exception +##% Attempts to encrypt using AES-CTR MUST fail. + +Justification: Same as above. AES-CTR is not available as an algorithm suite option, so it cannot be used for encryption. + +--- + +## GCM Auth Tag Appending + +##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key +##= type=exception +##% The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + +Justification: Python's `cryptography` library (`AESGCM.encrypt`) automatically appends the GCM authentication tag to the ciphertext. No manual appending is needed. + +--- + +##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf +##= type=exception +##% The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + +Justification: Python's `cryptography` library (`AESGCM.encrypt`) automatically appends the GCM authentication tag to the ciphertext. No manual appending is needed. + +--- + +## Cipher Initialization Validation + +##= specification/s3-encryption/encryption.md#cipher-initialization +##= type=exception +##% The client SHOULD validate that the generated IV or Message ID is not zeros. + +Justification: This SHOULD-level validation is not yet implemented. The IV and Message ID are generated using `os.urandom()`, which is cryptographically secure and extremely unlikely to produce all-zero output. This validation is planned for a future release. + +--- + +## Plaintext Length Validation + +##= specification/s3-encryption/encryption.md#content-encryption +##= type=exception +##% The client MUST validate that the length of the plaintext bytes does not exceed the algorithm suite's cipher's maximum content length in bytes. + +Justification: Maximum plaintext length validation is not yet implemented. For AES-GCM with a 12-byte IV, the maximum plaintext size is approximately 64 GiB, which exceeds practical S3 single-object upload limits. This validation is planned for a future release. + +--- diff --git a/compliance_exceptions/key_commitment_exceptions.md b/compliance_exceptions/key_commitment_exceptions.md new file mode 100644 index 00000000..104330ef --- /dev/null +++ b/compliance_exceptions/key_commitment_exceptions.md @@ -0,0 +1,39 @@ +# Compliance Exceptions for Key Commitment Policy — Encryption Side + +## Summary + +The Python S3 Encryption Client does not yet explicitly validate the commitment policy +against the configured algorithm suite on the encryption path. The client defaults to the +key-committing algorithm suite (`ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY`) and validates +that legacy (CBC) suites cannot be configured, but does not enforce the full matrix of +commitment policy vs. algorithm suite at encryption time. + +## FORBID_ENCRYPT_ALLOW_DECRYPT — Encrypt Restriction + +##= specification/s3-encryption/key-commitment.md#commitment-policy +##= type=exception +##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. + +Justification: The encryption path does not validate the commitment policy against the algorithm suite. A caller who configures `FORBID_ENCRYPT_ALLOW_DECRYPT` but leaves the default committing algorithm suite would incorrectly encrypt with a committing suite. This validation is planned for a future release. + +--- + +## REQUIRE_ENCRYPT_ALLOW_DECRYPT — Encrypt Restriction + +##= specification/s3-encryption/key-commitment.md#commitment-policy +##= type=exception +##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + +Justification: The encryption path does not explicitly validate that the algorithm suite supports key commitment when the policy is `REQUIRE_ENCRYPT_ALLOW_DECRYPT`. In practice, the default algorithm suite is the committing suite, so this is satisfied by default. However, there is no guard preventing a caller from overriding the algorithm suite to a non-committing one. This validation is planned for a future release. + +--- + +## REQUIRE_ENCRYPT_REQUIRE_DECRYPT — Encrypt Restriction + +##= specification/s3-encryption/key-commitment.md#commitment-policy +##= type=exception +##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + +Justification: Same as above. The default algorithm suite is the committing suite, so this is satisfied by default, but there is no explicit validation preventing a non-committing suite from being configured. This validation is planned for a future release. + +--- diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index cbd4b7db..70442e7c 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -71,22 +71,42 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by A tuple of (derived_encryption_key, commit_key). """ ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% - The hash function MUST be specified by the algorithm suite commitment settings. ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key) ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% The input info MUST be a concatenation of the algorithm suite ID as bytes + ##= type=implementation + ##% - The DEK input pseudorandom key MUST be the output from the extract step. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string DERIVEKEY as UTF8 encoded bytes. derived_encryption_key = _hkdf_expand( prk, info=SUITE_ID_BYTES + b"DERIVEKEY", length=ENCRYPTION_KEY_LENGTH ) ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% The input info MUST be a concatenation of the algorithm suite ID as bytes + ##= type=implementation + ##% - The CK input pseudorandom key MUST be the output from the extract step. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string COMMITKEY as UTF8 encoded bytes. commit_key = _hkdf_expand(prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index d1821bfc..13897d4a 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -60,6 +60,11 @@ def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): if algorithm_suite is None: algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The S3EC MUST use the encryption algorithm configured during + ##% [client](./client.md) initialization. + # Create encryption materials request with encryption context copy enc_mats_request = EncryptionMaterials( algorithm_suite=algorithm_suite, @@ -83,10 +88,21 @@ def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, ##% with the plaintext data key, the generated IV, and the tag length defined ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The generated IV or Message ID MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. iv = os.urandom(12) aesgcm = AESGCM(enc_mats.plaintext_data_key) encrypted_data = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) @@ -107,19 +123,38 @@ def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (V3 format).""" ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation ##% The client MUST generate an IV or Message ID using the length of the IV ##% or Message ID defined in the algorithm suite. + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The generated IV or Message ID MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. message_id = os.urandom(MESSAGE_ID_LENGTH) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation ##% The client MUST use HKDF to derive the key commitment value and the derived ##% encrypting key as described in [Key Derivation](key-derivation.md). + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation + ##% The derived key commitment value MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. derived_encryption_key, commit_key = derive_keys(enc_mats.plaintext_data_key, message_id) ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) encrypted_data = aesgcm.encrypt( @@ -340,6 +375,9 @@ def decrypt( ##% If the commitment policy requires decryption using a committing algorithm suite, ##% and the algorithm suite associated with the object does not support key commitment, ##% then the S3EC MUST throw an exception. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=implementation + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. if ( self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT and dec_materials.algorithm_suite @@ -352,12 +390,22 @@ def decrypt( "to allow decryption of non-committing objects." ) + # The FORBID_ENCRYPT_ALLOW_DECRYPT and REQUIRE_ENCRYPT_ALLOW_DECRYPT policies + # allow decryption with non-committing algorithm suites — no additional check needed. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=implementation + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=implementation + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + # Perform decryption based on algorithm suite match dec_materials.algorithm_suite: case AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: return self._decrypt_cbc_content(dec_materials, encrypted_data) case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=implementation ##% The client MUST NOT provide any AAD when encrypting with ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. aesgcm = AESGCM(dec_materials.plaintext_data_key) @@ -510,6 +558,7 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): stored_commitment = base64.b64decode(metadata.key_commitment_v3) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation ##% The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). derived_encryption_key, derived_commitment = derive_keys( dec_materials.plaintext_data_key, message_id @@ -527,9 +576,18 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): verify_commitment(stored_commitment, derived_commitment) ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) return aesgcm.decrypt(nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES) diff --git a/test/test_encryption.py b/test/test_encryption.py new file mode 100644 index 00000000..7484ee0a --- /dev/null +++ b/test/test_encryption.py @@ -0,0 +1,250 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for encryption specification compliance annotations. + +Each test in this module corresponds to a MUST/SHOULD requirement from +specification/s3-encryption/encryption.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import base64 +import os +from unittest.mock import MagicMock + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from s3_encryption.key_derivation import ( + KC_GCM_IV, + MESSAGE_ID_LENGTH, + SUITE_ID_BYTES, + derive_keys, +) +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.materials import AlgorithmSuite, EncryptionMaterials +from s3_encryption.pipelines import PutEncryptedObjectPipeline + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _mock_cmm(plaintext_key=None, encrypted_key=b"encrypted-key"): + """Return a CMM backed by a mock keyring that returns the given keys.""" + if plaintext_key is None: + plaintext_key = os.urandom(32) + + mock_keyring = MagicMock() + mock_keyring.on_encrypt.side_effect = lambda mats: _fill_materials( + mats, plaintext_key, encrypted_key + ) + return DefaultCryptoMaterialsManager(mock_keyring), plaintext_key + + +def _fill_materials(mats, plaintext_key, encrypted_key): + mats.plaintext_data_key = plaintext_key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=encrypted_key, + ) + return mats + + +# --------------------------------------------------------------------------- +# Content Encryption — General +# --------------------------------------------------------------------------- + +class TestContentEncryption: + """Tests for specification/s3-encryption/encryption.md#content-encryption.""" + + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=test + ##% The S3EC MUST use the encryption algorithm configured during + ##% [client](./client.md) initialization. + def test_uses_configured_algorithm_suite(self): + """The pipeline MUST encrypt using the algorithm suite passed to encrypt().""" + cmm, key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + plaintext = b"test data" + + # V2 (GCM no KDF) + _, meta_v2 = pipeline.encrypt( + plaintext, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + assert "x-amz-cek-alg" in meta_v2 + assert meta_v2["x-amz-cek-alg"] == "AES/GCM/NoPadding" + + # V3 (KC GCM) + _, meta_v3 = pipeline.encrypt( + plaintext, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + assert "x-amz-c" in meta_v3 + assert meta_v3["x-amz-c"] == "115" + + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=test + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + def test_iv_generated_with_correct_length_gcm(self): + """GCM encryption MUST produce a 12-byte IV.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + + _, meta = pipeline.encrypt( + b"test", + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + iv_bytes = base64.b64decode(meta["x-amz-iv"]) + assert len(iv_bytes) == 12 + + def test_message_id_generated_with_correct_length_kc(self): + """KC-GCM encryption MUST produce a 28-byte Message ID.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + + _, meta = pipeline.encrypt( + b"test", + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + message_id_bytes = base64.b64decode(meta["x-amz-i"]) + assert len(message_id_bytes) == MESSAGE_ID_LENGTH + + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=test + ##% The generated IV or Message ID MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + def test_iv_included_in_metadata_gcm(self): + """GCM encryption MUST include the IV in the returned metadata.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + + _, meta = pipeline.encrypt( + b"test", + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + assert "x-amz-iv" in meta + + def test_message_id_included_in_metadata_kc(self): + """KC-GCM encryption MUST include the Message ID in the returned metadata.""" + cmm, _ = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + + _, meta = pipeline.encrypt( + b"test", + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + assert "x-amz-i" in meta + + +# --------------------------------------------------------------------------- +# ALG_AES_256_GCM_IV12_TAG16_NO_KDF +# --------------------------------------------------------------------------- + +class TestGcmNoKdf: + """Tests for specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf.""" + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=test + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, + ##% with the plaintext data key, the generated IV, and the tag length defined + ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf + ##= type=test + ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + def test_gcm_encrypt_decrypt_roundtrip_no_aad(self): + """GCM encryption MUST use the data key, generated IV, and no AAD.""" + cmm, key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + plaintext = b"roundtrip test for GCM no KDF" + + ciphertext, meta = pipeline.encrypt( + plaintext, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + + # Decrypt with the same key, IV, and no AAD + iv = base64.b64decode(meta["x-amz-iv"]) + aesgcm = AESGCM(key) + decrypted = aesgcm.decrypt(nonce=iv, data=ciphertext, associated_data=None) + assert decrypted == plaintext + + def test_gcm_decrypt_fails_with_aad(self): + """Ciphertext produced with no AAD MUST NOT decrypt with AAD.""" + cmm, key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + + ciphertext, meta = pipeline.encrypt( + b"test", + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + + iv = base64.b64decode(meta["x-amz-iv"]) + aesgcm = AESGCM(key) + with pytest.raises(Exception): + aesgcm.decrypt(nonce=iv, data=ciphertext, associated_data=b"unexpected-aad") + + +# --------------------------------------------------------------------------- +# ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +# --------------------------------------------------------------------------- + +class TestKcGcm: + """Tests for specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key.""" + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=test + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key as described in [Key Derivation](key-derivation.md). + def test_kc_gcm_uses_hkdf_derived_key(self): + """KC-GCM encryption MUST use HKDF-derived keys, not the raw data key.""" + cmm, raw_key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + plaintext = b"roundtrip test for KC GCM" + + ciphertext, meta = pipeline.encrypt( + plaintext, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + + message_id = base64.b64decode(meta["x-amz-i"]) + derived_key, _ = derive_keys(raw_key, message_id) + + # Decrypt with the HKDF-derived key, fixed IV, and suite ID as AAD + aesgcm = AESGCM(derived_key) + decrypted = aesgcm.decrypt( + nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES + ) + assert decrypted == plaintext + + # Decrypting with the raw key must fail + aesgcm_raw = AESGCM(raw_key) + with pytest.raises(Exception): + aesgcm_raw.decrypt( + nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES + ) + + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=test + ##% The derived key commitment value MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + def test_kc_gcm_commitment_in_metadata(self): + """KC-GCM encryption MUST include the key commitment in metadata.""" + cmm, raw_key = _mock_cmm() + pipeline = PutEncryptedObjectPipeline(cmm) + + _, meta = pipeline.encrypt( + b"test", + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + + assert "x-amz-d" in meta + commitment_bytes = base64.b64decode(meta["x-amz-d"]) + + # Verify the commitment matches what HKDF would produce + message_id = base64.b64decode(meta["x-amz-i"]) + _, expected_commitment = derive_keys(raw_key, message_id) + assert commitment_bytes == expected_commitment diff --git a/test/test_key_commitment.py b/test/test_key_commitment.py new file mode 100644 index 00000000..6ddc953d --- /dev/null +++ b/test/test_key_commitment.py @@ -0,0 +1,149 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for key commitment policy specification compliance annotations. + +Each test in this module corresponds to a MUST/SHOULD requirement from +specification/s3-encryption/key-commitment.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import base64 +import os +from io import BytesIO +from unittest.mock import Mock + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.key_derivation import KC_GCM_IV, SUITE_ID_BYTES, derive_keys +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, +) +from s3_encryption.pipelines import GetEncryptedObjectPipeline + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_pipeline(commitment_policy, keyring_return=None): + """Create a GetEncryptedObjectPipeline with a mocked CMM/keyring.""" + mock_keyring = Mock(spec=S3Keyring) + if keyring_return is not None: + mock_keyring.on_decrypt.return_value = keyring_return + cmm = DefaultCryptoMaterialsManager(mock_keyring) + return GetEncryptedObjectPipeline( + cmm, + commitment_policy=commitment_policy, + enable_legacy_unauthenticated_modes=True, + ) + + +def _v2_gcm_response(key, plaintext=b"test data"): + """Create a V2 GCM-encrypted response with real ciphertext.""" + iv = os.urandom(12) + ciphertext = AESGCM(key).encrypt(iv, plaintext, None) + metadata = { + "x-amz-iv": base64.b64encode(iv).decode(), + "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": "{}", + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + dec_mats = DecryptionMaterials( + iv=iv, + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ) + return {"Body": BytesIO(ciphertext), "Metadata": metadata}, dec_mats, plaintext + + +def _v3_kc_gcm_response(key, plaintext=b"test data"): + """Create a V3 KC-GCM-encrypted response with real ciphertext.""" + message_id = os.urandom(28) + derived_key, commitment = derive_keys(key, message_id) + ciphertext = AESGCM(derived_key).encrypt(KC_GCM_IV, plaintext, SUITE_ID_BYTES) + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key").decode(), + "x-amz-w": "12", + "x-amz-t": "{}", + "x-amz-d": base64.b64encode(commitment).decode(), + "x-amz-i": base64.b64encode(message_id).decode(), + } + dec_mats = DecryptionMaterials( + plaintext_data_key=key, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ) + return {"Body": BytesIO(ciphertext), "Metadata": metadata}, dec_mats, plaintext + + +# --------------------------------------------------------------------------- +# Commitment Policy Tests +# --------------------------------------------------------------------------- + +class TestCommitmentPolicy: + """Tests for specification/s3-encryption/key-commitment.md#commitment-policy.""" + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + def test_forbid_encrypt_allows_non_committing_decrypt(self): + """FORBID_ENCRYPT_ALLOW_DECRYPT MUST allow decryption with non-committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v2_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response) + assert result == plaintext + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST allow decryption using algorithm suites which do not support key commitment. + def test_require_encrypt_allow_decrypt_allows_non_committing_decrypt(self): + """REQUIRE_ENCRYPT_ALLOW_DECRYPT MUST allow decryption with non-committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v2_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response) + assert result == plaintext + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. + def test_require_require_rejects_non_committing_decrypt(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST reject non-committing algorithm suites.""" + key = os.urandom(32) + response, dec_mats, _ = _v2_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + with pytest.raises(S3EncryptionClientError, match="cannot decrypt non-key-committing"): + pipeline.decrypt(response) + + def test_require_require_allows_committing_decrypt(self): + """REQUIRE_ENCRYPT_REQUIRE_DECRYPT MUST allow decryption with committing suites.""" + key = os.urandom(32) + response, dec_mats, plaintext = _v3_kc_gcm_response(key) + + pipeline = _make_pipeline( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + keyring_return=dec_mats, + ) + result = pipeline.decrypt(response) + assert result == plaintext diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py new file mode 100644 index 00000000..cfaa724a --- /dev/null +++ b/test/test_key_derivation.py @@ -0,0 +1,258 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for key derivation specification compliance annotations. + +Each test in this module corresponds to a MUST requirement from +specification/s3-encryption/key-derivation.md and carries a type=test annotation +that mirrors the type=implementation annotation in the source code. +""" + +import hmac + +import pytest +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.hashes import SHA512 +from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand + +from s3_encryption.key_derivation import ( + COMMIT_KEY_LENGTH, + ENCRYPTION_KEY_LENGTH, + KC_GCM_IV, + MESSAGE_ID_LENGTH, + SUITE_ID_BYTES, + derive_keys, +) + + +# --------------------------------------------------------------------------- +# HKDF Extract / Expand +# --------------------------------------------------------------------------- + +class TestHkdfOperation: + """Tests for specification/s3-encryption/key-derivation.md#hkdf-operation.""" + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The hash function MUST be specified by the algorithm suite commitment settings. + def test_hash_function_is_sha512(self): + """HKDF extract MUST use HMAC-SHA512 as the hash function.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + # Manual HMAC-SHA512 extract + prk = hmac.new(msg_id, pdk, "sha512").digest() + assert len(prk) == 64 # SHA-512 output + + # derive_keys should produce deterministic output consistent with SHA-512 + key1, ck1 = derive_keys(pdk, msg_id) + key2, ck2 = derive_keys(pdk, msg_id) + assert key1 == key2 + assert ck1 == ck2 + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + def test_ikm_is_plaintext_data_key(self): + """Different plaintext data keys MUST produce different derived keys.""" + import os + msg_id = os.urandom(MESSAGE_ID_LENGTH) + pdk_a = os.urandom(32) + pdk_b = os.urandom(32) + + key_a, _ = derive_keys(pdk_a, msg_id) + key_b, _ = derive_keys(pdk_b, msg_id) + assert key_a != key_b + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + def test_ikm_length_is_32_bytes(self): + """The plaintext data key (IKM) MUST be 32 bytes for AES-256.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + assert len(pdk) == 32 + # Should succeed with 32-byte key + key, ck = derive_keys(pdk, msg_id) + assert len(key) == ENCRYPTION_KEY_LENGTH + assert len(ck) == COMMIT_KEY_LENGTH + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. + def test_salt_is_message_id(self): + """Different Message IDs (salts) MUST produce different derived keys.""" + import os + pdk = os.urandom(32) + msg_id_a = os.urandom(MESSAGE_ID_LENGTH) + msg_id_b = os.urandom(MESSAGE_ID_LENGTH) + + key_a, _ = derive_keys(pdk, msg_id_a) + key_b, _ = derive_keys(pdk, msg_id_b) + assert key_a != key_b + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The DEK input pseudorandom key MUST be the output from the extract step. + def test_dek_uses_prk_from_extract(self): + """The DEK expand step MUST use the PRK from the extract step.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + # Manual extract + prk = hmac.new(msg_id, pdk, "sha512").digest() + # Manual expand for DEK + expected_dek = HKDFExpand( + algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, + info=SUITE_ID_BYTES + b"DERIVEKEY", + ).derive(prk) + + actual_dek, _ = derive_keys(pdk, msg_id) + assert actual_dek == expected_dek + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + def test_dek_output_length(self): + """The derived encryption key MUST be 32 bytes (256 bits).""" + import os + key, _ = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH)) + assert len(key) == ENCRYPTION_KEY_LENGTH + assert ENCRYPTION_KEY_LENGTH == 32 + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. + def test_dek_info_is_suite_id_plus_derivekey(self): + """DEK expand info MUST be suite_id_bytes + b'DERIVEKEY'.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + prk = hmac.new(msg_id, pdk, "sha512").digest() + + # Correct info + correct_dek = HKDFExpand( + algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, + info=SUITE_ID_BYTES + b"DERIVEKEY", + ).derive(prk) + + # Wrong info should produce different output + wrong_dek = HKDFExpand( + algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, + info=SUITE_ID_BYTES + b"WRONGKEY", + ).derive(prk) + + actual_dek, _ = derive_keys(pdk, msg_id) + assert actual_dek == correct_dek + assert actual_dek != wrong_dek + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The CK input pseudorandom key MUST be the output from the extract step. + def test_ck_uses_prk_from_extract(self): + """The CK expand step MUST use the PRK from the extract step.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + prk = hmac.new(msg_id, pdk, "sha512").digest() + expected_ck = HKDFExpand( + algorithm=SHA512(), length=COMMIT_KEY_LENGTH, + info=SUITE_ID_BYTES + b"COMMITKEY", + ).derive(prk) + + _, actual_ck = derive_keys(pdk, msg_id) + assert actual_ck == expected_ck + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + def test_ck_output_length(self): + """The commit key MUST be 28 bytes (224 bits).""" + import os + _, ck = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH)) + assert len(ck) == COMMIT_KEY_LENGTH + assert COMMIT_KEY_LENGTH == 28 + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. + def test_ck_info_is_suite_id_plus_commitkey(self): + """CK expand info MUST be suite_id_bytes + b'COMMITKEY'.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + + prk = hmac.new(msg_id, pdk, "sha512").digest() + + correct_ck = HKDFExpand( + algorithm=SHA512(), length=COMMIT_KEY_LENGTH, + info=SUITE_ID_BYTES + b"COMMITKEY", + ).derive(prk) + + wrong_ck = HKDFExpand( + algorithm=SHA512(), length=COMMIT_KEY_LENGTH, + info=SUITE_ID_BYTES + b"WRONGKEY", + ).derive(prk) + + _, actual_ck = derive_keys(pdk, msg_id) + assert actual_ck == correct_ck + assert actual_ck != wrong_ck + + +# --------------------------------------------------------------------------- +# IV and AAD for KC-GCM +# --------------------------------------------------------------------------- + +class TestKcGcmCipherParams: + """Tests for KC-GCM cipher initialization parameters.""" + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + def test_kc_gcm_iv_is_all_0x01(self): + """The KC-GCM IV MUST consist entirely of 0x01 bytes.""" + assert all(b == 0x01 for b in KC_GCM_IV) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + def test_kc_gcm_iv_length_is_12(self): + """The KC-GCM IV MUST be 12 bytes (AES-GCM standard nonce length).""" + assert len(KC_GCM_IV) == 12 + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, + ##% and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): + """KC-GCM MUST encrypt/decrypt with derived key, 0x01 IV, and suite ID as AAD.""" + import os + pdk = os.urandom(32) + msg_id = os.urandom(MESSAGE_ID_LENGTH) + plaintext = b"key derivation roundtrip test" + + derived_key, _ = derive_keys(pdk, msg_id) + + # Encrypt with derived key, KC_GCM_IV, and SUITE_ID_BYTES as AAD + aesgcm = AESGCM(derived_key) + ciphertext = aesgcm.encrypt(KC_GCM_IV, plaintext, SUITE_ID_BYTES) + + # Decrypt with same parameters + decrypted = aesgcm.decrypt(KC_GCM_IV, ciphertext, SUITE_ID_BYTES) + assert decrypted == plaintext + + # Decrypting with wrong AAD must fail + with pytest.raises(Exception): + aesgcm.decrypt(KC_GCM_IV, ciphertext, b"\x00\x00") + + # Decrypting with wrong IV must fail + with pytest.raises(Exception): + aesgcm.decrypt(b"\x00" * 12, ciphertext, SUITE_ID_BYTES) From aab46cb025f7327e513b0d81dfda0d072b414a4f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 21:35:35 -0800 Subject: [PATCH 17/32] add TestServer test that fails when encrypt validation is missing --- ...eyCommitmentPolicyEncryptFailureTests.java | 106 ++++++++++++++++++ .../amazon/encryption/s3/TestUtils.java | 37 ++++++ 2 files changed, 143 insertions(+) create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java new file mode 100644 index 00000000..6a444815 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java @@ -0,0 +1,106 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * Key Commitment Policy — Encryption Failure Tests + * + * Per the specification (key-commitment.md): + * "When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, + * the S3EC MUST only encrypt using an algorithm suite which supports key commitment." + * "When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + * the S3EC MUST only encrypt using an algorithm suite which supports key commitment." + * + * These tests verify that attempting to encrypt with a non-committing algorithm + * (ALG_AES_256_GCM_IV12_TAG16_NO_KDF) while using a REQUIRE_ENCRYPT_* commitment + * policy is rejected by the S3EC — either at client creation or at PutObject time. + * + * Similarly, per the specification: + * "When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, + * the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment." + * + * These tests also verify that attempting to encrypt with a committing algorithm + * (ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) while using FORBID_ENCRYPT_ALLOW_DECRYPT + * is rejected. + */ +@DisplayName("Key Commitment Policy — Encrypt Failures") +public class KeyCommitmentPolicyEncryptFailureTests { + + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // ========================================================================= + // REQUIRE_ENCRYPT_ALLOW_DECRYPT + non-committing GCM → MUST fail + // ========================================================================= + + @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_ALLOW_DECRYPT with non-committing GCM MUST fail to encrypt") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build(); + + TestUtils.Encrypt_fails(client, config, + appendTestSuffix("test-kc-policy-fail-REAC-gcm-" + language.getLanguageName())); + } + + // ========================================================================= + // REQUIRE_ENCRYPT_REQUIRE_DECRYPT + non-committing GCM → MUST fail + // ========================================================================= + + @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_REQUIRE_DECRYPT with non-committing GCM MUST fail to encrypt") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build(); + + TestUtils.Encrypt_fails(client, config, + appendTestSuffix("test-kc-policy-fail-RERD-gcm-" + language.getLanguageName())); + } + + // ========================================================================= + // FORBID_ENCRYPT_ALLOW_DECRYPT + committing GCM → MUST fail + // ========================================================================= + + @ParameterizedTest(name = "{0}: FORBID_ENCRYPT_ALLOW_DECRYPT with committing GCM MUST fail to encrypt") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .build(); + + TestUtils.Encrypt_fails(client, config, + appendTestSuffix("test-kc-policy-fail-FEAD-kc-gcm-" + language.getLanguageName())); + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index c3c804aa..34ac0dcb 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -35,12 +35,15 @@ import org.junit.jupiter.params.provider.Arguments; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; import software.amazon.encryption.s3.model.EncryptionAlgorithm; import software.amazon.encryption.s3.model.GetObjectInput; import software.amazon.encryption.s3.model.GetObjectOutput; import software.amazon.encryption.s3.model.KeyMaterial; import software.amazon.encryption.s3.model.PutObjectInput; import software.amazon.encryption.s3.model.PutObjectOutput; +import software.amazon.encryption.s3.model.S3ECConfig; import software.amazon.encryption.s3.model.S3EncryptionClientError; import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol; import software.amazon.smithy.java.client.core.ClientConfig; @@ -666,6 +669,40 @@ public static void DecryptWithMaterialsDescription( } } + /** + * Attempts to encrypt an object and expects the operation to fail with an S3EncryptionClientError. + * This is used for negative tests where the client configuration should prevent encryption + * (e.g., commitment policy violations). + * + * The failure may occur during client creation (CreateClient) or during the PutObject call, + * depending on when the server-side S3EC validates the configuration. + */ + public static void Encrypt_fails( + S3ECTestServerClient client, + S3ECConfig config, + String objectKey + ) { + try { + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(config) + .build()); + String S3ECId = clientOutput.getClientId(); + + client.putObject(PutObjectInput.builder() + .clientID(S3ECId) + .key(objectKey) + .bucket(TestUtils.BUCKET) + .body(ByteBuffer.wrap(objectKey.getBytes(StandardCharsets.UTF_8))) + .build()); + + fail("Encryption should have failed for object: " + objectKey + + " with config commitmentPolicy=" + config.getCommitmentPolicy() + + " encryptionAlgorithm=" + config.getEncryptionAlgorithm()); + } catch (S3EncryptionClientError e) { + // Expected - the S3EC should reject this configuration + } + } + public static void Decrypt_fails( S3ECTestServerClient client, String S3ECId, List crossLanguageObjects, From f7a37514b1fc9f60f6d3b5abba5b3ca808f7264a Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 22:31:07 -0800 Subject: [PATCH 18/32] investigate commit policy validation errors --- ...eyCommitmentPolicyEncryptFailureTests.java | 233 ++++++++++++++++-- 1 file changed, 215 insertions(+), 18 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java index 6a444815..61f8fba3 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java @@ -7,14 +7,23 @@ import static software.amazon.encryption.s3.TestUtils.*; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import software.amazon.encryption.s3.client.S3ECTestServerClient; import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.PutObjectInput; import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3EncryptionClientError; /** * Key Commitment Policy — Encryption Failure Tests @@ -24,18 +33,17 @@ * the S3EC MUST only encrypt using an algorithm suite which supports key commitment." * "When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, * the S3EC MUST only encrypt using an algorithm suite which supports key commitment." - * - * These tests verify that attempting to encrypt with a non-committing algorithm - * (ALG_AES_256_GCM_IV12_TAG16_NO_KDF) while using a REQUIRE_ENCRYPT_* commitment - * policy is rejected by the S3EC — either at client creation or at PutObject time. - * - * Similarly, per the specification: * "When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, * the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment." * - * These tests also verify that attempting to encrypt with a committing algorithm - * (ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) while using FORBID_ENCRYPT_ALLOW_DECRYPT - * is rejected. + * These tests verify that attempting to encrypt with an algorithm that conflicts + * with the commitment policy is rejected by the S3EC. + * + * The "experimental" tests go further: when a server does NOT reject the + * misconfiguration (i.e. the bug exists), they inspect the S3 object to + * determine what algorithm was actually used and whether the ciphertext + * can be decrypted. This helps characterize the severity of the bug + * across different language implementations. */ @DisplayName("Key Commitment Policy — Encrypt Failures") public class KeyCommitmentPolicyEncryptFailureTests { @@ -45,7 +53,7 @@ public class KeyCommitmentPolicyEncryptFailureTests { .build(); // ========================================================================= - // REQUIRE_ENCRYPT_ALLOW_DECRYPT + non-committing GCM → MUST fail + // Strict tests — these MUST fail for a compliant implementation // ========================================================================= @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_ALLOW_DECRYPT with non-committing GCM MUST fail to encrypt") @@ -64,10 +72,6 @@ void improved_require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( appendTestSuffix("test-kc-policy-fail-REAC-gcm-" + language.getLanguageName())); } - // ========================================================================= - // REQUIRE_ENCRYPT_REQUIRE_DECRYPT + non-committing GCM → MUST fail - // ========================================================================= - @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_REQUIRE_DECRYPT with non-committing GCM MUST fail to encrypt") @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") void improved_require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( @@ -84,10 +88,6 @@ void improved_require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( appendTestSuffix("test-kc-policy-fail-RERD-gcm-" + language.getLanguageName())); } - // ========================================================================= - // FORBID_ENCRYPT_ALLOW_DECRYPT + committing GCM → MUST fail - // ========================================================================= - @ParameterizedTest(name = "{0}: FORBID_ENCRYPT_ALLOW_DECRYPT with committing GCM MUST fail to encrypt") @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") void improved_forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( @@ -103,4 +103,201 @@ void improved_forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( TestUtils.Encrypt_fails(client, config, appendTestSuffix("test-kc-policy-fail-FEAD-kc-gcm-" + language.getLanguageName())); } + + // ========================================================================= + // Experimental diagnostic tests + // + // These tests probe what actually happens when a server does NOT reject + // the misconfigured commitment policy + algorithm combination. + // They always pass (they are diagnostic), but print detailed findings: + // - Did client creation or PutObject fail? (correct behavior) + // - If not, what algorithm suite was actually written to S3? + // - Can the object be decrypted by a permissive client? + // + // This helps us understand whether a buggy server: + // (a) silently ignores the requested algorithm and uses the policy-implied one, or + // (b) actually encrypts with the wrong algorithm, violating the policy. + // ========================================================================= + + /** + * Shared diagnostic logic for all three policy/algorithm mismatch scenarios. + * + * @param language the server under test + * @param policy the commitment policy to configure + * @param requestedAlgorithm the algorithm that conflicts with the policy + * @param label short label for log output + */ + private void diagnoseEncryptWithMismatchedPolicy( + TestUtils.LanguageServerTarget language, + CommitmentPolicy policy, + EncryptionAlgorithm requestedAlgorithm, + String label + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + String objectKey = appendTestSuffix("test-kc-diag-" + label + "-" + language.getLanguageName()); + String plaintext = objectKey; // convention: plaintext == object key + + S3ECConfig config = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(policy) + .encryptionAlgorithm(requestedAlgorithm) + .build(); + + // Phase 1: attempt the misconfigured encrypt + String s3ecId; + try { + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(config) + .build()); + s3ecId = clientOutput.getClientId(); + } catch (S3EncryptionClientError e) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — CORRECTLY rejected at CreateClient: " + e.getMessage()); + return; // correct behavior, nothing more to check + } + + try { + client.putObject(PutObjectInput.builder() + .clientID(s3ecId) + .key(objectKey) + .bucket(TestUtils.BUCKET) + .body(ByteBuffer.wrap(plaintext.getBytes(StandardCharsets.UTF_8))) + .build()); + } catch (S3EncryptionClientError e) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — CORRECTLY rejected at PutObject: " + e.getMessage()); + return; // correct behavior + } + + // If we get here, the server did NOT reject the misconfiguration — this is the bug. + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — BUG: encryption succeeded when it should have been rejected." + + " policy=" + policy + " requestedAlgorithm=" + requestedAlgorithm); + + // Phase 2: inspect what algorithm was actually written to S3 + EncryptionAlgorithm actualAlgorithm = null; + try { + actualAlgorithm = TestUtils.GetEncryptionAlgorithm(objectKey); + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Actual algorithm on S3 object: " + actualAlgorithm); + + if (actualAlgorithm.equals(requestedAlgorithm)) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Server used the REQUESTED (wrong) algorithm. The policy was fully ignored."); + } else { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Server used a DIFFERENT algorithm than requested." + + " It may have derived the algorithm from the policy instead."); + } + } catch (Exception e) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Could not determine actual algorithm: " + e.getMessage()); + } + + // Phase 3: attempt to decrypt with a permissive client + // Use FORBID_ENCRYPT_ALLOW_DECRYPT which allows decrypting both committing and non-committing + tryDecryptWithPermissiveClient(client, language, objectKey, plaintext, actualAlgorithm, label); + } + + /** + * Attempts to decrypt the object using a permissive client configuration + * (FORBID_ENCRYPT_ALLOW_DECRYPT allows decrypting any algorithm). + */ + private void tryDecryptWithPermissiveClient( + S3ECTestServerClient client, + TestUtils.LanguageServerTarget language, + String objectKey, + String expectedPlaintext, + EncryptionAlgorithm actualAlgorithm, + String label + ) { + // Build a permissive decrypt client — FORBID_ENCRYPT_ALLOW_DECRYPT allows + // decrypting both committing and non-committing ciphertexts. + // We also set the algorithm to match what was actually written, if known. + S3ECConfig.Builder decryptConfigBuilder = S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT); + + if (actualAlgorithm != null) { + decryptConfigBuilder.encryptionAlgorithm(actualAlgorithm); + } + + try { + CreateClientOutput decryptClientOutput = client.createClient(CreateClientInput.builder() + .config(decryptConfigBuilder.build()) + .build()); + String decryptId = decryptClientOutput.getClientId(); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(decryptId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + String decryptedText = new String(output.getBody().array(), StandardCharsets.UTF_8); + boolean plaintextMatches = expectedPlaintext.equals(decryptedText); + + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Decrypt with permissive client: SUCCESS" + + " | plaintext matches: " + plaintextMatches); + + if (!plaintextMatches) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Expected plaintext length=" + expectedPlaintext.length() + + " got length=" + decryptedText.length()); + } + } catch (S3EncryptionClientError e) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Decrypt with permissive client: FAILED (S3EncryptionClientError): " + e.getMessage()); + } catch (Exception e) { + System.out.println("[DIAG " + label + "] " + language.getLanguageName() + + " — Decrypt with permissive client: FAILED (" + e.getClass().getSimpleName() + "): " + + e.getMessage()); + } + } + + // --- Experimental: REQUIRE_ENCRYPT_ALLOW_DECRYPT + non-committing GCM --- + + @ParameterizedTest(name = "{0}: [EXPERIMENTAL] REQUIRE_ENCRYPT_ALLOW_DECRYPT + non-committing GCM — diagnose behavior") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void experimental_require_encrypt_allow_decrypt_with_non_committing_gcm( + TestUtils.LanguageServerTarget language + ) { + diagnoseEncryptWithMismatchedPolicy( + language, + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "REAC+GCM" + ); + } + + // --- Experimental: REQUIRE_ENCRYPT_REQUIRE_DECRYPT + non-committing GCM --- + + @ParameterizedTest(name = "{0}: [EXPERIMENTAL] REQUIRE_ENCRYPT_REQUIRE_DECRYPT + non-committing GCM — diagnose behavior") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void experimental_require_encrypt_require_decrypt_with_non_committing_gcm( + TestUtils.LanguageServerTarget language + ) { + diagnoseEncryptWithMismatchedPolicy( + language, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "RERD+GCM" + ); + } + + // --- Experimental: FORBID_ENCRYPT_ALLOW_DECRYPT + committing GCM --- + + @ParameterizedTest(name = "{0}: [EXPERIMENTAL] FORBID_ENCRYPT_ALLOW_DECRYPT + committing GCM — diagnose behavior") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void experimental_forbid_encrypt_allow_decrypt_with_committing_gcm( + TestUtils.LanguageServerTarget language + ) { + diagnoseEncryptWithMismatchedPolicy( + language, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + "FEAD+KC_GCM" + ); + } } From e1ab926742a43dc700e971e079cf2255b7f95d71 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 23:23:49 -0800 Subject: [PATCH 19/32] add validation, only test Python in TestServer, add integ tests --- .../key_commitment_exceptions.md | 39 --- src/s3_encryption/__init__.py | 28 +++ src/s3_encryption/materials/materials.py | 5 + ...eyCommitmentPolicyEncryptFailureTests.java | 232 +----------------- .../amazon/encryption/s3/TestUtils.java | 11 + test/test_key_commitment_encrypt.py | 108 ++++++++ 6 files changed, 162 insertions(+), 261 deletions(-) delete mode 100644 compliance_exceptions/key_commitment_exceptions.md create mode 100644 test/test_key_commitment_encrypt.py diff --git a/compliance_exceptions/key_commitment_exceptions.md b/compliance_exceptions/key_commitment_exceptions.md deleted file mode 100644 index 104330ef..00000000 --- a/compliance_exceptions/key_commitment_exceptions.md +++ /dev/null @@ -1,39 +0,0 @@ -# Compliance Exceptions for Key Commitment Policy — Encryption Side - -## Summary - -The Python S3 Encryption Client does not yet explicitly validate the commitment policy -against the configured algorithm suite on the encryption path. The client defaults to the -key-committing algorithm suite (`ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY`) and validates -that legacy (CBC) suites cannot be configured, but does not enforce the full matrix of -commitment policy vs. algorithm suite at encryption time. - -## FORBID_ENCRYPT_ALLOW_DECRYPT — Encrypt Restriction - -##= specification/s3-encryption/key-commitment.md#commitment-policy -##= type=exception -##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. - -Justification: The encryption path does not validate the commitment policy against the algorithm suite. A caller who configures `FORBID_ENCRYPT_ALLOW_DECRYPT` but leaves the default committing algorithm suite would incorrectly encrypt with a committing suite. This validation is planned for a future release. - ---- - -## REQUIRE_ENCRYPT_ALLOW_DECRYPT — Encrypt Restriction - -##= specification/s3-encryption/key-commitment.md#commitment-policy -##= type=exception -##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. - -Justification: The encryption path does not explicitly validate that the algorithm suite supports key commitment when the policy is `REQUIRE_ENCRYPT_ALLOW_DECRYPT`. In practice, the default algorithm suite is the committing suite, so this is satisfied by default. However, there is no guard preventing a caller from overriding the algorithm suite to a non-committing one. This validation is planned for a future release. - ---- - -## REQUIRE_ENCRYPT_REQUIRE_DECRYPT — Encrypt Restriction - -##= specification/s3-encryption/key-commitment.md#commitment-policy -##= type=exception -##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. - -Justification: Same as above. The default algorithm suite is the committing suite, so this is satisfied by default, but there is no explicit validation preventing a non-committing suite from being configured. This validation is planned for a future release. - ---- diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 0604a209..0fdb8fad 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -68,6 +68,10 @@ def _default_cmm_for_keyring(self): ##% The S3EC MUST validate that the configured encryption algorithm is not legacy. ##= specification/s3-encryption/client.md#encryption-algorithm ##% If the configured encryption algorithm is legacy, then the S3EC MUST throw an exception. + ##= specification/s3-encryption/client.md#key-commitment + ##% The S3EC MUST validate the configured Encryption Algorithm against the provided key commitment policy. + ##= specification/s3-encryption/client.md#key-commitment + ##% If the configured Encryption Algorithm is incompatible with the key commitment policy, then it MUST throw an exception. def __attrs_post_init__(self): if self.algorithm_suite.is_legacy: raise S3EncryptionClientError( @@ -76,6 +80,30 @@ def __attrs_post_init__(self): f"supported for decryption (and enable_legacy_unauthenticated_modes is True)." ) + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + if self.commitment_policy in ( + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) and not self.algorithm_suite.supports_key_commitment: + raise S3EncryptionClientError( + f"Commitment policy {self.commitment_policy.name} requires a key-committing " + f"algorithm suite, but {self.algorithm_suite.name} does not support key commitment." + ) + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. + if ( + self.commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT + and self.algorithm_suite.supports_key_commitment + ): + raise S3EncryptionClientError( + f"Commitment policy {self.commitment_policy.name} forbids key-committing " + f"algorithm suites, but {self.algorithm_suite.name} supports key commitment." + ) + class S3EncryptionClientPlugin: """Plugin that adds encryption/decryption capabilities to a boto3 S3 client. diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 39c6add6..f0314e00 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -27,6 +27,11 @@ def is_legacy(self) -> bool: """Return True if this algorithm suite is a legacy unauthenticated mode.""" return self == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + @property + def supports_key_commitment(self) -> bool: + """Return True if this algorithm suite supports key commitment.""" + return self == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + class CommitmentPolicy(Enum): """Commitment policies controlling key-commitment behavior.""" diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java index 61f8fba3..46df7de1 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KeyCommitmentPolicyEncryptFailureTests.java @@ -7,23 +7,14 @@ import static software.amazon.encryption.s3.TestUtils.*; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import software.amazon.encryption.s3.client.S3ECTestServerClient; import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.GetObjectInput; -import software.amazon.encryption.s3.model.GetObjectOutput; import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.PutObjectInput; import software.amazon.encryption.s3.model.S3ECConfig; -import software.amazon.encryption.s3.model.S3EncryptionClientError; /** * Key Commitment Policy — Encryption Failure Tests @@ -37,13 +28,11 @@ * the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment." * * These tests verify that attempting to encrypt with an algorithm that conflicts - * with the commitment policy is rejected by the S3EC. + * with the commitment policy is rejected by the S3EC — either at client creation + * or at PutObject time. * - * The "experimental" tests go further: when a server does NOT reject the - * misconfiguration (i.e. the bug exists), they inspect the S3 object to - * determine what algorithm was actually used and whether the ciphertext - * can be decrypted. This helps characterize the severity of the bug - * across different language implementations. + * Currently scoped to Python V3 only. Other languages can be enabled by + * switching the MethodSource to a broader provider (e.g. improvedClientsForTest). */ @DisplayName("Key Commitment Policy — Encrypt Failures") public class KeyCommitmentPolicyEncryptFailureTests { @@ -52,13 +41,9 @@ public class KeyCommitmentPolicyEncryptFailureTests { .kmsKeyId(TestUtils.KMS_KEY_ARN) .build(); - // ========================================================================= - // Strict tests — these MUST fail for a compliant implementation - // ========================================================================= - @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_ALLOW_DECRYPT with non-committing GCM MUST fail to encrypt") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + void require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( TestUtils.LanguageServerTarget language ) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); @@ -73,8 +58,8 @@ void improved_require_encrypt_allow_decrypt_with_non_committing_gcm_must_fail( } @ParameterizedTest(name = "{0}: REQUIRE_ENCRYPT_REQUIRE_DECRYPT with non-committing GCM MUST fail to encrypt") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + void require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( TestUtils.LanguageServerTarget language ) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); @@ -89,8 +74,8 @@ void improved_require_encrypt_require_decrypt_with_non_committing_gcm_must_fail( } @ParameterizedTest(name = "{0}: FORBID_ENCRYPT_ALLOW_DECRYPT with committing GCM MUST fail to encrypt") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( + @MethodSource("software.amazon.encryption.s3.TestUtils#pythonV3ClientForTest") + void forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( TestUtils.LanguageServerTarget language ) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); @@ -103,201 +88,4 @@ void improved_forbid_encrypt_allow_decrypt_with_committing_gcm_must_fail( TestUtils.Encrypt_fails(client, config, appendTestSuffix("test-kc-policy-fail-FEAD-kc-gcm-" + language.getLanguageName())); } - - // ========================================================================= - // Experimental diagnostic tests - // - // These tests probe what actually happens when a server does NOT reject - // the misconfigured commitment policy + algorithm combination. - // They always pass (they are diagnostic), but print detailed findings: - // - Did client creation or PutObject fail? (correct behavior) - // - If not, what algorithm suite was actually written to S3? - // - Can the object be decrypted by a permissive client? - // - // This helps us understand whether a buggy server: - // (a) silently ignores the requested algorithm and uses the policy-implied one, or - // (b) actually encrypts with the wrong algorithm, violating the policy. - // ========================================================================= - - /** - * Shared diagnostic logic for all three policy/algorithm mismatch scenarios. - * - * @param language the server under test - * @param policy the commitment policy to configure - * @param requestedAlgorithm the algorithm that conflicts with the policy - * @param label short label for log output - */ - private void diagnoseEncryptWithMismatchedPolicy( - TestUtils.LanguageServerTarget language, - CommitmentPolicy policy, - EncryptionAlgorithm requestedAlgorithm, - String label - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - String objectKey = appendTestSuffix("test-kc-diag-" + label + "-" + language.getLanguageName()); - String plaintext = objectKey; // convention: plaintext == object key - - S3ECConfig config = S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(policy) - .encryptionAlgorithm(requestedAlgorithm) - .build(); - - // Phase 1: attempt the misconfigured encrypt - String s3ecId; - try { - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(config) - .build()); - s3ecId = clientOutput.getClientId(); - } catch (S3EncryptionClientError e) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — CORRECTLY rejected at CreateClient: " + e.getMessage()); - return; // correct behavior, nothing more to check - } - - try { - client.putObject(PutObjectInput.builder() - .clientID(s3ecId) - .key(objectKey) - .bucket(TestUtils.BUCKET) - .body(ByteBuffer.wrap(plaintext.getBytes(StandardCharsets.UTF_8))) - .build()); - } catch (S3EncryptionClientError e) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — CORRECTLY rejected at PutObject: " + e.getMessage()); - return; // correct behavior - } - - // If we get here, the server did NOT reject the misconfiguration — this is the bug. - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — BUG: encryption succeeded when it should have been rejected." - + " policy=" + policy + " requestedAlgorithm=" + requestedAlgorithm); - - // Phase 2: inspect what algorithm was actually written to S3 - EncryptionAlgorithm actualAlgorithm = null; - try { - actualAlgorithm = TestUtils.GetEncryptionAlgorithm(objectKey); - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Actual algorithm on S3 object: " + actualAlgorithm); - - if (actualAlgorithm.equals(requestedAlgorithm)) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Server used the REQUESTED (wrong) algorithm. The policy was fully ignored."); - } else { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Server used a DIFFERENT algorithm than requested." - + " It may have derived the algorithm from the policy instead."); - } - } catch (Exception e) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Could not determine actual algorithm: " + e.getMessage()); - } - - // Phase 3: attempt to decrypt with a permissive client - // Use FORBID_ENCRYPT_ALLOW_DECRYPT which allows decrypting both committing and non-committing - tryDecryptWithPermissiveClient(client, language, objectKey, plaintext, actualAlgorithm, label); - } - - /** - * Attempts to decrypt the object using a permissive client configuration - * (FORBID_ENCRYPT_ALLOW_DECRYPT allows decrypting any algorithm). - */ - private void tryDecryptWithPermissiveClient( - S3ECTestServerClient client, - TestUtils.LanguageServerTarget language, - String objectKey, - String expectedPlaintext, - EncryptionAlgorithm actualAlgorithm, - String label - ) { - // Build a permissive decrypt client — FORBID_ENCRYPT_ALLOW_DECRYPT allows - // decrypting both committing and non-committing ciphertexts. - // We also set the algorithm to match what was actually written, if known. - S3ECConfig.Builder decryptConfigBuilder = S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT); - - if (actualAlgorithm != null) { - decryptConfigBuilder.encryptionAlgorithm(actualAlgorithm); - } - - try { - CreateClientOutput decryptClientOutput = client.createClient(CreateClientInput.builder() - .config(decryptConfigBuilder.build()) - .build()); - String decryptId = decryptClientOutput.getClientId(); - - GetObjectOutput output = client.getObject(GetObjectInput.builder() - .clientID(decryptId) - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); - - String decryptedText = new String(output.getBody().array(), StandardCharsets.UTF_8); - boolean plaintextMatches = expectedPlaintext.equals(decryptedText); - - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Decrypt with permissive client: SUCCESS" - + " | plaintext matches: " + plaintextMatches); - - if (!plaintextMatches) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Expected plaintext length=" + expectedPlaintext.length() - + " got length=" + decryptedText.length()); - } - } catch (S3EncryptionClientError e) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Decrypt with permissive client: FAILED (S3EncryptionClientError): " + e.getMessage()); - } catch (Exception e) { - System.out.println("[DIAG " + label + "] " + language.getLanguageName() - + " — Decrypt with permissive client: FAILED (" + e.getClass().getSimpleName() + "): " - + e.getMessage()); - } - } - - // --- Experimental: REQUIRE_ENCRYPT_ALLOW_DECRYPT + non-committing GCM --- - - @ParameterizedTest(name = "{0}: [EXPERIMENTAL] REQUIRE_ENCRYPT_ALLOW_DECRYPT + non-committing GCM — diagnose behavior") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void experimental_require_encrypt_allow_decrypt_with_non_committing_gcm( - TestUtils.LanguageServerTarget language - ) { - diagnoseEncryptWithMismatchedPolicy( - language, - CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - "REAC+GCM" - ); - } - - // --- Experimental: REQUIRE_ENCRYPT_REQUIRE_DECRYPT + non-committing GCM --- - - @ParameterizedTest(name = "{0}: [EXPERIMENTAL] REQUIRE_ENCRYPT_REQUIRE_DECRYPT + non-committing GCM — diagnose behavior") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void experimental_require_encrypt_require_decrypt_with_non_committing_gcm( - TestUtils.LanguageServerTarget language - ) { - diagnoseEncryptWithMismatchedPolicy( - language, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - "RERD+GCM" - ); - } - - // --- Experimental: FORBID_ENCRYPT_ALLOW_DECRYPT + committing GCM --- - - @ParameterizedTest(name = "{0}: [EXPERIMENTAL] FORBID_ENCRYPT_ALLOW_DECRYPT + committing GCM — diagnose behavior") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void experimental_forbid_encrypt_allow_decrypt_with_committing_gcm( - TestUtils.LanguageServerTarget language - ) { - diagnoseEncryptWithMismatchedPolicy( - language, - CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - "FEAD+KC_GCM" - ); - } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 34ac0dcb..d488fd2d 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -361,6 +361,17 @@ public static Stream improvedClientsForTest() { .map(Arguments::of); } + /** + * Get stream of arguments for the Python V3 client only. + * Other languages can be added to this set as their commitment policy + * validation is confirmed. + */ + public static Stream pythonV3ClientForTest() { + return serverMap.values().stream() + .filter(target -> PYTHON_V3.equals(target.getLanguageName())) + .map(Arguments::of); + } + /** * Get stream of arguments for clients that support RAW AES (includes CPP). */ diff --git a/test/test_key_commitment_encrypt.py b/test/test_key_commitment_encrypt.py new file mode 100644 index 00000000..e343927c --- /dev/null +++ b/test/test_key_commitment_encrypt.py @@ -0,0 +1,108 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for key commitment policy enforcement on the encryption path. + +Per specification/s3-encryption/key-commitment.md#commitment-policy: + - REQUIRE_ENCRYPT_ALLOW_DECRYPT: the S3EC MUST only encrypt using an + algorithm suite which supports key commitment. + - REQUIRE_ENCRYPT_REQUIRE_DECRYPT: the S3EC MUST only encrypt using an + algorithm suite which supports key commitment. + - FORBID_ENCRYPT_ALLOW_DECRYPT: the S3EC MUST NOT encrypt using an + algorithm suite which supports key commitment. + +Per specification/s3-encryption/client.md#key-commitment: + - The S3EC MUST validate the configured Encryption Algorithm against the + provided key commitment policy. + - If the configured Encryption Algorithm is incompatible with the key + commitment policy, then it MUST throw an exception. + +These tests verify that the S3EC rejects mismatched commitment policy and +algorithm suite configurations. The rejection may occur at client config +creation time or at encrypt time. +""" + +import os +from unittest.mock import MagicMock + +import pytest + +from s3_encryption import S3EncryptionClientConfig +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy +from s3_encryption.pipelines import PutEncryptedObjectPipeline + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _mock_keyring(): + """Return a mock keyring that populates encryption materials.""" + key = os.urandom(32) + mock = MagicMock() + + def on_encrypt(mats): + mats.plaintext_data_key = key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + return mats + + mock.on_encrypt.side_effect = on_encrypt + return mock + + +# --------------------------------------------------------------------------- +# REQUIRE_ENCRYPT_* with non-committing algorithm → MUST fail +# --------------------------------------------------------------------------- + +class TestRequireEncryptRejectsNonCommitting: + """Configuring REQUIRE_ENCRYPT_* with a non-committing algorithm MUST fail.""" + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + def test_require_encrypt_allow_decrypt_rejects_non_committing_gcm(self): + keyring = _mock_keyring() + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring=keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + ) + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. + def test_require_encrypt_require_decrypt_rejects_non_committing_gcm(self): + keyring = _mock_keyring() + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring=keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + + +# --------------------------------------------------------------------------- +# FORBID_ENCRYPT_ALLOW_DECRYPT with committing algorithm → MUST fail +# --------------------------------------------------------------------------- + +class TestForbidEncryptRejectsCommitting: + """Configuring FORBID_ENCRYPT_ALLOW_DECRYPT with a committing algorithm MUST fail.""" + + ##= specification/s3-encryption/key-commitment.md#commitment-policy + ##= type=test + ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. + def test_forbid_encrypt_allow_decrypt_rejects_committing_gcm(self): + keyring = _mock_keyring() + with pytest.raises(S3EncryptionClientError): + S3EncryptionClientConfig( + keyring=keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) From 96d093e54f85ea50086351e459f2f6fe6edd7af1 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Thu, 5 Mar 2026 23:37:11 -0800 Subject: [PATCH 20/32] fix integ tests --- test/integration/test_i_s3_encryption_instruction_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 2f14d9dd..8833f128 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -40,6 +40,7 @@ def test_decrypt_v1_instruction_file(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, enable_legacy_unauthenticated_modes=True, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) @@ -153,6 +154,7 @@ def test_decrypt_v2_instruction_file_custom_suffix(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, instruction_file_suffix=".custom-suffix-instruction", commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) From dbf7b12e35ea3b1a0e20327d4b2b52610f2cf53a Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Tue, 10 Mar 2026 18:06:00 -0700 Subject: [PATCH 21/32] consolidate crypto params into AlgorithmSuite --- src/s3_encryption/key_derivation.py | 65 ++++++---- src/s3_encryption/materials/kms_keyring.py | 6 +- src/s3_encryption/materials/materials.py | 138 ++++++++++++++++++++- src/s3_encryption/pipelines.py | 36 +++--- test/test_decryption.py | 4 +- test/test_encryption.py | 16 +-- test/test_key_commitment.py | 8 +- test/test_key_derivation.py | 41 +++--- 8 files changed, 235 insertions(+), 79 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index 70442e7c..c96f057d 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -11,65 +11,86 @@ - Expand (Commit Key): info = suite_id_bytes + b"COMMITKEY", output = 28 bytes """ +from __future__ import annotations + import hmac +from typing import TYPE_CHECKING from cryptography.hazmat.primitives.hashes import SHA512 from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand -from .exceptions import S3EncryptionClientSecurityError - -# Algorithm suite ID for S3EC.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY -SUITE_ID_BYTES = b"\x00\x73" +from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError -# Output lengths -ENCRYPTION_KEY_LENGTH = 32 # 256 bits -COMMIT_KEY_LENGTH = 28 # 224 bits -MESSAGE_ID_LENGTH = 28 # 224 bits (used as HKDF salt) +if TYPE_CHECKING: + from .materials.materials import AlgorithmSuite -# Fixed IV for KC GCM: 12 bytes of 0x01 -KC_GCM_IV = b"\x01" * 12 +# Map of supported KDF hash algorithm names to cryptography hash classes. +_HASH_ALGORITHMS = { + "sha512": SHA512, +} -def _hkdf_extract(salt: bytes, ikm: bytes) -> bytes: - """HKDF extract step using HMAC-SHA512. +def _hkdf_extract(salt: bytes, ikm: bytes, hash_algorithm: str) -> bytes: + """HKDF extract step using HMAC. Args: salt: The salt value (Message ID). ikm: Input keying material (plaintext data key). + hash_algorithm: Hash algorithm name (e.g. "sha512"). Returns: The pseudorandom key (PRK). """ - return hmac.new(salt, ikm, "sha512").digest() + return hmac.new(salt, ikm, hash_algorithm).digest() -def _hkdf_expand(prk: bytes, info: bytes, length: int) -> bytes: - """HKDF expand step using SHA-512. +def _hkdf_expand(prk: bytes, info: bytes, length: int, hash_algorithm: str) -> bytes: + """HKDF expand step. Args: prk: Pseudorandom key from extract step. info: Context/application-specific info string. length: Desired output length in bytes. + hash_algorithm: Hash algorithm name (e.g. "sha512"). Returns: Output keying material of the requested length. + + Raises: + S3EncryptionClientError: If the hash algorithm is not supported. """ - hkdf = HKDFExpand(algorithm=SHA512(), length=length, info=info) + hash_cls = _HASH_ALGORITHMS.get(hash_algorithm) + if hash_cls is None: + raise S3EncryptionClientError( + f"Unsupported KDF hash algorithm: {hash_algorithm}" + ) + hkdf = HKDFExpand(algorithm=hash_cls(), length=length, info=info) return hkdf.derive(prk) -def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, bytes]: +def derive_keys( + plaintext_data_key: bytes, + message_id: bytes, + algorithm_suite: AlgorithmSuite, +) -> tuple[bytes, bytes]: """Derive the encryption key and commitment key from a plaintext data key. Uses HKDF with SHA-512 as specified in the S3EC key derivation spec. Args: - plaintext_data_key: The plaintext data key from the keyring (32 bytes). - message_id: The generated Message ID used as the HKDF salt (28 bytes). + plaintext_data_key: The plaintext data key from the keyring. + message_id: The generated Message ID used as the HKDF salt. + algorithm_suite: The algorithm suite whose parameters drive key lengths + and info strings. Returns: A tuple of (derived_encryption_key, commit_key). """ + suite_id = algorithm_suite.suite_id_bytes + enc_key_len = algorithm_suite.data_key_length_bytes + commit_key_len = algorithm_suite.commitment_length_bytes + hash_alg = algorithm_suite.kdf_hash_algorithm + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation ##% - The hash function MUST be specified by the algorithm suite commitment settings. @@ -82,7 +103,7 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. - prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key) + prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key, hash_algorithm=hash_alg) ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation @@ -95,7 +116,7 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string DERIVEKEY as UTF8 encoded bytes. derived_encryption_key = _hkdf_expand( - prk, info=SUITE_ID_BYTES + b"DERIVEKEY", length=ENCRYPTION_KEY_LENGTH + prk, info=suite_id + b"DERIVEKEY", length=enc_key_len, hash_algorithm=hash_alg ) ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -108,7 +129,7 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by ##= type=implementation ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string COMMITKEY as UTF8 encoded bytes. - commit_key = _hkdf_expand(prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH) + commit_key = _hkdf_expand(prk, info=suite_id + b"COMMITKEY", length=commit_key_len, hash_algorithm=hash_alg) return derived_encryption_key, commit_key diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 821a0012..85df14a7 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -84,9 +84,11 @@ def on_encrypt(self, enc_materials): enc_materials.algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ): - encryption_context["aws:x-amz-cek-alg"] = "115" + encryption_context["aws:x-amz-cek-alg"] = str( + enc_materials.algorithm_suite.suite_id + ) else: - encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" + encryption_context["aws:x-amz-cek-alg"] = enc_materials.algorithm_suite.cipher_name # Python implementation uses KMS GenerateDataKey instead of the spec's # EncryptDataKey pattern diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index f0314e00..e146ab12 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -16,21 +16,147 @@ class AlgorithmSuite(Enum): - """Algorithm suites supported by the S3 Encryption Client.""" + """Algorithm suites supported by the S3 Encryption Client. - ALG_AES_256_CBC_IV16_NO_KDF = "AES/CBC/PKCS5Padding" - ALG_AES_256_GCM_IV12_TAG16_NO_KDF = "AES/GCM/NoPadding" - ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = "AES/GCM/HKDF/CommitKey" + Each member consolidates all cryptographic parameters for a given suite, + modeled after the Java reference implementation. The tuple values are: + + (id, is_legacy, data_key_algorithm, data_key_length_bits, + cipher_name, cipher_block_size_bits, cipher_iv_length_bits, + cipher_tag_length_bits, is_committing, commitment_length_bits, + commitment_nonce_length_bits, kdf_hash_algorithm, suite_id_bytes) + """ + + ALG_AES_256_CBC_IV16_NO_KDF = ( + 0x0070, # id + True, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits + "AES/CBC/PKCS5Padding", # cipher_name + 128, # cipher_block_size_bits + 128, # cipher_iv_length_bits (16 bytes) + 0, # cipher_tag_length_bits (CBC has no auth tag) + False, # is_committing + 0, # commitment_length_bits + 0, # commitment_nonce_length_bits + None, # kdf_hash_algorithm + b"", # suite_id_bytes + ) + + ALG_AES_256_GCM_IV12_TAG16_NO_KDF = ( + 0x0072, # id + False, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits + "AES/GCM/NoPadding", # cipher_name + 128, # cipher_block_size_bits + 96, # cipher_iv_length_bits (12 bytes) + 128, # cipher_tag_length_bits (16 bytes) + False, # is_committing + 0, # commitment_length_bits + 0, # commitment_nonce_length_bits + None, # kdf_hash_algorithm + b"", # suite_id_bytes + ) + + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = ( + 0x0073, # id + False, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits + "AES/GCM/HKDF/CommitKey", # cipher_name + 128, # cipher_block_size_bits + 96, # cipher_iv_length_bits (12 bytes) + 128, # cipher_tag_length_bits (16 bytes) + True, # is_committing + 224, # commitment_length_bits (28 bytes) + 224, # commitment_nonce_length_bits (28 bytes = message_id) + "sha512", # kdf_hash_algorithm + b"\x00\x73", # suite_id_bytes + ) + + def __init__( + self, + id: int, + is_legacy: bool, + data_key_algorithm: str, + data_key_length_bits: int, + cipher_name: str, + cipher_block_size_bits: int, + cipher_iv_length_bits: int, + cipher_tag_length_bits: int, + is_committing: bool, + commitment_length_bits: int, + commitment_nonce_length_bits: int, + kdf_hash_algorithm: str | None, + suite_id_bytes: bytes, + ): + self._id = id + self._is_legacy = is_legacy + self._data_key_algorithm = data_key_algorithm + self._data_key_length_bits = data_key_length_bits + self._cipher_name = cipher_name + self._cipher_block_size_bits = cipher_block_size_bits + self._cipher_iv_length_bits = cipher_iv_length_bits + self._cipher_tag_length_bits = cipher_tag_length_bits + self._is_committing = is_committing + self._commitment_length_bits = commitment_length_bits + self._commitment_nonce_length_bits = commitment_nonce_length_bits + self._kdf_hash_algorithm = kdf_hash_algorithm + self._suite_id_bytes = suite_id_bytes + + # --- Convenience properties --- + + @property + def suite_id(self) -> int: + return self._id @property def is_legacy(self) -> bool: """Return True if this algorithm suite is a legacy unauthenticated mode.""" - return self == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + return self._is_legacy @property def supports_key_commitment(self) -> bool: """Return True if this algorithm suite supports key commitment.""" - return self == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + return self._is_committing + + @property + def data_key_length_bytes(self) -> int: + return self._data_key_length_bits // 8 + + @property + def cipher_name(self) -> str: + return self._cipher_name + + @property + def cipher_iv_length_bytes(self) -> int: + return self._cipher_iv_length_bits // 8 + + @property + def commitment_length_bytes(self) -> int: + return self._commitment_length_bits // 8 + + @property + def commitment_nonce_length_bytes(self) -> int: + """Length of the message ID / HKDF salt in bytes.""" + return self._commitment_nonce_length_bits // 8 + + @property + def suite_id_bytes(self) -> bytes: + return self._suite_id_bytes + + @property + def kdf_hash_algorithm(self) -> str | None: + """Hash algorithm name for HKDF, usable with hmac (e.g. 'sha512').""" + return self._kdf_hash_algorithm + + @property + def kc_gcm_iv(self) -> bytes: + """Fixed IV for key-committing GCM: all 0x01 bytes of cipher_iv_length.""" + if not self._is_committing: + raise ValueError(f"{self.name} does not support key commitment") + return b"\x01" * self.cipher_iv_length_bytes class CommitmentPolicy(Enum): diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 13897d4a..e1db3755 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -17,13 +17,7 @@ from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from .instruction_file import fetch_instruction_file -from .key_derivation import ( - KC_GCM_IV, - MESSAGE_ID_LENGTH, - SUITE_ID_BYTES, - derive_keys, - verify_commitment, -) +from .key_derivation import derive_keys, verify_commitment from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey from .materials.materials import ( @@ -103,7 +97,7 @@ def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): ##= type=implementation ##% The generated IV or Message ID MUST be set or returned from the encryption ##% process such that it can be included in the content metadata. - iv = os.urandom(12) + iv = os.urandom(enc_mats.algorithm_suite.cipher_iv_length_bytes) aesgcm = AESGCM(enc_mats.plaintext_data_key) encrypted_data = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) @@ -130,7 +124,8 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##= type=implementation ##% The generated IV or Message ID MUST be set or returned from the encryption ##% process such that it can be included in the content metadata. - message_id = os.urandom(MESSAGE_ID_LENGTH) + algorithm_suite = enc_mats.algorithm_suite + message_id = os.urandom(algorithm_suite.commitment_nonce_length_bytes) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key ##= type=implementation @@ -140,7 +135,9 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##= type=implementation ##% The derived key commitment value MUST be set or returned from the encryption ##% process such that it can be included in the content metadata. - derived_encryption_key, commit_key = derive_keys(enc_mats.plaintext_data_key, message_id) + derived_encryption_key, commit_key = derive_keys( + enc_mats.plaintext_data_key, message_id, algorithm_suite + ) ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation @@ -158,7 +155,7 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) encrypted_data = aesgcm.encrypt( - nonce=KC_GCM_IV, data=plaintext, associated_data=SUITE_ID_BYTES + nonce=algorithm_suite.kc_gcm_iv, data=plaintext, associated_data=algorithm_suite.suite_id_bytes ) b64_edk = base64.b64encode(edk_bytes).decode("utf-8") @@ -166,14 +163,14 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): b64_commit_key = base64.b64encode(commit_key).decode("utf-8") # V3 metadata format - # x-amz-c: content cipher identifier (compressed algorithm suite) + # x-amz-c: content cipher identifier (compressed algorithm suite ID) # x-amz-w: wrapping algorithm identifier (12 = kms+context) # x-amz-3: encrypted data key # x-amz-i: message ID # x-amz-d: key commitment # x-amz-t: encryption context (for kms+context wrapping) metadata = ObjectMetadata( - content_cipher_v3="115", + content_cipher_v3=str(algorithm_suite.suite_id), encrypted_data_key_algorithm_v3="12", encrypted_data_key_v3=b64_edk, message_id_v3=b64_message_id, @@ -347,7 +344,7 @@ def decrypt( ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. - if algorithm_suite == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: + if algorithm_suite.is_legacy: ##= specification/s3-encryption/decryption.md#legacy-decryption ##= type=implementation ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites @@ -380,8 +377,7 @@ def decrypt( ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST NOT allow decryption using algorithm suites which do not support key commitment. if ( self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - and dec_materials.algorithm_suite - != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + and not dec_materials.algorithm_suite.supports_key_commitment ): raise S3EncryptionClientError( "Configuration conflict: cannot decrypt non-key-committing object " @@ -561,7 +557,7 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): ##= type=implementation ##% The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). derived_encryption_key, derived_commitment = derive_keys( - dec_materials.plaintext_data_key, message_id + dec_materials.plaintext_data_key, message_id, dec_materials.algorithm_suite ) ##= specification/s3-encryption/decryption.md#decrypting-with-commitment @@ -590,4 +586,8 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): ##= type=implementation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) - return aesgcm.decrypt(nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES) + return aesgcm.decrypt( + nonce=dec_materials.algorithm_suite.kc_gcm_iv, + data=encrypted_data, + associated_data=dec_materials.algorithm_suite.suite_id_bytes, + ) diff --git a/test/test_decryption.py b/test/test_decryption.py index 211adea2..f2b76931 100644 --- a/test/test_decryption.py +++ b/test/test_decryption.py @@ -213,7 +213,7 @@ def test_commitment_verified_against_stored_metadata(self): """The derived commitment MUST match the stored commitment from metadata.""" key = os.urandom(32) message_id = os.urandom(28) - _, correct_commitment = derive_keys(key, message_id) + _, correct_commitment = derive_keys(key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) # Should not raise verify_commitment(correct_commitment, correct_commitment) @@ -257,7 +257,7 @@ def test_commitment_verified_before_content_decryption(self): """Commitment verification MUST happen before content decryption is attempted.""" key = os.urandom(32) message_id = os.urandom(28) - _, real_commitment = derive_keys(key, message_id) + _, real_commitment = derive_keys(key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) # Build V3 metadata with a wrong commitment wrong_commitment = os.urandom(28) diff --git a/test/test_encryption.py b/test/test_encryption.py index 7484ee0a..12c7ca4a 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -14,15 +14,15 @@ import pytest from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from s3_encryption.key_derivation import ( - KC_GCM_IV, - MESSAGE_ID_LENGTH, - SUITE_ID_BYTES, - derive_keys, -) +from s3_encryption.key_derivation import derive_keys from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.encrypted_data_key import EncryptedDataKey from s3_encryption.materials.materials import AlgorithmSuite, EncryptionMaterials + +_KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +KC_GCM_IV = _KC_SUITE.kc_gcm_iv +MESSAGE_ID_LENGTH = _KC_SUITE.commitment_nonce_length_bytes +SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes from s3_encryption.pipelines import PutEncryptedObjectPipeline @@ -211,7 +211,7 @@ def test_kc_gcm_uses_hkdf_derived_key(self): ) message_id = base64.b64decode(meta["x-amz-i"]) - derived_key, _ = derive_keys(raw_key, message_id) + derived_key, _ = derive_keys(raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) # Decrypt with the HKDF-derived key, fixed IV, and suite ID as AAD aesgcm = AESGCM(derived_key) @@ -246,5 +246,5 @@ def test_kc_gcm_commitment_in_metadata(self): # Verify the commitment matches what HKDF would produce message_id = base64.b64decode(meta["x-amz-i"]) - _, expected_commitment = derive_keys(raw_key, message_id) + _, expected_commitment = derive_keys(raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) assert commitment_bytes == expected_commitment diff --git a/test/test_key_commitment.py b/test/test_key_commitment.py index 6ddc953d..b7736b5b 100644 --- a/test/test_key_commitment.py +++ b/test/test_key_commitment.py @@ -16,7 +16,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from s3_encryption.exceptions import S3EncryptionClientError -from s3_encryption.key_derivation import KC_GCM_IV, SUITE_ID_BYTES, derive_keys +from s3_encryption.key_derivation import derive_keys from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.keyring import S3Keyring from s3_encryption.materials.materials import ( @@ -26,6 +26,10 @@ ) from s3_encryption.pipelines import GetEncryptedObjectPipeline +_KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +KC_GCM_IV = _KC_SUITE.kc_gcm_iv +SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes + # --------------------------------------------------------------------------- # Helpers @@ -67,7 +71,7 @@ def _v2_gcm_response(key, plaintext=b"test data"): def _v3_kc_gcm_response(key, plaintext=b"test data"): """Create a V3 KC-GCM-encrypted response with real ciphertext.""" message_id = os.urandom(28) - derived_key, commitment = derive_keys(key, message_id) + derived_key, commitment = derive_keys(key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) ciphertext = AESGCM(derived_key).encrypt(KC_GCM_IV, plaintext, SUITE_ID_BYTES) metadata = { "x-amz-c": "115", diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py index cfaa724a..fb26cd1c 100644 --- a/test/test_key_derivation.py +++ b/test/test_key_derivation.py @@ -15,13 +15,16 @@ from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand from s3_encryption.key_derivation import ( - COMMIT_KEY_LENGTH, - ENCRYPTION_KEY_LENGTH, - KC_GCM_IV, - MESSAGE_ID_LENGTH, - SUITE_ID_BYTES, derive_keys, ) +from s3_encryption.materials.materials import AlgorithmSuite + +_KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes +ENCRYPTION_KEY_LENGTH = _KC_SUITE.data_key_length_bytes +COMMIT_KEY_LENGTH = _KC_SUITE.commitment_length_bytes +MESSAGE_ID_LENGTH = _KC_SUITE.commitment_nonce_length_bytes +KC_GCM_IV = _KC_SUITE.kc_gcm_iv # --------------------------------------------------------------------------- @@ -45,8 +48,8 @@ def test_hash_function_is_sha512(self): assert len(prk) == 64 # SHA-512 output # derive_keys should produce deterministic output consistent with SHA-512 - key1, ck1 = derive_keys(pdk, msg_id) - key2, ck2 = derive_keys(pdk, msg_id) + key1, ck1 = derive_keys(pdk, msg_id, _KC_SUITE) + key2, ck2 = derive_keys(pdk, msg_id, _KC_SUITE) assert key1 == key2 assert ck1 == ck2 @@ -60,8 +63,8 @@ def test_ikm_is_plaintext_data_key(self): pdk_a = os.urandom(32) pdk_b = os.urandom(32) - key_a, _ = derive_keys(pdk_a, msg_id) - key_b, _ = derive_keys(pdk_b, msg_id) + key_a, _ = derive_keys(pdk_a, msg_id, _KC_SUITE) + key_b, _ = derive_keys(pdk_b, msg_id, _KC_SUITE) assert key_a != key_b ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -74,7 +77,7 @@ def test_ikm_length_is_32_bytes(self): msg_id = os.urandom(MESSAGE_ID_LENGTH) assert len(pdk) == 32 # Should succeed with 32-byte key - key, ck = derive_keys(pdk, msg_id) + key, ck = derive_keys(pdk, msg_id, _KC_SUITE) assert len(key) == ENCRYPTION_KEY_LENGTH assert len(ck) == COMMIT_KEY_LENGTH @@ -88,8 +91,8 @@ def test_salt_is_message_id(self): msg_id_a = os.urandom(MESSAGE_ID_LENGTH) msg_id_b = os.urandom(MESSAGE_ID_LENGTH) - key_a, _ = derive_keys(pdk, msg_id_a) - key_b, _ = derive_keys(pdk, msg_id_b) + key_a, _ = derive_keys(pdk, msg_id_a, _KC_SUITE) + key_b, _ = derive_keys(pdk, msg_id_b, _KC_SUITE) assert key_a != key_b ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -109,7 +112,7 @@ def test_dek_uses_prk_from_extract(self): info=SUITE_ID_BYTES + b"DERIVEKEY", ).derive(prk) - actual_dek, _ = derive_keys(pdk, msg_id) + actual_dek, _ = derive_keys(pdk, msg_id, _KC_SUITE) assert actual_dek == expected_dek ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -118,7 +121,7 @@ def test_dek_uses_prk_from_extract(self): def test_dek_output_length(self): """The derived encryption key MUST be 32 bytes (256 bits).""" import os - key, _ = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH)) + key, _ = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) assert len(key) == ENCRYPTION_KEY_LENGTH assert ENCRYPTION_KEY_LENGTH == 32 @@ -145,7 +148,7 @@ def test_dek_info_is_suite_id_plus_derivekey(self): info=SUITE_ID_BYTES + b"WRONGKEY", ).derive(prk) - actual_dek, _ = derive_keys(pdk, msg_id) + actual_dek, _ = derive_keys(pdk, msg_id, _KC_SUITE) assert actual_dek == correct_dek assert actual_dek != wrong_dek @@ -164,7 +167,7 @@ def test_ck_uses_prk_from_extract(self): info=SUITE_ID_BYTES + b"COMMITKEY", ).derive(prk) - _, actual_ck = derive_keys(pdk, msg_id) + _, actual_ck = derive_keys(pdk, msg_id, _KC_SUITE) assert actual_ck == expected_ck ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -173,7 +176,7 @@ def test_ck_uses_prk_from_extract(self): def test_ck_output_length(self): """The commit key MUST be 28 bytes (224 bits).""" import os - _, ck = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH)) + _, ck = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) assert len(ck) == COMMIT_KEY_LENGTH assert COMMIT_KEY_LENGTH == 28 @@ -198,7 +201,7 @@ def test_ck_info_is_suite_id_plus_commitkey(self): info=SUITE_ID_BYTES + b"WRONGKEY", ).derive(prk) - _, actual_ck = derive_keys(pdk, msg_id) + _, actual_ck = derive_keys(pdk, msg_id, _KC_SUITE) assert actual_ck == correct_ck assert actual_ck != wrong_ck @@ -239,7 +242,7 @@ def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): msg_id = os.urandom(MESSAGE_ID_LENGTH) plaintext = b"key derivation roundtrip test" - derived_key, _ = derive_keys(pdk, msg_id) + derived_key, _ = derive_keys(pdk, msg_id, _KC_SUITE) # Encrypt with derived key, KC_GCM_IV, and SUITE_ID_BYTES as AAD aesgcm = AESGCM(derived_key) From 7763e0efa6d4e96338ffe38a34aaa67732402f4f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 11 Mar 2026 14:52:47 -0700 Subject: [PATCH 22/32] rename algorithm_suite to encryption_algorithm in public config API --- src/s3_encryption/__init__.py | 16 ++++++++-------- test-server/python-v3-server/src/main.py | 2 +- test/integration/test_i_s3_encryption.py | 2 +- .../test_i_s3_encryption_instruction_file.py | 6 +++--- .../test_i_s3_encryption_multithreaded.py | 6 +++--- test/test_key_commitment_encrypt.py | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 0fdb8fad..d81a14e7 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -37,7 +37,7 @@ class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" keyring: AbstractKeyring - algorithm_suite: AlgorithmSuite = field( + encryption_algorithm: AlgorithmSuite = field( default=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ) commitment_policy: CommitmentPolicy = field( @@ -73,10 +73,10 @@ def _default_cmm_for_keyring(self): ##= specification/s3-encryption/client.md#key-commitment ##% If the configured Encryption Algorithm is incompatible with the key commitment policy, then it MUST throw an exception. def __attrs_post_init__(self): - if self.algorithm_suite.is_legacy: + if self.encryption_algorithm.is_legacy: raise S3EncryptionClientError( f"Cannot configure S3 Encryption Client with legacy algorithm suite " - f"{self.algorithm_suite.name}. Legacy algorithm suites are only " + f"{self.encryption_algorithm.name}. Legacy algorithm suites are only " f"supported for decryption (and enable_legacy_unauthenticated_modes is True)." ) @@ -87,21 +87,21 @@ def __attrs_post_init__(self): if self.commitment_policy in ( CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - ) and not self.algorithm_suite.supports_key_commitment: + ) and not self.encryption_algorithm.supports_key_commitment: raise S3EncryptionClientError( f"Commitment policy {self.commitment_policy.name} requires a key-committing " - f"algorithm suite, but {self.algorithm_suite.name} does not support key commitment." + f"algorithm suite, but {self.encryption_algorithm.name} does not support key commitment." ) ##= specification/s3-encryption/key-commitment.md#commitment-policy ##% When the commitment policy is FORBID_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST NOT encrypt using an algorithm suite which supports key commitment. if ( self.commitment_policy == CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT - and self.algorithm_suite.supports_key_commitment + and self.encryption_algorithm.supports_key_commitment ): raise S3EncryptionClientError( f"Commitment policy {self.commitment_policy.name} forbids key-committing " - f"algorithm suites, but {self.algorithm_suite.name} supports key commitment." + f"algorithm suites, but {self.encryption_algorithm.name} supports key commitment." ) @@ -157,7 +157,7 @@ def on_put_object_before_call(self, params, **kwargs): encrypted_data, encryption_metadata = pipeline.encrypt( body_bytes, encryption_context=encryption_context, - algorithm_suite=self.config.algorithm_suite, + algorithm_suite=self.config.encryption_algorithm, ) params["body"] = encrypted_data diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v3-server/src/main.py index 3f32f389..cee2ab4e 100755 --- a/test-server/python-v3-server/src/main.py +++ b/test-server/python-v3-server/src/main.py @@ -211,7 +211,7 @@ async def client_endpoint(request: Request): if encryption_algorithm is not None: if encryption_algorithm not in _ALGORITHM_SUITE_MAP: raise ValueError(f"Unknown encryption algorithm: {encryption_algorithm}") - config_kwargs["algorithm_suite"] = _ALGORITHM_SUITE_MAP[encryption_algorithm] + config_kwargs["encryption_algorithm"] = _ALGORITHM_SUITE_MAP[encryption_algorithm] commitment_policy = config_data.get("commitmentPolicy") if commitment_policy is not None: diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index d4d5bcaa..17f6c682 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -41,7 +41,7 @@ def _make_client(algorithm_suite, commitment_policy): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=algorithm_suite, + encryption_algorithm=algorithm_suite, commitment_policy=commitment_policy, ) return S3EncryptionClient(wrapped_client, config) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 8833f128..f4f70704 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -40,7 +40,7 @@ def test_decrypt_v1_instruction_file(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, enable_legacy_unauthenticated_modes=True, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) @@ -66,7 +66,7 @@ def test_decrypt_v2_instruction_file(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) @@ -154,7 +154,7 @@ def test_decrypt_v2_instruction_file_custom_suffix(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, instruction_file_suffix=".custom-suffix-instruction", commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index f04d7fe5..e71a17df 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -41,7 +41,7 @@ def test_multithreaded_encryption_context_isolation(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) @@ -157,7 +157,7 @@ def test_multithreaded_rapid_context_switching(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) @@ -239,7 +239,7 @@ def test_multithreaded_mixed_with_and_without_context(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) diff --git a/test/test_key_commitment_encrypt.py b/test/test_key_commitment_encrypt.py index e343927c..dcd7b67d 100644 --- a/test/test_key_commitment_encrypt.py +++ b/test/test_key_commitment_encrypt.py @@ -71,7 +71,7 @@ def test_require_encrypt_allow_decrypt_rejects_non_committing_gcm(self): with pytest.raises(S3EncryptionClientError): S3EncryptionClientConfig( keyring=keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, ) @@ -83,7 +83,7 @@ def test_require_encrypt_require_decrypt_rejects_non_committing_gcm(self): with pytest.raises(S3EncryptionClientError): S3EncryptionClientConfig( keyring=keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, ) @@ -103,6 +103,6 @@ def test_forbid_encrypt_allow_decrypt_rejects_committing_gcm(self): with pytest.raises(S3EncryptionClientError): S3EncryptionClientConfig( keyring=keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, ) From 44c1a5b43562f4d833b38d4ae738fc86aafbe30b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 11 Mar 2026 15:59:50 -0700 Subject: [PATCH 23/32] revise annotations, improve test quality --- src/s3_encryption/key_derivation.py | 26 ++++---- test/test_key_derivation.py | 92 +++++++++++++++++------------ 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index c96f057d..d42258bb 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -87,19 +87,31 @@ def derive_keys( A tuple of (derived_encryption_key, commit_key). """ suite_id = algorithm_suite.suite_id_bytes + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. enc_key_len = algorithm_suite.data_key_length_bytes + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. commit_key_len = algorithm_suite.commitment_length_bytes - hash_alg = algorithm_suite.kdf_hash_algorithm - ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation ##% - The hash function MUST be specified by the algorithm suite commitment settings. + hash_alg = algorithm_suite.kdf_hash_algorithm + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation - ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + if len(plaintext_data_key) != enc_key_len: + raise S3EncryptionClientError( + f"Plaintext data key length ({len(plaintext_data_key)}) does not match " + f"the key derivation input length ({enc_key_len}) specified by the algorithm suite." + ) + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation - ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. @@ -110,9 +122,6 @@ def derive_keys( ##% - The DEK input pseudorandom key MUST be the output from the extract step. ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation - ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string DERIVEKEY as UTF8 encoded bytes. derived_encryption_key = _hkdf_expand( @@ -124,9 +133,6 @@ def derive_keys( ##% - The CK input pseudorandom key MUST be the output from the extract step. ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation - ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string COMMITKEY as UTF8 encoded bytes. commit_key = _hkdf_expand(prk, info=suite_id + b"COMMITKEY", length=commit_key_len, hash_algorithm=hash_alg) diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py index fb26cd1c..9eea6996 100644 --- a/test/test_key_derivation.py +++ b/test/test_key_derivation.py @@ -38,20 +38,24 @@ class TestHkdfOperation: ##= type=test ##% - The hash function MUST be specified by the algorithm suite commitment settings. def test_hash_function_is_sha512(self): - """HKDF extract MUST use HMAC-SHA512 as the hash function.""" + """HKDF extract MUST use the hash function specified by the algorithm suite.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) - # Manual HMAC-SHA512 extract - prk = hmac.new(msg_id, pdk, "sha512").digest() - assert len(prk) == 64 # SHA-512 output + # Manual extract using the algorithm suite's configured hash + hash_alg = _KC_SUITE.kdf_hash_algorithm + prk = hmac.new(msg_id, pdk, hash_alg).digest() - # derive_keys should produce deterministic output consistent with SHA-512 - key1, ck1 = derive_keys(pdk, msg_id, _KC_SUITE) - key2, ck2 = derive_keys(pdk, msg_id, _KC_SUITE) - assert key1 == key2 - assert ck1 == ck2 + # Expand with the same hash to get expected DEK + expected_dek = HKDFExpand( + algorithm=SHA512(), length=_KC_SUITE.data_key_length_bytes, + info=SUITE_ID_BYTES + b"DERIVEKEY", + ).derive(prk) + + # derive_keys using the suite must match + actual_dek, _ = derive_keys(pdk, msg_id, _KC_SUITE) + assert actual_dek == expected_dek ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=test @@ -60,8 +64,8 @@ def test_ikm_is_plaintext_data_key(self): """Different plaintext data keys MUST produce different derived keys.""" import os msg_id = os.urandom(MESSAGE_ID_LENGTH) - pdk_a = os.urandom(32) - pdk_b = os.urandom(32) + pdk_a = os.urandom(_KC_SUITE.data_key_length_bytes) + pdk_b = os.urandom(_KC_SUITE.data_key_length_bytes) key_a, _ = derive_keys(pdk_a, msg_id, _KC_SUITE) key_b, _ = derive_keys(pdk_b, msg_id, _KC_SUITE) @@ -71,23 +75,39 @@ def test_ikm_is_plaintext_data_key(self): ##= type=test ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. def test_ikm_length_is_32_bytes(self): - """The plaintext data key (IKM) MUST be 32 bytes for AES-256.""" + """The plaintext data key (IKM) length MUST equal the algorithm suite's data key length.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) - assert len(pdk) == 32 - # Should succeed with 32-byte key + assert len(pdk) == _KC_SUITE.data_key_length_bytes + # Should succeed with correct-length key key, ck = derive_keys(pdk, msg_id, _KC_SUITE) assert len(key) == ENCRYPTION_KEY_LENGTH assert len(ck) == COMMIT_KEY_LENGTH + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=test + ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + def test_ikm_wrong_length_raises(self): + """derive_keys MUST raise when the plaintext data key length doesn't match the suite.""" + import os + from s3_encryption.exceptions import S3EncryptionClientError + + msg_id = os.urandom(MESSAGE_ID_LENGTH) + # Too short + with pytest.raises(S3EncryptionClientError): + derive_keys(os.urandom(16), msg_id, _KC_SUITE) + # Too long + with pytest.raises(S3EncryptionClientError): + derive_keys(os.urandom(64), msg_id, _KC_SUITE) + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=test ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. def test_salt_is_message_id(self): """Different Message IDs (salts) MUST produce different derived keys.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id_a = os.urandom(MESSAGE_ID_LENGTH) msg_id_b = os.urandom(MESSAGE_ID_LENGTH) @@ -101,11 +121,11 @@ def test_salt_is_message_id(self): def test_dek_uses_prk_from_extract(self): """The DEK expand step MUST use the PRK from the extract step.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) # Manual extract - prk = hmac.new(msg_id, pdk, "sha512").digest() + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() # Manual expand for DEK expected_dek = HKDFExpand( algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, @@ -119,11 +139,10 @@ def test_dek_uses_prk_from_extract(self): ##= type=test ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. def test_dek_output_length(self): - """The derived encryption key MUST be 32 bytes (256 bits).""" + """The derived encryption key MUST match the encryption key length from the algorithm suite.""" import os - key, _ = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) - assert len(key) == ENCRYPTION_KEY_LENGTH - assert ENCRYPTION_KEY_LENGTH == 32 + key, _ = derive_keys(os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) + assert len(key) == _KC_SUITE.data_key_length_bytes ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=test @@ -131,10 +150,10 @@ def test_dek_output_length(self): def test_dek_info_is_suite_id_plus_derivekey(self): """DEK expand info MUST be suite_id_bytes + b'DERIVEKEY'.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) - prk = hmac.new(msg_id, pdk, "sha512").digest() + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() # Correct info correct_dek = HKDFExpand( @@ -158,10 +177,10 @@ def test_dek_info_is_suite_id_plus_derivekey(self): def test_ck_uses_prk_from_extract(self): """The CK expand step MUST use the PRK from the extract step.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) - prk = hmac.new(msg_id, pdk, "sha512").digest() + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() expected_ck = HKDFExpand( algorithm=SHA512(), length=COMMIT_KEY_LENGTH, info=SUITE_ID_BYTES + b"COMMITKEY", @@ -174,11 +193,10 @@ def test_ck_uses_prk_from_extract(self): ##= type=test ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. def test_ck_output_length(self): - """The commit key MUST be 28 bytes (224 bits).""" + """The commit key length MUST match the algorithm suite's commitment length.""" import os - _, ck = derive_keys(os.urandom(32), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) - assert len(ck) == COMMIT_KEY_LENGTH - assert COMMIT_KEY_LENGTH == 28 + _, ck = derive_keys(os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) + assert len(ck) == _KC_SUITE.commitment_length_bytes ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=test @@ -186,10 +204,10 @@ def test_ck_output_length(self): def test_ck_info_is_suite_id_plus_commitkey(self): """CK expand info MUST be suite_id_bytes + b'COMMITKEY'.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) - prk = hmac.new(msg_id, pdk, "sha512").digest() + prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() correct_ck = HKDFExpand( algorithm=SHA512(), length=COMMIT_KEY_LENGTH, @@ -225,8 +243,8 @@ def test_kc_gcm_iv_is_all_0x01(self): ##= type=test ##% The IV's total length MUST match the IV length defined by the algorithm suite. def test_kc_gcm_iv_length_is_12(self): - """The KC-GCM IV MUST be 12 bytes (AES-GCM standard nonce length).""" - assert len(KC_GCM_IV) == 12 + """The KC-GCM IV length MUST match the IV length defined by the algorithm suite.""" + assert len(KC_GCM_IV) == _KC_SUITE.cipher_iv_length_bytes ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=test @@ -238,7 +256,7 @@ def test_kc_gcm_iv_length_is_12(self): def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): """KC-GCM MUST encrypt/decrypt with derived key, 0x01 IV, and suite ID as AAD.""" import os - pdk = os.urandom(32) + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) plaintext = b"key derivation roundtrip test" @@ -258,4 +276,4 @@ def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): # Decrypting with wrong IV must fail with pytest.raises(Exception): - aesgcm.decrypt(b"\x00" * 12, ciphertext, SUITE_ID_BYTES) + aesgcm.decrypt(b"\x00" * _KC_SUITE.cipher_iv_length_bytes, ciphertext, SUITE_ID_BYTES) From 79a69bf090468e632aad07c4f0aa8fee79d0af3d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 11 Mar 2026 16:14:10 -0700 Subject: [PATCH 24/32] formatting --- src/s3_encryption/__init__.py | 12 ++-- src/s3_encryption/key_derivation.py | 8 +-- src/s3_encryption/materials/materials.py | 70 ++++++++++++------------ src/s3_encryption/pipelines.py | 8 ++- test/test_decryption.py | 28 +++++++--- test/test_encryption.py | 23 ++++---- test/test_key_commitment.py | 6 +- test/test_key_commitment_encrypt.py | 6 +- test/test_key_derivation.py | 43 ++++++++++++--- test/test_pipelines.py | 1 + 10 files changed, 128 insertions(+), 77 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index d81a14e7..bbe17a94 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -84,10 +84,14 @@ def __attrs_post_init__(self): ##% When the commitment policy is REQUIRE_ENCRYPT_ALLOW_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. ##= specification/s3-encryption/key-commitment.md#commitment-policy ##% When the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT, the S3EC MUST only encrypt using an algorithm suite which supports key commitment. - if self.commitment_policy in ( - CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, - CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, - ) and not self.encryption_algorithm.supports_key_commitment: + if ( + self.commitment_policy + in ( + CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + and not self.encryption_algorithm.supports_key_commitment + ): raise S3EncryptionClientError( f"Commitment policy {self.commitment_policy.name} requires a key-committing " f"algorithm suite, but {self.encryption_algorithm.name} does not support key commitment." diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index d42258bb..8183f5f3 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -61,9 +61,7 @@ def _hkdf_expand(prk: bytes, info: bytes, length: int, hash_algorithm: str) -> b """ hash_cls = _HASH_ALGORITHMS.get(hash_algorithm) if hash_cls is None: - raise S3EncryptionClientError( - f"Unsupported KDF hash algorithm: {hash_algorithm}" - ) + raise S3EncryptionClientError(f"Unsupported KDF hash algorithm: {hash_algorithm}") hkdf = HKDFExpand(algorithm=hash_cls(), length=length, info=info) return hkdf.derive(prk) @@ -135,7 +133,9 @@ def derive_keys( ##= type=implementation ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string COMMITKEY as UTF8 encoded bytes. - commit_key = _hkdf_expand(prk, info=suite_id + b"COMMITKEY", length=commit_key_len, hash_algorithm=hash_alg) + commit_key = _hkdf_expand( + prk, info=suite_id + b"COMMITKEY", length=commit_key_len, hash_algorithm=hash_alg + ) return derived_encryption_key, commit_key diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index e146ab12..6483beec 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -28,50 +28,50 @@ class AlgorithmSuite(Enum): """ ALG_AES_256_CBC_IV16_NO_KDF = ( - 0x0070, # id - True, # is_legacy - "AES", # data_key_algorithm - 256, # data_key_length_bits + 0x0070, # id + True, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits "AES/CBC/PKCS5Padding", # cipher_name - 128, # cipher_block_size_bits - 128, # cipher_iv_length_bits (16 bytes) - 0, # cipher_tag_length_bits (CBC has no auth tag) - False, # is_committing - 0, # commitment_length_bits - 0, # commitment_nonce_length_bits - None, # kdf_hash_algorithm - b"", # suite_id_bytes + 128, # cipher_block_size_bits + 128, # cipher_iv_length_bits (16 bytes) + 0, # cipher_tag_length_bits (CBC has no auth tag) + False, # is_committing + 0, # commitment_length_bits + 0, # commitment_nonce_length_bits + None, # kdf_hash_algorithm + b"", # suite_id_bytes ) ALG_AES_256_GCM_IV12_TAG16_NO_KDF = ( - 0x0072, # id - False, # is_legacy - "AES", # data_key_algorithm - 256, # data_key_length_bits + 0x0072, # id + False, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits "AES/GCM/NoPadding", # cipher_name - 128, # cipher_block_size_bits - 96, # cipher_iv_length_bits (12 bytes) - 128, # cipher_tag_length_bits (16 bytes) - False, # is_committing - 0, # commitment_length_bits - 0, # commitment_nonce_length_bits - None, # kdf_hash_algorithm - b"", # suite_id_bytes + 128, # cipher_block_size_bits + 96, # cipher_iv_length_bits (12 bytes) + 128, # cipher_tag_length_bits (16 bytes) + False, # is_committing + 0, # commitment_length_bits + 0, # commitment_nonce_length_bits + None, # kdf_hash_algorithm + b"", # suite_id_bytes ) ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = ( - 0x0073, # id - False, # is_legacy - "AES", # data_key_algorithm - 256, # data_key_length_bits + 0x0073, # id + False, # is_legacy + "AES", # data_key_algorithm + 256, # data_key_length_bits "AES/GCM/HKDF/CommitKey", # cipher_name - 128, # cipher_block_size_bits - 96, # cipher_iv_length_bits (12 bytes) - 128, # cipher_tag_length_bits (16 bytes) - True, # is_committing - 224, # commitment_length_bits (28 bytes) - 224, # commitment_nonce_length_bits (28 bytes = message_id) - "sha512", # kdf_hash_algorithm + 128, # cipher_block_size_bits + 96, # cipher_iv_length_bits (12 bytes) + 128, # cipher_tag_length_bits (16 bytes) + True, # is_committing + 224, # commitment_length_bits (28 bytes) + 224, # commitment_nonce_length_bits (28 bytes = message_id) + "sha512", # kdf_hash_algorithm b"\x00\x73", # suite_id_bytes ) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index e1db3755..6340b2a8 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -155,7 +155,9 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) encrypted_data = aesgcm.encrypt( - nonce=algorithm_suite.kc_gcm_iv, data=plaintext, associated_data=algorithm_suite.suite_id_bytes + nonce=algorithm_suite.kc_gcm_iv, + data=plaintext, + associated_data=algorithm_suite.suite_id_bytes, ) b64_edk = base64.b64encode(edk_bytes).decode("utf-8") @@ -294,7 +296,9 @@ def decrypt( ObjectMetadata.KEY_COMMITMENT_V3, ObjectMetadata.MESSAGE_ID_V3, } - forbidden_keys_in_instruction = set(instruction_metadata.keys()) & v3_object_metadata_exclusive_keys + forbidden_keys_in_instruction = ( + set(instruction_metadata.keys()) & v3_object_metadata_exclusive_keys + ) if forbidden_keys_in_instruction: raise S3EncryptionClientError( "Instruction file is tampered, instruction file contains object metadata " diff --git a/test/test_decryption.py b/test/test_decryption.py index f2b76931..6d22d439 100644 --- a/test/test_decryption.py +++ b/test/test_decryption.py @@ -8,17 +8,15 @@ """ import base64 -import hmac import os from io import BytesIO -from unittest.mock import MagicMock, Mock +from unittest.mock import Mock import pytest from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from s3_encryption.key_derivation import derive_keys, verify_commitment from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager -from s3_encryption.materials.encrypted_data_key import EncryptedDataKey from s3_encryption.materials.keyring import S3Keyring from s3_encryption.materials.materials import ( AlgorithmSuite, @@ -27,11 +25,11 @@ ) from s3_encryption.pipelines import GetEncryptedObjectPipeline - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _make_pipeline( commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, enable_legacy=False, @@ -69,7 +67,7 @@ def _v2_gcm_metadata(): "x-amz-iv": base64.b64encode(os.urandom(12)).decode(), "x-amz-key-v2": base64.b64encode(b"encrypted-key").decode(), "x-amz-wrap-alg": "kms+context", - "x-amz-matdesc": '{}', + "x-amz-matdesc": "{}", "x-amz-cek-alg": "AES/GCM/NoPadding", "x-amz-tag-len": "128", } @@ -83,6 +81,7 @@ def _response(metadata, body=b"ciphertext"): # CBC Decryption # --------------------------------------------------------------------------- + class TestCBCDecryption: """Tests for specification/s3-encryption/decryption.md#cbc-decryption.""" @@ -201,6 +200,7 @@ def test_cbc_decryption_fails_with_wrong_key(self): # Decrypting with Commitment # --------------------------------------------------------------------------- + class TestDecryptingWithCommitment: """Tests for specification/s3-encryption/decryption.md#decrypting-with-commitment.""" @@ -213,7 +213,9 @@ def test_commitment_verified_against_stored_metadata(self): """The derived commitment MUST match the stored commitment from metadata.""" key = os.urandom(32) message_id = os.urandom(28) - _, correct_commitment = derive_keys(key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + _, correct_commitment = derive_keys( + key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) # Should not raise verify_commitment(correct_commitment, correct_commitment) @@ -246,7 +248,9 @@ def test_commitment_mismatch_throws_exception(self): stored = os.urandom(28) derived = os.urandom(28) - with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): verify_commitment(stored, derived) ##= specification/s3-encryption/decryption.md#decrypting-with-commitment @@ -257,7 +261,9 @@ def test_commitment_verified_before_content_decryption(self): """Commitment verification MUST happen before content decryption is attempted.""" key = os.urandom(32) message_id = os.urandom(28) - _, real_commitment = derive_keys(key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + _, real_commitment = derive_keys( + key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) # Build V3 metadata with a wrong commitment wrong_commitment = os.urandom(28) @@ -280,7 +286,9 @@ def test_commitment_verified_before_content_decryption(self): ) # Must fail at commitment check, not at AES-GCM decryption - with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): pipeline.decrypt(_response(metadata, b"fake-ciphertext")) @@ -288,6 +296,7 @@ def test_commitment_verified_before_content_decryption(self): # Key Commitment Policy # --------------------------------------------------------------------------- + class TestKeyCommitmentPolicy: """Tests for specification/s3-encryption/decryption.md#key-commitment.""" @@ -351,6 +360,7 @@ def test_allow_decrypt_accepts_non_committing_suite(self): # Legacy Decryption # --------------------------------------------------------------------------- + class TestLegacyDecryption: """Tests for specification/s3-encryption/decryption.md#legacy-decryption.""" diff --git a/test/test_encryption.py b/test/test_encryption.py index 12c7ca4a..871a8360 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -17,7 +17,7 @@ from s3_encryption.key_derivation import derive_keys from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.encrypted_data_key import EncryptedDataKey -from s3_encryption.materials.materials import AlgorithmSuite, EncryptionMaterials +from s3_encryption.materials.materials import AlgorithmSuite _KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY KC_GCM_IV = _KC_SUITE.kc_gcm_iv @@ -25,11 +25,11 @@ SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes from s3_encryption.pipelines import PutEncryptedObjectPipeline - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _mock_cmm(plaintext_key=None, encrypted_key=b"encrypted-key"): """Return a CMM backed by a mock keyring that returns the given keys.""" if plaintext_key is None: @@ -56,6 +56,7 @@ def _fill_materials(mats, plaintext_key, encrypted_key): # Content Encryption — General # --------------------------------------------------------------------------- + class TestContentEncryption: """Tests for specification/s3-encryption/encryption.md#content-encryption.""" @@ -144,6 +145,7 @@ def test_message_id_included_in_metadata_kc(self): # ALG_AES_256_GCM_IV12_TAG16_NO_KDF # --------------------------------------------------------------------------- + class TestGcmNoKdf: """Tests for specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf.""" @@ -192,6 +194,7 @@ def test_gcm_decrypt_fails_with_aad(self): # ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY # --------------------------------------------------------------------------- + class TestKcGcm: """Tests for specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key.""" @@ -211,21 +214,19 @@ def test_kc_gcm_uses_hkdf_derived_key(self): ) message_id = base64.b64decode(meta["x-amz-i"]) - derived_key, _ = derive_keys(raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + derived_key, _ = derive_keys( + raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) # Decrypt with the HKDF-derived key, fixed IV, and suite ID as AAD aesgcm = AESGCM(derived_key) - decrypted = aesgcm.decrypt( - nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES - ) + decrypted = aesgcm.decrypt(nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES) assert decrypted == plaintext # Decrypting with the raw key must fail aesgcm_raw = AESGCM(raw_key) with pytest.raises(Exception): - aesgcm_raw.decrypt( - nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES - ) + aesgcm_raw.decrypt(nonce=KC_GCM_IV, data=ciphertext, associated_data=SUITE_ID_BYTES) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key ##= type=test @@ -246,5 +247,7 @@ def test_kc_gcm_commitment_in_metadata(self): # Verify the commitment matches what HKDF would produce message_id = base64.b64decode(meta["x-amz-i"]) - _, expected_commitment = derive_keys(raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + _, expected_commitment = derive_keys( + raw_key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) assert commitment_bytes == expected_commitment diff --git a/test/test_key_commitment.py b/test/test_key_commitment.py index b7736b5b..a82fc9fd 100644 --- a/test/test_key_commitment.py +++ b/test/test_key_commitment.py @@ -35,6 +35,7 @@ # Helpers # --------------------------------------------------------------------------- + def _make_pipeline(commitment_policy, keyring_return=None): """Create a GetEncryptedObjectPipeline with a mocked CMM/keyring.""" mock_keyring = Mock(spec=S3Keyring) @@ -71,7 +72,9 @@ def _v2_gcm_response(key, plaintext=b"test data"): def _v3_kc_gcm_response(key, plaintext=b"test data"): """Create a V3 KC-GCM-encrypted response with real ciphertext.""" message_id = os.urandom(28) - derived_key, commitment = derive_keys(key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + derived_key, commitment = derive_keys( + key, message_id, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) ciphertext = AESGCM(derived_key).encrypt(KC_GCM_IV, plaintext, SUITE_ID_BYTES) metadata = { "x-amz-c": "115", @@ -92,6 +95,7 @@ def _v3_kc_gcm_response(key, plaintext=b"test data"): # Commitment Policy Tests # --------------------------------------------------------------------------- + class TestCommitmentPolicy: """Tests for specification/s3-encryption/key-commitment.md#commitment-policy.""" diff --git a/test/test_key_commitment_encrypt.py b/test/test_key_commitment_encrypt.py index dcd7b67d..ed8e8f3e 100644 --- a/test/test_key_commitment_encrypt.py +++ b/test/test_key_commitment_encrypt.py @@ -28,16 +28,14 @@ from s3_encryption import S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError -from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.encrypted_data_key import EncryptedDataKey from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy -from s3_encryption.pipelines import PutEncryptedObjectPipeline - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _mock_keyring(): """Return a mock keyring that populates encryption materials.""" key = os.urandom(32) @@ -60,6 +58,7 @@ def on_encrypt(mats): # REQUIRE_ENCRYPT_* with non-committing algorithm → MUST fail # --------------------------------------------------------------------------- + class TestRequireEncryptRejectsNonCommitting: """Configuring REQUIRE_ENCRYPT_* with a non-committing algorithm MUST fail.""" @@ -92,6 +91,7 @@ def test_require_encrypt_require_decrypt_rejects_non_committing_gcm(self): # FORBID_ENCRYPT_ALLOW_DECRYPT with committing algorithm → MUST fail # --------------------------------------------------------------------------- + class TestForbidEncryptRejectsCommitting: """Configuring FORBID_ENCRYPT_ALLOW_DECRYPT with a committing algorithm MUST fail.""" diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py index 9eea6996..3843edfd 100644 --- a/test/test_key_derivation.py +++ b/test/test_key_derivation.py @@ -31,6 +31,7 @@ # HKDF Extract / Expand # --------------------------------------------------------------------------- + class TestHkdfOperation: """Tests for specification/s3-encryption/key-derivation.md#hkdf-operation.""" @@ -40,6 +41,7 @@ class TestHkdfOperation: def test_hash_function_is_sha512(self): """HKDF extract MUST use the hash function specified by the algorithm suite.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -49,7 +51,8 @@ def test_hash_function_is_sha512(self): # Expand with the same hash to get expected DEK expected_dek = HKDFExpand( - algorithm=SHA512(), length=_KC_SUITE.data_key_length_bytes, + algorithm=SHA512(), + length=_KC_SUITE.data_key_length_bytes, info=SUITE_ID_BYTES + b"DERIVEKEY", ).derive(prk) @@ -63,6 +66,7 @@ def test_hash_function_is_sha512(self): def test_ikm_is_plaintext_data_key(self): """Different plaintext data keys MUST produce different derived keys.""" import os + msg_id = os.urandom(MESSAGE_ID_LENGTH) pdk_a = os.urandom(_KC_SUITE.data_key_length_bytes) pdk_b = os.urandom(_KC_SUITE.data_key_length_bytes) @@ -77,6 +81,7 @@ def test_ikm_is_plaintext_data_key(self): def test_ikm_length_is_32_bytes(self): """The plaintext data key (IKM) length MUST equal the algorithm suite's data key length.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) assert len(pdk) == _KC_SUITE.data_key_length_bytes @@ -91,6 +96,7 @@ def test_ikm_length_is_32_bytes(self): def test_ikm_wrong_length_raises(self): """derive_keys MUST raise when the plaintext data key length doesn't match the suite.""" import os + from s3_encryption.exceptions import S3EncryptionClientError msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -107,6 +113,7 @@ def test_ikm_wrong_length_raises(self): def test_salt_is_message_id(self): """Different Message IDs (salts) MUST produce different derived keys.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id_a = os.urandom(MESSAGE_ID_LENGTH) msg_id_b = os.urandom(MESSAGE_ID_LENGTH) @@ -121,6 +128,7 @@ def test_salt_is_message_id(self): def test_dek_uses_prk_from_extract(self): """The DEK expand step MUST use the PRK from the extract step.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -128,7 +136,8 @@ def test_dek_uses_prk_from_extract(self): prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() # Manual expand for DEK expected_dek = HKDFExpand( - algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, + algorithm=SHA512(), + length=ENCRYPTION_KEY_LENGTH, info=SUITE_ID_BYTES + b"DERIVEKEY", ).derive(prk) @@ -141,7 +150,10 @@ def test_dek_uses_prk_from_extract(self): def test_dek_output_length(self): """The derived encryption key MUST match the encryption key length from the algorithm suite.""" import os - key, _ = derive_keys(os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) + + key, _ = derive_keys( + os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE + ) assert len(key) == _KC_SUITE.data_key_length_bytes ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -150,6 +162,7 @@ def test_dek_output_length(self): def test_dek_info_is_suite_id_plus_derivekey(self): """DEK expand info MUST be suite_id_bytes + b'DERIVEKEY'.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -157,13 +170,15 @@ def test_dek_info_is_suite_id_plus_derivekey(self): # Correct info correct_dek = HKDFExpand( - algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, + algorithm=SHA512(), + length=ENCRYPTION_KEY_LENGTH, info=SUITE_ID_BYTES + b"DERIVEKEY", ).derive(prk) # Wrong info should produce different output wrong_dek = HKDFExpand( - algorithm=SHA512(), length=ENCRYPTION_KEY_LENGTH, + algorithm=SHA512(), + length=ENCRYPTION_KEY_LENGTH, info=SUITE_ID_BYTES + b"WRONGKEY", ).derive(prk) @@ -177,12 +192,14 @@ def test_dek_info_is_suite_id_plus_derivekey(self): def test_ck_uses_prk_from_extract(self): """The CK expand step MUST use the PRK from the extract step.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() expected_ck = HKDFExpand( - algorithm=SHA512(), length=COMMIT_KEY_LENGTH, + algorithm=SHA512(), + length=COMMIT_KEY_LENGTH, info=SUITE_ID_BYTES + b"COMMITKEY", ).derive(prk) @@ -195,7 +212,10 @@ def test_ck_uses_prk_from_extract(self): def test_ck_output_length(self): """The commit key length MUST match the algorithm suite's commitment length.""" import os - _, ck = derive_keys(os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE) + + _, ck = derive_keys( + os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE + ) assert len(ck) == _KC_SUITE.commitment_length_bytes ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -204,18 +224,21 @@ def test_ck_output_length(self): def test_ck_info_is_suite_id_plus_commitkey(self): """CK expand info MUST be suite_id_bytes + b'COMMITKEY'.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) prk = hmac.new(msg_id, pdk, _KC_SUITE.kdf_hash_algorithm).digest() correct_ck = HKDFExpand( - algorithm=SHA512(), length=COMMIT_KEY_LENGTH, + algorithm=SHA512(), + length=COMMIT_KEY_LENGTH, info=SUITE_ID_BYTES + b"COMMITKEY", ).derive(prk) wrong_ck = HKDFExpand( - algorithm=SHA512(), length=COMMIT_KEY_LENGTH, + algorithm=SHA512(), + length=COMMIT_KEY_LENGTH, info=SUITE_ID_BYTES + b"WRONGKEY", ).derive(prk) @@ -228,6 +251,7 @@ def test_ck_info_is_suite_id_plus_commitkey(self): # IV and AAD for KC-GCM # --------------------------------------------------------------------------- + class TestKcGcmCipherParams: """Tests for KC-GCM cipher initialization parameters.""" @@ -256,6 +280,7 @@ def test_kc_gcm_iv_length_is_12(self): def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): """KC-GCM MUST encrypt/decrypt with derived key, 0x01 IV, and suite ID as AAD.""" import os + pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) plaintext = b"key derivation roundtrip test" diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 251ce26e..d3a643f5 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -242,6 +242,7 @@ def test_decrypt_with_custom_instruction_file_suffix(self): mock_s3_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test-key.custom-suffix" ) + def test_decrypt_v3_unsupported_wrap_alg(self): """Test that V3 decryption with unsupported wrapping algorithm is rejected by the keyring.""" # V3 metadata with AES/GCM wrapping (02) — not supported by the KMS keyring From f7ce7e531b0965c42b33bc21be2119c44380cd1d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 11 Mar 2026 16:24:48 -0700 Subject: [PATCH 25/32] linter --- src/s3_encryption/__init__.py | 1 + src/s3_encryption/materials/materials.py | 11 +++++++++-- src/s3_encryption/pipelines.py | 19 ++++++++++--------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index bbe17a94..f00e6052 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -73,6 +73,7 @@ def _default_cmm_for_keyring(self): ##= specification/s3-encryption/client.md#key-commitment ##% If the configured Encryption Algorithm is incompatible with the key commitment policy, then it MUST throw an exception. def __attrs_post_init__(self): + """Validate algorithm suite and commitment policy configuration.""" if self.encryption_algorithm.is_legacy: raise S3EncryptionClientError( f"Cannot configure S3 Encryption Client with legacy algorithm suite " diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 6483beec..8ea852bc 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -77,7 +77,7 @@ class AlgorithmSuite(Enum): def __init__( self, - id: int, + suite_id: int, is_legacy: bool, data_key_algorithm: str, data_key_length_bits: int, @@ -91,7 +91,8 @@ def __init__( kdf_hash_algorithm: str | None, suite_id_bytes: bytes, ): - self._id = id + """Initialize algorithm suite parameters from the enum tuple.""" + self._id = suite_id self._is_legacy = is_legacy self._data_key_algorithm = data_key_algorithm self._data_key_length_bits = data_key_length_bits @@ -109,6 +110,7 @@ def __init__( @property def suite_id(self) -> int: + """Numeric identifier for this algorithm suite.""" return self._id @property @@ -123,18 +125,22 @@ def supports_key_commitment(self) -> bool: @property def data_key_length_bytes(self) -> int: + """Data key length in bytes.""" return self._data_key_length_bits // 8 @property def cipher_name(self) -> str: + """Cipher transformation string (e.g. 'AES/GCM/NoPadding').""" return self._cipher_name @property def cipher_iv_length_bytes(self) -> int: + """Initialization vector length in bytes.""" return self._cipher_iv_length_bits // 8 @property def commitment_length_bytes(self) -> int: + """Key commitment value length in bytes.""" return self._commitment_length_bits // 8 @property @@ -144,6 +150,7 @@ def commitment_nonce_length_bytes(self) -> int: @property def suite_id_bytes(self) -> bytes: + """Algorithm suite ID as raw bytes for use in HKDF info strings.""" return self._suite_id_bytes @property diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 6340b2a8..04b469c0 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -348,7 +348,9 @@ def decrypt( ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. - if algorithm_suite.is_legacy: + if ( + algorithm_suite.is_legacy and not self.enable_legacy_unauthenticated_modes + ): # noqa: SIM102 ##= specification/s3-encryption/decryption.md#legacy-decryption ##= type=implementation ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites @@ -358,14 +360,13 @@ def decrypt( ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, ##% the client MUST throw an exception when attempting to decrypt an object encrypted ##% with a legacy unauthenticated algorithm suite. - if not self.enable_legacy_unauthenticated_modes: - raise S3EncryptionClientError( - "Cannot decrypt object encrypted with ALG_AES_256_CBC_IV16_NO_KDF. " - "The S3 Encryption Client is not configured to decrypt objects using " - "legacy unauthenticated algorithm suites. " - "Set enable_legacy_unauthenticated_modes=True to allow decryption " - "of objects encrypted with CBC." - ) + raise S3EncryptionClientError( + "Cannot decrypt object encrypted with ALG_AES_256_CBC_IV16_NO_KDF. " + "The S3 Encryption Client is not configured to decrypt objects using " + "legacy unauthenticated algorithm suites. " + "Set enable_legacy_unauthenticated_modes=True to allow decryption " + "of objects encrypted with CBC." + ) ##= specification/s3-encryption/decryption.md#key-commitment ##= type=implementation From b219535a71cc943a519db90860d213b682661be2 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 13 Mar 2026 11:59:23 -0700 Subject: [PATCH 26/32] default to key committing encryption algorithm --- src/s3_encryption/__init__.py | 2 +- src/s3_encryption/pipelines.py | 12 ++- test/test_default_algorithm_commitment.py | 89 +++++++++++++++++++++++ test/test_encryption.py | 20 ++--- 4 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 test/test_default_algorithm_commitment.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index f00e6052..0a0933b4 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -162,7 +162,7 @@ def on_put_object_before_call(self, params, **kwargs): encrypted_data, encryption_metadata = pipeline.encrypt( body_bytes, encryption_context=encryption_context, - algorithm_suite=self.config.encryption_algorithm, + encryption_algorithm=self.config.encryption_algorithm, ) params["body"] = encrypted_data diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 04b469c0..cd9c188b 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -39,20 +39,24 @@ class PutEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() - def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): + def encrypt(self, plaintext, encryption_context=None, encryption_algorithm=None): """Encrypt the data before it is stored in S3. Args: plaintext (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption - algorithm_suite (AlgorithmSuite, optional): Algorithm suite to use + encryption_algorithm (AlgorithmSuite): Algorithm suite to use Returns: bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ - if algorithm_suite is None: - algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + if encryption_algorithm is None: + raise S3EncryptionClientError( + "encryption_algorithm is required for encryption." + ) + + algorithm_suite = encryption_algorithm ##= specification/s3-encryption/encryption.md#content-encryption ##= type=implementation diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py new file mode 100644 index 00000000..cd108b2f --- /dev/null +++ b/test/test_default_algorithm_commitment.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Integration test: the default encryption algorithm MUST use key commitment. + +When S3EncryptionClientConfig is created with no explicit encryption_algorithm, +the default (ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) MUST produce ciphertext +that is decryptable under REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy. +""" + +import base64 +import os +from io import BytesIO +from unittest.mock import MagicMock, Mock + +from s3_encryption import S3EncryptionClientConfig +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.encrypted_data_key import EncryptedDataKey +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, +) +from s3_encryption.pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline + + +def _mock_keyring(key=None): + """Return a mock keyring that populates encryption/decryption materials.""" + if key is None: + key = os.urandom(32) + mock = MagicMock(spec=S3Keyring) + + def on_encrypt(mats): + mats.plaintext_data_key = key + mats.encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info="kms+context", + encrypted_data_key=b"encrypted-key", + ) + return mats + + def on_decrypt(mats, encrypted_data_keys=None): + mats.plaintext_data_key = key + return mats + + mock.on_encrypt.side_effect = on_encrypt + mock.on_decrypt.side_effect = on_decrypt + return mock, key + + +class TestDefaultAlgorithmUsesKeyCommitment: + """The default encryption algorithm MUST be key-committing.""" + + def test_default_config_encrypts_with_committing_algorithm(self): + """S3EncryptionClientConfig with no explicit algorithm MUST default to a + key-committing suite.""" + keyring, _ = _mock_keyring() + config = S3EncryptionClientConfig(keyring=keyring) + assert config.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + + def test_default_encryption_decryptable_with_require_decrypt(self): + """Ciphertext produced with the default algorithm MUST be decryptable + when the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT.""" + keyring, key = _mock_keyring() + config = S3EncryptionClientConfig(keyring=keyring) + cmm = DefaultCryptoMaterialsManager(keyring) + + # Encrypt using the default algorithm (no override) + pipeline = PutEncryptedObjectPipeline(cmm) + plaintext = b"integration test: default algorithm uses key commitment" + ciphertext, metadata = pipeline.encrypt( + plaintext, + encryption_algorithm=config.encryption_algorithm, + ) + + # Build a response dict as if we fetched this object from S3 + response = { + "Body": BytesIO(ciphertext), + "Metadata": metadata, + } + + # Decrypt with REQUIRE_ENCRYPT_REQUIRE_DECRYPT — this will reject + # non-committing algorithm suites, so success proves the default commits. + decrypt_pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) + result = decrypt_pipeline.decrypt(response) + assert result == plaintext diff --git a/test/test_encryption.py b/test/test_encryption.py index 871a8360..86640aa0 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -73,7 +73,7 @@ def test_uses_configured_algorithm_suite(self): # V2 (GCM no KDF) _, meta_v2 = pipeline.encrypt( plaintext, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) assert "x-amz-cek-alg" in meta_v2 assert meta_v2["x-amz-cek-alg"] == "AES/GCM/NoPadding" @@ -81,7 +81,7 @@ def test_uses_configured_algorithm_suite(self): # V3 (KC GCM) _, meta_v3 = pipeline.encrypt( plaintext, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) assert "x-amz-c" in meta_v3 assert meta_v3["x-amz-c"] == "115" @@ -97,7 +97,7 @@ def test_iv_generated_with_correct_length_gcm(self): _, meta = pipeline.encrypt( b"test", - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) iv_bytes = base64.b64decode(meta["x-amz-iv"]) assert len(iv_bytes) == 12 @@ -109,7 +109,7 @@ def test_message_id_generated_with_correct_length_kc(self): _, meta = pipeline.encrypt( b"test", - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) message_id_bytes = base64.b64decode(meta["x-amz-i"]) assert len(message_id_bytes) == MESSAGE_ID_LENGTH @@ -125,7 +125,7 @@ def test_iv_included_in_metadata_gcm(self): _, meta = pipeline.encrypt( b"test", - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) assert "x-amz-iv" in meta @@ -136,7 +136,7 @@ def test_message_id_included_in_metadata_kc(self): _, meta = pipeline.encrypt( b"test", - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) assert "x-amz-i" in meta @@ -165,7 +165,7 @@ def test_gcm_encrypt_decrypt_roundtrip_no_aad(self): ciphertext, meta = pipeline.encrypt( plaintext, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) # Decrypt with the same key, IV, and no AAD @@ -181,7 +181,7 @@ def test_gcm_decrypt_fails_with_aad(self): ciphertext, meta = pipeline.encrypt( b"test", - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) iv = base64.b64decode(meta["x-amz-iv"]) @@ -210,7 +210,7 @@ def test_kc_gcm_uses_hkdf_derived_key(self): ciphertext, meta = pipeline.encrypt( plaintext, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) message_id = base64.b64decode(meta["x-amz-i"]) @@ -239,7 +239,7 @@ def test_kc_gcm_commitment_in_metadata(self): _, meta = pipeline.encrypt( b"test", - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) assert "x-amz-d" in meta From 94724200febd4cc80ed28f8af9a69f08960ada9e Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 13 Mar 2026 12:50:52 -0700 Subject: [PATCH 27/32] PR feedback --- src/s3_encryption/materials/kms_keyring.py | 6 +- src/s3_encryption/materials/materials.py | 11 +++- src/s3_encryption/pipelines.py | 55 +++++++------------ test/test_default_algorithm_commitment.py | 8 +++ test/test_encryption_materials_integration.py | 2 +- test/test_key_derivation.py | 13 +---- test/test_kms_keyring.py | 2 +- 7 files changed, 43 insertions(+), 54 deletions(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 85df14a7..46c04340 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -81,14 +81,14 @@ def on_encrypt(self, enc_materials): # value is the algorithm suite ID as a string ("115"), not the cipher name. # For non-committing suites (V2), use the cipher name ("AES/GCM/NoPadding"). if ( - enc_materials.algorithm_suite + enc_materials.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ): encryption_context["aws:x-amz-cek-alg"] = str( - enc_materials.algorithm_suite.suite_id + enc_materials.encryption_algorithm.suite_id ) else: - encryption_context["aws:x-amz-cek-alg"] = enc_materials.algorithm_suite.cipher_name + encryption_context["aws:x-amz-cek-alg"] = enc_materials.encryption_algorithm.cipher_name # Python implementation uses KMS GenerateDataKey instead of the spec's # EncryptDataKey pattern diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 8ea852bc..f2e8fd4f 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -163,6 +163,13 @@ def kc_gcm_iv(self) -> bytes: """Fixed IV for key-committing GCM: all 0x01 bytes of cipher_iv_length.""" if not self._is_committing: raise ValueError(f"{self.name} does not support key commitment") + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% The IV's total length MUST match the IV length defined by the algorithm suite. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##= type=implementation + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. return b"\x01" * self.cipher_iv_length_bytes @@ -187,8 +194,8 @@ class EncryptionMaterials: plaintext_data_key (Optional[bytes]): The plaintext data key """ - algorithm_suite: AlgorithmSuite = field( - default=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + encryption_algorithm: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ) encryption_context: dict[str, str] = field(factory=dict) encrypted_data_key: EncryptedDataKey | None = field(default=None) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index cd9c188b..ba55bd22 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -65,7 +65,7 @@ def encrypt(self, plaintext, encryption_context=None, encryption_algorithm=None) # Create encryption materials request with encryption context copy enc_mats_request = EncryptionMaterials( - algorithm_suite=algorithm_suite, + encryption_algorithm=algorithm_suite, encryption_context={} if encryption_context is None else encryption_context.copy(), ) @@ -85,29 +85,28 @@ def encrypt(self, plaintext, encryption_context=None, encryption_algorithm=None) def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + iv = os.urandom(enc_mats.encryption_algorithm.cipher_iv_length_bytes) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##= type=implementation ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, ##% with the plaintext data key, the generated IV, and the tag length defined ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + aesgcm = AESGCM(enc_mats.plaintext_data_key) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##= type=implementation ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. - ##= specification/s3-encryption/encryption.md#content-encryption - ##= type=implementation - ##% The client MUST generate an IV or Message ID using the length of the IV - ##% or Message ID defined in the algorithm suite. - ##= specification/s3-encryption/encryption.md#content-encryption - ##= type=implementation - ##% The generated IV or Message ID MUST be set or returned from the encryption - ##% process such that it can be included in the content metadata. - iv = os.urandom(enc_mats.algorithm_suite.cipher_iv_length_bytes) - aesgcm = AESGCM(enc_mats.plaintext_data_key) encrypted_data = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) b64_iv = base64.b64encode(iv).decode("utf-8") b64_edk = base64.b64encode(edk_bytes).decode("utf-8") + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The generated IV or Message ID MUST be set or returned from the encryption metadata = ObjectMetadata( encrypted_data_key_v2=b64_edk, encrypted_data_key_algorithm="kms+context", @@ -124,32 +123,17 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##= type=implementation ##% The client MUST generate an IV or Message ID using the length of the IV ##% or Message ID defined in the algorithm suite. - ##= specification/s3-encryption/encryption.md#content-encryption - ##= type=implementation - ##% The generated IV or Message ID MUST be set or returned from the encryption - ##% process such that it can be included in the content metadata. - algorithm_suite = enc_mats.algorithm_suite + algorithm_suite = enc_mats.encryption_algorithm message_id = os.urandom(algorithm_suite.commitment_nonce_length_bytes) ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key ##= type=implementation ##% The client MUST use HKDF to derive the key commitment value and the derived ##% encrypting key as described in [Key Derivation](key-derivation.md). - ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key - ##= type=implementation - ##% The derived key commitment value MUST be set or returned from the encryption - ##% process such that it can be included in the content metadata. derived_encryption_key, commit_key = derive_keys( enc_mats.plaintext_data_key, message_id, algorithm_suite ) - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation - ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##= type=implementation - ##% The IV's total length MUST match the IV length defined by the algorithm suite. ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##= type=implementation ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only bytes with the value 0x01, @@ -168,13 +152,14 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): b64_message_id = base64.b64encode(message_id).decode("utf-8") b64_commit_key = base64.b64encode(commit_key).decode("utf-8") - # V3 metadata format - # x-amz-c: content cipher identifier (compressed algorithm suite ID) - # x-amz-w: wrapping algorithm identifier (12 = kms+context) - # x-amz-3: encrypted data key - # x-amz-i: message ID - # x-amz-d: key commitment - # x-amz-t: encryption context (for kms+context wrapping) + ##= specification/s3-encryption/encryption.md#content-encryption + ##= type=implementation + ##% The generated IV or Message ID MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##= type=implementation + ##% The derived key commitment value MUST be set or returned from the encryption + ##% process such that it can be included in the content metadata. metadata = ObjectMetadata( content_cipher_v3=str(algorithm_suite.suite_id), encrypted_data_key_algorithm_v3="12", @@ -467,6 +452,7 @@ def _decrypt_v1_v2( def _decrypt_cbc_content(self, dec_materials, encrypted_data): """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF. + """ ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=implementation ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and @@ -474,7 +460,6 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or ##% PKCS7Padding compatible padding for a 16-byte block cipher ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). - """ ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=implementation ##% If the cipher object cannot be created as described above, diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py index cd108b2f..9c7fcfd8 100644 --- a/test/test_default_algorithm_commitment.py +++ b/test/test_default_algorithm_commitment.py @@ -58,6 +58,14 @@ def test_default_config_encrypts_with_committing_algorithm(self): config = S3EncryptionClientConfig(keyring=keyring) assert config.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + def test_encryption_materials_defaults_to_committing_algorithm(self): + """EncryptionMaterials with no explicit algorithm MUST default to a + key-committing suite.""" + from s3_encryption.materials.materials import EncryptionMaterials + + mats = EncryptionMaterials() + assert mats.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + def test_default_encryption_decryptable_with_require_decrypt(self): """Ciphertext produced with the default algorithm MUST be decryptable when the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT.""" diff --git a/test/test_encryption_materials_integration.py b/test/test_encryption_materials_integration.py index 4b6bfefd..e9e59023 100644 --- a/test/test_encryption_materials_integration.py +++ b/test/test_encryption_materials_integration.py @@ -35,7 +35,7 @@ def test_keyring_on_encrypt(self): assert isinstance(result, EncryptionMaterials) assert result.encryption_context == { "key1": "value1", - "aws:x-amz-cek-alg": "AES/GCM/NoPadding", + "aws:x-amz-cek-alg": "115", } def test_cmm_get_encryption_materials_with_dict(self): diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py index 3843edfd..2dc7028a 100644 --- a/test/test_key_derivation.py +++ b/test/test_key_derivation.py @@ -8,6 +8,7 @@ """ import hmac +import os import pytest from cryptography.hazmat.primitives.ciphers.aead import AESGCM @@ -40,7 +41,6 @@ class TestHkdfOperation: ##% - The hash function MUST be specified by the algorithm suite commitment settings. def test_hash_function_is_sha512(self): """HKDF extract MUST use the hash function specified by the algorithm suite.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -65,7 +65,6 @@ def test_hash_function_is_sha512(self): ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. def test_ikm_is_plaintext_data_key(self): """Different plaintext data keys MUST produce different derived keys.""" - import os msg_id = os.urandom(MESSAGE_ID_LENGTH) pdk_a = os.urandom(_KC_SUITE.data_key_length_bytes) @@ -80,7 +79,6 @@ def test_ikm_is_plaintext_data_key(self): ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. def test_ikm_length_is_32_bytes(self): """The plaintext data key (IKM) length MUST equal the algorithm suite's data key length.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -95,7 +93,6 @@ def test_ikm_length_is_32_bytes(self): ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. def test_ikm_wrong_length_raises(self): """derive_keys MUST raise when the plaintext data key length doesn't match the suite.""" - import os from s3_encryption.exceptions import S3EncryptionClientError @@ -112,7 +109,6 @@ def test_ikm_wrong_length_raises(self): ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. def test_salt_is_message_id(self): """Different Message IDs (salts) MUST produce different derived keys.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id_a = os.urandom(MESSAGE_ID_LENGTH) @@ -127,7 +123,6 @@ def test_salt_is_message_id(self): ##% - The DEK input pseudorandom key MUST be the output from the extract step. def test_dek_uses_prk_from_extract(self): """The DEK expand step MUST use the PRK from the extract step.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -149,7 +144,6 @@ def test_dek_uses_prk_from_extract(self): ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. def test_dek_output_length(self): """The derived encryption key MUST match the encryption key length from the algorithm suite.""" - import os key, _ = derive_keys( os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE @@ -161,7 +155,6 @@ def test_dek_output_length(self): ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. def test_dek_info_is_suite_id_plus_derivekey(self): """DEK expand info MUST be suite_id_bytes + b'DERIVEKEY'.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -191,7 +184,6 @@ def test_dek_info_is_suite_id_plus_derivekey(self): ##% - The CK input pseudorandom key MUST be the output from the extract step. def test_ck_uses_prk_from_extract(self): """The CK expand step MUST use the PRK from the extract step.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -211,7 +203,6 @@ def test_ck_uses_prk_from_extract(self): ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. def test_ck_output_length(self): """The commit key length MUST match the algorithm suite's commitment length.""" - import os _, ck = derive_keys( os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE @@ -223,7 +214,6 @@ def test_ck_output_length(self): ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. def test_ck_info_is_suite_id_plus_commitkey(self): """CK expand info MUST be suite_id_bytes + b'COMMITKEY'.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -279,7 +269,6 @@ def test_kc_gcm_iv_length_is_12(self): ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): """KC-GCM MUST encrypt/decrypt with derived key, 0x01 IV, and suite ID as AAD.""" - import os pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) diff --git a/test/test_kms_keyring.py b/test/test_kms_keyring.py index d613cbf9..0a5d66de 100644 --- a/test/test_kms_keyring.py +++ b/test/test_kms_keyring.py @@ -128,7 +128,7 @@ def test_on_encrypt_adds_kms_context_algorithm(self): result = keyring.on_encrypt(enc_materials) call_args = mock_kms_client.generate_data_key.call_args - assert call_args.kwargs["EncryptionContext"]["aws:x-amz-cek-alg"] == "AES/GCM/NoPadding" + assert call_args.kwargs["EncryptionContext"]["aws:x-amz-cek-alg"] == "115" def test_on_encrypt_sets_encrypted_data_key(self): """Test that on_encrypt sets the encrypted data key from KMS response.""" From 2a11683d35479ba126af7ce5113be08b4ee4215d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 13 Mar 2026 12:54:47 -0700 Subject: [PATCH 28/32] lint --- src/s3_encryption/materials/kms_keyring.py | 4 +++- src/s3_encryption/pipelines.py | 8 ++------ test/test_default_algorithm_commitment.py | 13 +++++++------ test/test_key_derivation.py | 12 ------------ 4 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 46c04340..edf1d27b 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -88,7 +88,9 @@ def on_encrypt(self, enc_materials): enc_materials.encryption_algorithm.suite_id ) else: - encryption_context["aws:x-amz-cek-alg"] = enc_materials.encryption_algorithm.cipher_name + encryption_context["aws:x-amz-cek-alg"] = ( + enc_materials.encryption_algorithm.cipher_name + ) # Python implementation uses KMS GenerateDataKey instead of the spec's # EncryptDataKey pattern diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index ba55bd22..668e3a4d 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -52,9 +52,7 @@ def encrypt(self, plaintext, encryption_context=None, encryption_algorithm=None) dict: Metadata about the encryption to be stored with the object """ if encryption_algorithm is None: - raise S3EncryptionClientError( - "encryption_algorithm is required for encryption." - ) + raise S3EncryptionClientError("encryption_algorithm is required for encryption.") algorithm_suite = encryption_algorithm @@ -450,9 +448,7 @@ def _decrypt_v1_v2( return self.cmm.decrypt_materials(dec_materials) def _decrypt_cbc_content(self, dec_materials, encrypted_data): - """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF. - - """ + """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF.""" ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=implementation ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py index 9c7fcfd8..01508bd6 100644 --- a/test/test_default_algorithm_commitment.py +++ b/test/test_default_algorithm_commitment.py @@ -7,10 +7,9 @@ that is decryptable under REQUIRE_ENCRYPT_REQUIRE_DECRYPT commitment policy. """ -import base64 import os from io import BytesIO -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock from s3_encryption import S3EncryptionClientConfig from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager @@ -19,7 +18,6 @@ from s3_encryption.materials.materials import ( AlgorithmSuite, CommitmentPolicy, - DecryptionMaterials, ) from s3_encryption.pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline @@ -53,14 +51,16 @@ class TestDefaultAlgorithmUsesKeyCommitment: def test_default_config_encrypts_with_committing_algorithm(self): """S3EncryptionClientConfig with no explicit algorithm MUST default to a - key-committing suite.""" + key-committing suite. + """ keyring, _ = _mock_keyring() config = S3EncryptionClientConfig(keyring=keyring) assert config.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY def test_encryption_materials_defaults_to_committing_algorithm(self): """EncryptionMaterials with no explicit algorithm MUST default to a - key-committing suite.""" + key-committing suite. + """ from s3_encryption.materials.materials import EncryptionMaterials mats = EncryptionMaterials() @@ -68,7 +68,8 @@ def test_encryption_materials_defaults_to_committing_algorithm(self): def test_default_encryption_decryptable_with_require_decrypt(self): """Ciphertext produced with the default algorithm MUST be decryptable - when the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT.""" + when the commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT. + """ keyring, key = _mock_keyring() config = S3EncryptionClientConfig(keyring=keyring) cmm = DefaultCryptoMaterialsManager(keyring) diff --git a/test/test_key_derivation.py b/test/test_key_derivation.py index 2dc7028a..0bec87f7 100644 --- a/test/test_key_derivation.py +++ b/test/test_key_derivation.py @@ -41,7 +41,6 @@ class TestHkdfOperation: ##% - The hash function MUST be specified by the algorithm suite commitment settings. def test_hash_function_is_sha512(self): """HKDF extract MUST use the hash function specified by the algorithm suite.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -65,7 +64,6 @@ def test_hash_function_is_sha512(self): ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. def test_ikm_is_plaintext_data_key(self): """Different plaintext data keys MUST produce different derived keys.""" - msg_id = os.urandom(MESSAGE_ID_LENGTH) pdk_a = os.urandom(_KC_SUITE.data_key_length_bytes) pdk_b = os.urandom(_KC_SUITE.data_key_length_bytes) @@ -79,7 +77,6 @@ def test_ikm_is_plaintext_data_key(self): ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. def test_ikm_length_is_32_bytes(self): """The plaintext data key (IKM) length MUST equal the algorithm suite's data key length.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) assert len(pdk) == _KC_SUITE.data_key_length_bytes @@ -93,7 +90,6 @@ def test_ikm_length_is_32_bytes(self): ##% - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. def test_ikm_wrong_length_raises(self): """derive_keys MUST raise when the plaintext data key length doesn't match the suite.""" - from s3_encryption.exceptions import S3EncryptionClientError msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -109,7 +105,6 @@ def test_ikm_wrong_length_raises(self): ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. def test_salt_is_message_id(self): """Different Message IDs (salts) MUST produce different derived keys.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id_a = os.urandom(MESSAGE_ID_LENGTH) msg_id_b = os.urandom(MESSAGE_ID_LENGTH) @@ -123,7 +118,6 @@ def test_salt_is_message_id(self): ##% - The DEK input pseudorandom key MUST be the output from the extract step. def test_dek_uses_prk_from_extract(self): """The DEK expand step MUST use the PRK from the extract step.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -144,7 +138,6 @@ def test_dek_uses_prk_from_extract(self): ##% - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. def test_dek_output_length(self): """The derived encryption key MUST match the encryption key length from the algorithm suite.""" - key, _ = derive_keys( os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE ) @@ -155,7 +148,6 @@ def test_dek_output_length(self): ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. def test_dek_info_is_suite_id_plus_derivekey(self): """DEK expand info MUST be suite_id_bytes + b'DERIVEKEY'.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -184,7 +176,6 @@ def test_dek_info_is_suite_id_plus_derivekey(self): ##% - The CK input pseudorandom key MUST be the output from the extract step. def test_ck_uses_prk_from_extract(self): """The CK expand step MUST use the PRK from the extract step.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -203,7 +194,6 @@ def test_ck_uses_prk_from_extract(self): ##% - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. def test_ck_output_length(self): """The commit key length MUST match the algorithm suite's commitment length.""" - _, ck = derive_keys( os.urandom(_KC_SUITE.data_key_length_bytes), os.urandom(MESSAGE_ID_LENGTH), _KC_SUITE ) @@ -214,7 +204,6 @@ def test_ck_output_length(self): ##% - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. def test_ck_info_is_suite_id_plus_commitkey(self): """CK expand info MUST be suite_id_bytes + b'COMMITKEY'.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) @@ -269,7 +258,6 @@ def test_kc_gcm_iv_length_is_12(self): ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. def test_kc_gcm_roundtrip_with_derived_key_iv_aad(self): """KC-GCM MUST encrypt/decrypt with derived key, 0x01 IV, and suite ID as AAD.""" - pdk = os.urandom(_KC_SUITE.data_key_length_bytes) msg_id = os.urandom(MESSAGE_ID_LENGTH) plaintext = b"key derivation roundtrip test" From 95977d2164a3611f90f2f907f15fe01a9663d7a9 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 18 Mar 2026 12:58:29 -0700 Subject: [PATCH 29/32] PR feedback - make pipelines require enc alg --- src/s3_encryption/__init__.py | 2 +- src/s3_encryption/pipelines.py | 9 ++++----- test/test_default_algorithm_commitment.py | 2 +- test/test_encryption.py | 20 ++++++++++---------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 0a0933b4..68774f3f 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -161,8 +161,8 @@ def on_put_object_before_call(self, params, **kwargs): pipeline = PutEncryptedObjectPipeline(self.config.cmm) encrypted_data, encryption_metadata = pipeline.encrypt( body_bytes, + self.config.encryption_algorithm, encryption_context=encryption_context, - encryption_algorithm=self.config.encryption_algorithm, ) params["body"] = encrypted_data diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 668e3a4d..39ad0e0d 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -39,21 +39,18 @@ class PutEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() - def encrypt(self, plaintext, encryption_context=None, encryption_algorithm=None): + def encrypt(self, plaintext, encryption_algorithm, encryption_context=None): """Encrypt the data before it is stored in S3. Args: plaintext (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption - encryption_algorithm (AlgorithmSuite): Algorithm suite to use + encryption_algorithm (AlgorithmSuite): Algorithm suite to use (required) Returns: bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ - if encryption_algorithm is None: - raise S3EncryptionClientError("encryption_algorithm is required for encryption.") - algorithm_suite = encryption_algorithm ##= specification/s3-encryption/encryption.md#content-encryption @@ -263,6 +260,8 @@ def decrypt( if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") + # TODO: we should validate that these parameters must be None + # when not in instruction file mode. if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py index 01508bd6..2079ba56 100644 --- a/test/test_default_algorithm_commitment.py +++ b/test/test_default_algorithm_commitment.py @@ -79,7 +79,7 @@ def test_default_encryption_decryptable_with_require_decrypt(self): plaintext = b"integration test: default algorithm uses key commitment" ciphertext, metadata = pipeline.encrypt( plaintext, - encryption_algorithm=config.encryption_algorithm, + config.encryption_algorithm, ) # Build a response dict as if we fetched this object from S3 diff --git a/test/test_encryption.py b/test/test_encryption.py index 86640aa0..aa21cf39 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -73,7 +73,7 @@ def test_uses_configured_algorithm_suite(self): # V2 (GCM no KDF) _, meta_v2 = pipeline.encrypt( plaintext, - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) assert "x-amz-cek-alg" in meta_v2 assert meta_v2["x-amz-cek-alg"] == "AES/GCM/NoPadding" @@ -81,7 +81,7 @@ def test_uses_configured_algorithm_suite(self): # V3 (KC GCM) _, meta_v3 = pipeline.encrypt( plaintext, - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) assert "x-amz-c" in meta_v3 assert meta_v3["x-amz-c"] == "115" @@ -97,7 +97,7 @@ def test_iv_generated_with_correct_length_gcm(self): _, meta = pipeline.encrypt( b"test", - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) iv_bytes = base64.b64decode(meta["x-amz-iv"]) assert len(iv_bytes) == 12 @@ -109,7 +109,7 @@ def test_message_id_generated_with_correct_length_kc(self): _, meta = pipeline.encrypt( b"test", - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) message_id_bytes = base64.b64decode(meta["x-amz-i"]) assert len(message_id_bytes) == MESSAGE_ID_LENGTH @@ -125,7 +125,7 @@ def test_iv_included_in_metadata_gcm(self): _, meta = pipeline.encrypt( b"test", - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) assert "x-amz-iv" in meta @@ -136,7 +136,7 @@ def test_message_id_included_in_metadata_kc(self): _, meta = pipeline.encrypt( b"test", - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) assert "x-amz-i" in meta @@ -165,7 +165,7 @@ def test_gcm_encrypt_decrypt_roundtrip_no_aad(self): ciphertext, meta = pipeline.encrypt( plaintext, - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) # Decrypt with the same key, IV, and no AAD @@ -181,7 +181,7 @@ def test_gcm_decrypt_fails_with_aad(self): ciphertext, meta = pipeline.encrypt( b"test", - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, ) iv = base64.b64decode(meta["x-amz-iv"]) @@ -210,7 +210,7 @@ def test_kc_gcm_uses_hkdf_derived_key(self): ciphertext, meta = pipeline.encrypt( plaintext, - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) message_id = base64.b64decode(meta["x-amz-i"]) @@ -239,7 +239,7 @@ def test_kc_gcm_commitment_in_metadata(self): _, meta = pipeline.encrypt( b"test", - encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, ) assert "x-amz-d" in meta From 6f5b380ff3f900a5decadb98bd23e2203ff21c2d Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 18 Mar 2026 13:06:18 -0700 Subject: [PATCH 30/32] PR feedback - make commit policy required in decrypt pipeline --- src/s3_encryption/__init__.py | 2 +- src/s3_encryption/pipelines.py | 4 +--- test/test_pipelines.py | 31 +++++++++++++++++++++++++------ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 68774f3f..e89be577 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -207,8 +207,8 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( self.config.cmm, - s3_client=getattr(self._context, _CTX_S3_CLIENT, None), commitment_policy=self.config.commitment_policy, + s3_client=getattr(self._context, _CTX_S3_CLIENT, None), enable_legacy_unauthenticated_modes=self.config.enable_legacy_unauthenticated_modes, ) decrypted_data = pipeline.decrypt( diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 39ad0e0d..01161807 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -178,10 +178,8 @@ class GetEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() + commitment_policy: CommitmentPolicy = field() s3_client: object = field(default=None) - commitment_policy: CommitmentPolicy = field( - default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - ) enable_legacy_unauthenticated_modes: bool = field(default=False) # Map content cipher metadata values to AlgorithmSuite diff --git a/test/test_pipelines.py b/test/test_pipelines.py index d3a643f5..3d32b8cc 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -11,7 +11,7 @@ from s3_encryption.exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.keyring import S3Keyring -from s3_encryption.materials.materials import DecryptionMaterials +from s3_encryption.materials.materials import CommitmentPolicy, DecryptionMaterials from s3_encryption.pipelines import GetEncryptedObjectPipeline @@ -47,7 +47,11 @@ def test_decrypt_v1_from_instruction_file(self): cmm = DefaultCryptoMaterialsManager(mock_keyring) # Create pipeline with mocked S3 client - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) # Create mock response mock_response = { @@ -102,7 +106,11 @@ def test_decrypt_v2_from_instruction_file(self): cmm = DefaultCryptoMaterialsManager(mock_keyring) # Create pipeline with mocked S3 client - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) # Create mock response mock_response = { @@ -158,7 +166,11 @@ def test_decrypt_v3_from_instruction_file(self): cmm = DefaultCryptoMaterialsManager(mock_keyring) # Create pipeline with mocked S3 client - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) # Create mock response with encrypted data iv = os.urandom(12) @@ -220,7 +232,11 @@ def test_decrypt_with_custom_instruction_file_suffix(self): mock_keyring = Mock(spec=S3Keyring) cmm = DefaultCryptoMaterialsManager(mock_keyring) - pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + s3_client=mock_s3_client, + ) mock_response = { "Body": BytesIO(b"encrypted-test-data"), @@ -261,7 +277,10 @@ def test_decrypt_v3_unsupported_wrap_alg(self): "AES/GCM is not a valid key wrapping algorithm!" ) cmm = DefaultCryptoMaterialsManager(mock_keyring) - pipeline = GetEncryptedObjectPipeline(cmm) + pipeline = GetEncryptedObjectPipeline( + cmm, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + ) mock_response = { "Body": BytesIO(b"encrypted-test-data"), From ff188fdc6e009632638fd152df5bf7986c157530 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 18 Mar 2026 13:29:56 -0700 Subject: [PATCH 31/32] PR feedback + refactor --- src/s3_encryption/__init__.py | 3 +- src/s3_encryption/pipelines.py | 11 ++- test/integration/test_i_s3_encryption.py | 31 +++++++++ test/test_default_algorithm_commitment.py | 7 +- test/test_encryption.py | 85 +++++++++-------------- 5 files changed, 70 insertions(+), 67 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index e89be577..f2af6d7a 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -158,10 +158,9 @@ def on_put_object_before_call(self, params, **kwargs): encryption_context = getattr(self._context, _CTX_ENCRYPTION_CONTEXT, None) - pipeline = PutEncryptedObjectPipeline(self.config.cmm) + pipeline = PutEncryptedObjectPipeline(self.config.cmm, self.config.encryption_algorithm) encrypted_data, encryption_metadata = pipeline.encrypt( body_bytes, - self.config.encryption_algorithm, encryption_context=encryption_context, ) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 01161807..e4a4cb09 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -38,29 +38,26 @@ class PutEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() + encryption_algorithm: AlgorithmSuite = field() - def encrypt(self, plaintext, encryption_algorithm, encryption_context=None): + def encrypt(self, plaintext, encryption_context=None): """Encrypt the data before it is stored in S3. Args: plaintext (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption - encryption_algorithm (AlgorithmSuite): Algorithm suite to use (required) Returns: bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ - algorithm_suite = encryption_algorithm ##= specification/s3-encryption/encryption.md#content-encryption ##= type=implementation ##% The S3EC MUST use the encryption algorithm configured during ##% [client](./client.md) initialization. - - # Create encryption materials request with encryption context copy enc_mats_request = EncryptionMaterials( - encryption_algorithm=algorithm_suite, + encryption_algorithm=self.encryption_algorithm, encryption_context={} if encryption_context is None else encryption_context.copy(), ) @@ -74,7 +71,7 @@ def encrypt(self, plaintext, encryption_algorithm, encryption_context=None): edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key - if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + if self.encryption_algorithm == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: return self._encrypt_kc_gcm(plaintext, enc_mats, edk_bytes) return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 17f6c682..15133c05 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -221,3 +221,34 @@ def test_encryption_context_missing_on_decrypt(algorithm_suite, commitment_polic with pytest.raises(S3EncryptionClientError): s3ec.get_object(Bucket=bucket, Key=key) + + +# Expected metadata key that identifies the content encryption algorithm, +# keyed by algorithm suite. +_EXPECTED_ALGORITHM_METADATA = { + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: ("x-amz-cek-alg", "AES/GCM/NoPadding"), + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: ("x-amz-c", "115"), +} + + +##= specification/s3-encryption/encryption.md#content-encryption +##= type=test +##% The S3EC MUST use the encryption algorithm configured during +##% [client](./client.md) initialization. +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_put_object_uses_configured_algorithm(algorithm_suite, commitment_policy): + """PutObject MUST encrypt using the algorithm suite configured at client init.""" + key = _unique_key("configured-alg-") + data = b"test configured algorithm" + + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + + # Read back with a plain S3 client to inspect the raw metadata + plain_s3 = boto3.client("s3") + response = plain_s3.head_object(Bucket=bucket, Key=key) + metadata = response.get("Metadata", {}) + + meta_key, expected_value = _EXPECTED_ALGORITHM_METADATA[algorithm_suite] + assert meta_key in metadata, f"Expected metadata key '{meta_key}' not found in {metadata}" + assert metadata[meta_key] == expected_value diff --git a/test/test_default_algorithm_commitment.py b/test/test_default_algorithm_commitment.py index 2079ba56..1640451a 100644 --- a/test/test_default_algorithm_commitment.py +++ b/test/test_default_algorithm_commitment.py @@ -75,12 +75,9 @@ def test_default_encryption_decryptable_with_require_decrypt(self): cmm = DefaultCryptoMaterialsManager(keyring) # Encrypt using the default algorithm (no override) - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, config.encryption_algorithm) plaintext = b"integration test: default algorithm uses key commitment" - ciphertext, metadata = pipeline.encrypt( - plaintext, - config.encryption_algorithm, - ) + ciphertext, metadata = pipeline.encrypt(plaintext) # Build a response dict as if we fetched this object from S3 response = { diff --git a/test/test_encryption.py b/test/test_encryption.py index aa21cf39..d2ee5826 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -17,12 +17,13 @@ from s3_encryption.key_derivation import derive_keys from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.encrypted_data_key import EncryptedDataKey -from s3_encryption.materials.materials import AlgorithmSuite +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy _KC_SUITE = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY KC_GCM_IV = _KC_SUITE.kc_gcm_iv MESSAGE_ID_LENGTH = _KC_SUITE.commitment_nonce_length_bytes SUITE_ID_BYTES = _KC_SUITE.suite_id_bytes +from s3_encryption import S3EncryptionClientConfig from s3_encryption.pipelines import PutEncryptedObjectPipeline # --------------------------------------------------------------------------- @@ -60,29 +61,31 @@ def _fill_materials(mats, plaintext_key, encrypted_key): class TestContentEncryption: """Tests for specification/s3-encryption/encryption.md#content-encryption.""" - ##= specification/s3-encryption/encryption.md#content-encryption - ##= type=test - ##% The S3EC MUST use the encryption algorithm configured during - ##% [client](./client.md) initialization. def test_uses_configured_algorithm_suite(self): - """The pipeline MUST encrypt using the algorithm suite passed to encrypt().""" + """The pipeline MUST encrypt using the algorithm suite configured in the client.""" cmm, key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) plaintext = b"test data" # V2 (GCM no KDF) - _, meta_v2 = pipeline.encrypt( - plaintext, - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + config_v2 = S3EncryptionClientConfig( + keyring=MagicMock(), + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + cmm=cmm, ) + pipeline_v2 = PutEncryptedObjectPipeline(config_v2.cmm, config_v2.encryption_algorithm) + _, meta_v2 = pipeline_v2.encrypt(plaintext) assert "x-amz-cek-alg" in meta_v2 assert meta_v2["x-amz-cek-alg"] == "AES/GCM/NoPadding" # V3 (KC GCM) - _, meta_v3 = pipeline.encrypt( - plaintext, - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + config_v3 = S3EncryptionClientConfig( + keyring=MagicMock(), + encryption_algorithm=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + cmm=cmm, ) + pipeline_v3 = PutEncryptedObjectPipeline(config_v3.cmm, config_v3.encryption_algorithm) + _, meta_v3 = pipeline_v3.encrypt(plaintext) assert "x-amz-c" in meta_v3 assert meta_v3["x-amz-c"] == "115" @@ -93,24 +96,18 @@ def test_uses_configured_algorithm_suite(self): def test_iv_generated_with_correct_length_gcm(self): """GCM encryption MUST produce a 12-byte IV.""" cmm, _ = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - _, meta = pipeline.encrypt( - b"test", - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - ) + _, meta = pipeline.encrypt(b"test") iv_bytes = base64.b64decode(meta["x-amz-iv"]) assert len(iv_bytes) == 12 def test_message_id_generated_with_correct_length_kc(self): """KC-GCM encryption MUST produce a 28-byte Message ID.""" cmm, _ = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - _, meta = pipeline.encrypt( - b"test", - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - ) + _, meta = pipeline.encrypt(b"test") message_id_bytes = base64.b64decode(meta["x-amz-i"]) assert len(message_id_bytes) == MESSAGE_ID_LENGTH @@ -121,23 +118,17 @@ def test_message_id_generated_with_correct_length_kc(self): def test_iv_included_in_metadata_gcm(self): """GCM encryption MUST include the IV in the returned metadata.""" cmm, _ = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - _, meta = pipeline.encrypt( - b"test", - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - ) + _, meta = pipeline.encrypt(b"test") assert "x-amz-iv" in meta def test_message_id_included_in_metadata_kc(self): """KC-GCM encryption MUST include the Message ID in the returned metadata.""" cmm, _ = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - _, meta = pipeline.encrypt( - b"test", - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - ) + _, meta = pipeline.encrypt(b"test") assert "x-amz-i" in meta @@ -160,13 +151,10 @@ class TestGcmNoKdf: def test_gcm_encrypt_decrypt_roundtrip_no_aad(self): """GCM encryption MUST use the data key, generated IV, and no AAD.""" cmm, key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) plaintext = b"roundtrip test for GCM no KDF" - ciphertext, meta = pipeline.encrypt( - plaintext, - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - ) + ciphertext, meta = pipeline.encrypt(plaintext) # Decrypt with the same key, IV, and no AAD iv = base64.b64decode(meta["x-amz-iv"]) @@ -177,12 +165,9 @@ def test_gcm_encrypt_decrypt_roundtrip_no_aad(self): def test_gcm_decrypt_fails_with_aad(self): """Ciphertext produced with no AAD MUST NOT decrypt with AAD.""" cmm, key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - ciphertext, meta = pipeline.encrypt( - b"test", - AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - ) + ciphertext, meta = pipeline.encrypt(b"test") iv = base64.b64decode(meta["x-amz-iv"]) aesgcm = AESGCM(key) @@ -205,13 +190,10 @@ class TestKcGcm: def test_kc_gcm_uses_hkdf_derived_key(self): """KC-GCM encryption MUST use HKDF-derived keys, not the raw data key.""" cmm, raw_key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) plaintext = b"roundtrip test for KC GCM" - ciphertext, meta = pipeline.encrypt( - plaintext, - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - ) + ciphertext, meta = pipeline.encrypt(plaintext) message_id = base64.b64decode(meta["x-amz-i"]) derived_key, _ = derive_keys( @@ -235,12 +217,9 @@ def test_kc_gcm_uses_hkdf_derived_key(self): def test_kc_gcm_commitment_in_metadata(self): """KC-GCM encryption MUST include the key commitment in metadata.""" cmm, raw_key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm) + pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - _, meta = pipeline.encrypt( - b"test", - AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - ) + _, meta = pipeline.encrypt(b"test") assert "x-amz-d" in meta commitment_bytes = base64.b64decode(meta["x-amz-d"]) From 907712549f63b5da14e678b2620f3a6076ab5710 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Wed, 18 Mar 2026 13:31:06 -0700 Subject: [PATCH 32/32] format --- src/s3_encryption/pipelines.py | 3 +-- test/test_encryption.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index e4a4cb09..d0e9ba79 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -51,7 +51,6 @@ def encrypt(self, plaintext, encryption_context=None): bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ - ##= specification/s3-encryption/encryption.md#content-encryption ##= type=implementation ##% The S3EC MUST use the encryption algorithm configured during @@ -255,7 +254,7 @@ def decrypt( if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") - # TODO: we should validate that these parameters must be None + # TODO: we should validate that these parameters must be None # when not in instruction file mode. if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") diff --git a/test/test_encryption.py b/test/test_encryption.py index d2ee5826..a384afbd 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -105,7 +105,9 @@ def test_iv_generated_with_correct_length_gcm(self): def test_message_id_generated_with_correct_length_kc(self): """KC-GCM encryption MUST produce a 28-byte Message ID.""" cmm, _ = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) _, meta = pipeline.encrypt(b"test") message_id_bytes = base64.b64decode(meta["x-amz-i"]) @@ -126,7 +128,9 @@ def test_iv_included_in_metadata_gcm(self): def test_message_id_included_in_metadata_kc(self): """KC-GCM encryption MUST include the Message ID in the returned metadata.""" cmm, _ = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) _, meta = pipeline.encrypt(b"test") assert "x-amz-i" in meta @@ -190,7 +194,9 @@ class TestKcGcm: def test_kc_gcm_uses_hkdf_derived_key(self): """KC-GCM encryption MUST use HKDF-derived keys, not the raw data key.""" cmm, raw_key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) plaintext = b"roundtrip test for KC GCM" ciphertext, meta = pipeline.encrypt(plaintext) @@ -217,7 +223,9 @@ def test_kc_gcm_uses_hkdf_derived_key(self): def test_kc_gcm_commitment_in_metadata(self): """KC-GCM encryption MUST include the key commitment in metadata.""" cmm, raw_key = _mock_cmm() - pipeline = PutEncryptedObjectPipeline(cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + pipeline = PutEncryptedObjectPipeline( + cmm, AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) _, meta = pipeline.encrypt(b"test")