From f05108c3414dcad8a3e1900a34f413334085caca Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 10:55:28 -0600 Subject: [PATCH 01/35] feat(metadata): Add V3 format support to ObjectMetadata - Add V3 metadata field constants (x-amz-c, x-amz-3, x-amz-w, etc.) - Add V3 field attributes to ObjectMetadata class - Update from_dict() and to_dict() to handle V3 fields - Add is_v1_format(), is_v2_format(), is_v3_format() methods - Add has_exclusive_key_collision() to detect version key conflicts This enables the client to recognize and parse V3 format encrypted objects, which use compressed metadata keys. --- src/s3_encryption/metadata.py | 102 ++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index f42feadb..1128f9ab 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -47,6 +47,15 @@ class ObjectMetadata: content_cipher_tag_length: str | None = field(default="128") # Marker for instruction files instruction_file: str | None = field(default=None) + + # V3 format fields (compressed) + content_cipher_v3: str | None = field(default=None) + encrypted_data_key_v3: str | None = field(default=None) + mat_desc_v3: str | None = field(default=None) + encryption_context_v3: str | None = field(default=None) + encrypted_data_key_algorithm_v3: str | None = field(default=None) + key_commitment_v3: str | None = field(default=None) + message_id_v3: str | None = field(default=None) # Constants for metadata keys ENCRYPTED_DATA_KEY_V1 = "x-amz-key" @@ -57,6 +66,15 @@ class ObjectMetadata: CONTENT_CIPHER = "x-amz-cek-alg" CONTENT_CIPHER_TAG_LENGTH = "x-amz-tag-len" INSTRUCTION_FILE = "x-amz-crypto-instr-file" + + # V3 format constants (compressed) + CONTENT_CIPHER_V3 = "x-amz-c" + ENCRYPTED_DATA_KEY_V3 = "x-amz-3" + MAT_DESC_V3 = "x-amz-m" + ENCRYPTION_CONTEXT_V3 = "x-amz-t" + ENCRYPTED_DATA_KEY_ALGORITHM_V3 = "x-amz-w" + KEY_COMMITMENT_V3 = "x-amz-d" + MESSAGE_ID_V3 = "x-amz-i" @classmethod def from_dict(cls, metadata_dict: dict[str, Any]) -> "ObjectMetadata": @@ -84,6 +102,13 @@ def from_dict(cls, metadata_dict: dict[str, Any]) -> "ObjectMetadata": content_cipher=metadata_dict.get(cls.CONTENT_CIPHER), content_cipher_tag_length=metadata_dict.get(cls.CONTENT_CIPHER_TAG_LENGTH), instruction_file=metadata_dict.get(cls.INSTRUCTION_FILE), + content_cipher_v3=metadata_dict.get(cls.CONTENT_CIPHER_V3), + encrypted_data_key_v3=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_V3), + mat_desc_v3=metadata_dict.get(cls.MAT_DESC_V3), + encryption_context_v3=metadata_dict.get(cls.ENCRYPTION_CONTEXT_V3), + encrypted_data_key_algorithm_v3=metadata_dict.get(cls.ENCRYPTED_DATA_KEY_ALGORITHM_V3), + key_commitment_v3=metadata_dict.get(cls.KEY_COMMITMENT_V3), + message_id_v3=metadata_dict.get(cls.MESSAGE_ID_V3), ) def to_dict(self) -> dict[str, str]: @@ -118,4 +143,81 @@ def to_dict(self) -> dict[str, str]: if self.instruction_file is not None: result[self.INSTRUCTION_FILE] = self.instruction_file + if self.content_cipher_v3 is not None: + result[self.CONTENT_CIPHER_V3] = self.content_cipher_v3 + + if self.encrypted_data_key_v3 is not None: + result[self.ENCRYPTED_DATA_KEY_V3] = self.encrypted_data_key_v3 + + if self.mat_desc_v3 is not None: + result[self.MAT_DESC_V3] = self.mat_desc_v3 + + if self.encryption_context_v3 is not None: + result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 + + if self.encrypted_data_key_algorithm_v3 is not None: + result[self.ENCRYPTED_DATA_KEY_ALGORITHM_V3] = self.encrypted_data_key_algorithm_v3 + + if self.key_commitment_v3 is not None: + result[self.KEY_COMMITMENT_V3] = self.key_commitment_v3 + + if self.message_id_v3 is not None: + result[self.MESSAGE_ID_V3] = self.message_id_v3 + return result + + def is_v1_format(self) -> bool: + """Check if metadata is in V1 format. + + Returns: + bool: True if metadata contains V1 keys and excludes V2/V3 keys + """ + return ( + self.content_iv is not None + and self.encrypted_data_key_context is not None + and self.encrypted_data_key_v1 is not None + and self.encrypted_data_key_v2 is None + ) + + def is_v2_format(self) -> bool: + """Check if metadata is in V2 format. + + Returns: + bool: True if metadata contains V2 keys and excludes V1/V3 keys + """ + return ( + self.content_cipher is not None + and self.content_iv is not None + and self.encrypted_data_key_algorithm is not None + and self.encrypted_data_key_v2 is not None + and self.encrypted_data_key_v1 is None + ) + + def is_v3_format(self) -> bool: + """Check if metadata is in V3 format. + + Returns: + bool: True if metadata contains V3 keys and excludes V1/V2 keys + """ + return ( + self.content_cipher_v3 is not None + and self.encrypted_data_key_algorithm_v3 is not None + and self.key_commitment_v3 is not None + and self.message_id_v3 is not None + and self.encrypted_data_key_v3 is not None + and self.encrypted_data_key_v2 is None + and self.encrypted_data_key_v1 is None + ) + + def has_exclusive_key_collision(self) -> bool: + """Check if metadata has multiple exclusive version keys. + + Returns: + bool: True if more than one version key (V1, V2, V3) is present + """ + has_v1_key = self.encrypted_data_key_v1 is not None + has_v2_key = self.encrypted_data_key_v2 is not None + has_v3_key = self.encrypted_data_key_v3 is not None + + exclusive_key_count = sum([has_v1_key, has_v2_key, has_v3_key]) + return exclusive_key_count > 1 From bd9509cabac66de38c1c41317295f7908d2ebc52 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 11:03:25 -0600 Subject: [PATCH 02/35] test(metadata): add comprehensive tests for ObjectMetadata V3 support - Add test_from_dict_v3_fields() to verify V3 field parsing - Add test_to_dict_v3_fields() to verify V3 field serialization - Add test_is_v1_format() to verify V1 format detection - Add test_is_v2_format() to verify V2 format detection - Add test_is_v3_format() to verify V3 format detection - Add test_has_exclusive_key_collision() to verify collision detection All tests verify correct behavior including exclusion of other version keys and proper collision detection across V1/V2/V3. --- test/test_metadata.py | 121 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/test/test_metadata.py b/test/test_metadata.py index a061c185..7dab2081 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -79,3 +79,124 @@ def test_roundtrip(self): # Verify that the result matches the original assert result_dict == original_dict + + def test_from_dict_v3_fields(self): + # Create a metadata dictionary with V3 fields + metadata_dict = { + "x-amz-c": "02", + "x-amz-3": "encrypted-key-v3", + "x-amz-w": "12", + "x-amz-d": "key-commitment", + "x-amz-i": "message-id", + "x-amz-m": "mat-desc", + "x-amz-t": "encryption-context", + } + + metadata = ObjectMetadata.from_dict(metadata_dict) + + assert metadata.content_cipher_v3 == "02" + assert metadata.encrypted_data_key_v3 == "encrypted-key-v3" + assert metadata.encrypted_data_key_algorithm_v3 == "12" + assert metadata.key_commitment_v3 == "key-commitment" + assert metadata.message_id_v3 == "message-id" + assert metadata.mat_desc_v3 == "mat-desc" + assert metadata.encryption_context_v3 == "encryption-context" + + def test_to_dict_v3_fields(self): + # Create an ObjectMetadata instance with V3 fields + metadata = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_v3="encrypted-key-v3", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="key-commitment", + message_id_v3="message-id", + mat_desc_v3="mat-desc", + encryption_context_v3="encryption-context", + ) + + metadata_dict = metadata.to_dict() + + assert metadata_dict["x-amz-c"] == "02" + assert metadata_dict["x-amz-3"] == "encrypted-key-v3" + assert metadata_dict["x-amz-w"] == "12" + assert metadata_dict["x-amz-d"] == "key-commitment" + assert metadata_dict["x-amz-i"] == "message-id" + assert metadata_dict["x-amz-m"] == "mat-desc" + assert metadata_dict["x-amz-t"] == "encryption-context" + + def test_is_v1_format(self): + metadata = ObjectMetadata( + content_iv="iv", + encrypted_data_key_context={"key": "value"}, + encrypted_data_key_v1="edk-v1", + ) + assert metadata.is_v1_format() is True + + # V2 key present should return False + metadata_v2 = ObjectMetadata( + content_iv="iv", + encrypted_data_key_context={"key": "value"}, + encrypted_data_key_v1="edk-v1", + encrypted_data_key_v2="edk-v2", + ) + assert metadata_v2.is_v1_format() is False + + def test_is_v2_format(self): + metadata = ObjectMetadata( + content_cipher="AES/GCM/NoPadding", + content_iv="iv", + encrypted_data_key_algorithm="kms+context", + encrypted_data_key_v2="edk-v2", + ) + assert metadata.is_v2_format() is True + + # V1 key present should return False + metadata_v1 = ObjectMetadata( + content_cipher="AES/GCM/NoPadding", + content_iv="iv", + encrypted_data_key_algorithm="kms+context", + encrypted_data_key_v2="edk-v2", + encrypted_data_key_v1="edk-v1", + ) + assert metadata_v1.is_v2_format() is False + + def test_is_v3_format(self): + metadata = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + encrypted_data_key_v3="edk-v3", + ) + assert metadata.is_v3_format() is True + + # V1 or V2 keys present should return False + metadata_v2 = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + encrypted_data_key_v3="edk-v3", + encrypted_data_key_v2="edk-v2", + ) + assert metadata_v2.is_v3_format() is False + + def test_has_exclusive_key_collision(self): + # No collision - only V2 + metadata_v2 = ObjectMetadata(encrypted_data_key_v2="edk-v2") + assert metadata_v2.has_exclusive_key_collision() is False + + # Collision - V1 and V2 + metadata_collision = ObjectMetadata( + encrypted_data_key_v1="edk-v1", + encrypted_data_key_v2="edk-v2", + ) + assert metadata_collision.has_exclusive_key_collision() is True + + # Collision - all three + metadata_all = ObjectMetadata( + encrypted_data_key_v1="edk-v1", + encrypted_data_key_v2="edk-v2", + encrypted_data_key_v3="edk-v3", + ) + assert metadata_all.has_exclusive_key_collision() is True From 96acff74cfa708031f86ef0d65f8db8feff639f5 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 11:11:18 -0600 Subject: [PATCH 03/35] style(metadata): apply Black formatting Remove trailing whitespace to comply with Black formatting rules. --- src/s3_encryption/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 1128f9ab..99d4dae5 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -47,7 +47,7 @@ class ObjectMetadata: content_cipher_tag_length: str | None = field(default="128") # Marker for instruction files instruction_file: str | None = field(default=None) - + # V3 format fields (compressed) content_cipher_v3: str | None = field(default=None) encrypted_data_key_v3: str | None = field(default=None) @@ -66,7 +66,7 @@ class ObjectMetadata: CONTENT_CIPHER = "x-amz-cek-alg" CONTENT_CIPHER_TAG_LENGTH = "x-amz-tag-len" INSTRUCTION_FILE = "x-amz-crypto-instr-file" - + # V3 format constants (compressed) CONTENT_CIPHER_V3 = "x-amz-c" ENCRYPTED_DATA_KEY_V3 = "x-amz-3" @@ -218,6 +218,6 @@ def has_exclusive_key_collision(self) -> bool: has_v1_key = self.encrypted_data_key_v1 is not None has_v2_key = self.encrypted_data_key_v2 is not None has_v3_key = self.encrypted_data_key_v3 is not None - + exclusive_key_count = sum([has_v1_key, has_v2_key, has_v3_key]) return exclusive_key_count > 1 From ebbe077f173336de55eda2a021aac490237f1f93 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:04:01 -0600 Subject: [PATCH 04/35] refactor(pipelines): refactor decrypt to use version detection methods - Add s3_client field to GetEncryptedObjectPipeline for instruction file support - Refactor decrypt() to use is_v1_format(), is_v2_format(), is_v3_format() - Extract version-specific logic into _decrypt_v1(), _decrypt_v2(), _decrypt_v3() - Separate materials preparation from decryption operation - Add return type annotations to _decrypt methods In metadata.py: - Add is_v3_in_object_metadata() to detect V3 with instruction file - Add should_use_instruction_file() to determine when to fetch instruction file This refactoring prepares the pipeline for instruction file support by making version detection explicit and separating concerns. --- src/s3_encryption/metadata.py | 31 +++++++++++ src/s3_encryption/pipelines.py | 96 ++++++++++++++++++++-------------- 2 files changed, 89 insertions(+), 38 deletions(-) diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 99d4dae5..ad668a64 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -221,3 +221,34 @@ def has_exclusive_key_collision(self) -> bool: exclusive_key_count = sum([has_v1_key, has_v2_key, has_v3_key]) return exclusive_key_count > 1 + + def is_v3_in_object_metadata(self) -> bool: + """Check if V3 content keys are in object metadata (without encrypted data key). + + Returns: + bool: True if V3 content keys present but no encrypted data key + """ + return ( + self.content_cipher_v3 is not None + and self.key_commitment_v3 is not None + and self.message_id_v3 is not None + and self.encrypted_data_key_v3 is None + ) + + def should_use_instruction_file(self) -> bool: + """Check if instruction file should be used for decryption. + + Returns: + bool: True if instruction file should be fetched + """ + # V3 with content keys but no encrypted data key -> instruction file + if self.is_v3_in_object_metadata(): + return True + + # No version keys at all -> try instruction file for V1/V2 + has_any_key = ( + self.encrypted_data_key_v1 is not None + or self.encrypted_data_key_v2 is not None + or self.encrypted_data_key_v3 is not None + ) + return not has_any_key diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 3a83a359..64802144 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -12,6 +12,7 @@ from attrs import define, field from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from .exceptions import S3EncryptionClientError from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey from .materials.materials import DecryptionMaterials, EncryptionMaterials @@ -90,6 +91,7 @@ class GetEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() + s3_client: object = field(default=None) def decrypt(self, response, encryption_context=None): """Decrypt the data after it is retrieved from S3. @@ -111,44 +113,17 @@ def decrypt(self, response, encryption_context=None): if encryption_context is None: encryption_context = {} - iv_b64 = metadata.content_iv - edk_b64 = metadata.encrypted_data_key_v2 - - # TODO: probably move this to ObjectMetadata - iv_bytes = base64.b64decode(iv_b64) - - # Create a list of encrypted data keys to try - encrypted_data_keys = [] - # Create an instance of EncryptedDataKey - if edk_b64: - edk_bytes = base64.b64decode(edk_b64) - encrypted_data_key = EncryptedDataKey( - key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=edk_bytes, + # Determine which format we're dealing with and get decryption materials + if metadata.is_v1_format(): + dec_materials = self._decrypt_v1(metadata, encryption_context) + elif metadata.is_v2_format(): + dec_materials = self._decrypt_v2(metadata, encryption_context) + elif metadata.is_v3_format(): + dec_materials = self._decrypt_v3(metadata, encryption_context) + else: + raise S3EncryptionClientError( + "Unable to determine S3 Encryption Client message format." ) - encrypted_data_keys.append(encrypted_data_key) - - # Also check for legacy encrypted data key (v1) if available - if metadata.encrypted_data_key_v1: - legacy_edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) - legacy_encrypted_data_key = EncryptedDataKey( - key_provider_id=b"S3Keyring", - key_provider_info=metadata.encrypted_data_key_algorithm, - encrypted_data_key=legacy_edk_bytes, - ) - encrypted_data_keys.append(legacy_encrypted_data_key) - - # Create a DecryptionMaterials instance - dec_materials = DecryptionMaterials( - iv=iv_bytes, - encrypted_data_keys=encrypted_data_keys, - encryption_context_stored=metadata.encrypted_data_key_context or {}, - encryption_context_from_request=encryption_context or {}, - ) - - # Get decryption materials from the crypto materials manager - dec_materials = self.cmm.decrypt_materials(dec_materials) ##= specification/s3-encryption/decryption.md#cbc-decryption ##= type=TODO @@ -157,6 +132,51 @@ def decrypt(self, response, encryption_context=None): ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + # Perform decryption aesgcm = AESGCM(dec_materials.plaintext_data_key) + return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) + + def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V2 decryption materials.""" + iv_bytes = base64.b64decode(metadata.content_iv) + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v2) + + encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info=metadata.encrypted_data_key_algorithm, + encrypted_data_key=edk_bytes, + ) + + dec_materials = DecryptionMaterials( + iv=iv_bytes, + encrypted_data_keys=[encrypted_data_key], + encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_from_request=encryption_context, + ) + + return self.cmm.decrypt_materials(dec_materials) + + def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V1 decryption materials.""" + iv_bytes = base64.b64decode(metadata.content_iv) + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v1) + + encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info=metadata.encrypted_data_key_algorithm, + encrypted_data_key=edk_bytes, + ) + + dec_materials = DecryptionMaterials( + iv=iv_bytes, + encrypted_data_keys=[encrypted_data_key], + encryption_context_stored=metadata.encrypted_data_key_context or {}, + encryption_context_from_request=encryption_context, + ) + + return self.cmm.decrypt_materials(dec_materials) - return aesgcm.decrypt(nonce=iv_bytes, data=encrypted_data, associated_data=None) + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: + """Prepare V3 decryption materials.""" + # TODO: Implement V3 decryption + raise NotImplementedError("V3 decryption not yet implemented") From 382063fe533f253b0c26fba5feab5a0b69f99597 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:01:17 -0600 Subject: [PATCH 05/35] test(pipelines): add instruction file tests for V1, V2, and V3 - Add test_decrypt_v1_from_instruction_file - Add test_decrypt_v2_from_instruction_file - Add test_decrypt_v3_from_instruction_file - Add json import to pipelines.py - Tests verify instruction file fetching works correctly --- src/s3_encryption/pipelines.py | 36 ++++++- test/test_pipelines.py | 177 +++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 test/test_pipelines.py diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 64802144..cf09381d 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -7,6 +7,7 @@ """ import base64 +import json import os from attrs import define, field @@ -93,12 +94,14 @@ class GetEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() s3_client: object = field(default=None) - def decrypt(self, response, encryption_context=None): + def decrypt(self, response, encryption_context=None, bucket=None, key=None): """Decrypt the data after it is retrieved from S3. Args: response (dict): The response from S3 containing the encrypted data and metadata encryption_context (dict, optional): Additional context for decryption + bucket (str, optional): S3 bucket name (required for instruction file) + key (str, optional): S3 object key (required for instruction file) Returns: bytes: The decrypted data @@ -113,6 +116,21 @@ def decrypt(self, response, encryption_context=None): if encryption_context is None: encryption_context = {} + # Check if we need to fetch instruction file + if metadata.should_use_instruction_file(): + if self.s3_client is None: + raise S3EncryptionClientError( + "S3 client required to fetch instruction file" + ) + if bucket is None or key is None: + raise S3EncryptionClientError( + "Bucket and key required to fetch instruction file" + ) + + instruction_metadata = self._fetch_instruction_file(bucket, key) + instruction_metadata.update(encryption_metadata) + metadata = ObjectMetadata.from_dict(instruction_metadata) + # Determine which format we're dealing with and get decryption materials if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) @@ -136,6 +154,22 @@ def decrypt(self, response, encryption_context=None): aesgcm = AESGCM(dec_materials.plaintext_data_key) return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) + def _fetch_instruction_file(self, bucket: str, key: str, suffix: str = ".instruction") -> dict: + """Fetch instruction file from S3. + + Args: + bucket: S3 bucket name + key: S3 object key + suffix: Instruction file suffix (default: .instruction) + + Returns: + dict: Parsed JSON metadata from instruction file + """ + instruction_key = key + suffix + response = self.s3_client.get_object(Bucket=bucket, Key=instruction_key) + instruction_data = response["Body"].read() + return json.loads(instruction_data) + def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" iv_bytes = base64.b64decode(metadata.content_iv) diff --git a/test/test_pipelines.py b/test/test_pipelines.py new file mode 100644 index 00000000..d4a64d54 --- /dev/null +++ b/test/test_pipelines.py @@ -0,0 +1,177 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import base64 +import json +import os +import sys +from io import BytesIO +from unittest.mock import MagicMock, Mock + +import pytest + +# Add the src directory to the Python path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src"))) + +from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager +from s3_encryption.materials.keyring import S3Keyring +from s3_encryption.pipelines import GetEncryptedObjectPipeline + + +class TestGetEncryptedObjectPipelineInstructionFile: + def test_decrypt_v1_from_instruction_file(self): + """Test decrypting V1 format with instruction file.""" + # V1: Object metadata is empty, all metadata in instruction file + object_metadata = {} + + # Instruction file contains all V1 metadata + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(16)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/CBC/PKCS5Padding", + } + + # Create mock S3 client + mock_s3_client = Mock() + instruction_file_body = BytesIO(json.dumps(instruction_file_metadata).encode("utf-8")) + mock_s3_client.get_object.return_value = { + "Body": instruction_file_body, + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Create mock keyring and CMM + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + # Create pipeline with mocked S3 client + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + # Create mock response + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + # Mock the keyring to raise an error so we don't actually decrypt + mock_keyring.on_decrypt.side_effect = Exception("Keyring called - instruction file was fetched") + + # Should fail when trying to decrypt (proving instruction file was fetched) + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + + # Verify instruction file was fetched + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) + + def test_decrypt_v2_from_instruction_file(self): + """Test decrypting V2 format with instruction file.""" + # V2: Object metadata is empty, all metadata in instruction file + object_metadata = {} + + # Instruction file contains all V2 metadata + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + } + + # Create mock S3 client + mock_s3_client = Mock() + instruction_file_body = BytesIO(json.dumps(instruction_file_metadata).encode("utf-8")) + mock_s3_client.get_object.return_value = { + "Body": instruction_file_body, + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Create mock keyring and CMM + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + # Create pipeline with mocked S3 client + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + # Create mock response + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + # Mock the keyring to raise an error so we don't actually decrypt + mock_keyring.on_decrypt.side_effect = Exception("Keyring called - instruction file was fetched") + + # Should fail when trying to decrypt (proving instruction file was fetched) + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + + # Verify instruction file was fetched + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) + + def test_decrypt_v3_from_instruction_file(self): + """Test decrypting V3 format with instruction file.""" + # Object metadata contains V3 content keys only + object_metadata = { + "x-amz-c": "115", # Compressed algorithm suite + "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), + "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), + } + + # Instruction file contains encrypted data key and wrapping algorithm + instruction_file_metadata = { + "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-w": "02", # AES/GCM + "x-amz-m": json.dumps({"test-instruction": "material-desc-instruction"}), + } + + # Create mock S3 client + mock_s3_client = Mock() + instruction_file_body = BytesIO(json.dumps(instruction_file_metadata).encode("utf-8")) + mock_s3_client.get_object.return_value = { + "Body": instruction_file_body, + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Create mock keyring and CMM + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + + # Create pipeline with mocked S3 client + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + # Create mock response with encrypted data + iv = os.urandom(12) + encrypted_data = b"encrypted-test-data" + + mock_response = { + "Body": BytesIO(encrypted_data), + "Metadata": object_metadata, + } + + # Mock the keyring to return decryption materials + from s3_encryption.materials.materials import DecryptionMaterials + plaintext_data_key = os.urandom(32) + + mock_dec_materials = DecryptionMaterials( + iv=iv, + encrypted_data_keys=[], + encryption_context_stored={}, + encryption_context_from_request={}, + ) + mock_dec_materials.plaintext_data_key = plaintext_data_key + + mock_keyring.on_decrypt.return_value = mock_dec_materials + + # This should fail with NotImplementedError since V3 decryption isn't implemented yet + with pytest.raises(NotImplementedError, match="V3 decryption not yet implemented"): + pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") + + # Verify instruction file was fetched + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.instruction" + ) From c12528142d4d08d73a3700d75fa2dcda26e03074 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:03:02 -0600 Subject: [PATCH 06/35] feat(client): pass s3_client, bucket, and key to decrypt pipeline - Pass s3_client to GetEncryptedObjectPipeline constructor - Pass bucket and key from params to pipeline.decrypt() - Required for instruction file support --- src/s3_encryption/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 064096bb..5d18e87b 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -114,12 +114,19 @@ def on_get_object_after_call(self, parsed, **kwargs): } # Create a pipeline and decrypt the data - pipeline = GetEncryptedObjectPipeline(self.config.cmm) - decrypted_data = pipeline.decrypt(response, encryption_context) + pipeline = GetEncryptedObjectPipeline( + self.config.cmm, + s3_client=getattr(self._context, "wrapped_s3_client", None)) + decrypted_data = pipeline.decrypt( + response, encryption_context, + bucket=kwargs.get("Bucket", None), key=kwargs.get("Key", None) + ) - # Replace body with decrypted data + # Create a new streaming body with the decrypted data stream = io.BytesIO(decrypted_data) streaming_body = StreamingBody(stream, len(decrypted_data)) + + # Replace body with decrypted data parsed["Body"] = streaming_body @@ -207,6 +214,8 @@ def get_object(self, **kwargs): # Store encryption context in thread-local storage for the event handler self._plugin._context.encryption_context = encryption_context + # Store wrapped client in thread-local storage for the event handler to fetch instruction files + self._plugin._context.wrapped_s3_client = self.wrapped_s3_client try: return self.wrapped_s3_client.get_object(**kwargs) From 95a293d3410766dcb67f9af1a7891526dbdd8f94 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:03:43 -0600 Subject: [PATCH 07/35] style: apply Black formatting --- src/s3_encryption/pipelines.py | 10 +++------- test/test_pipelines.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index cf09381d..5861fb8b 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -119,14 +119,10 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): # Check if we need to fetch instruction file if metadata.should_use_instruction_file(): if self.s3_client is None: - raise S3EncryptionClientError( - "S3 client required to fetch instruction file" - ) + raise S3EncryptionClientError("S3 client required to fetch instruction file") if bucket is None or key is None: - raise S3EncryptionClientError( - "Bucket and key required to fetch instruction file" - ) - + raise S3EncryptionClientError("Bucket and key required to fetch instruction file") + instruction_metadata = self._fetch_instruction_file(bucket, key) instruction_metadata.update(encryption_metadata) metadata = ObjectMetadata.from_dict(instruction_metadata) diff --git a/test/test_pipelines.py b/test/test_pipelines.py index d4a64d54..7ac9d043 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -5,7 +5,7 @@ import os import sys from io import BytesIO -from unittest.mock import MagicMock, Mock +from unittest.mock import Mock import pytest @@ -54,7 +54,9 @@ def test_decrypt_v1_from_instruction_file(self): } # Mock the keyring to raise an error so we don't actually decrypt - mock_keyring.on_decrypt.side_effect = Exception("Keyring called - instruction file was fetched") + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) # Should fail when trying to decrypt (proving instruction file was fetched) with pytest.raises(Exception, match="Keyring called"): @@ -102,7 +104,9 @@ def test_decrypt_v2_from_instruction_file(self): } # Mock the keyring to raise an error so we don't actually decrypt - mock_keyring.on_decrypt.side_effect = Exception("Keyring called - instruction file was fetched") + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) # Should fail when trying to decrypt (proving instruction file was fetched) with pytest.raises(Exception, match="Keyring called"): @@ -147,7 +151,7 @@ def test_decrypt_v3_from_instruction_file(self): # Create mock response with encrypted data iv = os.urandom(12) encrypted_data = b"encrypted-test-data" - + mock_response = { "Body": BytesIO(encrypted_data), "Metadata": object_metadata, @@ -155,8 +159,9 @@ def test_decrypt_v3_from_instruction_file(self): # Mock the keyring to return decryption materials from s3_encryption.materials.materials import DecryptionMaterials + plaintext_data_key = os.urandom(32) - + mock_dec_materials = DecryptionMaterials( iv=iv, encrypted_data_keys=[], @@ -164,7 +169,7 @@ def test_decrypt_v3_from_instruction_file(self): encryption_context_from_request={}, ) mock_dec_materials.plaintext_data_key = plaintext_data_key - + mock_keyring.on_decrypt.return_value = mock_dec_materials # This should fail with NotImplementedError since V3 decryption isn't implemented yet From 3d9ad73783dabc248cdc0db25135acf8fccee247 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:09:27 -0600 Subject: [PATCH 08/35] chore: formatting --- src/s3_encryption/__init__.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 5d18e87b..c01cfa67 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -115,11 +115,13 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( - self.config.cmm, - s3_client=getattr(self._context, "wrapped_s3_client", None)) + self.config.cmm, s3_client=getattr(self._context, "wrapped_s3_client", None) + ) decrypted_data = pipeline.decrypt( - response, encryption_context, - bucket=kwargs.get("Bucket", None), key=kwargs.get("Key", None) + response, + encryption_context, + bucket=kwargs.get("Bucket"), + key=kwargs.get("Key"), ) # Create a new streaming body with the decrypted data @@ -214,7 +216,8 @@ def get_object(self, **kwargs): # Store encryption context in thread-local storage for the event handler self._plugin._context.encryption_context = encryption_context - # Store wrapped client in thread-local storage for the event handler to fetch instruction files + # Store wrapped client in thread-local storage for + # the event handler to fetch instruction files self._plugin._context.wrapped_s3_client = self.wrapped_s3_client try: From de77fbd2e10f3c19adc8b5dcc35615831ad8c33c Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:03:56 -0600 Subject: [PATCH 09/35] test(instructionFiles): working Ins. implementation --- pyproject.toml | 1 + src/s3_encryption/__init__.py | 24 +++++-- src/s3_encryption/pipelines.py | 12 ++-- .../test_i_s3_encryption_instruction_file.py | 68 +++++++++++++++++++ 4 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 test/integration/test_i_s3_encryption_instruction_file.py diff --git a/pyproject.toml b/pyproject.toml index b7489a03..1876da9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ test = [ dev = [ "black>=24.3.0,<27.0.0", "ruff>=0.3.0", + "boto3-stubs~=1.42.49", ] [build-system] diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index c01cfa67..fa8c464b 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -115,13 +115,15 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( - self.config.cmm, s3_client=getattr(self._context, "wrapped_s3_client", None) + self.config.cmm, + # TODO(instructionFile): Refactor Instruction File Support to use config + instruction_file_client=getattr(self._context, "instruction_file_client", None), ) decrypted_data = pipeline.decrypt( response, encryption_context, - bucket=kwargs.get("Bucket"), - key=kwargs.get("Key"), + bucket=getattr(self._context, "bucket", None), + key=getattr(self._context, "key", None), ) # Create a new streaming body with the decrypted data @@ -146,6 +148,8 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() _plugin: S3EncryptionClientPlugin = field(init=False) + # TODO(instructionFile): Refactor Instruction File Support to use config + instruction_file_client = field(default=None) def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" @@ -218,7 +222,10 @@ def get_object(self, **kwargs): self._plugin._context.encryption_context = encryption_context # Store wrapped client in thread-local storage for # the event handler to fetch instruction files - self._plugin._context.wrapped_s3_client = self.wrapped_s3_client + # TODO(instructionFile): Refactor Instruction File Support to use config + self._plugin._context.instruction_file_client = self.instruction_file_client + self._plugin._context.bucket = kwargs.get("Bucket") + self._plugin._context.key = kwargs.get("Key") try: return self.wrapped_s3_client.get_object(**kwargs) @@ -229,6 +236,9 @@ def get_object(self, **kwargs): # Wrap any unexpected errors during decryption raise S3EncryptionClientError(f"Failed to decrypt object: {str(e)}") from e finally: - # Clean up thread-local storage - if hasattr(self._plugin._context, "encryption_context"): - delattr(self._plugin._context, "encryption_context") + # Clean up thread-local storage; + # do not clean up the client as it is not thread local only + attrs = ["encryption_context", "Bucket", "Key"] + for attr in attrs: + if hasattr(self._plugin._context, attr): + delattr(self._plugin._context, attr) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 5861fb8b..993ab259 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -92,7 +92,7 @@ class GetEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() - s3_client: object = field(default=None) + instruction_file_client: object = field(default=None) def decrypt(self, response, encryption_context=None, bucket=None, key=None): """Decrypt the data after it is retrieved from S3. @@ -117,9 +117,13 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): encryption_context = {} # Check if we need to fetch instruction file + # TODO(instructionFile): Refactor Instruction File Support to use config if metadata.should_use_instruction_file(): - if self.s3_client is None: - raise S3EncryptionClientError("S3 client required to fetch instruction file") + if self.instruction_file_client is None: + raise S3EncryptionClientError( + "instruction_file_client argument required to use instruction file;" + " pass unique S3 Client as instruction_file_client." + ) if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") @@ -162,7 +166,7 @@ def _fetch_instruction_file(self, bucket: str, key: str, suffix: str = ".instruc dict: Parsed JSON metadata from instruction file """ instruction_key = key + suffix - response = self.s3_client.get_object(Bucket=bucket, Key=instruction_key) + response = self.instruction_file_client.get_object(Bucket=bucket, Key=instruction_key) instruction_data = response["Body"].read() return json.loads(instruction_data) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py new file mode 100644 index 00000000..3ec9e3e0 --- /dev/null +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -0,0 +1,68 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import os + +import boto3 +import pytest + +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption.materials.kms_keyring import KmsKeyring + +# TODO(instructionFiles): Create a Static Bucket for Instruction File Messages +# TODO(instructionFiles): Add Static Bucket for Instruction File Messages to test env +bucket = os.environ.get("CI_S3_INSTRUCTION_BUCKET", "s3ec-github-test-bucket") +region = os.environ.get("CI_AWS_REGION", "us-west-2") +# TODO(instructionFiles): Add INS FILES KMS Key to test env +kms_key_id = os.environ.get( + "CI_KMS_KEY_INSTRUCTION_FILES", + "arn:aws:kms:us-west-2:370957321024:key/c3eafb5f-e87d-4584-9400-cf419ce5d782", +) + +# Test keys for objects encrypted by Java S3EC with instruction files +TEST_OBJECTS = { + # TODO(instructionFiles): V1 Instruction File + "v1_instruction_file": "test-v1-cbc-instruction", + # TODO(instructionFiles): Proper V2 Instruction File + "v2_instruction_file": "kms-instruction-file-test-260220-105428-19668", + # TODO(instructionFiles): V3 Instruction File + "v3_instruction_file": "test-v3-instruction", +} + + +@pytest.mark.skip(reason="Requires pre-existing test objects encrypted by Java S3EC") +def test_decrypt_v1_instruction_file(): + """Test decrypting V1 object with instruction file.""" + key = TEST_OBJECTS["v1_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "test data v1 cbc" + print("Success! V1 instruction file decryption completed.") + + +# @pytest.mark.skip(reason="Requires pre-existing test objects encrypted by Java S3EC") +def test_decrypt_v2_instruction_file(): + """Test decrypting V2 object with instruction file.""" + key = TEST_OBJECTS["v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + instruction_file_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_client=instruction_file_client + ) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "Testing encryption of instruction file with KMS Keyring" + print("Success! V2 instruction file decryption completed.") From f8d14eb3c0a84aba492dc3a2c5700025544b3ca9 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:36:02 -0600 Subject: [PATCH 10/35] refactor(instructionFiles): Require Disable XOR Client --- src/s3_encryption/__init__.py | 30 +++++++++++++++++-- test/integration/test_i_s3_encryption.py | 28 +++++++++-------- .../test_i_s3_encryption_instruction_file.py | 21 +++++++++++-- .../test_i_s3_encryption_multithreaded.py | 10 +++---- 4 files changed, 66 insertions(+), 23 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index fa8c464b..4392c783 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -4,6 +4,8 @@ import io import threading +from enum import Enum +from typing import Any from attrs import define, field from botocore.response import StreamingBody @@ -19,6 +21,19 @@ S3_METADATA_PREFIX = "x-amz-meta-" +class InstructionFileSetting(Enum): + """ + Instruction file setting for the S3 Encryption Client. + + DISABLE: Do not use instruction files. + + To enable use of Instruction Files, + do not pass this, + and only pass instruction_file_client. + """ + + DISABLE = "disable" + @define class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" @@ -147,9 +162,20 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() - _plugin: S3EncryptionClientPlugin = field(init=False) # TODO(instructionFile): Refactor Instruction File Support to use config - instruction_file_client = field(default=None) + instruction_file_setting: (InstructionFileSetting|None) = field(default=None) + instruction_file_client: (Any|None) = field(default=None) + _plugin: S3EncryptionClientPlugin = field(init=False) + + @instruction_file_setting.validator + def _validate_instruction_file_setting(self, attribute, value): + if not isinstance(value, InstructionFileSetting) and not value is None: + raise TypeError(f"instruction_file_setting must be InstructionFileSetting or None, got {type(value)}") + if value != InstructionFileSetting.DISABLE and not self.instruction_file_client: + raise ValueError("instruction_file_client required when instruction_file_setting is not DISABLE") + if value == InstructionFileSetting.DISABLE and self.instruction_file_client: + raise ValueError("instruction_file_client must be None when instruction_file_setting is DISABLE") + def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 616f8da4..cca46ffa 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -6,7 +6,7 @@ import boto3 import pytest -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -29,7 +29,9 @@ def test_simple_roundtrip_ascii_string(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -56,7 +58,7 @@ def test_empty_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -84,7 +86,7 @@ def test_no_body_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Call put_object without providing a Body parameter s3ec.put_object(Bucket=bucket, Key=key) @@ -118,7 +120,7 @@ def test_unicode_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -153,7 +155,7 @@ def test_specific_encoding_utf8_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -190,7 +192,7 @@ def test_specific_encoding_latin1_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -224,7 +226,7 @@ def test_binary_data_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Pass the binary data directly s3ec.put_object(Bucket=bucket, Key=key, Body=data) @@ -254,7 +256,7 @@ def test_invalid_body_types(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Test with integer with pytest.raises(S3EncryptionClientError) as excinfo: @@ -307,7 +309,7 @@ def test_user_metadata_preservation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with user metadata s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) @@ -367,7 +369,7 @@ def test_encryption_context_roundtrip(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -407,7 +409,7 @@ def test_encryption_context_mismatch(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -447,7 +449,7 @@ def test_encryption_context_missing_on_decrypt(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 3ec9e3e0..1b2f0e3d 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -5,7 +5,7 @@ import boto3 import pytest -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting from s3_encryption.materials.kms_keyring import KmsKeyring # TODO(instructionFiles): Create a Static Bucket for Instruction File Messages @@ -55,8 +55,8 @@ def test_decrypt_v2_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - instruction_file_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) + instruction_file_client = boto3.client("s3") s3ec = S3EncryptionClient( wrapped_client, config, instruction_file_client=instruction_file_client ) @@ -66,3 +66,20 @@ def test_decrypt_v2_instruction_file(): assert output == "Testing encryption of instruction file with KMS Keyring" print("Success! V2 instruction file decryption completed.") + +def test_decrypt_instruction_file_disable_fails(): + """Test that decryption fails when instruction_file_setting is DISABLE.""" + key = TEST_OBJECTS["v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + instruction_file_client = boto3.client("s3") + config = S3EncryptionClientConfig( + keyring + ) + + with pytest.raises(ValueError): + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE, instruction_file_client=instruction_file_client + ) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 419ca7ea..7330a1c5 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -13,7 +13,7 @@ import boto3 -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -39,7 +39,7 @@ def test_multithreaded_encryption_context_isolation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Number of threads to test with num_threads = 10 @@ -151,8 +151,7 @@ def test_multithreaded_rapid_context_switching(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) num_iterations = 20 errors = [] @@ -229,8 +228,7 @@ def test_multithreaded_mixed_with_and_without_context(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) errors = [] def worker_with_context(thread_id): From fda6240dc19bb47dbfc34dc08baa5bba16c28580 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 20 Feb 2026 14:45:32 -0600 Subject: [PATCH 11/35] format --- src/s3_encryption/__init__.py | 23 ++++++---- test/integration/test_i_s3_encryption.py | 46 ++++++++++++++----- .../test_i_s3_encryption_instruction_file.py | 12 +++-- .../test_i_s3_encryption_multithreaded.py | 14 ++++-- 4 files changed, 65 insertions(+), 30 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 4392c783..63cb13b9 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -22,8 +22,7 @@ class InstructionFileSetting(Enum): - """ - Instruction file setting for the S3 Encryption Client. + """Instruction file setting for the S3 Encryption Client. DISABLE: Do not use instruction files. @@ -34,6 +33,7 @@ class InstructionFileSetting(Enum): DISABLE = "disable" + @define class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" @@ -163,19 +163,24 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() # TODO(instructionFile): Refactor Instruction File Support to use config - instruction_file_setting: (InstructionFileSetting|None) = field(default=None) - instruction_file_client: (Any|None) = field(default=None) + instruction_file_setting: InstructionFileSetting | None = field(default=None) + instruction_file_client: Any | None = field(default=None) _plugin: S3EncryptionClientPlugin = field(init=False) @instruction_file_setting.validator def _validate_instruction_file_setting(self, attribute, value): - if not isinstance(value, InstructionFileSetting) and not value is None: - raise TypeError(f"instruction_file_setting must be InstructionFileSetting or None, got {type(value)}") + if not isinstance(value, InstructionFileSetting) and value is not None: + raise TypeError( + f"instruction_file_setting must be InstructionFileSetting or None, got {type(value)}" + ) if value != InstructionFileSetting.DISABLE and not self.instruction_file_client: - raise ValueError("instruction_file_client required when instruction_file_setting is not DISABLE") + raise ValueError( + "instruction_file_client required when instruction_file_setting is not DISABLE" + ) if value == InstructionFileSetting.DISABLE and self.instruction_file_client: - raise ValueError("instruction_file_client must be None when instruction_file_setting is DISABLE") - + raise ValueError( + "instruction_file_client must be None when instruction_file_setting is DISABLE" + ) def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index cca46ffa..39a738ee 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -6,7 +6,7 @@ import boto3 import pytest -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting +from s3_encryption import InstructionFileSetting, S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -58,7 +58,9 @@ def test_empty_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -86,7 +88,9 @@ def test_no_body_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Call put_object without providing a Body parameter s3ec.put_object(Bucket=bucket, Key=key) @@ -120,7 +124,9 @@ def test_unicode_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -155,7 +161,9 @@ def test_specific_encoding_utf8_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -192,7 +200,9 @@ def test_specific_encoding_latin1_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -226,7 +236,9 @@ def test_binary_data_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Pass the binary data directly s3ec.put_object(Bucket=bucket, Key=key, Body=data) @@ -256,7 +268,9 @@ def test_invalid_body_types(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Test with integer with pytest.raises(S3EncryptionClientError) as excinfo: @@ -309,7 +323,9 @@ def test_user_metadata_preservation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Put object with user metadata s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) @@ -369,7 +385,9 @@ def test_encryption_context_roundtrip(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -409,7 +427,9 @@ def test_encryption_context_mismatch(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -449,7 +469,9 @@ def test_encryption_context_missing_on_decrypt(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 1b2f0e3d..93d7f7b9 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -5,7 +5,7 @@ import boto3 import pytest -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting +from s3_encryption import InstructionFileSetting, S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring # TODO(instructionFiles): Create a Static Bucket for Instruction File Messages @@ -67,6 +67,7 @@ def test_decrypt_v2_instruction_file(): assert output == "Testing encryption of instruction file with KMS Keyring" print("Success! V2 instruction file decryption completed.") + def test_decrypt_instruction_file_disable_fails(): """Test that decryption fails when instruction_file_setting is DISABLE.""" key = TEST_OBJECTS["v2_instruction_file"] @@ -75,11 +76,12 @@ def test_decrypt_instruction_file_disable_fails(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") instruction_file_client = boto3.client("s3") - config = S3EncryptionClientConfig( - keyring - ) + config = S3EncryptionClientConfig(keyring) with pytest.raises(ValueError): s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE, instruction_file_client=instruction_file_client + wrapped_client, + config, + instruction_file_setting=InstructionFileSetting.DISABLE, + instruction_file_client=instruction_file_client, ) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 7330a1c5..49ad3886 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -13,7 +13,7 @@ import boto3 -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting +from s3_encryption import InstructionFileSetting, S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -39,7 +39,9 @@ def test_multithreaded_encryption_context_isolation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) # Number of threads to test with num_threads = 10 @@ -151,7 +153,9 @@ def test_multithreaded_rapid_context_switching(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) num_iterations = 20 errors = [] @@ -228,7 +232,9 @@ def test_multithreaded_mixed_with_and_without_context(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient( + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE + ) errors = [] def worker_with_context(thread_id): From 713928a6984eaaae208b8aa9505567e75aa86787 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 23 Feb 2026 10:14:41 -0600 Subject: [PATCH 12/35] Revert "format" This reverts commit fda6240dc19bb47dbfc34dc08baa5bba16c28580. --- src/s3_encryption/__init__.py | 23 ++++------ test/integration/test_i_s3_encryption.py | 46 +++++-------------- .../test_i_s3_encryption_instruction_file.py | 12 ++--- .../test_i_s3_encryption_multithreaded.py | 14 ++---- 4 files changed, 30 insertions(+), 65 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 63cb13b9..4392c783 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -22,7 +22,8 @@ class InstructionFileSetting(Enum): - """Instruction file setting for the S3 Encryption Client. + """ + Instruction file setting for the S3 Encryption Client. DISABLE: Do not use instruction files. @@ -33,7 +34,6 @@ class InstructionFileSetting(Enum): DISABLE = "disable" - @define class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" @@ -163,24 +163,19 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() # TODO(instructionFile): Refactor Instruction File Support to use config - instruction_file_setting: InstructionFileSetting | None = field(default=None) - instruction_file_client: Any | None = field(default=None) + instruction_file_setting: (InstructionFileSetting|None) = field(default=None) + instruction_file_client: (Any|None) = field(default=None) _plugin: S3EncryptionClientPlugin = field(init=False) @instruction_file_setting.validator def _validate_instruction_file_setting(self, attribute, value): - if not isinstance(value, InstructionFileSetting) and value is not None: - raise TypeError( - f"instruction_file_setting must be InstructionFileSetting or None, got {type(value)}" - ) + if not isinstance(value, InstructionFileSetting) and not value is None: + raise TypeError(f"instruction_file_setting must be InstructionFileSetting or None, got {type(value)}") if value != InstructionFileSetting.DISABLE and not self.instruction_file_client: - raise ValueError( - "instruction_file_client required when instruction_file_setting is not DISABLE" - ) + raise ValueError("instruction_file_client required when instruction_file_setting is not DISABLE") if value == InstructionFileSetting.DISABLE and self.instruction_file_client: - raise ValueError( - "instruction_file_client must be None when instruction_file_setting is DISABLE" - ) + raise ValueError("instruction_file_client must be None when instruction_file_setting is DISABLE") + def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 39a738ee..cca46ffa 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -6,7 +6,7 @@ import boto3 import pytest -from s3_encryption import InstructionFileSetting, S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -58,9 +58,7 @@ def test_empty_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -88,9 +86,7 @@ def test_no_body_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Call put_object without providing a Body parameter s3ec.put_object(Bucket=bucket, Key=key) @@ -124,9 +120,7 @@ def test_unicode_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -161,9 +155,7 @@ def test_specific_encoding_utf8_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -200,9 +192,7 @@ def test_specific_encoding_latin1_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -236,9 +226,7 @@ def test_binary_data_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Pass the binary data directly s3ec.put_object(Bucket=bucket, Key=key, Body=data) @@ -268,9 +256,7 @@ def test_invalid_body_types(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Test with integer with pytest.raises(S3EncryptionClientError) as excinfo: @@ -323,9 +309,7 @@ def test_user_metadata_preservation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with user metadata s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) @@ -385,9 +369,7 @@ def test_encryption_context_roundtrip(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -427,9 +409,7 @@ def test_encryption_context_mismatch(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -469,9 +449,7 @@ def test_encryption_context_missing_on_decrypt(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 93d7f7b9..1b2f0e3d 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -5,7 +5,7 @@ import boto3 import pytest -from s3_encryption import InstructionFileSetting, S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting from s3_encryption.materials.kms_keyring import KmsKeyring # TODO(instructionFiles): Create a Static Bucket for Instruction File Messages @@ -67,7 +67,6 @@ def test_decrypt_v2_instruction_file(): assert output == "Testing encryption of instruction file with KMS Keyring" print("Success! V2 instruction file decryption completed.") - def test_decrypt_instruction_file_disable_fails(): """Test that decryption fails when instruction_file_setting is DISABLE.""" key = TEST_OBJECTS["v2_instruction_file"] @@ -76,12 +75,11 @@ def test_decrypt_instruction_file_disable_fails(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") instruction_file_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring + ) with pytest.raises(ValueError): s3ec = S3EncryptionClient( - wrapped_client, - config, - instruction_file_setting=InstructionFileSetting.DISABLE, - instruction_file_client=instruction_file_client, + wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE, instruction_file_client=instruction_file_client ) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 49ad3886..7330a1c5 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -13,7 +13,7 @@ import boto3 -from s3_encryption import InstructionFileSetting, S3EncryptionClient, S3EncryptionClientConfig +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -39,9 +39,7 @@ def test_multithreaded_encryption_context_isolation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) # Number of threads to test with num_threads = 10 @@ -153,9 +151,7 @@ def test_multithreaded_rapid_context_switching(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) num_iterations = 20 errors = [] @@ -232,9 +228,7 @@ def test_multithreaded_mixed_with_and_without_context(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) errors = [] def worker_with_context(thread_id): From bb7131882546e770f14c4bcc473355a605ab3d1b Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 23 Feb 2026 10:17:15 -0600 Subject: [PATCH 13/35] Revert "refactor(instructionFiles): Require Disable XOR Client" This reverts commit f8d14eb3c0a84aba492dc3a2c5700025544b3ca9. --- src/s3_encryption/__init__.py | 30 ++----------------- test/integration/test_i_s3_encryption.py | 28 ++++++++--------- .../test_i_s3_encryption_instruction_file.py | 21 ++----------- .../test_i_s3_encryption_multithreaded.py | 10 ++++--- 4 files changed, 23 insertions(+), 66 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 4392c783..fa8c464b 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -4,8 +4,6 @@ import io import threading -from enum import Enum -from typing import Any from attrs import define, field from botocore.response import StreamingBody @@ -21,19 +19,6 @@ S3_METADATA_PREFIX = "x-amz-meta-" -class InstructionFileSetting(Enum): - """ - Instruction file setting for the S3 Encryption Client. - - DISABLE: Do not use instruction files. - - To enable use of Instruction Files, - do not pass this, - and only pass instruction_file_client. - """ - - DISABLE = "disable" - @define class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" @@ -162,20 +147,9 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() - # TODO(instructionFile): Refactor Instruction File Support to use config - instruction_file_setting: (InstructionFileSetting|None) = field(default=None) - instruction_file_client: (Any|None) = field(default=None) _plugin: S3EncryptionClientPlugin = field(init=False) - - @instruction_file_setting.validator - def _validate_instruction_file_setting(self, attribute, value): - if not isinstance(value, InstructionFileSetting) and not value is None: - raise TypeError(f"instruction_file_setting must be InstructionFileSetting or None, got {type(value)}") - if value != InstructionFileSetting.DISABLE and not self.instruction_file_client: - raise ValueError("instruction_file_client required when instruction_file_setting is not DISABLE") - if value == InstructionFileSetting.DISABLE and self.instruction_file_client: - raise ValueError("instruction_file_client must be None when instruction_file_setting is DISABLE") - + # TODO(instructionFile): Refactor Instruction File Support to use config + instruction_file_client = field(default=None) def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index cca46ffa..616f8da4 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -6,7 +6,7 @@ import boto3 import pytest -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -29,9 +29,7 @@ def test_simple_roundtrip_ascii_string(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE - ) + s3ec = S3EncryptionClient(wrapped_client, config) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -58,7 +56,7 @@ def test_empty_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -86,7 +84,7 @@ def test_no_body_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Call put_object without providing a Body parameter s3ec.put_object(Bucket=bucket, Key=key) @@ -120,7 +118,7 @@ def test_unicode_string_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) s3ec.put_object(Bucket=bucket, Key=key, Body=data) get_req = {"Bucket": bucket, "Key": key} response = s3ec.get_object(**get_req) @@ -155,7 +153,7 @@ def test_specific_encoding_utf8_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -192,7 +190,7 @@ def test_specific_encoding_latin1_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Pass the pre-encoded bytes to put_object s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) @@ -226,7 +224,7 @@ def test_binary_data_roundtrip(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Pass the binary data directly s3ec.put_object(Bucket=bucket, Key=key, Body=data) @@ -256,7 +254,7 @@ def test_invalid_body_types(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Test with integer with pytest.raises(S3EncryptionClientError) as excinfo: @@ -309,7 +307,7 @@ def test_user_metadata_preservation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Put object with user metadata s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) @@ -369,7 +367,7 @@ def test_encryption_context_roundtrip(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -409,7 +407,7 @@ def test_encryption_context_mismatch(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) @@ -449,7 +447,7 @@ def test_encryption_context_missing_on_decrypt(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Put object with encryption context s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 1b2f0e3d..3ec9e3e0 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -5,7 +5,7 @@ import boto3 import pytest -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring # TODO(instructionFiles): Create a Static Bucket for Instruction File Messages @@ -55,8 +55,8 @@ def test_decrypt_v2_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) instruction_file_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) s3ec = S3EncryptionClient( wrapped_client, config, instruction_file_client=instruction_file_client ) @@ -66,20 +66,3 @@ def test_decrypt_v2_instruction_file(): assert output == "Testing encryption of instruction file with KMS Keyring" print("Success! V2 instruction file decryption completed.") - -def test_decrypt_instruction_file_disable_fails(): - """Test that decryption fails when instruction_file_setting is DISABLE.""" - key = TEST_OBJECTS["v2_instruction_file"] - - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - instruction_file_client = boto3.client("s3") - config = S3EncryptionClientConfig( - keyring - ) - - with pytest.raises(ValueError): - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE, instruction_file_client=instruction_file_client - ) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 7330a1c5..419ca7ea 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -13,7 +13,7 @@ import boto3 -from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig, InstructionFileSetting +from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring @@ -39,7 +39,7 @@ def test_multithreaded_encryption_context_isolation(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) # Number of threads to test with num_threads = 10 @@ -151,7 +151,8 @@ def test_multithreaded_rapid_context_switching(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) + num_iterations = 20 errors = [] @@ -228,7 +229,8 @@ def test_multithreaded_mixed_with_and_without_context(): keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config, instruction_file_setting=InstructionFileSetting.DISABLE) + s3ec = S3EncryptionClient(wrapped_client, config) + errors = [] def worker_with_context(thread_id): From bcef24ca5e4dac897ba6d252285de84f5081b2ae Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 23 Feb 2026 11:01:13 -0600 Subject: [PATCH 14/35] feat(instruction-file): add plaintext mode and strict validation - Add S3EC_INTERNAL_PLAINTEXT_MODE constant for internal plaintext fetching - Implement plaintext mode check in on_get_object_after_call to skip decryption - Create instruction_file.py module with strict validation: - Validates instruction file is valid JSON - Ensures only S3EC metadata keys are present - Verifies x-amz-crypto-instr-file marker in response metadata - Fetches instruction files in plaintext mode to prevent recursive decryption - Update TODO comments to reference plaintext_mode refactoring This enables secure instruction file fetching without attempting to decrypt the instruction file itself, preventing infinite recursion. --- src/s3_encryption/__init__.py | 14 +++- src/s3_encryption/instruction_file.py | 96 +++++++++++++++++++++++++++ src/s3_encryption/pipelines.py | 2 +- 3 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 src/s3_encryption/instruction_file.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index fa8c464b..9d68fce0 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -17,6 +17,7 @@ from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline S3_METADATA_PREFIX = "x-amz-meta-" +S3EC_INTERNAL_PLAINTEXT_MODE = "s3ec_internal_plaintext_mode" @define @@ -56,6 +57,8 @@ def on_put_object_before_call(self, params, **kwargs): params: Dictionary of parameters for the PutObject call (after serialization) **kwargs: Additional event arguments """ + # TODO(instructionFile): ensure if S3EC_INTERNAL_PLAINTEXT_MODE error is thrown. + # At this point, boto3 has already serialized the Body # Extract the serialized body from the request body = params.get("body") @@ -101,6 +104,11 @@ def on_get_object_after_call(self, parsed, **kwargs): parsed: Dictionary containing the parsed response **kwargs: Additional event arguments (includes 'params' with request parameters) """ + # Check if plaintext mode is enabled + if S3EC_INTERNAL_PLAINTEXT_MODE in kwargs: + # Skip decryption in plaintext mode + return + # Get encryption context from thread-local storage (set by get_object wrapper) encryption_context = getattr(self._context, "encryption_context", None) @@ -116,7 +124,7 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( self.config.cmm, - # TODO(instructionFile): Refactor Instruction File Support to use config + # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode instruction_file_client=getattr(self._context, "instruction_file_client", None), ) decrypted_data = pipeline.decrypt( @@ -148,7 +156,7 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() _plugin: S3EncryptionClientPlugin = field(init=False) - # TODO(instructionFile): Refactor Instruction File Support to use config + # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode instruction_file_client = field(default=None) def __attrs_post_init__(self): @@ -222,7 +230,7 @@ def get_object(self, **kwargs): self._plugin._context.encryption_context = encryption_context # Store wrapped client in thread-local storage for # the event handler to fetch instruction files - # TODO(instructionFile): Refactor Instruction File Support to use config + # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode self._plugin._context.instruction_file_client = self.instruction_file_client self._plugin._context.bucket = kwargs.get("Bucket") self._plugin._context.key = kwargs.get("Key") diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py new file mode 100644 index 00000000..a456eb81 --- /dev/null +++ b/src/s3_encryption/instruction_file.py @@ -0,0 +1,96 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Instruction file handling for S3 Encryption Client. + +This module provides utilities for fetching and parsing instruction files +that contain encryption metadata for S3 objects. +""" + +import json +from typing import Any + +from .exceptions import S3EncryptionClientError + +# Valid S3 Encryption Client metadata keys +VALID_S3EC_METADATA_KEYS = { + # V1/V2 format keys + "x-amz-key", + "x-amz-key-v2", + "x-amz-wrap-alg", + "x-amz-matdesc", + "x-amz-iv", + "x-amz-cek-alg", + "x-amz-tag-len", + "x-amz-crypto-instr-file", + # V3 format keys (compressed) + "x-amz-c", + "x-amz-3", + "x-amz-m", + "x-amz-t", + "x-amz-w", + "x-amz-d", + "x-amz-i", +} + + +def fetch_instruction_file( + s3_client, bucket: str, key: str, suffix: str = ".instruction" +) -> dict[str, Any]: + """Fetch and parse an instruction file from S3. + + This function strictly validates that: + 1. The instruction file response metadata contains the x-amz-crypto-instr-file marker + 2. The instruction file body is valid JSON + 3. The JSON contains only S3 Encryption Client metadata keys + + Args: + s3_client: Boto3 S3 client to use for fetching + bucket: S3 bucket name + key: S3 object key + suffix: Instruction file suffix (default: .instruction) + + Returns: + dict: Parsed JSON metadata from instruction file + + Raises: + S3EncryptionClientError: If the instruction file marker is missing, + the instruction file is not valid JSON, or contains non-S3EC metadata keys + """ + instruction_key = key + suffix + response = s3_client.get_object( + Bucket=bucket, Key=instruction_key, s3ec_internal_plaintext_mode=True + ) + + # Verify instruction file marker is present in response metadata + response_metadata = response.get("Metadata", {}) + if "x-amz-crypto-instr-file" not in response_metadata: + raise S3EncryptionClientError( + f"Instruction file metadata does not contain " + f"x-amz-crypto-instr-file marker: {instruction_key}" + ) + + instruction_data = response["Body"].read() + + # Validate JSON format + try: + metadata = json.loads(instruction_data) + except json.JSONDecodeError as e: + raise S3EncryptionClientError( + f"Instruction file is not valid JSON: {instruction_key}" + ) from e + + # Validate that it's a dictionary + if not isinstance(metadata, dict): + raise S3EncryptionClientError( + f"Instruction file must contain a JSON object, " + f"got {type(metadata).__name__}: {instruction_key}" + ) + + # Validate that all keys are S3EC metadata keys + invalid_keys = set(metadata.keys()) - VALID_S3EC_METADATA_KEYS + if invalid_keys: + raise S3EncryptionClientError( + f"Instruction file contains invalid keys: {invalid_keys} in {instruction_key}" + ) + + return metadata diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 993ab259..150b441d 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -117,7 +117,7 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): encryption_context = {} # Check if we need to fetch instruction file - # TODO(instructionFile): Refactor Instruction File Support to use config + # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode if metadata.should_use_instruction_file(): if self.instruction_file_client is None: raise S3EncryptionClientError( From ce1c99e5e0fcb476c40d7d5f5c4f0d2a0a3fd9f0 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 23 Feb 2026 11:05:53 -0600 Subject: [PATCH 15/35] format --- src/s3_encryption/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 9d68fce0..b221bf47 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -58,7 +58,7 @@ def on_put_object_before_call(self, params, **kwargs): **kwargs: Additional event arguments """ # TODO(instructionFile): ensure if S3EC_INTERNAL_PLAINTEXT_MODE error is thrown. - + # At this point, boto3 has already serialized the Body # Extract the serialized body from the request body = params.get("body") From 9c567751ff9148857a41494fc6ddc94beba646c8 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 23 Feb 2026 14:16:44 -0600 Subject: [PATCH 16/35] refactor(instruction-file): use thread-local plaintext mode instead of separate client - Remove instruction_file_client parameter from S3EncryptionClient - Implement thread-local plaintext_mode flag for instruction file fetching - Expose plugin context on wrapped client for instruction file access - Event handler validates instruction files and replaces body stream - Update GetEncryptedObjectPipeline to use s3_client instead of instruction_file_client - Refactor fetch_instruction_file to set/clear plaintext mode flag - Split instruction file logic into parse_instruction_file and fetch_instruction_file - Update integration and unit tests to remove instruction_file_client parameter - Add test stub for invalid instruction file parsing This enables instruction file fetching without requiring a separate S3 client, preventing infinite recursion by using thread-local context instead of passing parameters through boto3's validation layer. --- src/s3_encryption/__init__.py | 24 ++++-- src/s3_encryption/instruction_file.py | 85 +++++++++++++------ src/s3_encryption/pipelines.py | 17 ++-- .../test_i_s3_encryption_instruction_file.py | 18 +++- 4 files changed, 91 insertions(+), 53 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index b221bf47..1e4dc8f5 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -9,6 +9,7 @@ from botocore.response import StreamingBody from .exceptions import S3EncryptionClientError +from .instruction_file import parse_instruction_file from .materials.crypto_materials_manager import ( AbstractCryptoMaterialsManager, DefaultCryptoMaterialsManager, @@ -104,9 +105,15 @@ def on_get_object_after_call(self, parsed, **kwargs): parsed: Dictionary containing the parsed response **kwargs: Additional event arguments (includes 'params' with request parameters) """ - # Check if plaintext mode is enabled - if S3EC_INTERNAL_PLAINTEXT_MODE in kwargs: - # Skip decryption in plaintext mode + # Check if plaintext mode is enabled via thread-local flag + if getattr(self._context, "plaintext_mode", False): + # Skip decryption in plaintext mode BUT validate that it is an instruction file + instruction_data = parsed.get("Body").read() + parse_instruction_file(instruction_data, getattr(self._context, "key", None)) + # Replace the body with a new stream so caller can read it again + stream = io.BytesIO(instruction_data) + streaming_body = StreamingBody(stream, len(instruction_data)) + parsed["Body"] = streaming_body return # Get encryption context from thread-local storage (set by get_object wrapper) @@ -124,8 +131,7 @@ def on_get_object_after_call(self, parsed, **kwargs): # Create a pipeline and decrypt the data pipeline = GetEncryptedObjectPipeline( self.config.cmm, - # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode - instruction_file_client=getattr(self._context, "instruction_file_client", None), + s3_client=getattr(self._context, "s3_client", None), ) decrypted_data = pipeline.decrypt( response, @@ -156,14 +162,15 @@ class S3EncryptionClient: wrapped_s3_client = field() config: S3EncryptionClientConfig = field() _plugin: S3EncryptionClientPlugin = field(init=False) - # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode - instruction_file_client = field(default=None) def __attrs_post_init__(self): """Install the encryption plugin on the wrapped client using boto3 events.""" # Create the plugin object.__setattr__(self, "_plugin", S3EncryptionClientPlugin(self.config)) + # Expose plugin context on wrapped client for instruction file fetching + self.wrapped_s3_client._s3ec_plugin_context = self._plugin._context + # Register event handlers using boto3's event system event_system = self.wrapped_s3_client.meta.events event_system.register("before-call.s3.PutObject", self._plugin.on_put_object_before_call) @@ -230,8 +237,7 @@ def get_object(self, **kwargs): self._plugin._context.encryption_context = encryption_context # Store wrapped client in thread-local storage for # the event handler to fetch instruction files - # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode - self._plugin._context.instruction_file_client = self.instruction_file_client + self._plugin._context.s3_client = self.wrapped_s3_client self._plugin._context.bucket = kwargs.get("Bucket") self._plugin._context.key = kwargs.get("Key") diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index a456eb81..bb3fdd9f 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -33,44 +33,24 @@ } -def fetch_instruction_file( - s3_client, bucket: str, key: str, suffix: str = ".instruction" -) -> dict[str, Any]: - """Fetch and parse an instruction file from S3. +def parse_instruction_file(instruction_data: bytes, instruction_key: str) -> dict[str, Any]: + """Parse and validate instruction file data. This function strictly validates that: - 1. The instruction file response metadata contains the x-amz-crypto-instr-file marker - 2. The instruction file body is valid JSON - 3. The JSON contains only S3 Encryption Client metadata keys + 1. The instruction file body is valid JSON + 2. The JSON contains only S3 Encryption Client metadata keys Args: - s3_client: Boto3 S3 client to use for fetching - bucket: S3 bucket name - key: S3 object key - suffix: Instruction file suffix (default: .instruction) + instruction_data: Raw bytes from instruction file body + instruction_key: Instruction file key (for error messages) Returns: dict: Parsed JSON metadata from instruction file Raises: - S3EncryptionClientError: If the instruction file marker is missing, - the instruction file is not valid JSON, or contains non-S3EC metadata keys + S3EncryptionClientError: If the instruction file is not valid JSON + or contains non-S3EC metadata keys """ - instruction_key = key + suffix - response = s3_client.get_object( - Bucket=bucket, Key=instruction_key, s3ec_internal_plaintext_mode=True - ) - - # Verify instruction file marker is present in response metadata - response_metadata = response.get("Metadata", {}) - if "x-amz-crypto-instr-file" not in response_metadata: - raise S3EncryptionClientError( - f"Instruction file metadata does not contain " - f"x-amz-crypto-instr-file marker: {instruction_key}" - ) - - instruction_data = response["Body"].read() - # Validate JSON format try: metadata = json.loads(instruction_data) @@ -94,3 +74,52 @@ def fetch_instruction_file( ) return metadata + + +def fetch_instruction_file( + s3_client, bucket: str, key: str, suffix: str = ".instruction" +) -> dict[str, Any]: + """Fetch and parse an instruction file from S3. + + This function: + 1. Fetches the instruction file in plaintext mode + 2. Verifies the x-amz-crypto-instr-file marker is present + 3. Parses and validates the instruction file content + + Args: + s3_client: Boto3 S3 client to use for fetching + bucket: S3 bucket name + key: S3 object key + suffix: Instruction file suffix (default: .instruction) + + Returns: + dict: Parsed JSON metadata from instruction file + + Raises: + S3EncryptionClientError: If the instruction file marker is missing, + the instruction file is not valid JSON, or contains non-S3EC metadata keys + """ + instruction_key = key + suffix + + # Set plaintext mode flag in thread-local context before calling get_object + # This will be checked by the event handler to skip decryption + if hasattr(s3_client, "_s3ec_plugin_context"): + s3_client._s3ec_plugin_context.plaintext_mode = True + + try: + response = s3_client.get_object(Bucket=bucket, Key=instruction_key) + finally: + # Clear the flag after the call + if hasattr(s3_client, "_s3ec_plugin_context"): + s3_client._s3ec_plugin_context.plaintext_mode = False + + # Verify instruction file marker is present in response metadata + response_metadata = response.get("Metadata", {}) + if "x-amz-crypto-instr-file" not in response_metadata: + raise S3EncryptionClientError( + f"Instruction file metadata does not contain " + f"x-amz-crypto-instr-file marker: {instruction_key}" + ) + + instruction_data = response["Body"].read() + return parse_instruction_file(instruction_data, instruction_key) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 150b441d..24d6716a 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -7,13 +7,13 @@ """ import base64 -import json import os from attrs import define, field from cryptography.hazmat.primitives.ciphers.aead import AESGCM from .exceptions import S3EncryptionClientError +from .instruction_file import fetch_instruction_file from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey from .materials.materials import DecryptionMaterials, EncryptionMaterials @@ -92,7 +92,7 @@ class GetEncryptedObjectPipeline: """ cmm: AbstractCryptoMaterialsManager = field() - instruction_file_client: object = field(default=None) + s3_client: object = field(default=None) def decrypt(self, response, encryption_context=None, bucket=None, key=None): """Decrypt the data after it is retrieved from S3. @@ -117,13 +117,9 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): encryption_context = {} # Check if we need to fetch instruction file - # TODO(instructionFile): Refactor Instruction File Support to use plaintext_mode if metadata.should_use_instruction_file(): - if self.instruction_file_client is None: - raise S3EncryptionClientError( - "instruction_file_client argument required to use instruction file;" - " pass unique S3 Client as instruction_file_client." - ) + if self.s3_client is None: + raise S3EncryptionClientError("s3_client required to fetch instruction file") if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") @@ -165,10 +161,7 @@ def _fetch_instruction_file(self, bucket: str, key: str, suffix: str = ".instruc Returns: dict: Parsed JSON metadata from instruction file """ - instruction_key = key + suffix - response = self.instruction_file_client.get_object(Bucket=bucket, Key=instruction_key) - instruction_data = response["Body"].read() - return json.loads(instruction_data) + return fetch_instruction_file(self.s3_client, bucket, key, suffix) def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 3ec9e3e0..71695062 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -55,14 +55,24 @@ def test_decrypt_v2_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - instruction_file_client = boto3.client("s3") config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient( - wrapped_client, config, instruction_file_client=instruction_file_client - ) + s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") assert output == "Testing encryption of instruction file with KMS Keyring" print("Success! V2 instruction file decryption completed.") + + +@pytest.mark.skip(reason="TODO: Implement test for invalid instruction file parsing") +def test_parse_invalid_instruction_file(): + """Test that parsing an invalid instruction file raises an error.""" + from s3_encryption.exceptions import S3EncryptionClientError + from s3_encryption.instruction_file import parse_instruction_file + + # TODO: Provide invalid instruction file data + invalid_data = b"" + + with pytest.raises(S3EncryptionClientError, match="file must contain a JSON object"): + parse_instruction_file(invalid_data, "test-key.instruction") From b0ed73f4ffbd6342f4e5163ecb3eee65c301490f Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:44:34 -0600 Subject: [PATCH 17/35] feat(cdk): add S3ECStaticTestObjectsBucket for cross-language testing - Add s3ec-static-test-objects bucket to CDK stack - Grant object-level permissions (GetObject, PutObject, DeleteObject) - Grant bucket-level permissions (ListBucket) - Accessible by GitHub Actions and ToolsDevelopment role - Supports static test objects created by Java S3EC for Python compatibility testing --- cdk/lib/cdk-stack.ts | 12 ++ cdk/package-lock.json | 262 +++++++++++++++++++++++++++--------------- cdk/package.json | 4 +- 3 files changed, 185 insertions(+), 93 deletions(-) diff --git a/cdk/lib/cdk-stack.ts b/cdk/lib/cdk-stack.ts index 1fad4b74..b5a28084 100644 --- a/cdk/lib/cdk-stack.ts +++ b/cdk/lib/cdk-stack.ts @@ -91,6 +91,16 @@ export class S3ECPythonGithub extends cdk.Stack { } ) + // New bucket for static test objects + const S3ECStaticTestObjectsBucket = new Bucket( + this, + "S3ECStaticTestObjectsBucket", + { + bucketName: "s3ec-static-test-objects", + blockPublicAccess: new BlockPublicAccess(AccessConfiguration) + } + ) + // S3 bucket policy const S3ECGithubS3BucketPolicy = new ManagedPolicy( this, @@ -110,6 +120,7 @@ export class S3ECPythonGithub extends cdk.Stack { resources: [ S3ECGithubTestS3Bucket.bucketArn + "/*", // object-level permissions need this extra path S3ECTestServerGithubBucket.bucketArn + "/*", // Add permissions for the new test-server bucket + S3ECStaticTestObjectsBucket.bucketArn + "/*", // Add permissions for static test objects bucket "arn:aws:s3:::aws-net-sdk-*/*" // permission for object inside S3EC .net bucket. For S3EC-NET repo ], }), @@ -125,6 +136,7 @@ export class S3ECPythonGithub extends cdk.Stack { resources: [ S3ECGithubTestS3Bucket.bucketArn, S3ECTestServerGithubBucket.bucketArn, // Add permissions for the new test-server bucket + S3ECStaticTestObjectsBucket.bucketArn, // Add permissions for static test objects bucket "arn:aws:s3:::aws-net-sdk-*", // permission for S3EC .net bucket. For S3EC-NET repo ], }), diff --git a/cdk/package-lock.json b/cdk/package-lock.json index 4f44562c..fa174491 100644 --- a/cdk/package-lock.json +++ b/cdk/package-lock.json @@ -8,7 +8,7 @@ "name": "cdk", "version": "0.1.0", "dependencies": { - "aws-cdk-lib": "2.92.0", + "aws-cdk-lib": "^2.240.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" }, @@ -40,16 +40,9 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.247", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.247.tgz", - "integrity": "sha512-PGFzztdu5YozUgoUd8gq5qi1FR3EYMjNrl5JFrAlYh2w1PcTfExEwqDzZy9z6uzogEJKwQJDgyhWe+OcZzQqFg==", - "license": "Apache-2.0" - }, - "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.4.tgz", - "integrity": "sha512-Ps2MkmjYgMyflagqQ4dgTElc7Vwpqj8spw8dQVFiSeaaMPsuDSNsPax3/HjuDuwqsmLdaCZc6umlxYLpL0kYDA==", - "license": "Apache-2.0" + "version": "2.2.263", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.263.tgz", + "integrity": "sha512-X9JvcJhYcb7PHs8R7m4zMablO5C9PGb/hYfLnxds9h/rKJu6l7MiXE/SabCibuehxPnuO/vk+sVVJiUWrccarQ==" }, "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { "version": "2.1.0", @@ -57,6 +50,41 @@ "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", "license": "Apache-2.0" }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "50.4.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-50.4.0.tgz", + "integrity": "sha512-9Cplwc5C+SNe3hMfqZET7gXeM68tiH2ytQytCi+zz31Bn7O3GAgAnC2dYe+HWnZAgVH788ZkkBwnYXkeqx7v4g==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.3" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.7.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", @@ -1230,11 +1258,12 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.92.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.92.0.tgz", - "integrity": "sha512-J+SUFSnOt9u2GbY5QIABgjGNiw8bL/v0S3zsPhhO1dVwK+G7oE+bhLcAi3iILrw2sIpirNWH9K3W0by9K+cyMw==", + "version": "2.240.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.240.0.tgz", + "integrity": "sha512-3dXmUnPB5kK0VgrNHOlV3jiQM4Dungukk/CV91nclO2lgNcrGyigauJdzmz9sOmI1gbKJJ2SRAotaXityzZMRw==", "bundleDependencies": [ "@balena/dockerignore", + "@aws-cdk/cloud-assembly-api", "case", "fs-extra", "ignore", @@ -1243,29 +1272,69 @@ "punycode", "semver", "table", - "yaml" + "yaml", + "mime-types" ], - "license": "Apache-2.0", "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.200", - "@aws-cdk/asset-kubectl-v20": "^2.1.2", - "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", + "@aws-cdk/asset-awscli-v1": "2.2.263", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", + "@aws-cdk/cloud-assembly-api": "^2.0.1", + "@aws-cdk/cloud-assembly-schema": "^50.3.0", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^11.1.1", - "ignore": "^5.2.4", - "jsonschema": "^1.4.1", - "minimatch": "^3.1.2", - "punycode": "^2.3.0", - "semver": "^7.5.4", - "table": "^6.8.1", + "fs-extra": "^11.3.3", + "ignore": "^5.3.2", + "jsonschema": "^1.5.0", + "mime-types": "^2.1.35", + "minimatch": "^10.2.1", + "punycode": "^2.3.1", + "semver": "^7.7.4", + "table": "^6.9.0", "yaml": "1.10.2" }, "engines": { - "node": ">= 14.15.0" + "node": ">= 18.0.0" }, "peerDependencies": { - "constructs": "^10.0.0" + "constructs": "^10.5.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api": { + "version": "2.0.1", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "inBundle": true, + "license": "Apache-2.0", + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.3" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "@aws-cdk/cloud-assembly-schema": ">=50.3.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/@aws-cdk/cloud-assembly-api/node_modules/semver": { + "version": "7.7.3", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { @@ -1274,14 +1343,14 @@ "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { - "version": "8.12.0", + "version": "8.18.0", "inBundle": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -1319,17 +1388,22 @@ } }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { - "version": "1.0.2", + "version": "4.0.4", "inBundle": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } }, "node_modules/aws-cdk-lib/node_modules/brace-expansion": { - "version": "1.1.11", + "version": "5.0.3", "inBundle": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" } }, "node_modules/aws-cdk-lib/node_modules/case": { @@ -1356,11 +1430,6 @@ "inBundle": true, "license": "MIT" }, - "node_modules/aws-cdk-lib/node_modules/concat-map": { - "version": "0.0.1", - "inBundle": true, - "license": "MIT" - }, "node_modules/aws-cdk-lib/node_modules/emoji-regex": { "version": "8.0.0", "inBundle": true, @@ -1371,8 +1440,23 @@ "inBundle": true, "license": "MIT" }, + "node_modules/aws-cdk-lib/node_modules/fast-uri": { + "version": "3.1.0", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "11.1.1", + "version": "11.3.3", "inBundle": true, "license": "MIT", "dependencies": { @@ -1390,7 +1474,7 @@ "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { - "version": "5.2.4", + "version": "5.3.2", "inBundle": true, "license": "MIT", "engines": { @@ -1411,7 +1495,7 @@ "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/jsonfile": { - "version": "6.1.0", + "version": "6.2.0", "inBundle": true, "license": "MIT", "dependencies": { @@ -1422,7 +1506,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/jsonschema": { - "version": "1.4.1", + "version": "1.5.0", "inBundle": true, "license": "MIT", "engines": { @@ -1434,30 +1518,41 @@ "inBundle": true, "license": "MIT" }, - "node_modules/aws-cdk-lib/node_modules/lru-cache": { - "version": "6.0.0", + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", "inBundle": true, - "license": "ISC", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", "dependencies": { - "yallist": "^4.0.0" + "mime-db": "1.52.0" }, "engines": { - "node": ">=10" + "node": ">= 0.6" } }, "node_modules/aws-cdk-lib/node_modules/minimatch": { - "version": "3.1.2", + "version": "10.2.2", "inBundle": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^5.0.2" }, "engines": { - "node": "*" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/aws-cdk-lib/node_modules/punycode": { - "version": "2.3.0", + "version": "2.3.1", "inBundle": true, "license": "MIT", "engines": { @@ -1473,12 +1568,9 @@ } }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.5.4", + "version": "7.7.4", "inBundle": true, "license": "ISC", - "dependencies": { - "lru-cache": "^6.0.0" - }, "bin": { "semver": "bin/semver.js" }, @@ -1527,7 +1619,7 @@ } }, "node_modules/aws-cdk-lib/node_modules/table": { - "version": "6.8.1", + "version": "6.9.0", "inBundle": true, "license": "BSD-3-Clause", "dependencies": { @@ -1542,26 +1634,13 @@ } }, "node_modules/aws-cdk-lib/node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", "inBundle": true, "license": "MIT", "engines": { "node": ">= 10.0.0" } }, - "node_modules/aws-cdk-lib/node_modules/uri-js": { - "version": "4.4.1", - "inBundle": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/aws-cdk-lib/node_modules/yallist": { - "version": "4.0.0", - "inBundle": true, - "license": "ISC" - }, "node_modules/aws-cdk-lib/node_modules/yaml": { "version": "1.10.2", "inBundle": true, @@ -1690,12 +1769,14 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, "license": "MIT" }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -1925,13 +2006,13 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, "license": "MIT" }, "node_modules/constructs": { - "version": "10.4.2", - "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.2.tgz", - "integrity": "sha512-wsNxBlAott2qg8Zv87q3eYZYgheb9lchtBfjHzzLHtXbttwSrHPs1NNQbBrmbb1YZvYg2+Vh0Dor76w4mFxJkA==", - "license": "Apache-2.0" + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.5.1.tgz", + "integrity": "sha512-f/TfFXiS3G/yVIXDjOQn9oTlyu9Wo7Fxyjj7lb8r92iO81jR2uST+9MstxZTmDGx/CgIbxCXkFXgupnLTNxQZg==" }, "node_modules/convert-source-map": { "version": "2.0.0", @@ -2038,11 +2119,10 @@ } }, "node_modules/diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", + "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", "dev": true, - "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -3209,11 +3289,10 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, - "license": "MIT", "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -3390,10 +3469,10 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "license": "ISC", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.3.tgz", + "integrity": "sha512-M2GCs7Vk83NxkUyQV1bkABc4yxgz9kILhHImZiBPAZ9ybuvCb0/H7lEl5XvIg3g+9d4eNotkZA5IWwYl0tibaA==", + "dev": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -3783,6 +3862,7 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/cdk/package.json b/cdk/package.json index f1e769db..7cc118ae 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -13,14 +13,14 @@ "devDependencies": { "@types/jest": "^29.5.3", "@types/node": "20.4.10", + "aws-cdk": "2.92.0", "jest": "^29.6.2", "ts-jest": "^29.1.1", - "aws-cdk": "2.92.0", "ts-node": "^10.9.1", "typescript": "~5.1.6" }, "dependencies": { - "aws-cdk-lib": "2.92.0", + "aws-cdk-lib": "^2.240.0", "constructs": "^10.0.0", "source-map-support": "^0.5.21" } From 3e9f5342cb78b48445ce159c2892ba7cb6171a3c Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:47:01 -0600 Subject: [PATCH 18/35] refactor(test): use static test objects for instruction file tests - Update bucket to s3ec-static-test-objects - Update KMS key to S3ECTestServerKMSKey (a3889cd9-99eb-4138-a93a-aea9d52ec2ef) - Use static V2 test object: static-v2-instruction-file-from-java-v4 - Use static V3 test object: static-v3-instruction-file-from-java-v4 - V2 test active and expects exact content match - V3 test skipped until V3 decryption implemented --- .../test_i_s3_encryption_instruction_file.py | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 71695062..f69e2e50 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -8,31 +8,29 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring -# TODO(instructionFiles): Create a Static Bucket for Instruction File Messages -# TODO(instructionFiles): Add Static Bucket for Instruction File Messages to test env -bucket = os.environ.get("CI_S3_INSTRUCTION_BUCKET", "s3ec-github-test-bucket") +# Static test objects bucket +bucket = os.environ.get("CI_S3_STATIC_TEST_BUCKET", "s3ec-static-test-objects") region = os.environ.get("CI_AWS_REGION", "us-west-2") -# TODO(instructionFiles): Add INS FILES KMS Key to test env +# KMS key used for static test objects (S3ECTestServerKMSKey) kms_key_id = os.environ.get( - "CI_KMS_KEY_INSTRUCTION_FILES", - "arn:aws:kms:us-west-2:370957321024:key/c3eafb5f-e87d-4584-9400-cf419ce5d782", + "CI_KMS_KEY_STATIC_TESTS", + "arn:aws:kms:us-west-2:370957321024:key/a3889cd9-99eb-4138-a93a-aea9d52ec2ef", ) -# Test keys for objects encrypted by Java S3EC with instruction files +# Static test object keys created by Java S3EC V4 TEST_OBJECTS = { - # TODO(instructionFiles): V1 Instruction File - "v1_instruction_file": "test-v1-cbc-instruction", - # TODO(instructionFiles): Proper V2 Instruction File - "v2_instruction_file": "kms-instruction-file-test-260220-105428-19668", - # TODO(instructionFiles): V3 Instruction File - "v3_instruction_file": "test-v3-instruction", + "v2_instruction_file": "static-v2-instruction-file-from-java-v4", + "v3_instruction_file": "static-v3-instruction-file-from-java-v4", } -@pytest.mark.skip(reason="Requires pre-existing test objects encrypted by Java S3EC") -def test_decrypt_v1_instruction_file(): - """Test decrypting V1 object with instruction file.""" - key = TEST_OBJECTS["v1_instruction_file"] +def test_decrypt_v2_instruction_file(): + """Test decrypting V2 object with instruction file. + + V2 format uses ALG_AES_256_GCM_IV12_TAG16_NO_KDF (no key commitment). + Object encrypted by Java S3EC V4 with instruction file enabled. + """ + key = TEST_OBJECTS["v2_instruction_file"] kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) @@ -43,14 +41,18 @@ def test_decrypt_v1_instruction_file(): response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - assert output == "test data v1 cbc" - print("Success! V1 instruction file decryption completed.") + assert output == "static-v2-instruction-file-from-java-v4" + print("Success! V2 instruction file decryption completed.") + +@pytest.mark.skip(reason="V3 decryption not yet implemented") +def test_decrypt_v3_instruction_file(): + """Test decrypting V3 object with instruction file. -# @pytest.mark.skip(reason="Requires pre-existing test objects encrypted by Java S3EC") -def test_decrypt_v2_instruction_file(): - """Test decrypting V2 object with instruction file.""" - key = TEST_OBJECTS["v2_instruction_file"] + V3 format uses ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (with key commitment). + Object encrypted by Java S3EC V4 with instruction file enabled. + """ + key = TEST_OBJECTS["v3_instruction_file"] kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) @@ -61,8 +63,8 @@ def test_decrypt_v2_instruction_file(): response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - assert output == "Testing encryption of instruction file with KMS Keyring" - print("Success! V2 instruction file decryption completed.") + assert output != "static-v3-instruction-file-from-java-v4" + print("Success! V3 instruction file decryption completed.") @pytest.mark.skip(reason="TODO: Implement test for invalid instruction file parsing") From eb5d1df45023df54e9ac7dce29934f62855a575c Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 27 Feb 2026 07:53:03 -0600 Subject: [PATCH 19/35] test(integration): add V1 and negative instruction file tests - Add V1 instruction file test (skipped until CBC decryption implemented) - Uses static-v1-instruction-file-from-java-v1 test object - Enables legacy wrapping algorithms for V1 format - Add negative test for invalid instruction file - Uses NEGATIVE-static-v2-instruction-file-test-from-java-v4 test object - Expects S3EncryptionClientError when instruction file is invalid - Prints error message for debugging - Replace skipped unit test stub with full integration test --- .../test_i_s3_encryption_instruction_file.py | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index f69e2e50..b76fe429 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -19,11 +19,35 @@ # Static test object keys created by Java S3EC V4 TEST_OBJECTS = { + "v1_instruction_file": "static-v1-instruction-file-from-java-v1", "v2_instruction_file": "static-v2-instruction-file-from-java-v4", "v3_instruction_file": "static-v3-instruction-file-from-java-v4", + "negative_v2_instruction_file": "NEGATIVE-static-v2-instruction-file-test-from-java-v4", } +@pytest.mark.skip(reason="V1 CBC decryption not yet implemented") +def test_decrypt_v1_instruction_file(): + """Test decrypting V1 object with instruction file. + + V1 format uses ALG_AES_256_CBC_IV16_NO_KDF (CBC mode, no key commitment). + Object encrypted by Java S3EC V1 with instruction file enabled. + """ + key = TEST_OBJECTS["v1_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + + assert output == "static-v1-instruction-file-from-java-v1" + print("Success! V1 instruction file decryption completed.") + + def test_decrypt_v2_instruction_file(): """Test decrypting V2 object with instruction file. @@ -67,14 +91,23 @@ def test_decrypt_v3_instruction_file(): print("Success! V3 instruction file decryption completed.") -@pytest.mark.skip(reason="TODO: Implement test for invalid instruction file parsing") -def test_parse_invalid_instruction_file(): - """Test that parsing an invalid instruction file raises an error.""" +def test_decrypt_invalid_instruction_file(): + """Test that decrypting with an invalid instruction file raises an error. + + The NEGATIVE test object has an invalid instruction file that should + cause the S3 Encryption Client to raise an exception during decryption. + """ from s3_encryption.exceptions import S3EncryptionClientError - from s3_encryption.instruction_file import parse_instruction_file - # TODO: Provide invalid instruction file data - invalid_data = b"" + key = TEST_OBJECTS["negative_v2_instruction_file"] + + kms_client = boto3.client("kms", region_name=region) + keyring = KmsKeyring(kms_client, kms_key_id) + wrapped_client = boto3.client("s3") + config = S3EncryptionClientConfig(keyring) + s3ec = S3EncryptionClient(wrapped_client, config) + + with pytest.raises(S3EncryptionClientError) as exc_info: + s3ec.get_object(Bucket=bucket, Key=key) - with pytest.raises(S3EncryptionClientError, match="file must contain a JSON object"): - parse_instruction_file(invalid_data, "test-key.instruction") + print(f"Error message: {exc_info.value}") From 8b9adc98c6af3405f77eb6019ce0d5618f75cfd4 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 27 Feb 2026 10:53:42 -0600 Subject: [PATCH 20/35] refactor(instruction-file): return metadata in Metadata field, not Body - Event handler now parses instruction file and places metadata in Metadata field - Clear Body when in plaintext mode (instruction files shouldn't return body content) - fetch_instruction_file returns metadata from response Metadata field - Add validation that instruction file metadata is not empty and contains S3EC keys - Update unit tests to mock new behavior (metadata in Metadata, empty Body) - All 37 unit tests passing --- src/s3_encryption/__init__.py | 24 +++++++++++++++---- src/s3_encryption/instruction_file.py | 34 +++++++++++++++++++-------- test/test_pipelines.py | 21 ++++++++++------- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 1e4dc8f5..539877b8 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -107,12 +107,26 @@ def on_get_object_after_call(self, parsed, **kwargs): """ # Check if plaintext mode is enabled via thread-local flag if getattr(self._context, "plaintext_mode", False): - # Skip decryption in plaintext mode BUT validate that it is an instruction file + # In plaintext mode, parse instruction file and append to metadata instruction_data = parsed.get("Body").read() - parse_instruction_file(instruction_data, getattr(self._context, "key", None)) - # Replace the body with a new stream so caller can read it again - stream = io.BytesIO(instruction_data) - streaming_body = StreamingBody(stream, len(instruction_data)) + instruction_key = getattr(self._context, "key", None) + instruction_metadata = parse_instruction_file(instruction_data, instruction_key) + + # Verify instruction file marker is present + if "x-amz-crypto-instr-file" not in instruction_metadata: + raise S3EncryptionClientError( + f"Instruction file does not contain " + f"x-amz-crypto-instr-file marker: {instruction_key}" + ) + + # Append parsed instruction file content to existing metadata + existing_metadata = parsed.get("Metadata", {}) + existing_metadata.update(instruction_metadata) + parsed["Metadata"] = existing_metadata + + # Clear the body since instruction files shouldn't return body content + stream = io.BytesIO(b"") + streaming_body = StreamingBody(stream, 0) parsed["Body"] = streaming_body return diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index bb3fdd9f..fe8e55bf 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -83,8 +83,12 @@ def fetch_instruction_file( This function: 1. Fetches the instruction file in plaintext mode - 2. Verifies the x-amz-crypto-instr-file marker is present - 3. Parses and validates the instruction file content + 2. Returns the parsed metadata from the response Metadata field + + The event handler (on_get_object_after_call) handles: + - Verifying the x-amz-crypto-instr-file marker is present + - Parsing and validating the instruction file content + - Placing parsed metadata in response["Metadata"] Args: s3_client: Boto3 S3 client to use for fetching @@ -105,21 +109,31 @@ def fetch_instruction_file( # This will be checked by the event handler to skip decryption if hasattr(s3_client, "_s3ec_plugin_context"): s3_client._s3ec_plugin_context.plaintext_mode = True + s3_client._s3ec_plugin_context.key = instruction_key try: response = s3_client.get_object(Bucket=bucket, Key=instruction_key) finally: - # Clear the flag after the call + # Clear the flags after the call if hasattr(s3_client, "_s3ec_plugin_context"): s3_client._s3ec_plugin_context.plaintext_mode = False + if hasattr(s3_client._s3ec_plugin_context, "key"): + delattr(s3_client._s3ec_plugin_context, "key") + + # In plaintext mode, the event handler places parsed metadata in Metadata field + metadata = response.get("Metadata", {}) + + # Verify metadata is not empty + if not metadata: + raise S3EncryptionClientError( + f"Instruction file returned empty metadata: {instruction_key}" + ) - # Verify instruction file marker is present in response metadata - response_metadata = response.get("Metadata", {}) - if "x-amz-crypto-instr-file" not in response_metadata: + # Verify metadata contains at least one S3EC key + has_s3ec_key = any(key in VALID_S3EC_METADATA_KEYS for key in metadata) + if not has_s3ec_key: raise S3EncryptionClientError( - f"Instruction file metadata does not contain " - f"x-amz-crypto-instr-file marker: {instruction_key}" + f"Instruction file metadata does not contain any S3EC keys: {instruction_key}" ) - instruction_data = response["Body"].read() - return parse_instruction_file(instruction_data, instruction_key) + return metadata diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 7ac9d043..9d1e5256 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -30,14 +30,15 @@ def test_decrypt_v1_from_instruction_file(self): "x-amz-wrap-alg": "kms", "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), "x-amz-cek-alg": "AES/CBC/PKCS5Padding", + "x-amz-crypto-instr-file": "", } # Create mock S3 client mock_s3_client = Mock() - instruction_file_body = BytesIO(json.dumps(instruction_file_metadata).encode("utf-8")) + # Mock returns parsed metadata (simulating event handler behavior) mock_s3_client.get_object.return_value = { - "Body": instruction_file_body, - "Metadata": {"x-amz-crypto-instr-file": ""}, + "Body": BytesIO(b""), # Body is cleared by event handler + "Metadata": instruction_file_metadata, } # Create mock keyring and CMM @@ -80,14 +81,15 @@ def test_decrypt_v2_from_instruction_file(self): "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), "x-amz-cek-alg": "AES/GCM/NoPadding", "x-amz-tag-len": "128", + "x-amz-crypto-instr-file": "", } # Create mock S3 client mock_s3_client = Mock() - instruction_file_body = BytesIO(json.dumps(instruction_file_metadata).encode("utf-8")) + # Mock returns parsed metadata (simulating event handler behavior) mock_s3_client.get_object.return_value = { - "Body": instruction_file_body, - "Metadata": {"x-amz-crypto-instr-file": ""}, + "Body": BytesIO(b""), # Body is cleared by event handler + "Metadata": instruction_file_metadata, } # Create mock keyring and CMM @@ -131,14 +133,15 @@ def test_decrypt_v3_from_instruction_file(self): "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), "x-amz-w": "02", # AES/GCM "x-amz-m": json.dumps({"test-instruction": "material-desc-instruction"}), + "x-amz-crypto-instr-file": "", } # Create mock S3 client mock_s3_client = Mock() - instruction_file_body = BytesIO(json.dumps(instruction_file_metadata).encode("utf-8")) + # Mock returns parsed metadata (simulating event handler behavior) mock_s3_client.get_object.return_value = { - "Body": instruction_file_body, - "Metadata": {"x-amz-crypto-instr-file": ""}, + "Body": BytesIO(b""), # Body is cleared by event handler + "Metadata": instruction_file_metadata, } # Create mock keyring and CMM From a07b25aaff43ccdc5657d51c5f8e427c6e96578e Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 27 Feb 2026 11:00:40 -0600 Subject: [PATCH 21/35] fix(instruction-file): check marker in S3 metadata, add event handler tests - Fix: Check x-amz-crypto-instr-file marker in S3 object metadata, not parsed JSON body - Add 5 unit tests for on_get_object_after_call instruction file behavior - Test successful parsing and metadata update - Test missing marker error - Test invalid JSON error - Test non-dict JSON error - Test invalid keys error - All 42 unit tests passing --- src/s3_encryption/__init__.py | 6 +- test/test_on_get_object_after_call.py | 166 ++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 test/test_on_get_object_after_call.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 539877b8..5b78a938 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -112,15 +112,15 @@ def on_get_object_after_call(self, parsed, **kwargs): instruction_key = getattr(self._context, "key", None) instruction_metadata = parse_instruction_file(instruction_data, instruction_key) - # Verify instruction file marker is present - if "x-amz-crypto-instr-file" not in instruction_metadata: + # Verify instruction file marker is present in S3 object metadata + existing_metadata = parsed.get("Metadata", {}) + if "x-amz-crypto-instr-file" not in existing_metadata: raise S3EncryptionClientError( f"Instruction file does not contain " f"x-amz-crypto-instr-file marker: {instruction_key}" ) # Append parsed instruction file content to existing metadata - existing_metadata = parsed.get("Metadata", {}) existing_metadata.update(instruction_metadata) parsed["Metadata"] = existing_metadata diff --git a/test/test_on_get_object_after_call.py b/test/test_on_get_object_after_call.py new file mode 100644 index 00000000..d03a4eb2 --- /dev/null +++ b/test/test_on_get_object_after_call.py @@ -0,0 +1,166 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for S3EncryptionClientPlugin event handler.""" + +import io +import json +from unittest.mock import Mock + +import pytest +from botocore.response import StreamingBody + +from s3_encryption import S3EncryptionClientConfig, S3EncryptionClientPlugin +from s3_encryption.exceptions import S3EncryptionClientError +from s3_encryption.materials.keyring import S3Keyring + + +class TestEventHandlerInstructionFile: + """Test event handler behavior for instruction files in plaintext mode.""" + + def test_plaintext_mode_parses_instruction_file(self): + """Test that plaintext mode parses instruction file and returns metadata.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.plaintext_mode = True + plugin._context.key = "test-key.instruction" + + # Create instruction file body + instruction_metadata = { + "x-amz-iv": "test-iv", + "x-amz-key-v2": "test-key", + "x-amz-wrap-alg": "kms+context", + "x-amz-cek-alg": "AES/GCM/NoPadding", + } + instruction_body = json.dumps(instruction_metadata).encode("utf-8") + + # Create parsed response with instruction file marker in S3 metadata + parsed = { + "Body": StreamingBody(io.BytesIO(instruction_body), len(instruction_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Call event handler + plugin.on_get_object_after_call(parsed) + + # Verify metadata was updated with parsed instruction file + assert parsed["Metadata"]["x-amz-iv"] == "test-iv" + assert parsed["Metadata"]["x-amz-key-v2"] == "test-key" + assert parsed["Metadata"]["x-amz-wrap-alg"] == "kms+context" + assert parsed["Metadata"]["x-amz-cek-alg"] == "AES/GCM/NoPadding" + assert parsed["Metadata"]["x-amz-crypto-instr-file"] == "" + + # Verify body was cleared + assert parsed["Body"].read() == b"" + + def test_plaintext_mode_missing_marker_raises_error(self): + """Test that missing instruction file marker raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.plaintext_mode = True + plugin._context.key = "test-key.instruction" + + # Create instruction file body + instruction_metadata = { + "x-amz-iv": "test-iv", + "x-amz-key-v2": "test-key", + } + instruction_body = json.dumps(instruction_metadata).encode("utf-8") + + # Create parsed response WITHOUT instruction file marker + parsed = { + "Body": StreamingBody(io.BytesIO(instruction_body), len(instruction_body)), + "Metadata": {}, # Missing x-amz-crypto-instr-file + } + + # Should raise error + with pytest.raises( + S3EncryptionClientError, + match="Instruction file does not contain x-amz-crypto-instr-file marker", + ): + plugin.on_get_object_after_call(parsed) + + def test_plaintext_mode_invalid_json_raises_error(self): + """Test that invalid JSON in instruction file raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.plaintext_mode = True + plugin._context.key = "test-key.instruction" + + # Create invalid JSON body + invalid_body = b"not valid json" + + # Create parsed response + parsed = { + "Body": StreamingBody(io.BytesIO(invalid_body), len(invalid_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Should raise error + with pytest.raises(S3EncryptionClientError, match="Instruction file is not valid JSON"): + plugin.on_get_object_after_call(parsed) + + def test_plaintext_mode_non_dict_json_raises_error(self): + """Test that non-dict JSON in instruction file raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.plaintext_mode = True + plugin._context.key = "test-key.instruction" + + # Create JSON array instead of object + invalid_body = json.dumps(["not", "a", "dict"]).encode("utf-8") + + # Create parsed response + parsed = { + "Body": StreamingBody(io.BytesIO(invalid_body), len(invalid_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Should raise error + with pytest.raises( + S3EncryptionClientError, match="Instruction file must contain a JSON object" + ): + plugin.on_get_object_after_call(parsed) + + def test_plaintext_mode_invalid_keys_raises_error(self): + """Test that invalid keys in instruction file raises error.""" + # Create plugin + mock_keyring = Mock(spec=S3Keyring) + config = S3EncryptionClientConfig(keyring=mock_keyring) + plugin = S3EncryptionClientPlugin(config) + + # Set plaintext mode + plugin._context.plaintext_mode = True + plugin._context.key = "test-key.instruction" + + # Create instruction file with invalid keys + instruction_metadata = { + "x-amz-iv": "test-iv", + "invalid-key": "should-not-be-here", + } + instruction_body = json.dumps(instruction_metadata).encode("utf-8") + + # Create parsed response + parsed = { + "Body": StreamingBody(io.BytesIO(instruction_body), len(instruction_body)), + "Metadata": {"x-amz-crypto-instr-file": ""}, + } + + # Should raise error + with pytest.raises(S3EncryptionClientError, match="Instruction file contains invalid keys"): + plugin.on_get_object_after_call(parsed) From 8a0cc50cfae09149bd0cb497d8b7ee9e012881b2 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 27 Feb 2026 13:58:10 -0600 Subject: [PATCH 22/35] refactor(instruction-file): extract process_instruction_file method and improve error handling - Extract instruction file processing logic into process_instruction_file method - Add docstring for process_instruction_file method - Add error check for plaintext mode in put_object (should never happen) - Move VALID_S3EC_METADATA_KEYS to metadata.py for better organization - Add error if plugin context not available when fetching instruction file - Remove _fetch_instruction_file wrapper method, call fetch_instruction_file directly - Rename test file: test_on_get_object_after_call.py -> test_s3_encryption_client_plugin.py - Add TODO comments for V1 CBC and V3 decryption in integration tests - Add x-amz-meta-x-amz-unencrypted-content-length to V1 test metadata - Fix linting: line length, missing docstring, remove commented code --- src/s3_encryption/__init__.py | 62 ++++++++++++------- src/s3_encryption/instruction_file.py | 31 +++------- src/s3_encryption/metadata.py | 22 +++++++ src/s3_encryption/pipelines.py | 15 +---- .../test_i_s3_encryption_instruction_file.py | 2 + test/test_pipelines.py | 7 +-- ...py => test_s3_encryption_client_plugin.py} | 6 +- 7 files changed, 74 insertions(+), 71 deletions(-) rename test/{test_on_get_object_after_call.py => test_s3_encryption_client_plugin.py} (97%) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 5b78a938..dc19994c 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -18,7 +18,6 @@ from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline S3_METADATA_PREFIX = "x-amz-meta-" -S3EC_INTERNAL_PLAINTEXT_MODE = "s3ec_internal_plaintext_mode" @define @@ -58,8 +57,11 @@ def on_put_object_before_call(self, params, **kwargs): params: Dictionary of parameters for the PutObject call (after serialization) **kwargs: Additional event arguments """ - # TODO(instructionFile): ensure if S3EC_INTERNAL_PLAINTEXT_MODE error is thrown. - + if getattr(self._context, "plaintext_mode", False): + raise S3EncryptionClientError( + "Plaintext mode is exclusively for reading instruction files " + "and not supported in put_object!" + ) # At this point, boto3 has already serialized the Body # Extract the serialized body from the request body = params.get("body") @@ -107,27 +109,7 @@ def on_get_object_after_call(self, parsed, **kwargs): """ # Check if plaintext mode is enabled via thread-local flag if getattr(self._context, "plaintext_mode", False): - # In plaintext mode, parse instruction file and append to metadata - instruction_data = parsed.get("Body").read() - instruction_key = getattr(self._context, "key", None) - instruction_metadata = parse_instruction_file(instruction_data, instruction_key) - - # Verify instruction file marker is present in S3 object metadata - existing_metadata = parsed.get("Metadata", {}) - if "x-amz-crypto-instr-file" not in existing_metadata: - raise S3EncryptionClientError( - f"Instruction file does not contain " - f"x-amz-crypto-instr-file marker: {instruction_key}" - ) - - # Append parsed instruction file content to existing metadata - existing_metadata.update(instruction_metadata) - parsed["Metadata"] = existing_metadata - - # Clear the body since instruction files shouldn't return body content - stream = io.BytesIO(b"") - streaming_body = StreamingBody(stream, 0) - parsed["Body"] = streaming_body + self.process_instruction_file(parsed) return # Get encryption context from thread-local storage (set by get_object wrapper) @@ -161,6 +143,38 @@ def on_get_object_after_call(self, parsed, **kwargs): # Replace body with decrypted data parsed["Body"] = streaming_body + def process_instruction_file(self, parsed): + """Process instruction file in plaintext mode. + + Validates the instruction file marker, parses the JSON body, + and updates the response metadata with parsed content. + + Args: + parsed: Dictionary containing the parsed response + """ + instruction_key = getattr(self._context, "key", None) + + # Verify instruction file marker is present in S3 object metadata + existing_metadata = parsed.get("Metadata", {}) + if "x-amz-crypto-instr-file" not in existing_metadata: + raise S3EncryptionClientError( + f"Instruction file does not contain " + f"x-amz-crypto-instr-file marker: {instruction_key}" + ) + + # In plaintext mode, parse instruction file and append to metadata + instruction_data = parsed.get("Body").read() + instruction_metadata = parse_instruction_file(instruction_data, instruction_key) + + # Append parsed instruction file content to existing metadata + existing_metadata.update(instruction_metadata) + parsed["Metadata"] = existing_metadata + + # Clear the body since instruction files shouldn't return body content + stream = io.BytesIO(b"") + streaming_body = StreamingBody(stream, 0) + parsed["Body"] = streaming_body + @define class S3EncryptionClient: diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index fe8e55bf..b4fe2eed 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -10,27 +10,7 @@ from typing import Any from .exceptions import S3EncryptionClientError - -# Valid S3 Encryption Client metadata keys -VALID_S3EC_METADATA_KEYS = { - # V1/V2 format keys - "x-amz-key", - "x-amz-key-v2", - "x-amz-wrap-alg", - "x-amz-matdesc", - "x-amz-iv", - "x-amz-cek-alg", - "x-amz-tag-len", - "x-amz-crypto-instr-file", - # V3 format keys (compressed) - "x-amz-c", - "x-amz-3", - "x-amz-m", - "x-amz-t", - "x-amz-w", - "x-amz-d", - "x-amz-i", -} +from .metadata import VALID_S3EC_METADATA_KEYS def parse_instruction_file(instruction_data: bytes, instruction_key: str) -> dict[str, Any]: @@ -85,7 +65,7 @@ def fetch_instruction_file( 1. Fetches the instruction file in plaintext mode 2. Returns the parsed metadata from the response Metadata field - The event handler (on_get_object_after_call) handles: + S3EncryptionClientPlugin's event handler (on_get_object_after_call) handles: - Verifying the x-amz-crypto-instr-file marker is present - Parsing and validating the instruction file content - Placing parsed metadata in response["Metadata"] @@ -110,6 +90,11 @@ def fetch_instruction_file( if hasattr(s3_client, "_s3ec_plugin_context"): s3_client._s3ec_plugin_context.plaintext_mode = True s3_client._s3ec_plugin_context.key = instruction_key + else: + raise S3EncryptionClientError( + f"Could not fetch instruction file without " + f"the S3 Encryption Client Plugin installed. Instruction key: {instruction_key}" + ) try: response = s3_client.get_object(Bucket=bucket, Key=instruction_key) @@ -117,8 +102,6 @@ def fetch_instruction_file( # Clear the flags after the call if hasattr(s3_client, "_s3ec_plugin_context"): s3_client._s3ec_plugin_context.plaintext_mode = False - if hasattr(s3_client._s3ec_plugin_context, "key"): - delattr(s3_client._s3ec_plugin_context, "key") # In plaintext mode, the event handler places parsed metadata in Metadata field metadata = response.get("Metadata", {}) diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index ad668a64..034626e7 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -252,3 +252,25 @@ def should_use_instruction_file(self) -> bool: or self.encrypted_data_key_v3 is not None ) return not has_any_key + + +# Valid S3 Encryption Client metadata keys +VALID_S3EC_METADATA_KEYS = { + # V1/V2 format keys + "x-amz-key", + "x-amz-key-v2", + "x-amz-wrap-alg", + "x-amz-matdesc", + "x-amz-iv", + "x-amz-cek-alg", + "x-amz-tag-len", + "x-amz-crypto-instr-file", + # V3 format keys (compressed) + "x-amz-c", + "x-amz-3", + "x-amz-m", + "x-amz-t", + "x-amz-w", + "x-amz-d", + "x-amz-i", +} diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 24d6716a..8e7dcea2 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -123,7 +123,7 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") - instruction_metadata = self._fetch_instruction_file(bucket, key) + instruction_metadata = fetch_instruction_file(self.s3_client, bucket, key) instruction_metadata.update(encryption_metadata) metadata = ObjectMetadata.from_dict(instruction_metadata) @@ -150,19 +150,6 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): aesgcm = AESGCM(dec_materials.plaintext_data_key) return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) - def _fetch_instruction_file(self, bucket: str, key: str, suffix: str = ".instruction") -> dict: - """Fetch instruction file from S3. - - Args: - bucket: S3 bucket name - key: S3 object key - suffix: Instruction file suffix (default: .instruction) - - Returns: - dict: Parsed JSON metadata from instruction file - """ - return fetch_instruction_file(self.s3_client, bucket, key, suffix) - def _decrypt_v2(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V2 decryption materials.""" iv_bytes = base64.b64decode(metadata.content_iv) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index b76fe429..9b1473e7 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -26,6 +26,7 @@ } +# TODO(cbc): enable once CBC decryption is implemented @pytest.mark.skip(reason="V1 CBC decryption not yet implemented") def test_decrypt_v1_instruction_file(): """Test decrypting V1 object with instruction file. @@ -69,6 +70,7 @@ def test_decrypt_v2_instruction_file(): print("Success! V2 instruction file decryption completed.") +# TODO(v3): enable once v3 is implemented @pytest.mark.skip(reason="V3 decryption not yet implemented") def test_decrypt_v3_instruction_file(): """Test decrypting V3 object with instruction file. diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 9d1e5256..7e7598b3 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -3,15 +3,11 @@ import base64 import json import os -import sys from io import BytesIO from unittest.mock import Mock import pytest -# Add the src directory to the Python path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../src"))) - from s3_encryption.materials.crypto_materials_manager import DefaultCryptoMaterialsManager from s3_encryption.materials.keyring import S3Keyring from s3_encryption.pipelines import GetEncryptedObjectPipeline @@ -20,8 +16,7 @@ class TestGetEncryptedObjectPipelineInstructionFile: def test_decrypt_v1_from_instruction_file(self): """Test decrypting V1 format with instruction file.""" - # V1: Object metadata is empty, all metadata in instruction file - object_metadata = {} + object_metadata = {"x-amz-meta-x-amz-unencrypted-content-length": "39"} # Instruction file contains all V1 metadata instruction_file_metadata = { diff --git a/test/test_on_get_object_after_call.py b/test/test_s3_encryption_client_plugin.py similarity index 97% rename from test/test_on_get_object_after_call.py rename to test/test_s3_encryption_client_plugin.py index d03a4eb2..fa1a0cae 100644 --- a/test/test_on_get_object_after_call.py +++ b/test/test_s3_encryption_client_plugin.py @@ -1,6 +1,6 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for S3EncryptionClientPlugin event handler.""" +"""Unit tests for S3EncryptionClientPlugin event handlers.""" import io import json @@ -14,8 +14,8 @@ from s3_encryption.materials.keyring import S3Keyring -class TestEventHandlerInstructionFile: - """Test event handler behavior for instruction files in plaintext mode.""" +class TestS3EncryptionClientPlugin: + """S3EncryptionClientPlugin event handler behavior.""" def test_plaintext_mode_parses_instruction_file(self): """Test that plaintext mode parses instruction file and returns metadata.""" From f613080e798d1c3d8bdf2051eb98a3329ab04039 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Fri, 27 Feb 2026 14:52:16 -0600 Subject: [PATCH 23/35] docs(duvet): add specification citations for instruction file handling - Add 7 Duvet citations across instruction_file.py, pipelines.py, and metadata.py - instruction_file.py: JSON serialization, default suffix, custom suffix support - pipelines.py: instruction file fallback, V1/V2 metadata storage - metadata.py: V3 instruction file split between object metadata and instruction file --- pyproject.toml | 1 + src/s3_encryption/instruction_file.py | 18 ++++++++++++++++++ src/s3_encryption/metadata.py | 7 +++++++ src/s3_encryption/pipelines.py | 10 ++++++++++ 4 files changed, 36 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1876da9a..ef2b0121 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,3 +57,4 @@ known-first-party = ["s3_encryption"] [tool.ruff.lint.per-file-ignores] "test/**/*.py" = ["D100", "D101", "D102", "D103", "D104", "E501"] +"src/s3_encryption/pipelines.py" = ["E501"] diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index b4fe2eed..a95bd14e 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -31,6 +31,14 @@ def parse_instruction_file(instruction_data: bytes, instruction_key: str) -> dic S3EncryptionClientError: If the instruction file is not valid JSON or contains non-S3EC metadata keys """ + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The content metadata stored in the Instruction File MUST be serialized to a JSON string. + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The serialized JSON string MUST be the only contents of the Instruction File. + # Validate JSON format try: metadata = json.loads(instruction_data) @@ -61,6 +69,11 @@ def fetch_instruction_file( ) -> dict[str, Any]: """Fetch and parse an instruction file from S3. + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The S3EC SHOULD support providing a custom Instruction File suffix + ##% on GetObject requests, regardless of whether or not re-encryption is supported. + This function: 1. Fetches the instruction file in plaintext mode 2. Returns the parsed metadata from the response Metadata field @@ -85,6 +98,11 @@ def fetch_instruction_file( """ instruction_key = key + suffix + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". + # Set plaintext mode flag in thread-local context before calling get_object # This will be checked by the event handler to skip decryption if hasattr(s3_client, "_s3ec_plugin_context"): diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 034626e7..6d8c0c09 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -225,6 +225,13 @@ def has_exclusive_key_collision(self) -> bool: def is_v3_in_object_metadata(self) -> bool: """Check if V3 content keys are in object metadata (without encrypted data key). + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=citation + ##% In the V3 message format, only the content metadata related to + ##% the encrypted data is stored in the Instruction File. + ##% In the V3 message format, the content metadata related to + ##% the encrypted content is stored in the Object Metadata. + Returns: bool: True if V3 content keys present but no encrypted data key """ diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 8e7dcea2..ae76bf0e 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -118,11 +118,21 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): # Check if we need to fetch instruction file if metadata.should_use_instruction_file(): + ##= specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + ##= type=citation + ##% If the object matches none of the V1/V2/V3 formats, + ##% the S3EC MUST attempt to get the instruction file. + if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") + ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files + ##= type=citation + ##% In the V1/V2 message format, all of the content metadata + ##% MUST be stored in the Instruction File. + instruction_metadata = fetch_instruction_file(self.s3_client, bucket, key) instruction_metadata.update(encryption_metadata) metadata = ObjectMetadata.from_dict(instruction_metadata) From 4d228a2523832e15815bb7e24df6b121421bf0fc Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 15:46:04 -0800 Subject: [PATCH 24/35] first pass at KC GCM enc/dec --- src/s3_encryption/__init__.py | 11 +- src/s3_encryption/key_derivation.py | 114 +++++ src/s3_encryption/materials/__init__.py | 4 +- src/s3_encryption/materials/materials.py | 20 +- src/s3_encryption/metadata.py | 7 +- src/s3_encryption/pipelines.py | 178 ++++++- test/integration/test_i_s3_encryption.py | 479 +++++------------- .../test_i_s3_encryption_instruction_file.py | 13 +- .../test_i_s3_encryption_multithreaded.py | 19 +- test/test_pipelines.py | 7 +- 10 files changed, 454 insertions(+), 398 deletions(-) create mode 100644 src/s3_encryption/key_derivation.py diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 1e4dc8f5..9c89c23d 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -15,6 +15,7 @@ DefaultCryptoMaterialsManager, ) from .materials.keyring import AbstractKeyring +from .materials.materials import AlgorithmSuite, CommitmentPolicy from .pipelines import GetEncryptedObjectPipeline, PutEncryptedObjectPipeline S3_METADATA_PREFIX = "x-amz-meta-" @@ -26,6 +27,12 @@ class S3EncryptionClientConfig: """Configuration object for the S3 Encryption Client.""" keyring: AbstractKeyring + algorithm_suite: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ) + commitment_policy: CommitmentPolicy = field( + default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) cmm: AbstractCryptoMaterialsManager = field() @cmm.default @@ -80,7 +87,9 @@ def on_put_object_before_call(self, params, **kwargs): pipeline = PutEncryptedObjectPipeline(self.config.cmm) encrypted_data, encryption_metadata = pipeline.encrypt( - body_bytes, encryption_context=encryption_context + body_bytes, + encryption_context=encryption_context, + algorithm_suite=self.config.algorithm_suite, ) params["body"] = encrypted_data diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py new file mode 100644 index 00000000..0d3514c0 --- /dev/null +++ b/src/s3_encryption/key_derivation.py @@ -0,0 +1,114 @@ +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Key derivation for S3 Encryption Client key-committing algorithm suites. + +Implements HKDF-based key derivation as specified in: + specification/s3-encryption/key-derivation.md + +For ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + - Extract: HKDF-SHA512, salt = Message ID (28 bytes), IKM = plaintext data key + - Expand (DEK): info = suite_id_bytes + b"DERIVEKEY", output = 32 bytes + - Expand (Commit Key): info = suite_id_bytes + b"COMMITKEY", output = 28 bytes +""" + +import hmac + +from cryptography.hazmat.primitives.hashes import SHA512 +from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +# Algorithm suite ID for S3EC.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +SUITE_ID_BYTES = b"\x00\x73" + +# Output lengths +ENCRYPTION_KEY_LENGTH = 32 # 256 bits +COMMIT_KEY_LENGTH = 28 # 224 bits +MESSAGE_ID_LENGTH = 28 # 224 bits (used as HKDF salt) + +# Fixed IV for KC GCM: 12 bytes of 0x01 +KC_GCM_IV = b"\x01" * 12 + + +def _hkdf_extract(salt: bytes, ikm: bytes) -> bytes: + """HKDF extract step using HMAC-SHA512. + + Args: + salt: The salt value (Message ID). + ikm: Input keying material (plaintext data key). + + Returns: + The pseudorandom key (PRK). + """ + return hmac.new(salt, ikm, "sha512").digest() + + +def _hkdf_expand(prk: bytes, info: bytes, length: int) -> bytes: + """HKDF expand step using SHA-512. + + Args: + prk: Pseudorandom key from extract step. + info: Context/application-specific info string. + length: Desired output length in bytes. + + Returns: + Output keying material of the requested length. + """ + hkdf = HKDFExpand(algorithm=SHA512(), length=length, info=info) + return hkdf.derive(prk) + + +def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, bytes]: + """Derive the encryption key and commitment key from a plaintext data key. + + Uses HKDF with SHA-512 as specified in the S3EC key derivation spec. + + Args: + plaintext_data_key: The plaintext data key from the keyring (32 bytes). + message_id: The generated Message ID used as the HKDF salt (28 bytes). + + Returns: + A tuple of (derived_encryption_key, commit_key). + """ + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The hash function MUST be specified by the algorithm suite commitment settings. + ##% The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + ##% The salt MUST be the Message ID with the length defined in the algorithm suite. + prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The input info MUST be a concatenation of the algorithm suite ID as bytes + ##% followed by the string DERIVEKEY as UTF8 encoded bytes. + derived_encryption_key = _hkdf_expand( + prk, info=SUITE_ID_BYTES + b"DERIVEKEY", length=ENCRYPTION_KEY_LENGTH + ) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The input info MUST be a concatenation of the algorithm suite ID as bytes + ##% followed by the string COMMITKEY as UTF8 encoded bytes. + commit_key = _hkdf_expand( + prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH + ) + + return derived_encryption_key, commit_key + + +def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> None: + """Verify key commitment in constant time. + + Args: + stored_commitment: The commitment value from the object metadata. + derived_commitment: The commitment value derived from the data key. + + Raises: + S3EncryptionClientSecurityError: If the commitment values do not match. + """ + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##% the verification of the derived key commitment value MUST be done in constant time. + ##% the client MUST throw an exception when the derived key commitment value + ##% and stored key commitment value do not match. + from .exceptions import S3EncryptionClientSecurityError + + if not hmac.compare_digest(stored_commitment, derived_commitment): + raise S3EncryptionClientSecurityError( + "Key commitment verification failed: stored commitment does not match derived commitment." + ) diff --git a/src/s3_encryption/materials/__init__.py b/src/s3_encryption/materials/__init__.py index c67d5802..c5cc7d6d 100644 --- a/src/s3_encryption/materials/__init__.py +++ b/src/s3_encryption/materials/__init__.py @@ -10,7 +10,7 @@ from .encrypted_data_key import EncryptedDataKey from .keyring import AbstractKeyring from .kms_keyring import KmsKeyring -from .materials import EncryptionMaterials +from .materials import AlgorithmSuite, CommitmentPolicy, EncryptionMaterials __all__ = [ "AbstractKeyring", @@ -18,5 +18,7 @@ "AbstractCryptoMaterialsManager", "DefaultCryptoMaterialsManager", "EncryptedDataKey", + "AlgorithmSuite", + "CommitmentPolicy", "EncryptionMaterials", ] diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 3966e17c..0bca41a4 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -8,11 +8,25 @@ """ from typing import Any - +from enum import Enum from attrs import define, field from .encrypted_data_key import EncryptedDataKey +class AlgorithmSuite(Enum): + """Algorithm suites supported by the S3 Encryption Client.""" + + ALG_AES_256_GCM_IV12_TAG16_NO_KDF = "AES/GCM/NoPadding" + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = "AES/GCM/HKDF/CommitKey" + + +class CommitmentPolicy(Enum): + """Commitment policies controlling key-commitment behavior.""" + + FORBID_ENCRYPT_ALLOW_DECRYPT = "ForbidEncryptAllowDecrypt" + REQUIRE_ENCRYPT_ALLOW_DECRYPT = "RequireEncryptAllowDecrypt" + REQUIRE_ENCRYPT_REQUIRE_DECRYPT = "RequireEncryptRequireDecrypt" + @define class EncryptionMaterials: @@ -27,6 +41,9 @@ class EncryptionMaterials: plaintext_data_key (Optional[bytes]): The plaintext data key """ + algorithm_suite: AlgorithmSuite = field( + default=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ) encryption_context: dict[str, str] = field(factory=dict) encrypted_data_key: EncryptedDataKey | None = field(default=None) plaintext_data_key: bytes | None = field(default=None) @@ -87,6 +104,7 @@ class DecryptionMaterials: encryption_context_stored: dict[str, str] = field(factory=dict) encryption_context_from_request: dict[str, str] = field(factory=dict) plaintext_data_key: bytes | None = field(default=None) + algorithm_suite: AlgorithmSuite | None = field(default=None) @classmethod def from_dict(cls, materials_dict: dict[str, Any]) -> "DecryptionMaterials": diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index ad668a64..12715ea0 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -51,7 +51,7 @@ class ObjectMetadata: # V3 format fields (compressed) content_cipher_v3: str | None = field(default=None) encrypted_data_key_v3: str | None = field(default=None) - mat_desc_v3: str | None = field(default=None) + mat_desc_v3: str | dict | None = field(default=None) encryption_context_v3: str | None = field(default=None) encrypted_data_key_algorithm_v3: str | None = field(default=None) key_commitment_v3: str | None = field(default=None) @@ -150,7 +150,10 @@ def to_dict(self) -> dict[str, str]: result[self.ENCRYPTED_DATA_KEY_V3] = self.encrypted_data_key_v3 if self.mat_desc_v3 is not None: - result[self.MAT_DESC_V3] = self.mat_desc_v3 + if isinstance(self.mat_desc_v3, dict): + result[self.MAT_DESC_V3] = json.dumps(self.mat_desc_v3) + else: + result[self.MAT_DESC_V3] = self.mat_desc_v3 if self.encryption_context_v3 is not None: result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 24d6716a..1c344aa5 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -14,9 +14,16 @@ from .exceptions import S3EncryptionClientError from .instruction_file import fetch_instruction_file +from .key_derivation import ( + KC_GCM_IV, + MESSAGE_ID_LENGTH, + SUITE_ID_BYTES, + derive_keys, + verify_commitment, +) from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import DecryptionMaterials, EncryptionMaterials +from .materials.materials import AlgorithmSuite, DecryptionMaterials, EncryptionMaterials from .metadata import ObjectMetadata @@ -30,45 +37,56 @@ class PutEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() - def encrypt(self, plaintext, encryption_context=None): + def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): """Encrypt the data before it is stored in S3. Args: plaintext (bytes or str): The data to be encrypted encryption_context (dict, optional): Additional context for encryption + algorithm_suite (AlgorithmSuite, optional): Algorithm suite to use Returns: bytes: The encrypted data dict: Metadata about the encryption to be stored with the object """ + if algorithm_suite is None: + algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + # Create encryption materials request with encryption context copy enc_mats_request = EncryptionMaterials( - encryption_context={} if encryption_context is None else encryption_context.copy() + algorithm_suite=algorithm_suite, + encryption_context={} if encryption_context is None else encryption_context.copy(), ) # Get encryption materials from the crypto materials manager enc_mats = self.cmm.get_encryption_materials(enc_mats_request) - # Generate initialization vector - iv = os.urandom(12) - - # Encrypt the data if enc_mats.plaintext_data_key is None: raise RuntimeError("No plaintext data key found!") - - aesgcm = AESGCM(enc_mats.plaintext_data_key) - ciphertext = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) - encrypted_data = ciphertext - b64_iv = base64.b64encode(iv).decode("utf-8") - - # Get the encrypted data key if enc_mats.encrypted_data_key is None: raise RuntimeError("No encrypted data key found!") edk_bytes = enc_mats.encrypted_data_key.encrypted_data_key + + if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + return self._encrypt_kc_gcm(plaintext, enc_mats, edk_bytes) + else: + return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) + + def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): + """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" + ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, + ##% with the plaintext data key, the generated IV, and the tag length defined + ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + ##% The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + iv = os.urandom(12) + aesgcm = AESGCM(enc_mats.plaintext_data_key) + encrypted_data = aesgcm.encrypt(nonce=iv, data=plaintext, associated_data=None) + + b64_iv = base64.b64encode(iv).decode("utf-8") b64_edk = base64.b64encode(edk_bytes).decode("utf-8") - # Create metadata using the ObjectMetadata class metadata = ObjectMetadata( encrypted_data_key_v2=b64_edk, encrypted_data_key_algorithm="kms+context", @@ -77,10 +95,50 @@ def encrypt(self, plaintext, encryption_context=None): encrypted_data_key_context=enc_mats.encryption_context, ) - # Convert to dictionary for storage in S3 metadata - encryption_metadata = metadata.to_dict() + return encrypted_data, metadata.to_dict() + + def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): + """Encrypt using ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY (V3 format).""" + ##= specification/s3-encryption/encryption.md#content-encryption + ##% The client MUST generate an IV or Message ID using the length of the IV + ##% or Message ID defined in the algorithm suite. + message_id = os.urandom(MESSAGE_ID_LENGTH) + + ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_hkdf_sha512_commit_key + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key as described in Key Derivation. + derived_encryption_key, commit_key = derive_keys(enc_mats.plaintext_data_key, message_id) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% the IV used in the AES-GCM content encryption/decryption MUST consist + ##% entirely of bytes with the value 0x01. + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + aesgcm = AESGCM(derived_encryption_key) + encrypted_data = aesgcm.encrypt( + nonce=KC_GCM_IV, data=plaintext, associated_data=SUITE_ID_BYTES + ) + + b64_edk = base64.b64encode(edk_bytes).decode("utf-8") + b64_message_id = base64.b64encode(message_id).decode("utf-8") + b64_commit_key = base64.b64encode(commit_key).decode("utf-8") + + # V3 metadata format + # x-amz-c: content cipher identifier (compressed algorithm suite) + # x-amz-w: wrapping algorithm identifier + # x-amz-3: encrypted data key + # x-amz-i: message ID + # x-amz-d: key commitment + # x-amz-m: material description (encryption context as JSON) + metadata = ObjectMetadata( + content_cipher_v3="115", + encrypted_data_key_algorithm_v3="02", + encrypted_data_key_v3=b64_edk, + message_id_v3=b64_message_id, + key_commitment_v3=b64_commit_key, + mat_desc_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, + ) - return encrypted_data, encryption_metadata + return encrypted_data, metadata.to_dict() @define @@ -131,9 +189,12 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) elif metadata.is_v2_format(): + # TODO: this is not how this works dec_materials = self._decrypt_v2(metadata, encryption_context) + dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF elif metadata.is_v3_format(): dec_materials = self._decrypt_v3(metadata, encryption_context) + dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY else: raise S3EncryptionClientError( "Unable to determine S3 Encryption Client message format." @@ -147,8 +208,20 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. # Perform decryption - aesgcm = AESGCM(dec_materials.plaintext_data_key) - return aesgcm.decrypt(nonce=dec_materials.iv, data=encrypted_data, associated_data=None) + # TODO: include CBC here too + match dec_materials.algorithm_suite: + case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: + ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##% The client MUST NOT provide any AAD when encrypting with + ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + aesgcm = AESGCM(dec_materials.plaintext_data_key) + return aesgcm.decrypt( + nonce=dec_materials.iv, data=encrypted_data, associated_data=None + ) + case AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + return self._decrypt_kc_gcm_content(dec_materials, encrypted_data, metadata) + case _: + raise S3EncryptionClientError("Unknown algorithm suite!") def _fetch_instruction_file(self, bucket: str, key: str, suffix: str = ".instruction") -> dict: """Fetch instruction file from S3. @@ -203,7 +276,68 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) + # V3 compressed wrapping algorithm identifiers → canonical names + _V3_WRAP_ALG_MAP = { + "02": "kms+context", + } + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V3 decryption materials.""" - # TODO: Implement V3 decryption - raise NotImplementedError("V3 decryption not yet implemented") + edk_bytes = base64.b64decode(metadata.encrypted_data_key_v3) + + # Map V3 compressed wrapping algorithm to canonical key_provider_info + raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "02" + wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) + + encrypted_data_key = EncryptedDataKey( + key_provider_id=b"S3Keyring", + key_provider_info=wrap_alg, + encrypted_data_key=edk_bytes, + ) + + # V3 stores encryption context in mat_desc_v3 (as dict or JSON string) + stored_context = {} + if metadata.mat_desc_v3 is not None: + if isinstance(metadata.mat_desc_v3, dict): + stored_context = metadata.mat_desc_v3 + elif isinstance(metadata.mat_desc_v3, str): + import json + + stored_context = json.loads(metadata.mat_desc_v3) + + dec_materials = DecryptionMaterials( + encrypted_data_keys=[encrypted_data_key], + encryption_context_stored=stored_context, + encryption_context_from_request=encryption_context, + ) + + return self.cmm.decrypt_materials(dec_materials) + + def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): + """Decrypt content encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + + Performs HKDF key derivation, key commitment verification, and AES-GCM decryption. + """ + message_id = base64.b64decode(metadata.message_id_v3) + stored_commitment = base64.b64decode(metadata.key_commitment_v3) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% The client MUST use HKDF to derive the key commitment value and the derived + ##% encrypting key. + derived_encryption_key, derived_commitment = derive_keys( + dec_materials.plaintext_data_key, message_id + ) + + ##= specification/s3-encryption/decryption.md#decrypting-with-commitment + ##% the client MUST verify the key commitment values match before deriving + ##% the derived encryption key. + verify_commitment(stored_commitment, derived_commitment) + + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% the IV used in the AES-GCM content encryption/decryption MUST consist + ##% entirely of bytes with the value 0x01. + ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. + aesgcm = AESGCM(derived_encryption_key) + return aesgcm.decrypt( + nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES + ) diff --git a/test/integration/test_i_s3_encryption.py b/test/integration/test_i_s3_encryption.py index 616f8da4..d4d5bcaa 100644 --- a/test/integration/test_i_s3_encryption.py +++ b/test/integration/test_i_s3_encryption.py @@ -9,6 +9,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -16,457 +17,207 @@ "CI_KMS_KEY_ALIAS", "arn:aws:kms:us-west-2:370957321024:alias/S3EC-Python-Github-KMS-Key" ) - -def test_simple_roundtrip_ascii_string(): - key = "simple-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - data = "test input for simple v3 round trip" - +# Parameterized algorithm suite configurations. +# Each entry is (algorithm_suite, commitment_policy, id_label). +# "default" uses the client defaults (KC GCM + Require/Require). +ALGORITHM_CONFIGS = [ + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + id="AES_GCM", + ), + pytest.param( + AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + id="KC_GCM", + ), +] + + +def _make_client(algorithm_suite, commitment_policy): + """Create an S3EncryptionClient with the given algorithm config.""" kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(input) - print("Output:") - print(output) - raise RuntimeError - print("Success!") - + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=algorithm_suite, + commitment_policy=commitment_policy, + ) + return S3EncryptionClient(wrapped_client, config) -def test_empty_string_roundtrip(): - key = "empty-string-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - data = "" # Empty string as test data +def _unique_key(prefix): + """Generate a unique S3 key with a timestamp suffix.""" + return prefix + datetime.now().strftime("%Y-%m-%d-%H:%M:%S-%f") - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_simple_roundtrip_ascii_string(algorithm_suite, commitment_policy): + key = _unique_key("simple-rt-") + data = "test input for simple v3 round trip" - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) # Using repr to clearly show it's an empty string - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Empty string encrypted and decrypted correctly.") + assert output == data -def test_no_body_roundtrip(): - key = "no-body-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_empty_string_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("empty-string-rt-") + data = "" - # Expected data when no Body is provided (empty bytes) - expected_data = b"" - - kms_client = boto3.client("kms", region_name=region) + s3ec = _make_client(algorithm_suite, commitment_policy) + s3ec.put_object(Bucket=bucket, Key=key, Body=data) + response = s3ec.get_object(Bucket=bucket, Key=key) + output = response["Body"].read().decode("utf-8") + assert output == data - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_no_body_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("no-body-rt-") + expected_data = b"" - # Call put_object without providing a Body parameter + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read() + assert output == expected_data - if output != expected_data: - print("Uh oh! Output doesn't match expected empty bytes!") - print("Expected:") - print(repr(expected_data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print( - "Success! Object with no Body parameter encrypted and decrypted correctly as empty bytes." - ) - - -def test_unicode_string_roundtrip(): - key = "unicode-string-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with unusual Unicode characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_unicode_string_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("unicode-string-rt-") data = "Unicode test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!, ½⅓¼⅕⅙⅐⅛⅑⅒⅔⅖⅗⅘⅙⅚⅜⅝⅞" - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Boto3 encodes to utf-8 in put_object but does not - # decode in get_object; do so manually to complete the - # round trip + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Unicode string encrypted and decrypted correctly.") - + assert output == data -def test_specific_encoding_utf8_roundtrip(): - key = "utf8-encoding-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - # String with mixed characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_specific_encoding_utf8_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("utf8-encoding-rt-") data = "UTF-8 encoding test: 你好, こんにちは, 안녕하세요, Привет, مرحبا, ¡Hola!" - - # Explicitly encode as UTF-8 before sending encoded_data = data.encode("utf-8") - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the pre-encoded bytes to put_object + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes and decode with the same encoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! UTF-8 encoded string encrypted and decrypted correctly.") + assert output == data -def test_specific_encoding_latin1_roundtrip(): - key = "latin1-encoding-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - # String with Latin-1 compatible characters +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_specific_encoding_latin1_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("latin1-encoding-rt-") data = "Latin-1 encoding test: éèêë àâäãåá çñ ¿¡ øæå ØÆÅÉÈÊËÀÂÄÃÅÁ" - - # Explicitly encode as Latin-1 before sending encoded_data = data.encode("latin-1") - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the pre-encoded bytes to put_object + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=encoded_data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes and decode with the same encoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("latin-1") - - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Latin-1 encoded string encrypted and decrypted correctly.") + assert output == data -def test_binary_data_roundtrip(): - key = "binary-data-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - # Create some binary data (not valid in any particular encoding) +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_binary_data_roundtrip(algorithm_suite, commitment_policy): + key = _unique_key("binary-data-rt-") data = bytes(range(256)) - kms_client = boto3.client("kms", region_name=region) - - keyring = KmsKeyring(kms_client, kms_key_id) - - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Pass the binary data directly + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data) - - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Read raw bytes without decoding + response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read() + assert output == data - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - print("Success! Binary data encrypted and decrypted correctly.") - -def test_invalid_body_types(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_invalid_body_types(algorithm_suite, commitment_policy): """Test that put_object raises an exception when given invalid body types.""" - key = "invalid-body-type" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) + key = _unique_key("invalid-body-type-") - # Test with integer - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=42) - assert "Invalid type for parameter Body" in str(excinfo.value) + s3ec = _make_client(algorithm_suite, commitment_policy) - # Test with float - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=3.14) - assert "Invalid type for parameter Body" in str(excinfo.value) + for body in [42, 3.14, [1, 2, 3], {"key": "value"}, True, None]: + with pytest.raises(S3EncryptionClientError) as excinfo: + s3ec.put_object(Bucket=bucket, Key=key, Body=body) + assert "Invalid type for parameter Body" in str(excinfo.value) - # Test with list - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=[1, 2, 3]) - assert "Invalid type for parameter Body" in str(excinfo.value) - # Test with dictionary - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body={"key": "value"}) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with boolean - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=True) - assert "Invalid type for parameter Body" in str(excinfo.value) - - # Test with None (also raises an exception) - with pytest.raises(S3EncryptionClientError) as excinfo: - s3ec.put_object(Bucket=bucket, Key=key, Body=None) - assert "Invalid type for parameter Body" in str(excinfo.value) - - print("Success! All invalid body types correctly raised exceptions.") - - -def test_user_metadata_preservation(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_user_metadata_preservation(algorithm_suite, commitment_policy): """Test that user-provided metadata is preserved during encryption.""" - key = "metadata-preservation-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - + key = _unique_key("metadata-preservation-rt-") data = "Test data with user metadata" - - # User metadata to include user_metadata = { "author": "test-user", "version": "1.0", "description": "Test object with custom metadata", } - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with user metadata + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, Metadata=user_metadata) + response = s3ec.get_object(Bucket=bucket, Key=key) - # Get the object back - get_req = {"Bucket": bucket, "Key": key} - response = s3ec.get_object(**get_req) - - # Verify the data decrypts correctly output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError - - # Verify user metadata is preserved - returned_metadata = response.get("Metadata", {}) + assert output == data + returned_metadata = response.get("Metadata", {}) for key_name, expected_value in user_metadata.items(): - if key_name not in returned_metadata: - print(f"Uh oh! User metadata key '{key_name}' is missing!") - print("Expected metadata:") - print(user_metadata) - print("Returned metadata:") - print(returned_metadata) - raise RuntimeError - - if returned_metadata[key_name] != expected_value: - print(f"Uh oh! User metadata value for '{key_name}' doesn't match!") - print(f"Expected: {expected_value}") - print(f"Got: {returned_metadata[key_name]}") - raise RuntimeError - - print("Success! User metadata preserved correctly during encryption/decryption.") - print(f"User metadata: {user_metadata}") - print(f"Returned metadata keys: {list(returned_metadata.keys())}") - - -def test_encryption_context_roundtrip(): - """Test that EncryptionContext is properly used during encryption and required for decryption.""" - key = "encryption-context-rt" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + assert key_name in returned_metadata, f"User metadata key '{key_name}' is missing" + assert returned_metadata[key_name] == expected_value - data = "Test data with encryption context" - # Encryption context to use for additional authenticated data +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_roundtrip(algorithm_suite, commitment_policy): + """Test that EncryptionContext is properly used during encryption and required for decryption.""" + key = _unique_key("encryption-context-rt-") + data = "Test data with encryption context" encryption_context = { "department": "engineering", "project": "s3-encryption", "environment": "test", } - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) + response = s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=encryption_context) - # Get the object back WITH the same encryption context - get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": encryption_context} - response = s3ec.get_object(**get_req) - - # Verify the data decrypts correctly output = response["Body"].read().decode("utf-8") - if output != data: - print("Uh oh! Input and output don't match!") - print("Input:") - print(repr(data)) - print("Output:") - print(repr(output)) - raise RuntimeError + assert output == data - print("Success! Encryption context used correctly during encryption/decryption.") - print(f"Encryption context: {encryption_context}") - -def test_encryption_context_mismatch(): +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_mismatch(algorithm_suite, commitment_policy): """Test that decryption fails when EncryptionContext doesn't match.""" - key = "encryption-context-mismatch" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - + key = _unique_key("encryption-context-mismatch-") data = "Test data with encryption context" - - # Original encryption context encryption_context = {"department": "engineering", "project": "s3-encryption"} - - # Wrong encryption context for decryption wrong_encryption_context = {"department": "marketing", "project": "s3-encryption"} - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) - # Try to get the object back with WRONG encryption context - should fail - get_req = {"Bucket": bucket, "Key": key, "EncryptionContext": wrong_encryption_context} - - try: - s3ec.get_object(**get_req) - # If we get here, the test failed - decryption should have failed - print("Uh oh! Decryption succeeded with wrong encryption context!") - print(f"Original context: {encryption_context}") - print(f"Wrong context used: {wrong_encryption_context}") - raise RuntimeError("Expected decryption to fail with mismatched encryption context") - except S3EncryptionClientError as e: - # This is expected - decryption should fail - print("Success! Decryption correctly failed with mismatched encryption context.") - print(f"Error message: {str(e)}") - except Exception as e: - # Some other error occurred - print(f"Unexpected error type: {type(e).__name__}") - print(f"Error message: {str(e)}") - raise - - -def test_encryption_context_missing_on_decrypt(): - """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" - key = "encryption-context-missing" - key += datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key, EncryptionContext=wrong_encryption_context) - data = "Test data with encryption context" - # Encryption context used during encryption +@pytest.mark.parametrize("algorithm_suite,commitment_policy", ALGORITHM_CONFIGS) +def test_encryption_context_missing_on_decrypt(algorithm_suite, commitment_policy): + """Test that decryption fails when encryption context is not provided for an object encrypted with context.""" + key = _unique_key("encryption-context-missing-") + data = "Test data with encryption context" encryption_context = {"department": "engineering", "project": "s3-encryption"} - kms_client = boto3.client("kms", region_name=region) - keyring = KmsKeyring(kms_client, kms_key_id) - wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) - s3ec = S3EncryptionClient(wrapped_client, config) - - # Put object with encryption context + s3ec = _make_client(algorithm_suite, commitment_policy) s3ec.put_object(Bucket=bucket, Key=key, Body=data, EncryptionContext=encryption_context) - # Try to get the object back WITHOUT encryption context - should fail - get_req = {"Bucket": bucket, "Key": key} - - try: - s3ec.get_object(**get_req) - # If we get here, the test failed - decryption should have failed - print("Uh oh! Decryption succeeded without providing required encryption context!") - print(f"Original context: {encryption_context}") - raise RuntimeError("Expected decryption to fail when encryption context not provided") - except S3EncryptionClientError as e: - # This is expected - decryption should fail - print("Success! Decryption correctly failed when encryption context was not provided.") - print(f"Error message: {str(e)}") - except Exception as e: - # Some other error occurred - print(f"Unexpected error type: {type(e).__name__}") - print(f"Error message: {str(e)}") - raise + with pytest.raises(S3EncryptionClientError): + s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index f69e2e50..58d19a66 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -7,6 +7,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy # Static test objects bucket bucket = os.environ.get("CI_S3_STATIC_TEST_BUCKET", "s3ec-static-test-objects") @@ -35,7 +36,11 @@ def test_decrypt_v2_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -57,7 +62,11 @@ def test_decrypt_v3_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/integration/test_i_s3_encryption_multithreaded.py b/test/integration/test_i_s3_encryption_multithreaded.py index 419ca7ea..f04d7fe5 100644 --- a/test/integration/test_i_s3_encryption_multithreaded.py +++ b/test/integration/test_i_s3_encryption_multithreaded.py @@ -16,6 +16,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy bucket = os.environ.get("CI_S3_BUCKET", "s3ec-python-github-test-bucket") region = os.environ.get("CI_AWS_REGION", "us-west-2") @@ -38,7 +39,11 @@ def test_multithreaded_encryption_context_isolation(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) # Number of threads to test with @@ -150,7 +155,11 @@ def test_multithreaded_rapid_context_switching(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) num_iterations = 20 @@ -228,7 +237,11 @@ def test_multithreaded_mixed_with_and_without_context(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig( + keyring, + algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) errors = [] diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 7ac9d043..9ec8b4f5 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -172,8 +172,11 @@ def test_decrypt_v3_from_instruction_file(self): mock_keyring.on_decrypt.return_value = mock_dec_materials - # This should fail with NotImplementedError since V3 decryption isn't implemented yet - with pytest.raises(NotImplementedError, match="V3 decryption not yet implemented"): + # V3 decryption is now implemented; with fake commitment data, + # key commitment verification will fail. + from s3_encryption.exceptions import S3EncryptionClientSecurityError + + with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") # Verify instruction file was fetched From a687103b96bc3667e267868d437a79418db6c5ab Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 15:52:27 -0800 Subject: [PATCH 25/35] first pass test server --- .../amazon/encryption/s3/TestUtils.java | 2 +- test-server/python-v3-server/src/main.py | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 2b9cd062..c3c804aa 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -155,7 +155,7 @@ public class TestUtils { public static final Set IMPROVED_VERSIONS = Set.of( JAVA_V4, - // PYTHON_V3, + PYTHON_V3, GO_V4, NET_V4, CPP_V3, diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v3-server/src/main.py index 6c57c6bd..eb3855b7 100755 --- a/test-server/python-v3-server/src/main.py +++ b/test-server/python-v3-server/src/main.py @@ -7,6 +7,7 @@ from s3_encryption import S3EncryptionClient, S3EncryptionClientConfig from s3_encryption.exceptions import S3EncryptionClientError from s3_encryption.materials.kms_keyring import KmsKeyring +from s3_encryption.materials.materials import AlgorithmSuite, CommitmentPolicy import boto3 import uvicorn import json @@ -67,6 +68,19 @@ def create_s3_encryption_client_error( ) +# Maps from Smithy model enum strings to Python AlgorithmSuite/CommitmentPolicy enums +_ALGORITHM_SUITE_MAP = { + "ALG_AES_256_GCM_IV12_TAG16_NO_KDF": AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY": AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, +} + +_COMMITMENT_POLICY_MAP = { + "FORBID_ENCRYPT_ALLOW_DECRYPT": CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + "REQUIRE_ENCRYPT_ALLOW_DECRYPT": CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT, + "REQUIRE_ENCRYPT_REQUIRE_DECRYPT": CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, +} + + @app.put("/object/{bucket}/{key}") async def put_object(bucket: str, key: str, request: Request): """ @@ -185,7 +199,23 @@ async def client_endpoint(request: Request): enable_legacy_wrapping_algorithms=enable_legacy_wrapping_algorithms, ) wrapped_client = boto3.client("s3") - client_config = S3EncryptionClientConfig(keyring) + + # Build config kwargs, only including algorithm_suite and commitment_policy if provided + config_kwargs = {"keyring": keyring} + + encryption_algorithm = config_data.get("encryptionAlgorithm") + if encryption_algorithm is not None: + if encryption_algorithm not in _ALGORITHM_SUITE_MAP: + raise ValueError(f"Unknown encryption algorithm: {encryption_algorithm}") + config_kwargs["algorithm_suite"] = _ALGORITHM_SUITE_MAP[encryption_algorithm] + + commitment_policy = config_data.get("commitmentPolicy") + if commitment_policy is not None: + if commitment_policy not in _COMMITMENT_POLICY_MAP: + raise ValueError(f"Unknown commitment policy: {commitment_policy}") + config_kwargs["commitment_policy"] = _COMMITMENT_POLICY_MAP[commitment_policy] + + client_config = S3EncryptionClientConfig(**config_kwargs) # Create S3EncryptionClient client = S3EncryptionClient(wrapped_client, client_config) From be192ad9c42d818d3d5fe2addf6c01568d1dda5b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 16:17:04 -0800 Subject: [PATCH 26/35] fix duvet --- src/s3_encryption/key_derivation.py | 12 +++++++----- src/s3_encryption/pipelines.py | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index 0d3514c0..0b0e8b6b 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -70,9 +70,11 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by A tuple of (derived_encryption_key, commit_key). """ ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% The hash function MUST be specified by the algorithm suite commitment settings. - ##% The input keying material MUST be the plaintext data key (PDK) generated by the key provider. - ##% The salt MUST be the Message ID with the length defined in the algorithm suite. + ##% - The hash function MUST be specified by the algorithm suite commitment settings. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation + ##% - The salt MUST be the Message ID with the length defined in the algorithm suite. prk = _hkdf_extract(salt=message_id, ikm=plaintext_data_key) ##= specification/s3-encryption/key-derivation.md#hkdf-operation @@ -103,8 +105,8 @@ def verify_commitment(stored_commitment: bytes, derived_commitment: bytes) -> No S3EncryptionClientSecurityError: If the commitment values do not match. """ ##= specification/s3-encryption/decryption.md#decrypting-with-commitment - ##% the verification of the derived key commitment value MUST be done in constant time. - ##% the client MUST throw an exception when the derived key commitment value + ##% When using an algorithm suite which supports key commitment, the verification of the derived key commitment value MUST be done in constant time. + ##% When using an algorithm suite which supports key commitment, the client MUST throw an exception when the derived key commitment value ##% and stored key commitment value do not match. from .exceptions import S3EncryptionClientSecurityError diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 1c344aa5..0df464fd 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -75,7 +75,7 @@ def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" - ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##% The client MUST initialize the cipher, or call an AES-GCM encryption API, ##% with the plaintext data key, the generated IV, and the tag length defined ##% in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. @@ -104,14 +104,15 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): ##% or Message ID defined in the algorithm suite. message_id = os.urandom(MESSAGE_ID_LENGTH) - ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_hkdf_sha512_commit_key + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key ##% The client MUST use HKDF to derive the key commitment value and the derived - ##% encrypting key as described in Key Derivation. + ##% encrypting key as described in [Key Derivation](key-derivation.md). derived_encryption_key, commit_key = derive_keys(enc_mats.plaintext_data_key, message_id) ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% the IV used in the AES-GCM content encryption/decryption MUST consist - ##% entirely of bytes with the value 0x01. + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) encrypted_data = aesgcm.encrypt( @@ -211,7 +212,7 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): # TODO: include CBC here too match dec_materials.algorithm_suite: case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: - ##= specification/s3-encryption/encryption.md#alg_aes_256_gcm_iv12_tag16_no_kdf + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##% The client MUST NOT provide any AAD when encrypting with ##% ALG_AES_256_GCM_IV12_TAG16_NO_KDF. aesgcm = AESGCM(dec_materials.plaintext_data_key) @@ -321,21 +322,21 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): message_id = base64.b64decode(metadata.message_id_v3) stored_commitment = base64.b64decode(metadata.key_commitment_v3) - ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% The client MUST use HKDF to derive the key commitment value and the derived - ##% encrypting key. + ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-hkdf-sha512-commit-key + ##% The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). derived_encryption_key, derived_commitment = derive_keys( dec_materials.plaintext_data_key, message_id ) ##= specification/s3-encryption/decryption.md#decrypting-with-commitment - ##% the client MUST verify the key commitment values match before deriving - ##% the derived encryption key. + ##% When using an algorithm suite which supports key commitment, the client MUST verify the key commitment values match before deriving + ##% the [derived encryption key](./key-derivation.md#hkdf-operation). verify_commitment(stored_commitment, derived_commitment) ##= specification/s3-encryption/key-derivation.md#hkdf-operation - ##% the IV used in the AES-GCM content encryption/decryption MUST consist - ##% entirely of bytes with the value 0x01. + ##% When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + ##% the IV used in the AES-GCM content encryption/decryption MUST consist entirely of bytes with the value 0x01. + ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) return aesgcm.decrypt( From bae7ee73563b1bb6009c66d0cff1c145d05bd34f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 16:29:33 -0800 Subject: [PATCH 27/35] fix wrap alg --- src/s3_encryption/metadata.py | 7 +++-- src/s3_encryption/pipelines.py | 52 +++++++++++++++++++++++++--------- test/test_pipelines.py | 33 +++++++++++++++++++-- 3 files changed, 74 insertions(+), 18 deletions(-) diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 12715ea0..a4cf7b24 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -52,7 +52,7 @@ class ObjectMetadata: content_cipher_v3: str | None = field(default=None) encrypted_data_key_v3: str | None = field(default=None) mat_desc_v3: str | dict | None = field(default=None) - encryption_context_v3: str | None = field(default=None) + encryption_context_v3: str | dict | None = field(default=None) encrypted_data_key_algorithm_v3: str | None = field(default=None) key_commitment_v3: str | None = field(default=None) message_id_v3: str | None = field(default=None) @@ -156,7 +156,10 @@ def to_dict(self) -> dict[str, str]: result[self.MAT_DESC_V3] = self.mat_desc_v3 if self.encryption_context_v3 is not None: - result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 + if isinstance(self.encryption_context_v3, dict): + result[self.ENCRYPTION_CONTEXT_V3] = json.dumps(self.encryption_context_v3) + else: + result[self.ENCRYPTION_CONTEXT_V3] = self.encryption_context_v3 if self.encrypted_data_key_algorithm_v3 is not None: result[self.ENCRYPTED_DATA_KEY_ALGORITHM_V3] = self.encrypted_data_key_algorithm_v3 diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 0df464fd..8ad589f5 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -125,18 +125,18 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): # V3 metadata format # x-amz-c: content cipher identifier (compressed algorithm suite) - # x-amz-w: wrapping algorithm identifier + # x-amz-w: wrapping algorithm identifier (12 = kms+context) # x-amz-3: encrypted data key # x-amz-i: message ID # x-amz-d: key commitment - # x-amz-m: material description (encryption context as JSON) + # x-amz-t: encryption context (for kms+context wrapping) metadata = ObjectMetadata( content_cipher_v3="115", - encrypted_data_key_algorithm_v3="02", + encrypted_data_key_algorithm_v3="12", encrypted_data_key_v3=b64_edk, message_id_v3=b64_message_id, key_commitment_v3=b64_commit_key, - mat_desc_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, + encryption_context_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, ) return encrypted_data, metadata.to_dict() @@ -277,34 +277,60 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) - # V3 compressed wrapping algorithm identifiers → canonical names + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The V3 format uses compression here such that each wrapping algorithm + ##% is represented by a two digit string. + ##% The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + ##% The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + ##% The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. _V3_WRAP_ALG_MAP = { - "02": "kms+context", + "02": "AES/GCM", + "12": "kms+context", + "22": "RSA-OAEP-SHA1", } + # Wrapping algorithms that this client currently supports for decryption + _SUPPORTED_WRAP_ALGS = {"kms+context", "kms"} + def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: """Prepare V3 decryption materials.""" edk_bytes = base64.b64decode(metadata.encrypted_data_key_v3) # Map V3 compressed wrapping algorithm to canonical key_provider_info - raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "02" + raw_wrap_alg = metadata.encrypted_data_key_algorithm_v3 or "12" wrap_alg = self._V3_WRAP_ALG_MAP.get(raw_wrap_alg, raw_wrap_alg) + if wrap_alg not in self._SUPPORTED_WRAP_ALGS: + raise S3EncryptionClientError( + f"Unsupported wrapping algorithm: {wrap_alg}. " + f"The S3 Encryption Client for Python currently supports: " + f"{', '.join(sorted(self._SUPPORTED_WRAP_ALGS))}." + ) + encrypted_data_key = EncryptedDataKey( key_provider_id=b"S3Keyring", key_provider_info=wrap_alg, encrypted_data_key=edk_bytes, ) - # V3 stores encryption context in mat_desc_v3 (as dict or JSON string) + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. + ##% The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). + # For kms+context, the stored context comes from x-amz-t (encryption_context_v3). + # For AES/GCM and RSA-OAEP-SHA1, it comes from x-amz-m (mat_desc_v3). stored_context = {} - if metadata.mat_desc_v3 is not None: - if isinstance(metadata.mat_desc_v3, dict): - stored_context = metadata.mat_desc_v3 - elif isinstance(metadata.mat_desc_v3, str): + if wrap_alg == "kms+context": + raw_ctx = metadata.encryption_context_v3 + else: + raw_ctx = metadata.mat_desc_v3 + + if raw_ctx is not None: + if isinstance(raw_ctx, dict): + stored_context = raw_ctx + elif isinstance(raw_ctx, str): import json - stored_context = json.loads(metadata.mat_desc_v3) + stored_context = json.loads(raw_ctx) dec_materials = DecryptionMaterials( encrypted_data_keys=[encrypted_data_key], diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 9ec8b4f5..98cba95e 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -118,7 +118,7 @@ def test_decrypt_v2_from_instruction_file(self): ) def test_decrypt_v3_from_instruction_file(self): - """Test decrypting V3 format with instruction file.""" + """Test decrypting V3 format with instruction file (kms+context wrapping).""" # Object metadata contains V3 content keys only object_metadata = { "x-amz-c": "115", # Compressed algorithm suite @@ -127,10 +127,11 @@ def test_decrypt_v3_from_instruction_file(self): } # Instruction file contains encrypted data key and wrapping algorithm + # Uses "12" (kms+context) with "x-amz-t" for encryption context instruction_file_metadata = { "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), - "x-amz-w": "02", # AES/GCM - "x-amz-m": json.dumps({"test-instruction": "material-desc-instruction"}), + "x-amz-w": "12", # kms+context + "x-amz-t": json.dumps({"aws:x-amz-cek-alg": "AES/GCM/NoPadding"}), } # Create mock S3 client @@ -183,3 +184,29 @@ def test_decrypt_v3_from_instruction_file(self): mock_s3_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test-key.instruction" ) + + def test_decrypt_v3_unsupported_wrap_alg(self): + """Test that V3 decryption with unsupported wrapping algorithm gives a useful error.""" + from s3_encryption.exceptions import S3EncryptionClientError + + # V3 metadata with AES/GCM wrapping (02) — not supported by this client + metadata = { + "x-amz-c": "115", + "x-amz-3": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-w": "02", # AES/GCM — unsupported + "x-amz-m": json.dumps({"some": "material-desc"}), + "x-amz-d": base64.b64encode(b"key-commitment-data").decode("utf-8"), + "x-amz-i": base64.b64encode(b"test-message-id").decode("utf-8"), + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline(cmm) + + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": metadata, + } + + with pytest.raises(S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM"): + pipeline.decrypt(mock_response) From 8d36d1eacc606d4fcd1660c562c7f2673eba3f1b Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 16:39:51 -0800 Subject: [PATCH 28/35] fix duvet --- src/s3_encryption/pipelines.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index c6c1e313..b6d88e21 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -275,11 +275,13 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) ##= specification/s3-encryption/data-format/content-metadata.md#v3-only - ##% The V3 format uses compression here such that each wrapping algorithm - ##% is represented by a two digit string. - ##% The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. - ##% The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. - ##% The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. + ##% The V3 format uses compression here such that each wrapping algorithm is represented by a two digit string. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only + ##% - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. _V3_WRAP_ALG_MAP = { "02": "AES/GCM", "12": "kms+context", @@ -312,6 +314,7 @@ def _decrypt_v3(self, metadata, encryption_context) -> DecryptionMaterials: ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). # For kms+context, the stored context comes from x-amz-t (encryption_context_v3). # For AES/GCM and RSA-OAEP-SHA1, it comes from x-amz-m (mat_desc_v3). From b67e1bbf272804b581777a5e18f44dbb2e4b5c7f Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 17:01:55 -0800 Subject: [PATCH 29/35] fixup --- src/s3_encryption/materials/kms_keyring.py | 10 +++++++++- src/s3_encryption/metadata.py | 2 +- test/test_metadata.py | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index f8bc4997..8caa1054 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -10,6 +10,7 @@ from botocore import client from ..exceptions import S3EncryptionClientError +from ..materials.materials import AlgorithmSuite from .encrypted_data_key import EncryptedDataKey from .keyring import S3Keyring @@ -47,7 +48,14 @@ def on_encrypt(self, enc_materials): enc_materials = super().on_encrypt(enc_materials) encryption_context = enc_materials.encryption_context - encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" + + # For committing algorithm suites (V3), the encryption context algorithm + # value is the algorithm suite ID as a string ("115"), not the cipher name. + # For non-committing suites (V2), use the cipher name ("AES/GCM/NoPadding"). + if enc_materials.algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + encryption_context["aws:x-amz-cek-alg"] = "115" + else: + encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" response = self.kms_client.generate_data_key( KeyId=self.kms_key_id, KeySpec="AES_256", EncryptionContext=encryption_context diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 336f6978..5fc8a6bb 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -137,7 +137,7 @@ def to_dict(self) -> dict[str, str]: if self.content_cipher is not None: result[self.CONTENT_CIPHER] = self.content_cipher - if self.content_cipher_tag_length is not None: + if self.content_cipher_tag_length is not None and not self.is_v3_format(): result[self.CONTENT_CIPHER_TAG_LENGTH] = self.content_cipher_tag_length if self.instruction_file is not None: diff --git a/test/test_metadata.py b/test/test_metadata.py index 7dab2081..fb7dbca1 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -56,7 +56,7 @@ def test_to_dict(self): # Verify that fields that are None are not included in the dictionary assert "x-amz-key" not in metadata_dict assert "x-amz-matdesc" not in metadata_dict - # Note: content_cipher_tag_length has a default value of "128" + # content_cipher_tag_length defaults to "128" for V1/V2 assert metadata_dict.get("x-amz-tag-len") == "128" assert "x-amz-crypto-instr-file" not in metadata_dict @@ -124,6 +124,9 @@ def test_to_dict_v3_fields(self): assert metadata_dict["x-amz-m"] == "mat-desc" assert metadata_dict["x-amz-t"] == "encryption-context" + # V3 metadata must NOT include V1/V2-only keys like x-amz-tag-len + assert "x-amz-tag-len" not in metadata_dict + def test_is_v1_format(self): metadata = ObjectMetadata( content_iv="iv", From 62a44f23c2acc5f3f93e2e432bce4076ea1c76e8 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 19:02:39 -0800 Subject: [PATCH 30/35] implement CBC --- src/s3_encryption/__init__.py | 19 +++ src/s3_encryption/materials/materials.py | 6 + src/s3_encryption/pipelines.py | 143 +++++++++++++++++- .../test_i_s3_encryption_instruction_file.py | 12 +- 4 files changed, 164 insertions(+), 16 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index f6987012..609ca4be 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -32,12 +32,29 @@ class S3EncryptionClientConfig: commitment_policy: CommitmentPolicy = field( default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT ) + ##= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + ##% The S3EC MUST support the option to enable or disable legacy unauthenticated modes (content encryption algorithms). + ##= specification/s3-encryption/client.md#enable-legacy-unauthenticated-modes + ##% The option to enable legacy unauthenticated modes MUST be set to false by default. + enable_legacy_unauthenticated_modes: bool = field(default=False) cmm: AbstractCryptoMaterialsManager = field() @cmm.default def _default_cmm_for_keyring(self): return DefaultCryptoMaterialsManager(self.keyring) + ##= specification/s3-encryption/client.md#encryption-algorithm + ##% The S3EC MUST validate that the configured encryption algorithm is not legacy. + ##= specification/s3-encryption/client.md#encryption-algorithm + ##% If the configured encryption algorithm is legacy, then the S3EC MUST throw an exception. + def __attrs_post_init__(self): + if self.algorithm_suite.is_legacy: + raise S3EncryptionClientError( + f"Cannot configure S3 Encryption Client with legacy algorithm suite " + f"{self.algorithm_suite.name}. Legacy algorithm suites are only " + f"supported for decryption (and enable_legacy_unauthenticated_modes is True)." + ) + class S3EncryptionClientPlugin: """Plugin that adds encryption/decryption capabilities to a boto3 S3 client. @@ -137,6 +154,8 @@ def on_get_object_after_call(self, parsed, **kwargs): pipeline = GetEncryptedObjectPipeline( self.config.cmm, s3_client=getattr(self._context, "s3_client", None), + commitment_policy=self.config.commitment_policy, + enable_legacy_unauthenticated_modes=self.config.enable_legacy_unauthenticated_modes, ) decrypted_data = pipeline.decrypt( response, diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index 0bca41a4..ecfcdb13 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -16,9 +16,15 @@ class AlgorithmSuite(Enum): """Algorithm suites supported by the S3 Encryption Client.""" + ALG_AES_256_CBC_IV16_NO_KDF = "AES/CBC/PKCS5Padding" ALG_AES_256_GCM_IV12_TAG16_NO_KDF = "AES/GCM/NoPadding" ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY = "AES/GCM/HKDF/CommitKey" + @property + def is_legacy(self) -> bool: + """Return True if this algorithm suite is a legacy unauthenticated mode.""" + return self == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + class CommitmentPolicy(Enum): """Commitment policies controlling key-commitment behavior.""" diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index b6d88e21..42f110bb 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -12,7 +12,7 @@ from attrs import define, field from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from .exceptions import S3EncryptionClientError +from .exceptions import S3EncryptionClientError, S3EncryptionClientSecurityError from .instruction_file import fetch_instruction_file from .key_derivation import ( KC_GCM_IV, @@ -23,7 +23,7 @@ ) from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import AlgorithmSuite, DecryptionMaterials, EncryptionMaterials +from .materials.materials import AlgorithmSuite, CommitmentPolicy, DecryptionMaterials, EncryptionMaterials from .metadata import ObjectMetadata @@ -152,6 +152,59 @@ class GetEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() s3_client: object = field(default=None) + commitment_policy: CommitmentPolicy = field( + default=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + enable_legacy_unauthenticated_modes: bool = field(default=False) + + # Map content cipher metadata values to AlgorithmSuite + _CONTENT_CIPHER_TO_ALGORITHM_SUITE = { + "AES/CBC/PKCS5Padding": AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF, + "AES/GCM/NoPadding": AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + "115": AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + } + + def _determine_algorithm_suite(self, metadata) -> AlgorithmSuite: + """Determine the algorithm suite from object metadata. + + V1 objects are always CBC. + V2/V3 objects check x-amz-cek-alg / x-amz-c to determine the content algorithm. + """ + if metadata.is_v1_format(): + ##= specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility + ##= type=citation + ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version. + return AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF + + if metadata.is_v2_format(): + cek_alg = metadata.content_cipher + if cek_alg is None: + raise S3EncryptionClientError( + "V2 format object missing required x-amz-cek-alg metadata." + ) + suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) + if suite is None: + raise S3EncryptionClientError( + f"Unknown content encryption algorithm: {cek_alg}" + ) + return suite + + if metadata.is_v3_format(): + cek_alg = metadata.content_cipher_v3 + if cek_alg is None: + raise S3EncryptionClientError( + "V3 format object missing required x-amz-c metadata." + ) + suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) + if suite is None: + raise S3EncryptionClientError( + f"Unknown content encryption algorithm: {cek_alg}" + ) + return suite + + raise S3EncryptionClientError( + "Unable to determine S3 Encryption Client message format." + ) def decrypt(self, response, encryption_context=None, bucket=None, key=None): """Decrypt the data after it is retrieved from S3. @@ -196,31 +249,67 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): instruction_metadata.update(encryption_metadata) metadata = ObjectMetadata.from_dict(instruction_metadata) + # Determine the algorithm suite from the metadata + algorithm_suite = self._determine_algorithm_suite(metadata) + # Determine which format we're dealing with and get decryption materials if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) elif metadata.is_v2_format(): - # TODO: this is not how this works dec_materials = self._decrypt_v2(metadata, encryption_context) - dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF elif metadata.is_v3_format(): dec_materials = self._decrypt_v3(metadata, encryption_context) - dec_materials.algorithm_suite = AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY else: raise S3EncryptionClientError( "Unable to determine S3 Encryption Client message format." ) + dec_materials.algorithm_suite = algorithm_suite + ##= specification/s3-encryption/decryption.md#cbc-decryption - ##= type=TODO ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is NOT enabled, ##% the S3EC MUST throw an error which details that client was ##% not configured to decrypt objects with ALG_AES_256_CBC_IV16_NO_KDF. + if algorithm_suite == AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##% The S3EC MUST NOT decrypt objects encrypted using legacy unauthenticated algorithm suites + ##% unless specifically configured to do so. + ##= specification/s3-encryption/decryption.md#legacy-decryption + ##% If the S3EC is not configured to enable legacy unauthenticated content decryption, + ##% the client MUST throw an exception when attempting to decrypt an object encrypted + ##% with a legacy unauthenticated algorithm suite. + if not self.enable_legacy_unauthenticated_modes: + raise S3EncryptionClientError( + "Cannot decrypt object encrypted with ALG_AES_256_CBC_IV16_NO_KDF. " + "The S3 Encryption Client is not configured to decrypt objects using " + "legacy unauthenticated algorithm suites. " + "Set enable_legacy_unauthenticated_modes=True to allow decryption " + "of objects encrypted with CBC." + ) - # Perform decryption - # TODO: include CBC here too + ##= specification/s3-encryption/decryption.md#key-commitment + ##% The S3EC MUST validate the algorithm suite used for decryption against the + ##% key commitment policy before attempting to decrypt the content ciphertext. + ##= specification/s3-encryption/decryption.md#key-commitment + ##% If the commitment policy requires decryption using a committing algorithm suite, + ##% and the algorithm suite associated with the object does not support key commitment, + ##% then the S3EC MUST throw an exception. + if ( + self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + and dec_materials.algorithm_suite != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ): + raise S3EncryptionClientError( + "Configuration conflict: cannot decrypt non-key-committing object " + "when commitment policy is REQUIRE_ENCRYPT_REQUIRE_DECRYPT. " + "Use REQUIRE_ENCRYPT_ALLOW_DECRYPT or FORBID_ENCRYPT_ALLOW_DECRYPT " + "to allow decryption of non-committing objects." + ) + + # Perform decryption based on algorithm suite match dec_materials.algorithm_suite: + case AlgorithmSuite.ALG_AES_256_CBC_IV16_NO_KDF: + return self._decrypt_cbc_content(dec_materials, encrypted_data) case AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF: ##= specification/s3-encryption/encryption.md#alg-aes-256-gcm-iv12-tag16-no-kdf ##% The client MUST NOT provide any AAD when encrypting with @@ -274,6 +363,44 @@ def _decrypt_v1(self, metadata, encryption_context) -> DecryptionMaterials: return self.cmm.decrypt_materials(dec_materials) + def _decrypt_cbc_content(self, dec_materials, encrypted_data): + """Decrypt content encrypted with ALG_AES_256_CBC_IV16_NO_KDF. + + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##% If an object is encrypted with ALG_AES_256_CBC_IV16_NO_KDF and + ##% [legacy unauthenticated algorithm suites](#legacy-decryption) is enabled, + ##% then the S3EC MUST create a cipher with AES in CBC Mode with PKCS5Padding or + ##% PKCS7Padding compatible padding for a 16-byte block cipher + ##% (example: for the Java JCE, this is "AES/CBC/PKCS5Padding"). + """ + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##% If the cipher object cannot be created as described above, + ##% Decryption MUST fail. + ##= specification/s3-encryption/decryption.md#cbc-decryption + ##% The error SHOULD detail why the cipher could not be initialized + ##% (such as CBC or PKCS5Padding is not supported by the underlying crypto provider). + try: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.primitives.padding import PKCS7 + + cipher = Cipher( + algorithms.AES(dec_materials.plaintext_data_key), + modes.CBC(dec_materials.iv), + ) + decryptor = cipher.decryptor() + padded_plaintext = decryptor.update(encrypted_data) + decryptor.finalize() + + # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) + unpadder = PKCS7(128).unpadder() + plaintext = unpadder.update(padded_plaintext) + unpadder.finalize() + + return plaintext + except Exception as e: + raise S3EncryptionClientSecurityError( + f"Failed to decrypt CBC content: {e}. " + "Ensure the underlying crypto provider supports AES/CBC/PKCS7Padding." + ) from e + ##= specification/s3-encryption/data-format/content-metadata.md#v3-only ##% The V3 format uses compression here such that each wrapping algorithm is represented by a two digit string. ##= specification/s3-encryption/data-format/content-metadata.md#v3-only diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index 590732f9..be0a06d5 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -27,8 +27,6 @@ } -# TODO(cbc): enable once CBC decryption is implemented -@pytest.mark.skip(reason="V1 CBC decryption not yet implemented") def test_decrypt_v1_instruction_file(): """Test decrypting V1 object with instruction file. @@ -40,7 +38,8 @@ def test_decrypt_v1_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring) + config = S3EncryptionClientConfig(keyring, enable_legacy_unauthenticated_modes=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) @@ -75,8 +74,6 @@ def test_decrypt_v2_instruction_file(): print("Success! V2 instruction file decryption completed.") -# TODO(v3): enable once v3 is implemented -@pytest.mark.skip(reason="V3 decryption not yet implemented") def test_decrypt_v3_instruction_file(): """Test decrypting V3 object with instruction file. @@ -90,15 +87,14 @@ def test_decrypt_v3_instruction_file(): wrapped_client = boto3.client("s3") config = S3EncryptionClientConfig( keyring, - algorithm_suite=AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF, - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT, ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) output = response["Body"].read().decode("utf-8") - assert output != "static-v3-instruction-file-from-java-v4" + assert output == "static-v3-instruction-file-from-java-v4" print("Success! V3 instruction file decryption completed.") From 68f31dac61fb024bede42e1b7193be9faad056c5 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Fri, 27 Feb 2026 19:04:57 -0800 Subject: [PATCH 31/35] format --- src/s3_encryption/key_derivation.py | 5 +-- src/s3_encryption/materials/kms_keyring.py | 5 ++- src/s3_encryption/materials/materials.py | 4 +- src/s3_encryption/pipelines.py | 41 ++++++++----------- .../test_i_s3_encryption_instruction_file.py | 7 +++- test/test_pipelines.py | 8 +++- 6 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/s3_encryption/key_derivation.py b/src/s3_encryption/key_derivation.py index 0b0e8b6b..58335a4b 100644 --- a/src/s3_encryption/key_derivation.py +++ b/src/s3_encryption/key_derivation.py @@ -15,7 +15,6 @@ from cryptography.hazmat.primitives.hashes import SHA512 from cryptography.hazmat.primitives.kdf.hkdf import HKDFExpand -from cryptography.hazmat.primitives.kdf.hkdf import HKDF # Algorithm suite ID for S3EC.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY SUITE_ID_BYTES = b"\x00\x73" @@ -87,9 +86,7 @@ def derive_keys(plaintext_data_key: bytes, message_id: bytes) -> tuple[bytes, by ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The input info MUST be a concatenation of the algorithm suite ID as bytes ##% followed by the string COMMITKEY as UTF8 encoded bytes. - commit_key = _hkdf_expand( - prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH - ) + commit_key = _hkdf_expand(prk, info=SUITE_ID_BYTES + b"COMMITKEY", length=COMMIT_KEY_LENGTH) return derived_encryption_key, commit_key diff --git a/src/s3_encryption/materials/kms_keyring.py b/src/s3_encryption/materials/kms_keyring.py index 8caa1054..bf2e91a6 100644 --- a/src/s3_encryption/materials/kms_keyring.py +++ b/src/s3_encryption/materials/kms_keyring.py @@ -52,7 +52,10 @@ def on_encrypt(self, enc_materials): # For committing algorithm suites (V3), the encryption context algorithm # value is the algorithm suite ID as a string ("115"), not the cipher name. # For non-committing suites (V2), use the cipher name ("AES/GCM/NoPadding"). - if enc_materials.algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: + if ( + enc_materials.algorithm_suite + == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ): encryption_context["aws:x-amz-cek-alg"] = "115" else: encryption_context["aws:x-amz-cek-alg"] = "AES/GCM/NoPadding" diff --git a/src/s3_encryption/materials/materials.py b/src/s3_encryption/materials/materials.py index ecfcdb13..39c6add6 100644 --- a/src/s3_encryption/materials/materials.py +++ b/src/s3_encryption/materials/materials.py @@ -7,12 +7,14 @@ and decryption operations. """ -from typing import Any from enum import Enum +from typing import Any + from attrs import define, field from .encrypted_data_key import EncryptedDataKey + class AlgorithmSuite(Enum): """Algorithm suites supported by the S3 Encryption Client.""" diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index 42f110bb..aef7e2de 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -23,7 +23,12 @@ ) from .materials.crypto_materials_manager import AbstractCryptoMaterialsManager from .materials.encrypted_data_key import EncryptedDataKey -from .materials.materials import AlgorithmSuite, CommitmentPolicy, DecryptionMaterials, EncryptionMaterials +from .materials.materials import ( + AlgorithmSuite, + CommitmentPolicy, + DecryptionMaterials, + EncryptionMaterials, +) from .metadata import ObjectMetadata @@ -70,8 +75,7 @@ def encrypt(self, plaintext, encryption_context=None, algorithm_suite=None): if algorithm_suite == AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY: return self._encrypt_kc_gcm(plaintext, enc_mats, edk_bytes) - else: - return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) + return self._encrypt_gcm(plaintext, enc_mats, edk_bytes) def _encrypt_gcm(self, plaintext, enc_mats, edk_bytes): """Encrypt using ALG_AES_256_GCM_IV12_TAG16_NO_KDF (V2 format).""" @@ -136,7 +140,9 @@ def _encrypt_kc_gcm(self, plaintext, enc_mats, edk_bytes): encrypted_data_key_v3=b64_edk, message_id_v3=b64_message_id, key_commitment_v3=b64_commit_key, - encryption_context_v3=enc_mats.encryption_context if enc_mats.encryption_context else None, + encryption_context_v3=( + enc_mats.encryption_context if enc_mats.encryption_context else None + ), ) return encrypted_data, metadata.to_dict() @@ -184,27 +190,19 @@ def _determine_algorithm_suite(self, metadata) -> AlgorithmSuite: ) suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) if suite is None: - raise S3EncryptionClientError( - f"Unknown content encryption algorithm: {cek_alg}" - ) + raise S3EncryptionClientError(f"Unknown content encryption algorithm: {cek_alg}") return suite if metadata.is_v3_format(): cek_alg = metadata.content_cipher_v3 if cek_alg is None: - raise S3EncryptionClientError( - "V3 format object missing required x-amz-c metadata." - ) + raise S3EncryptionClientError("V3 format object missing required x-amz-c metadata.") suite = self._CONTENT_CIPHER_TO_ALGORITHM_SUITE.get(cek_alg) if suite is None: - raise S3EncryptionClientError( - f"Unknown content encryption algorithm: {cek_alg}" - ) + raise S3EncryptionClientError(f"Unknown content encryption algorithm: {cek_alg}") return suite - raise S3EncryptionClientError( - "Unable to determine S3 Encryption Client message format." - ) + raise S3EncryptionClientError("Unable to determine S3 Encryption Client message format.") def decrypt(self, response, encryption_context=None, bucket=None, key=None): """Decrypt the data after it is retrieved from S3. @@ -297,7 +295,8 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): ##% then the S3EC MUST throw an exception. if ( self.commitment_policy == CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT - and dec_materials.algorithm_suite != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + and dec_materials.algorithm_suite + != AlgorithmSuite.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ): raise S3EncryptionClientError( "Configuration conflict: cannot decrypt non-key-committing object " @@ -392,9 +391,7 @@ def _decrypt_cbc_content(self, dec_materials, encrypted_data): # Remove PKCS7 padding (compatible with PKCS5Padding for 16-byte block ciphers) unpadder = PKCS7(128).unpadder() - plaintext = unpadder.update(padded_plaintext) + unpadder.finalize() - - return plaintext + return unpadder.update(padded_plaintext) + unpadder.finalize() except Exception as e: raise S3EncryptionClientSecurityError( f"Failed to decrypt CBC content: {e}. " @@ -492,6 +489,4 @@ def _decrypt_kc_gcm_content(self, dec_materials, encrypted_data, metadata): ##= specification/s3-encryption/key-derivation.md#hkdf-operation ##% The client MUST set the AAD to the Algorithm Suite ID represented as bytes. aesgcm = AESGCM(derived_encryption_key) - return aesgcm.decrypt( - nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES - ) + return aesgcm.decrypt(nonce=KC_GCM_IV, data=encrypted_data, associated_data=SUITE_ID_BYTES) diff --git a/test/integration/test_i_s3_encryption_instruction_file.py b/test/integration/test_i_s3_encryption_instruction_file.py index be0a06d5..c762fff2 100644 --- a/test/integration/test_i_s3_encryption_instruction_file.py +++ b/test/integration/test_i_s3_encryption_instruction_file.py @@ -38,8 +38,11 @@ def test_decrypt_v1_instruction_file(): kms_client = boto3.client("kms", region_name=region) keyring = KmsKeyring(kms_client, kms_key_id, enable_legacy_wrapping_algorithms=True) wrapped_client = boto3.client("s3") - config = S3EncryptionClientConfig(keyring, enable_legacy_unauthenticated_modes=True, - commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + config = S3EncryptionClientConfig( + keyring, + enable_legacy_unauthenticated_modes=True, + commitment_policy=CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT, + ) s3ec = S3EncryptionClient(wrapped_client, config) response = s3ec.get_object(Bucket=bucket, Key=key) diff --git a/test/test_pipelines.py b/test/test_pipelines.py index bd75a8b8..4f90c6c2 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -174,7 +174,9 @@ def test_decrypt_v3_from_instruction_file(self): # key commitment verification will fail. from s3_encryption.exceptions import S3EncryptionClientSecurityError - with pytest.raises(S3EncryptionClientSecurityError, match="Key commitment verification failed"): + with pytest.raises( + S3EncryptionClientSecurityError, match="Key commitment verification failed" + ): pipeline.decrypt(mock_response, bucket="test-bucket", key="test-key") # Verify instruction file was fetched @@ -205,5 +207,7 @@ def test_decrypt_v3_unsupported_wrap_alg(self): "Metadata": metadata, } - with pytest.raises(S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM"): + with pytest.raises( + S3EncryptionClientError, match="Unsupported wrapping algorithm: AES/GCM" + ): pipeline.decrypt(mock_response) From 5bf5ca467dd9428c01f4cc630812ba56f03f3234 Mon Sep 17 00:00:00 2001 From: Kess Plasmeier Date: Mon, 2 Mar 2026 10:06:37 -0800 Subject: [PATCH 32/35] test-server legacy unauth --- test-server/python-v3-server/src/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-server/python-v3-server/src/main.py b/test-server/python-v3-server/src/main.py index eb3855b7..3f32f389 100755 --- a/test-server/python-v3-server/src/main.py +++ b/test-server/python-v3-server/src/main.py @@ -189,6 +189,7 @@ async def client_endpoint(request: Request): key_material = config_data.get("keyMaterial", {}) enable_legacy_wrapping_algorithms = config_data.get("enableLegacyWrappingAlgorithms", False) + enable_legacy_unauthenticated_modes = config_data.get("enableLegacyUnauthenticatedModes", False) # TODO pull region from ARN kms_client = boto3.client("kms", region_name="us-west-2") @@ -201,7 +202,10 @@ async def client_endpoint(request: Request): wrapped_client = boto3.client("s3") # Build config kwargs, only including algorithm_suite and commitment_policy if provided - config_kwargs = {"keyring": keyring} + config_kwargs = { + "keyring": keyring, + "enable_legacy_unauthenticated_modes": enable_legacy_unauthenticated_modes, + } encryption_algorithm = config_data.get("encryptionAlgorithm") if encryption_algorithm is not None: From ce906566b3dfd2a315c524ba543bfe4ac681ce36 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 2 Mar 2026 14:13:52 -0600 Subject: [PATCH 33/35] fix(instruction-file): address PR review comments from texastony - Add instruction_file_suffix config field with duvet citations - Pass instruction_suffix through to decrypt pipeline - Build instruction key in pipeline instead of instruction_file.py - Remove suffix param from fetch_instruction_file (receives full key) - Add V1/V2 object metadata validation check for instruction files - Move duvet citations to appropriate locations - Fix instruction_suffix -> instruction_file_suffix typo in __init__.py --- src/s3_encryption/__init__.py | 11 ++++++ src/s3_encryption/instruction_file.py | 50 ++++++++------------------- src/s3_encryption/metadata.py | 4 +++ src/s3_encryption/pipelines.py | 35 +++++++++++++------ 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index dc19994c..56528cda 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -26,6 +26,16 @@ class S3EncryptionClientConfig: keyring: AbstractKeyring cmm: AbstractCryptoMaterialsManager = field() + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The S3EC SHOULD support providing a custom Instruction File suffix + ##% on GetObject requests, regardless of whether or not re-encryption is supported. + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". + instruction_file_suffix: str = field(default=".instruction") @cmm.default def _default_cmm_for_keyring(self): @@ -134,6 +144,7 @@ def on_get_object_after_call(self, parsed, **kwargs): encryption_context, bucket=getattr(self._context, "bucket", None), key=getattr(self._context, "key", None), + instruction_suffix=self.config.instruction_file_suffix, ) # Create a new streaming body with the decrypted data diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index a95bd14e..a49a7e97 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -13,7 +13,7 @@ from .metadata import VALID_S3EC_METADATA_KEYS -def parse_instruction_file(instruction_data: bytes, instruction_key: str) -> dict[str, Any]: +def parse_instruction_file(instruction_data: bytes, key: str) -> dict[str, Any]: """Parse and validate instruction file data. This function strictly validates that: @@ -22,7 +22,7 @@ def parse_instruction_file(instruction_data: bytes, instruction_key: str) -> dic Args: instruction_data: Raw bytes from instruction file body - instruction_key: Instruction file key (for error messages) + key: Instruction file key (for error messages) Returns: dict: Parsed JSON metadata from instruction file @@ -35,45 +35,34 @@ def parse_instruction_file(instruction_data: bytes, instruction_key: str) -> dic ##= type=citation ##% The content metadata stored in the Instruction File MUST be serialized to a JSON string. - ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file - ##= type=citation - ##% The serialized JSON string MUST be the only contents of the Instruction File. - # Validate JSON format try: metadata = json.loads(instruction_data) except json.JSONDecodeError as e: - raise S3EncryptionClientError( - f"Instruction file is not valid JSON: {instruction_key}" - ) from e + raise S3EncryptionClientError(f"Instruction file is not valid JSON: {key}") from e # Validate that it's a dictionary if not isinstance(metadata, dict): raise S3EncryptionClientError( - f"Instruction file must contain a JSON object, " - f"got {type(metadata).__name__}: {instruction_key}" + f"Instruction file must contain a JSON object, " f"got {type(metadata).__name__}: {key}" ) # Validate that all keys are S3EC metadata keys + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=citation + ##% The serialized JSON string MUST be the only contents of the Instruction File. invalid_keys = set(metadata.keys()) - VALID_S3EC_METADATA_KEYS if invalid_keys: raise S3EncryptionClientError( - f"Instruction file contains invalid keys: {invalid_keys} in {instruction_key}" + f"Instruction file contains invalid keys: {invalid_keys} in {key}" ) return metadata -def fetch_instruction_file( - s3_client, bucket: str, key: str, suffix: str = ".instruction" -) -> dict[str, Any]: +def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: """Fetch and parse an instruction file from S3. - ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file - ##= type=citation - ##% The S3EC SHOULD support providing a custom Instruction File suffix - ##% on GetObject requests, regardless of whether or not re-encryption is supported. - This function: 1. Fetches the instruction file in plaintext mode 2. Returns the parsed metadata from the response Metadata field @@ -87,8 +76,6 @@ def fetch_instruction_file( s3_client: Boto3 S3 client to use for fetching bucket: S3 bucket name key: S3 object key - suffix: Instruction file suffix (default: .instruction) - Returns: dict: Parsed JSON metadata from instruction file @@ -96,26 +83,19 @@ def fetch_instruction_file( S3EncryptionClientError: If the instruction file marker is missing, the instruction file is not valid JSON, or contains non-S3EC metadata keys """ - instruction_key = key + suffix - - ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file - ##= type=citation - ##% The default Instruction File behavior uses the same S3 object key - ##% as its associated object suffixed with ".instruction". - # Set plaintext mode flag in thread-local context before calling get_object # This will be checked by the event handler to skip decryption if hasattr(s3_client, "_s3ec_plugin_context"): s3_client._s3ec_plugin_context.plaintext_mode = True - s3_client._s3ec_plugin_context.key = instruction_key + s3_client._s3ec_plugin_context.key = key else: raise S3EncryptionClientError( f"Could not fetch instruction file without " - f"the S3 Encryption Client Plugin installed. Instruction key: {instruction_key}" + f"the S3 Encryption Client Plugin installed. Instruction key: {key}" ) try: - response = s3_client.get_object(Bucket=bucket, Key=instruction_key) + response = s3_client.get_object(Bucket=bucket, Key=key) finally: # Clear the flags after the call if hasattr(s3_client, "_s3ec_plugin_context"): @@ -126,15 +106,13 @@ def fetch_instruction_file( # Verify metadata is not empty if not metadata: - raise S3EncryptionClientError( - f"Instruction file returned empty metadata: {instruction_key}" - ) + raise S3EncryptionClientError(f"Instruction file returned empty metadata: {key}") # Verify metadata contains at least one S3EC key has_s3ec_key = any(key in VALID_S3EC_METADATA_KEYS for key in metadata) if not has_s3ec_key: raise S3EncryptionClientError( - f"Instruction file metadata does not contain any S3EC keys: {instruction_key}" + f"Instruction file metadata does not contain any S3EC keys: {key}" ) return metadata diff --git a/src/s3_encryption/metadata.py b/src/s3_encryption/metadata.py index 6d8c0c09..bd93c047 100644 --- a/src/s3_encryption/metadata.py +++ b/src/s3_encryption/metadata.py @@ -242,6 +242,10 @@ def is_v3_in_object_metadata(self) -> bool: and self.encrypted_data_key_v3 is None ) + ##= specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + ##= type=citation + ##% If the object matches none of the V1/V2/V3 formats, + ##% the S3EC MUST attempt to get the instruction file. def should_use_instruction_file(self) -> bool: """Check if instruction file should be used for decryption. diff --git a/src/s3_encryption/pipelines.py b/src/s3_encryption/pipelines.py index ae76bf0e..12459671 100644 --- a/src/s3_encryption/pipelines.py +++ b/src/s3_encryption/pipelines.py @@ -94,7 +94,14 @@ class GetEncryptedObjectPipeline: cmm: AbstractCryptoMaterialsManager = field() s3_client: object = field(default=None) - def decrypt(self, response, encryption_context=None, bucket=None, key=None): + def decrypt( + self, + response, + encryption_context=None, + bucket=None, + key=None, + instruction_suffix=".instruction", + ): """Decrypt the data after it is retrieved from S3. Args: @@ -102,6 +109,7 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): encryption_context (dict, optional): Additional context for decryption bucket (str, optional): S3 bucket name (required for instruction file) key (str, optional): S3 object key (required for instruction file) + instruction_suffix(str, optional): suffix for instruction file; defaults to ".instruction". Returns: bytes: The decrypted data @@ -118,25 +126,32 @@ def decrypt(self, response, encryption_context=None, bucket=None, key=None): # Check if we need to fetch instruction file if metadata.should_use_instruction_file(): - ##= specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status - ##= type=citation - ##% If the object matches none of the V1/V2/V3 formats, - ##% the S3EC MUST attempt to get the instruction file. if self.s3_client is None: raise S3EncryptionClientError("s3_client required to fetch instruction file") if bucket is None or key is None: raise S3EncryptionClientError("Bucket and key required to fetch instruction file") + instruction_key = key + instruction_suffix + instruction_metadata = fetch_instruction_file(self.s3_client, bucket, instruction_key) + instruction_metadata.update(encryption_metadata) + metadata = ObjectMetadata.from_dict(instruction_metadata) ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files ##= type=citation ##% In the V1/V2 message format, all of the content metadata ##% MUST be stored in the Instruction File. - - instruction_metadata = fetch_instruction_file(self.s3_client, bucket, key) - instruction_metadata.update(encryption_metadata) - metadata = ObjectMetadata.from_dict(instruction_metadata) - + if metadata.is_v1_format() or metadata.is_v2_format(): + object_metadata = ObjectMetadata.from_dict(encryption_metadata) + if not ( + object_metadata.content_cipher is None + and object_metadata.content_iv is None + and object_metadata.encrypted_data_key_algorithm is None + ): + raise S3EncryptionClientError( + "Content metadata found in object metadata for V1 or V2 message format " + "BUT Instruction File is being used. This is an illegal combination. " + f"bucket: {bucket}\n key:{key}\n instruction_file:{instruction_key}" + ) # Determine which format we're dealing with and get decryption materials if metadata.is_v1_format(): dec_materials = self._decrypt_v1(metadata, encryption_context) From d9c2f2c92199bd65be4921a6a8e06a486d76fdc9 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 2 Mar 2026 14:36:47 -0600 Subject: [PATCH 34/35] test(duvet): add missing test citations for instruction file spec coverage - Add test for should_use_instruction_file with duvet citation - Add test for custom instruction_file_suffix with duvet citation - Annotate existing tests with duvet citations for default suffix, JSON serialization, V1/V2 instruction file, and V3 instruction file - All 7 instruction file spec citations now have type=test annotations --- test/test_metadata.py | 36 ++++++++++++++ test/test_pipelines.py | 61 ++++++++++++++++++++++++ test/test_s3_encryption_client_plugin.py | 6 +++ 3 files changed, 103 insertions(+) diff --git a/test/test_metadata.py b/test/test_metadata.py index 7dab2081..ba783bf5 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -200,3 +200,39 @@ def test_has_exclusive_key_collision(self): encrypted_data_key_v3="edk-v3", ) assert metadata_all.has_exclusive_key_collision() is True + + ##= specification/s3-encryption/data-format/content-metadata.md#determining-s3ec-object-status + ##= type=test + ##% If the object matches none of the V1/V2/V3 formats, + ##% the S3EC MUST attempt to get the instruction file. + def test_should_use_instruction_file(self): + # No keys at all -> should use instruction file + metadata_empty = ObjectMetadata() + assert metadata_empty.should_use_instruction_file() is True + + # V3 in object metadata (has content keys but no EDK) -> instruction file + metadata_v3_partial = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + ) + assert metadata_v3_partial.should_use_instruction_file() is True + + # V1 with EDK -> no instruction file needed + metadata_v1 = ObjectMetadata(encrypted_data_key_v1="edk-v1") + assert metadata_v1.should_use_instruction_file() is False + + # V2 with EDK -> no instruction file needed + metadata_v2 = ObjectMetadata(encrypted_data_key_v2="edk-v2") + assert metadata_v2.should_use_instruction_file() is False + + # V3 with EDK -> no instruction file needed + metadata_v3 = ObjectMetadata( + content_cipher_v3="02", + encrypted_data_key_algorithm_v3="12", + key_commitment_v3="commitment", + message_id_v3="msg-id", + encrypted_data_key_v3="edk-v3", + ) + assert metadata_v3.should_use_instruction_file() is False diff --git a/test/test_pipelines.py b/test/test_pipelines.py index 7e7598b3..9f40cd5c 100644 --- a/test/test_pipelines.py +++ b/test/test_pipelines.py @@ -14,6 +14,10 @@ class TestGetEncryptedObjectPipelineInstructionFile: + ##= specification/s3-encryption/data-format/metadata-strategy.md#v1-v2-instruction-files + ##= type=test + ##% In the V1/V2 message format, all of the content metadata + ##% MUST be stored in the Instruction File. def test_decrypt_v1_from_instruction_file(self): """Test decrypting V1 format with instruction file.""" object_metadata = {"x-amz-meta-x-amz-unencrypted-content-length": "39"} @@ -63,6 +67,10 @@ def test_decrypt_v1_from_instruction_file(self): Bucket="test-bucket", Key="test-key.instruction" ) + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The default Instruction File behavior uses the same S3 object key + ##% as its associated object suffixed with ".instruction". def test_decrypt_v2_from_instruction_file(self): """Test decrypting V2 format with instruction file.""" # V2: Object metadata is empty, all metadata in instruction file @@ -114,6 +122,10 @@ def test_decrypt_v2_from_instruction_file(self): Bucket="test-bucket", Key="test-key.instruction" ) + ##= specification/s3-encryption/data-format/metadata-strategy.md#v3-instruction-files + ##= type=test + ##% In the V3 message format, only the content metadata related to + ##% the encrypted data is stored in the Instruction File. def test_decrypt_v3_from_instruction_file(self): """Test decrypting V3 format with instruction file.""" # Object metadata contains V3 content keys only @@ -178,3 +190,52 @@ def test_decrypt_v3_from_instruction_file(self): mock_s3_client.get_object.assert_called_once_with( Bucket="test-bucket", Key="test-key.instruction" ) + + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The S3EC SHOULD support providing a custom Instruction File suffix + ##% on GetObject requests, regardless of whether or not re-encryption is supported. + def test_decrypt_with_custom_instruction_file_suffix(self): + """Test that a custom instruction file suffix is used when provided.""" + object_metadata = {} + + instruction_file_metadata = { + "x-amz-iv": base64.b64encode(os.urandom(12)).decode("utf-8"), + "x-amz-key-v2": base64.b64encode(b"encrypted-key-data").decode("utf-8"), + "x-amz-wrap-alg": "kms+context", + "x-amz-matdesc": json.dumps({"kms_cmk_id": "test-key-id"}), + "x-amz-cek-alg": "AES/GCM/NoPadding", + "x-amz-tag-len": "128", + "x-amz-crypto-instr-file": "", + } + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = { + "Body": BytesIO(b""), + "Metadata": instruction_file_metadata, + } + + mock_keyring = Mock(spec=S3Keyring) + cmm = DefaultCryptoMaterialsManager(mock_keyring) + pipeline = GetEncryptedObjectPipeline(cmm, mock_s3_client) + + mock_response = { + "Body": BytesIO(b"encrypted-test-data"), + "Metadata": object_metadata, + } + + mock_keyring.on_decrypt.side_effect = Exception( + "Keyring called - instruction file was fetched" + ) + + with pytest.raises(Exception, match="Keyring called"): + pipeline.decrypt( + mock_response, + bucket="test-bucket", + key="test-key", + instruction_suffix=".custom-suffix", + ) + + mock_s3_client.get_object.assert_called_once_with( + Bucket="test-bucket", Key="test-key.custom-suffix" + ) diff --git a/test/test_s3_encryption_client_plugin.py b/test/test_s3_encryption_client_plugin.py index fa1a0cae..b51edf1a 100644 --- a/test/test_s3_encryption_client_plugin.py +++ b/test/test_s3_encryption_client_plugin.py @@ -87,6 +87,9 @@ def test_plaintext_mode_missing_marker_raises_error(self): ): plugin.on_get_object_after_call(parsed) + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The content metadata stored in the Instruction File MUST be serialized to a JSON string. def test_plaintext_mode_invalid_json_raises_error(self): """Test that invalid JSON in instruction file raises error.""" # Create plugin @@ -137,6 +140,9 @@ def test_plaintext_mode_non_dict_json_raises_error(self): ): plugin.on_get_object_after_call(parsed) + ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file + ##= type=test + ##% The serialized JSON string MUST be the only contents of the Instruction File. def test_plaintext_mode_invalid_keys_raises_error(self): """Test that invalid keys in instruction file raises error.""" # Create plugin From 5a595fe3c4d767fa8833cb8dc3ed4f71b7a3a096 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Mon, 2 Mar 2026 14:50:02 -0600 Subject: [PATCH 35/35] fix(instruction-file): address PR review comments from kessplas - Remove strict x-amz-crypto-instr-file marker check for Ruby/PHP compat - Rename plaintext_mode to instruction_file_mode - Remove marker validation test - Update docstrings to reflect relaxed validation --- src/s3_encryption/__init__.py | 15 ++------ src/s3_encryption/instruction_file.py | 9 ++--- test/test_s3_encryption_client_plugin.py | 47 ++++-------------------- 3 files changed, 16 insertions(+), 55 deletions(-) diff --git a/src/s3_encryption/__init__.py b/src/s3_encryption/__init__.py index 56528cda..19895406 100644 --- a/src/s3_encryption/__init__.py +++ b/src/s3_encryption/__init__.py @@ -67,9 +67,9 @@ def on_put_object_before_call(self, params, **kwargs): params: Dictionary of parameters for the PutObject call (after serialization) **kwargs: Additional event arguments """ - if getattr(self._context, "plaintext_mode", False): + if getattr(self._context, "instruction_file_mode", False): raise S3EncryptionClientError( - "Plaintext mode is exclusively for reading instruction files " + "Instruction file mode is exclusively for reading instruction files " "and not supported in put_object!" ) # At this point, boto3 has already serialized the Body @@ -118,7 +118,7 @@ def on_get_object_after_call(self, parsed, **kwargs): **kwargs: Additional event arguments (includes 'params' with request parameters) """ # Check if plaintext mode is enabled via thread-local flag - if getattr(self._context, "plaintext_mode", False): + if getattr(self._context, "instruction_file_mode", False): self.process_instruction_file(parsed) return @@ -165,15 +165,8 @@ def process_instruction_file(self, parsed): """ instruction_key = getattr(self._context, "key", None) - # Verify instruction file marker is present in S3 object metadata - existing_metadata = parsed.get("Metadata", {}) - if "x-amz-crypto-instr-file" not in existing_metadata: - raise S3EncryptionClientError( - f"Instruction file does not contain " - f"x-amz-crypto-instr-file marker: {instruction_key}" - ) - # In plaintext mode, parse instruction file and append to metadata + existing_metadata = parsed.get("Metadata", {}) instruction_data = parsed.get("Body").read() instruction_metadata = parse_instruction_file(instruction_data, instruction_key) diff --git a/src/s3_encryption/instruction_file.py b/src/s3_encryption/instruction_file.py index a49a7e97..8e0d33b4 100644 --- a/src/s3_encryption/instruction_file.py +++ b/src/s3_encryption/instruction_file.py @@ -68,7 +68,6 @@ def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: 2. Returns the parsed metadata from the response Metadata field S3EncryptionClientPlugin's event handler (on_get_object_after_call) handles: - - Verifying the x-amz-crypto-instr-file marker is present - Parsing and validating the instruction file content - Placing parsed metadata in response["Metadata"] @@ -80,13 +79,13 @@ def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: dict: Parsed JSON metadata from instruction file Raises: - S3EncryptionClientError: If the instruction file marker is missing, - the instruction file is not valid JSON, or contains non-S3EC metadata keys + S3EncryptionClientError: If the instruction file is not valid JSON, + or contains non-S3EC metadata keys """ # Set plaintext mode flag in thread-local context before calling get_object # This will be checked by the event handler to skip decryption if hasattr(s3_client, "_s3ec_plugin_context"): - s3_client._s3ec_plugin_context.plaintext_mode = True + s3_client._s3ec_plugin_context.instruction_file_mode = True s3_client._s3ec_plugin_context.key = key else: raise S3EncryptionClientError( @@ -99,7 +98,7 @@ def fetch_instruction_file(s3_client, bucket: str, key: str) -> dict[str, Any]: finally: # Clear the flags after the call if hasattr(s3_client, "_s3ec_plugin_context"): - s3_client._s3ec_plugin_context.plaintext_mode = False + s3_client._s3ec_plugin_context.instruction_file_mode = False # In plaintext mode, the event handler places parsed metadata in Metadata field metadata = response.get("Metadata", {}) diff --git a/test/test_s3_encryption_client_plugin.py b/test/test_s3_encryption_client_plugin.py index b51edf1a..bdc48c79 100644 --- a/test/test_s3_encryption_client_plugin.py +++ b/test/test_s3_encryption_client_plugin.py @@ -17,7 +17,7 @@ class TestS3EncryptionClientPlugin: """S3EncryptionClientPlugin event handler behavior.""" - def test_plaintext_mode_parses_instruction_file(self): + def test_instruction_file_mode_parses_instruction_file(self): """Test that plaintext mode parses instruction file and returns metadata.""" # Create plugin mock_keyring = Mock(spec=S3Keyring) @@ -25,7 +25,7 @@ def test_plaintext_mode_parses_instruction_file(self): plugin = S3EncryptionClientPlugin(config) # Set plaintext mode - plugin._context.plaintext_mode = True + plugin._context.instruction_file_mode = True plugin._context.key = "test-key.instruction" # Create instruction file body @@ -56,41 +56,10 @@ def test_plaintext_mode_parses_instruction_file(self): # Verify body was cleared assert parsed["Body"].read() == b"" - def test_plaintext_mode_missing_marker_raises_error(self): - """Test that missing instruction file marker raises error.""" - # Create plugin - mock_keyring = Mock(spec=S3Keyring) - config = S3EncryptionClientConfig(keyring=mock_keyring) - plugin = S3EncryptionClientPlugin(config) - - # Set plaintext mode - plugin._context.plaintext_mode = True - plugin._context.key = "test-key.instruction" - - # Create instruction file body - instruction_metadata = { - "x-amz-iv": "test-iv", - "x-amz-key-v2": "test-key", - } - instruction_body = json.dumps(instruction_metadata).encode("utf-8") - - # Create parsed response WITHOUT instruction file marker - parsed = { - "Body": StreamingBody(io.BytesIO(instruction_body), len(instruction_body)), - "Metadata": {}, # Missing x-amz-crypto-instr-file - } - - # Should raise error - with pytest.raises( - S3EncryptionClientError, - match="Instruction file does not contain x-amz-crypto-instr-file marker", - ): - plugin.on_get_object_after_call(parsed) - ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file ##= type=test ##% The content metadata stored in the Instruction File MUST be serialized to a JSON string. - def test_plaintext_mode_invalid_json_raises_error(self): + def test_instruction_file_mode_invalid_json_raises_error(self): """Test that invalid JSON in instruction file raises error.""" # Create plugin mock_keyring = Mock(spec=S3Keyring) @@ -98,7 +67,7 @@ def test_plaintext_mode_invalid_json_raises_error(self): plugin = S3EncryptionClientPlugin(config) # Set plaintext mode - plugin._context.plaintext_mode = True + plugin._context.instruction_file_mode = True plugin._context.key = "test-key.instruction" # Create invalid JSON body @@ -114,7 +83,7 @@ def test_plaintext_mode_invalid_json_raises_error(self): with pytest.raises(S3EncryptionClientError, match="Instruction file is not valid JSON"): plugin.on_get_object_after_call(parsed) - def test_plaintext_mode_non_dict_json_raises_error(self): + def test_instruction_file_mode_non_dict_json_raises_error(self): """Test that non-dict JSON in instruction file raises error.""" # Create plugin mock_keyring = Mock(spec=S3Keyring) @@ -122,7 +91,7 @@ def test_plaintext_mode_non_dict_json_raises_error(self): plugin = S3EncryptionClientPlugin(config) # Set plaintext mode - plugin._context.plaintext_mode = True + plugin._context.instruction_file_mode = True plugin._context.key = "test-key.instruction" # Create JSON array instead of object @@ -143,7 +112,7 @@ def test_plaintext_mode_non_dict_json_raises_error(self): ##= specification/s3-encryption/data-format/metadata-strategy.md#instruction-file ##= type=test ##% The serialized JSON string MUST be the only contents of the Instruction File. - def test_plaintext_mode_invalid_keys_raises_error(self): + def test_instruction_file_mode_invalid_keys_raises_error(self): """Test that invalid keys in instruction file raises error.""" # Create plugin mock_keyring = Mock(spec=S3Keyring) @@ -151,7 +120,7 @@ def test_plaintext_mode_invalid_keys_raises_error(self): plugin = S3EncryptionClientPlugin(config) # Set plaintext mode - plugin._context.plaintext_mode = True + plugin._context.instruction_file_mode = True plugin._context.key = "test-key.instruction" # Create instruction file with invalid keys