From 2b8f937d185c9e0481eb9fc2407e9a5634848a86 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 14 Nov 2025 13:54:27 -0800 Subject: [PATCH 01/40] ruby spec updates --- test-server/ruby-v2-server/.duvet/config.toml | 15 +++++++++++++-- test-server/ruby-v2-server/local-ruby-sdk | 2 +- test-server/ruby-v3-server/local-ruby-sdk | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/test-server/ruby-v2-server/.duvet/config.toml b/test-server/ruby-v2-server/.duvet/config.toml index 7118cd70..7a34c0ff 100644 --- a/test-server/ruby-v2-server/.duvet/config.toml +++ b/test-server/ruby-v2-server/.duvet/config.toml @@ -4,15 +4,26 @@ pattern = "local-ruby-sdk/gems/aws-sdk-s3/lib/**/*.rb" comment-style = { meta = "##=", content = "##%" } +[[source]] +pattern = "local-ruby-sdk/gems/aws-sdk-s3/spec/**/*.rb" +comment-style = { meta = "##=", content = "##%" } + # Include required specifications here [[specification]] -source = "../specification/s3-encryption/data-format/content-metadata.md" +source = "../specification/s3-encryption/client.md" [[specification]] -source = "../specification/s3-encryption/data-format/metadata-strategy.md" +source = "../specification/s3-encryption/decryption.md" [[specification]] source = "../specification/s3-encryption/encryption.md" [[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + [report.html] enabled = true diff --git a/test-server/ruby-v2-server/local-ruby-sdk b/test-server/ruby-v2-server/local-ruby-sdk index 582e0241..d6b93925 160000 --- a/test-server/ruby-v2-server/local-ruby-sdk +++ b/test-server/ruby-v2-server/local-ruby-sdk @@ -1 +1 @@ -Subproject commit 582e02418ac2c5540c48dd089a7db506712e6f94 +Subproject commit d6b93925c65e0ad4b9410ade709c63ca874634c3 diff --git a/test-server/ruby-v3-server/local-ruby-sdk b/test-server/ruby-v3-server/local-ruby-sdk index 582e0241..d6b93925 160000 --- a/test-server/ruby-v3-server/local-ruby-sdk +++ b/test-server/ruby-v3-server/local-ruby-sdk @@ -1 +1 @@ -Subproject commit 582e02418ac2c5540c48dd089a7db506712e6f94 +Subproject commit d6b93925c65e0ad4b9410ade709c63ca874634c3 From d0f16e00b726d1f93a9c1da669d1a813dd246e4a Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 14 Nov 2025 16:57:08 -0800 Subject: [PATCH 02/40] Start on instruction failures --- .../s3/InstructionFileFailures.java | 329 ++++++++++++++++++ .../amazon/encryption/s3/TestUtils.java | 29 +- 2 files changed, 351 insertions(+), 7 deletions(-) create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java new file mode 100644 index 00000000..0d458a3b --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -0,0 +1,329 @@ +/* +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.opentest4j.TestAbortedException; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** +* Exhaustive tests for S3 Encryption Client round-trip operations. +* These tests cover various combinations of client versions, commitment policies, and encryption modes. +* +* Tests are based on the exhaustive test matrix defined at: +* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 +* +*/ + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class InstructionFileFailures { + private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; + private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-kms-instruction-file"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + private static final List crossLanguageObjects = new ArrayList<>(); + private static KeyPair RSA_KEY_PAIR_1; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + } + + public static Stream improvedClientsCanPutKMSWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream clientsCanGetKMSWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + @Order(1) + @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutKMSWithInstructionFile") + void encrypt_with_instruction_files_kms_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(2) + @Test + void make_good_copies_to_verify_we_can() throws Exception { + // Create a plaintext S3 client to copy objects with instruction files + try (S3Client ptS3Client = S3Client.create()) { + for (String objectKey : crossLanguageObjects) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Get the instruction file + String instructionFileKey = objectKey + ".instruction"; + ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + + String instructionFileJson = instructionFile.asUtf8String(); + Map objectMetadata = encryptedObject.response().metadata(); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + "-good-copy", + encryptedObject.asByteArray(), + objectMetadata, + instructionFileJson + ); + + ObjectMapper mapper = new ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); + instructionFileMap.put("x-amz-matdesc", objectMetadata.get("x-amz-matdesc")); + + String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + "-bad-both-meta-and-instruction", + encryptedObject.asByteArray(), + objectMetadata, + instructionFileWithCommitmentValues + ); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + "-bad-only-instruction", + encryptedObject.asByteArray(), + Map.of(), + instructionFileWithCommitmentValues + ); + + + } + } + } + + void putObjectWithInstructionFile( + S3Client ptS3Client, + String newObjectKey, + byte[] objectData, + Map objectMetadata, + String instructionFileJson + ) { + + // Put the encrypted object copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + + // Put the instruction file copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey + ".instruction") + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + + + @Order(10) + @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjects + ); + } + + @Order(11) + @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjects + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(12) + @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjects + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(13) + @ParameterizedTest(name = "{0}: Fail to decrypt duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjects + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(14) + @ParameterizedTest(name = "{0}: Fail to decrypt instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjects + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index d3d25d58..294f128b 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -492,18 +492,33 @@ public static void Encrypt( public static void Decrypt( S3ECTestServerClient client, - String S3ECId, List crossLanguageObjects, + String S3ECId, + List crossLanguageObjects, EncryptionAlgorithm expectedEncryptionAlgorithm ) { - for (String objectKey : crossLanguageObjects) { + // Call 5-parameter version with crossLanguageObjects as expectedPlaintexts + Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, crossLanguageObjects); + } + + public static void Decrypt( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts + ) { + for (int i = 0; i < crossLanguageObjects.size(); i++) { + String objectKey = crossLanguageObjects.get(i); + String expectedPlaintext = expectedPlaintexts.get(i); + GetObjectOutput output = client.getObject(GetObjectInput.builder() - .clientID(S3ECId) - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); // Then: Pass - assertEquals(objectKey, new String(output.getBody().array())); + assertEquals(expectedPlaintext, new String(output.getBody().array())); assertEquals( expectedEncryptionAlgorithm, GetEncryptionAlgorithm(objectKey), From 496a8db14fc55fce16fa5787dc96a7b58f2f8ad9 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Sun, 16 Nov 2025 16:31:08 -0800 Subject: [PATCH 03/40] Instruction files that pass and fail --- .../amazon/encryption/s3/TestUtils.java | 60 +++++++++++++++---- test-server/ruby-v2-server/app.rb | 4 +- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 294f128b..dc07a2d4 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -17,10 +17,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.amazonaws.services.s3.model.S3Object; +import com.fasterxml.jackson.databind.ObjectMapper; + import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -449,22 +453,58 @@ public static String appendTestSuffix(final String s) { private static AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); public static EncryptionAlgorithm GetEncryptionAlgorithm(String objectKey) { + // Lambda to determine encryption algorithm from a metadata map + java.util.function.Function, Optional> getAlgorithmFromMap = (map) -> { + if (map.containsKey("x-amz-c")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else if (map.containsKey("x-amz-cek-alg")) { + String cek = (String) map.get("x-amz-cek-alg"); + if (cek.contains("CBC")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } else if (cek.contains("GCM")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + } + return Optional.empty(); + }; + ObjectMetadata metadata = s3Client.getObjectMetadata(TestUtils.BUCKET, objectKey); Map userMetadata = metadata.getUserMetadata(); - // This is optimized to not need to go to the instruction files for commit_key - if (userMetadata.containsKey("x-amz-c")) { - return EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY; - } else if (userMetadata.containsKey("x-amz-cek-alg")) { - String cek = userMetadata.get("x-amz-cek-alg"); - if (cek.contains("CBC")) { - return EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF; - } else if (cek.contains("GCM")) { - return EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF; + // Try to get algorithm from object metadata + Optional algorithm = getAlgorithmFromMap.apply(userMetadata); + if (algorithm.isPresent()) { + return algorithm.get(); + } + + // Check instruction file + try { + String instructionFileKey = objectKey + ".instruction"; + com.amazonaws.services.s3.model.S3Object instructionFileObject = + s3Client.getObject(TestUtils.BUCKET, instructionFileKey); + + // Read instruction file content + java.io.InputStream inputStream = instructionFileObject.getObjectContent(); + String instructionFileJson = new String( + inputStream.readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8 + ); + inputStream.close(); + + // Parse JSON to get metadata + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + // Try to get algorithm from instruction file + algorithm = getAlgorithmFromMap.apply(instructionFileMap); + if (algorithm.isPresent()) { + return algorithm.get(); } + } catch (Exception e) { + // Instruction file doesn't exist or couldn't be read } - throw new RuntimeException("Need to support instruction files!"); + throw new RuntimeException("Could not determine encryption algorithm from object metadata or instruction file!"); } public static void Encrypt( diff --git a/test-server/ruby-v2-server/app.rb b/test-server/ruby-v2-server/app.rb index cde757a3..5a39e2ea 100644 --- a/test-server/ruby-v2-server/app.rb +++ b/test-server/ruby-v2-server/app.rb @@ -132,7 +132,7 @@ def initialize metadata: response_metadata }.to_json - rescue Aws::S3::EncryptionV2::Errors::EncryptionError => e + rescue Aws::S3::EncryptionV2::Errors::EncryptionError, Aws::S3::EncryptionV3::Errors::EncryptionError => e S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) ErrorHandlers.send_s3_encryption_client_error(self, e.message) rescue StandardError => e @@ -201,7 +201,7 @@ def initialize content_type 'application/octet-stream' body - rescue Aws::S3::EncryptionV2::Errors::DecryptionError => e + rescue Aws::S3::EncryptionV2::Errors::DecryptionError, Aws::S3::EncryptionV3::Errors::DecryptionError => e S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) ErrorHandlers.send_s3_encryption_client_error(self, e.message) rescue StandardError => e From 54853fafd056411fcaf3369d81f24cbba413ca33 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 10:19:54 -0800 Subject: [PATCH 04/40] starting to add raw --- .../s3/InstructionFileFailures.java | 132 +++++++++++++++--- 1 file changed, 113 insertions(+), 19 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java index 0d458a3b..aa00abac 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -17,6 +17,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -54,19 +57,34 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class InstructionFileFailures { - private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; - private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-kms-instruction-file"; + private static final String sharedObjectKeyBaseMetaDataMode = "test-instruction-files-cases"; private static KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(TestUtils.KMS_KEY_ARN) .build(); - private static final List crossLanguageObjects = new ArrayList<>(); - private static KeyPair RSA_KEY_PAIR_1; + private static final List crossLanguageObjectsKms = new ArrayList<>(); + private static final List crossLanguageObjectsRsa = new ArrayList<>(); + private static final List crossLanguageObjectsAes = new ArrayList<>(); + + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; @BeforeAll static void setupKeys() throws Exception { KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); keyPairGen.initialize(2048); - RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + RSA_KEY = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) + .build(); + + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesSecretKey = keyGen.generateKey(); + + AES_KEY = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) + .build(); } public static Stream improvedClientsCanPutKMSWithInstructionFile() { @@ -75,6 +93,12 @@ public static Stream improvedClientsCanPutKMSWithInstructionFile() { .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); } + public static Stream improvedClientsCanPutRawWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + public static Stream clientsCanGetKMSWithInstructionFile() { Stream improved = improvedClientsForTest() .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); @@ -85,6 +109,16 @@ public static Stream clientsCanGetKMSWithInstructionFile() { return Stream.concat(improved, transition); } + public static Stream clientsCanGetRawWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + @Order(1) @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutKMSWithInstructionFile") @@ -93,7 +127,6 @@ void encrypt_with_instruction_files_kms_kc_gcm(TestUtils.LanguageServerTarget la CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) .instructionFileConfig( InstructionFileConfig.builder() .enableInstructionFilePutObject(true) @@ -106,18 +139,77 @@ void encrypt_with_instruction_files_kms_kc_gcm(TestUtils.LanguageServerTarget la TestUtils.Encrypt( client, S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), - crossLanguageObjects, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-kms" + language.getLanguageName()), + crossLanguageObjectsKms, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ); } @Order(2) + @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutRawWithInstructionFile") + void encrypt_with_instruction_files_rsa_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-rsa" + language.getLanguageName()), + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(3) + @ParameterizedTest(name = "{0}: Encrypt AES KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutRawWithInstructionFile") + void encrypt_with_instruction_files_aes_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(9) @Test - void make_good_copies_to_verify_we_can() throws Exception { + void make_copies_to_verify_things() throws Exception { // Create a plaintext S3 client to copy objects with instruction files try (S3Client ptS3Client = S3Client.create()) { - for (String objectKey : crossLanguageObjects) { + List allCrossLanguageObjects = Stream.of( + crossLanguageObjectsKms.stream(), + crossLanguageObjectsRsa.stream(), + crossLanguageObjectsAes.stream() + ).flatMap(s -> s).collect(Collectors.toList()); + for (String objectKey : allCrossLanguageObjects) { // Get the encrypted object ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder .bucket(TestUtils.BUCKET) @@ -134,6 +226,7 @@ void make_good_copies_to_verify_we_can() throws Exception { String instructionFileJson = instructionFile.asUtf8String(); Map objectMetadata = encryptedObject.response().metadata(); + // Put a strict copy, to verify that we know how to do this putObjectWithInstructionFile( ptS3Client, objectKey + "-good-copy", @@ -146,10 +239,12 @@ void make_good_copies_to_verify_we_can() throws Exception { Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); - instructionFileMap.put("x-amz-matdesc", objectMetadata.get("x-amz-matdesc")); + instructionFileMap.put("x-amz-d", objectMetadata.get("x-amz-d")); + instructionFileMap.put("x-amz-i", objectMetadata.get("x-amz-i")); String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); + // Put instruction files that should fail: putObjectWithInstructionFile( ptS3Client, objectKey + "-bad-both-meta-and-instruction", @@ -166,7 +261,6 @@ void make_good_copies_to_verify_we_can() throws Exception { instructionFileWithCommitmentValues ); - } } } @@ -212,19 +306,19 @@ void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTar TestUtils.Decrypt( client, S3ECId, - crossLanguageObjects, + crossLanguageObjectsKms, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ); TestUtils.Decrypt( client, S3ECId, - crossLanguageObjects + crossLanguageObjectsKms .stream() .map(key -> key + "-good-copy") .collect(Collectors.toList()), EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjects + crossLanguageObjectsKms ); } @@ -244,7 +338,7 @@ void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUti TestUtils.Decrypt_fails( client, S3ECId, - crossLanguageObjects + crossLanguageObjectsKms .stream() .map(key -> key + "-bad-both-meta-and-instruction") .collect(Collectors.toList()), @@ -268,7 +362,7 @@ void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageSe TestUtils.Decrypt_fails( client, S3ECId, - crossLanguageObjects + crossLanguageObjectsKms .stream() .map(key -> key + "-bad-only-instruction") .collect(Collectors.toList()), @@ -293,7 +387,7 @@ void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.Langua TestUtils.Decrypt_fails( client, S3ECId, - crossLanguageObjects + crossLanguageObjectsKms .stream() .map(key -> key + "-bad-both-meta-and-instruction") .collect(Collectors.toList()), @@ -318,7 +412,7 @@ void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils TestUtils.Decrypt_fails( client, S3ECId, - crossLanguageObjects + crossLanguageObjectsKms .stream() .map(key -> key + "-bad-only-instruction") .collect(Collectors.toList()), From f0aa6f7845fc7c6a10da473371a414784670fdae Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 10:56:41 -0800 Subject: [PATCH 05/40] adding ruby to raw support --- .../s3/InstructionFileFailures.java | 268 +++++++++++++++++- .../amazon/encryption/s3/TestUtils.java | 1 + .../ruby-v2-server/lib/client_manager.rb | 36 ++- .../ruby-v3-server/lib/client_manager.rb | 38 ++- 4 files changed, 329 insertions(+), 14 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java index aa00abac..5456d344 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -194,7 +194,7 @@ void encrypt_with_instruction_files_aes_kc_gcm(TestUtils.LanguageServerTarget la client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), - crossLanguageObjectsRsa, + crossLanguageObjectsAes, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY ); } @@ -289,6 +289,7 @@ void putObjectWithInstructionFile( software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); } + // KMS instruction files decrypt @Order(10) @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") @@ -420,4 +421,269 @@ void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils ); } + // RSA instruction file decrypt + + @Order(20) + @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsRsa + ); + } + + @Order(21) + @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(22) + @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(23) + @ParameterizedTest(name = "{0}: Fail to decrypt duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(24) + @ParameterizedTest(name = "{0}: Fail to decrypt instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // AES instruction file decrypt + + @Order(30) + @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsAes + ); + } + + @Order(31) + @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(32) + @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(33) + @ParameterizedTest(name = "{0}: Fail to decrypt duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @Order(34) + @ParameterizedTest(name = "{0}: Fail to decrypt instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") + void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index dc07a2d4..90995dde 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -103,6 +103,7 @@ public class TestUtils { public static final Set RAW_SUPPORTED = Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 + , RUBY_V2_TRANSITION, RUBY_V3 ); // .NET only supports decrypting instruction files using AES and RSA. diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb index 717003bf..85a91038 100644 --- a/test-server/ruby-v2-server/lib/client_manager.rb +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -2,6 +2,8 @@ require 'securerandom' require 'aws-sdk-s3' require 'aws-sdk-kms' +require 'openssl' +require 'base64' require_relative 'logger' # Manages S3 Encryption Client instances @@ -14,20 +16,42 @@ def initialize # Create a new S3 encryption client and return its ID def create_client(config) - # Extract configuration + # Extract all key material types kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + rsa_key_blob = config.dig('keyMaterial', 'rsaKey') + aes_key_blob = config.dig('keyMaterial', 'aesKey') inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') - raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? + # Validate that only one key type is provided + key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count + raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 # Create S3 encryption client configuration encryption_config = { - kms_key_id: kms_key_id, - kms_client: @kms_client, - key_wrap_schema: :kms_context, content_encryption_schema: :aes_gcm_no_padding, envelope_location: inst_file_put ? :instruction_file : :metadata - }.tap do |hash| + } + + # Configure based on key type + if kms_key_id + encryption_config[:kms_key_id] = kms_key_id + encryption_config[:kms_client] = @kms_client + encryption_config[:key_wrap_schema] = :kms_context + elsif rsa_key_blob + # Parse RSA private key from PKCS8 format + key_bytes = Base64.decode64(rsa_key_blob) + rsa_key = OpenSSL::PKey::RSA.new(key_bytes) + encryption_config[:encryption_key] = rsa_key + encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 + elsif aes_key_blob + # Extract AES key bytes + key_bytes = Base64.decode64(aes_key_blob) + encryption_config[:encryption_key] = key_bytes + encryption_config[:key_wrap_schema] = :aes_gcm + end + + # Apply legacy settings + encryption_config.tap do |hash| if !config['enableLegacyWrappingAlgorithms'].nil? || !config['enableLegacyUnauthenticatedModes'].nil? legacy_modes = config['enableLegacyWrappingAlgorithms'] || config['enableLegacyUnauthenticatedModes'] # Set security profile based on legacy wrapping algorithms setting diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb index a6fb551f..115183b2 100644 --- a/test-server/ruby-v3-server/lib/client_manager.rb +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -2,6 +2,8 @@ require 'securerandom' require 'aws-sdk-s3' require 'aws-sdk-kms' +require 'openssl' +require 'base64' require_relative 'logger' # Manages S3 Encryption Client instances @@ -14,11 +16,17 @@ def initialize # Create a new S3 encryption client and return its ID def create_client(config) - # Extract configuration + # Extract all key material types kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + rsa_key_blob = config.dig('keyMaterial', 'rsaKey') + aes_key_blob = config.dig('keyMaterial', 'aesKey') inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') content_alg = config.dig('encryptionAlgorithm') + # Validate that only one key type is provided + key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count + raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 + # translate between canonical AlgSuite and Ruby symbols if content_alg.nil? || content_alg == 'ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY' content_alg = :alg_aes_256_gcm_hkdf_sha512_commit_key @@ -28,16 +36,32 @@ def create_client(config) raise 'Unknown content encryption algorithm provided: ' + content_alg end - raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? - # Create S3 encryption client configuration encryption_config = { - kms_key_id: kms_key_id, - kms_client: @kms_client, - key_wrap_schema: :kms_context, envelope_location: inst_file_put ? :instruction_file : :metadata, content_encryption_schema: content_alg - }.tap do |hash| + } + + # Configure based on key type + if kms_key_id + encryption_config[:kms_key_id] = kms_key_id + encryption_config[:kms_client] = @kms_client + encryption_config[:key_wrap_schema] = :kms_context + elsif rsa_key_blob + # Parse RSA private key from PKCS8 format + key_bytes = Base64.decode64(rsa_key_blob) + rsa_key = OpenSSL::PKey::RSA.new(key_bytes) + encryption_config[:encryption_key] = rsa_key + encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 + elsif aes_key_blob + # Extract AES key bytes + key_bytes = Base64.decode64(aes_key_blob) + encryption_config[:encryption_key] = key_bytes + encryption_config[:key_wrap_schema] = :aes_gcm + end + + # Apply additional configuration + encryption_config.tap do |hash| if !config['commitmentPolicy'].nil? hash[:commitment_policy] = case config['commitmentPolicy'] when 'FORBID_ENCRYPT_ALLOW_DECRYPT' From 7c4ddb868d8a36bb0613e2cc9f697e97c85c345f Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 11:01:01 -0800 Subject: [PATCH 06/40] update names --- .../s3/InstructionFileFailures.java | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java index 5456d344..0433366b 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -292,9 +292,9 @@ void putObjectWithInstructionFile( // KMS instruction files decrypt @Order(10) - @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") + @ParameterizedTest(name = "{0}: Successfully decrypt KMS encrypted original and good-copy objects") @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + void decrypt_kms_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -324,9 +324,9 @@ void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTar } @Order(11) - @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is duplicated in metadata and instruction file") + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is duplicated in metadata and instruction file") @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + void decrypt_kms_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -348,9 +348,9 @@ void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUti } @Order(12) - @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is only in instruction file") + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is only in instruction file") @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + void decrypt_kms_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -372,9 +372,9 @@ void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageSe } @Order(13) - @ParameterizedTest(name = "{0}: Fail to decrypt duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @ParameterizedTest(name = "{0}: Fail to decrypt KMS duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + void decrypt_kms_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -397,9 +397,9 @@ void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.Langua } @Order(14) - @ParameterizedTest(name = "{0}: Fail to decrypt instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @ParameterizedTest(name = "{0}: Fail to decrypt KMS instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + void decrypt_kms_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -424,9 +424,9 @@ void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils // RSA instruction file decrypt @Order(20) - @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Successfully decrypt RSA encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_rsa_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -456,9 +456,9 @@ void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTar } @Order(21) - @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_rsa_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -480,9 +480,9 @@ void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUti } @Order(22) - @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_rsa_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -504,9 +504,9 @@ void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageSe } @Order(23) - @ParameterizedTest(name = "{0}: Fail to decrypt duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt RSA duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_rsa_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -529,9 +529,9 @@ void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.Langua } @Order(24) - @ParameterizedTest(name = "{0}: Fail to decrypt instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt RSA instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_rsa_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -556,9 +556,9 @@ void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils // AES instruction file decrypt @Order(30) - @ParameterizedTest(name = "{0}: Successfully decrypt original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Successfully decrypt AES encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_aes_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -588,9 +588,9 @@ void decrypt_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTar } @Order(31) - @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_aes_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -612,9 +612,9 @@ void decrypt_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUti } @Order(32) - @ParameterizedTest(name = "{0}: Fail to decrypt when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_aes_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -636,9 +636,9 @@ void decrypt_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageSe } @Order(33) - @ParameterizedTest(name = "{0}: Fail to decrypt duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt AES duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_aes_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() @@ -661,9 +661,9 @@ void decrypt_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.Langua } @Order(34) - @ParameterizedTest(name = "{0}: Fail to decrypt instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + @ParameterizedTest(name = "{0}: Fail to decrypt AES instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") + void decrypt_aes_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { S3ECTestServerClient client = TestUtils.testServerClientFor(language); CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() From aeb359c08f62596659fa6fff67d4da04390584b5 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 11:15:45 -0800 Subject: [PATCH 07/40] speed up the build --- test-server/Makefile | 2 +- test-server/java-v3-server/Makefile | 2 +- test-server/java-v3-transition-server/Makefile | 2 +- test-server/java-v4-server/Makefile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 9b18b857..255323a1 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -82,7 +82,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --info --parallel integ -Dtest.filter.servers="$(FILTER)" + ./gradlew --build-cache --info --parallel --no-daemon integ -Dtest.filter.servers="$(FILTER)" @echo "Tests completed successfully" # Stop the servers diff --git a/test-server/java-v3-server/Makefile b/test-server/java-v3-server/Makefile index 692e80b3..59dcdff5 100644 --- a/test-server/java-v3-server/Makefile +++ b/test-server/java-v3-server/Makefile @@ -7,7 +7,7 @@ PORT := 8080 build-server: @echo "Building Java V3 server..." - ./gradlew --build-cache --parallel build + ./gradlew --build-cache --parallel --no-daemon build start-server: @echo "Starting Java V3 server..." diff --git a/test-server/java-v3-transition-server/Makefile b/test-server/java-v3-transition-server/Makefile index 5a25a8aa..81726b59 100644 --- a/test-server/java-v3-transition-server/Makefile +++ b/test-server/java-v3-transition-server/Makefile @@ -10,7 +10,7 @@ build-server: cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." @echo "Building Java V3 Transition server..." - ./gradlew --build-cache --parallel build + ./gradlew --build-cache --parallel --no-daemon build start-server: @echo "Starting Java V3 Transition server..." diff --git a/test-server/java-v4-server/Makefile b/test-server/java-v4-server/Makefile index 418e0127..3d1aae2a 100644 --- a/test-server/java-v4-server/Makefile +++ b/test-server/java-v4-server/Makefile @@ -10,7 +10,7 @@ build-server: cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." @echo "Building Java V4 server..." - ./gradlew --build-cache --parallel build + ./gradlew --build-cache --parallel --no-daemon build start-server: @echo "Starting Java V4 server..." From 270f2d56d2cffc5f558f65d2270463e78d128a19 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 11:42:58 -0800 Subject: [PATCH 08/40] speed up wait --- .github/workflows/test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 880ca3f4..ef47fb14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -171,9 +171,7 @@ jobs: - name: Wait for servers to start run: cd test-server && make wait-all-servers env: - AWS_REGION: us-west-2 - TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} - TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} - name: Run run-tests run: cd test-server && make run-tests From 271fcbaf88fb6e467798148f5a508367a0d5d84d Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 12:35:16 -0800 Subject: [PATCH 09/40] update client building --- .../amazon/encryption/s3/InstructionFileFailures.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java index 0433366b..4f86a90b 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -381,6 +381,7 @@ void decrypt_kms_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.La .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn) .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String S3ECId = clientOutput.getClientId(); @@ -406,6 +407,7 @@ void decrypt_kms_with_instruction_file_commitment_fails_with_forbid_policy(TestU .config(S3ECConfig.builder() .keyMaterial(kmsKeyArn) .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String S3ECId = clientOutput.getClientId(); @@ -513,6 +515,7 @@ void decrypt_rsa_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.La .config(S3ECConfig.builder() .keyMaterial(RSA_KEY) .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String S3ECId = clientOutput.getClientId(); @@ -538,6 +541,7 @@ void decrypt_rsa_with_instruction_file_commitment_fails_with_forbid_policy(TestU .config(S3ECConfig.builder() .keyMaterial(RSA_KEY) .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String S3ECId = clientOutput.getClientId(); @@ -645,6 +649,7 @@ void decrypt_aes_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.La .config(S3ECConfig.builder() .keyMaterial(AES_KEY) .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String S3ECId = clientOutput.getClientId(); @@ -670,6 +675,7 @@ void decrypt_aes_with_instruction_file_commitment_fails_with_forbid_policy(TestU .config(S3ECConfig.builder() .keyMaterial(AES_KEY) .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) .build()) .build()); String S3ECId = clientOutput.getClientId(); From 231e3ad8d5769942080e7204cc457ec78a6b55d7 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Mon, 17 Nov 2025 13:54:46 -0800 Subject: [PATCH 10/40] fix: Go in instruction files tests (#104) Update Go for instruction file failures --- test-server/go-v3-transition-server/local-go-s3ec | 2 +- test-server/go-v4-server/local-go-s3ec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index f51a4402..e59a38ca 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit f51a4402c741cd989c7984336de560e9c54baf17 +Subproject commit e59a38caeddfcfbf41e064e125b5783cdfce3878 diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec index f51a4402..e59a38ca 160000 --- a/test-server/go-v4-server/local-go-s3ec +++ b/test-server/go-v4-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit f51a4402c741cd989c7984336de560e9c54baf17 +Subproject commit e59a38caeddfcfbf41e064e125b5783cdfce3878 From 0a4ef95f13898686818806fe48d8d4097f039abb Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Mon, 17 Nov 2025 15:37:43 -0800 Subject: [PATCH 11/40] better failures? --- .../amazon/encryption/s3/RoundTripTests.java | 2 +- .../amazon/encryption/s3/TestUtils.java | 46 +++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 468fc708..347c44e5 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -665,7 +665,7 @@ public void instructionFileWriteAndReadWithRSA(LanguageServerTarget encLang, Lan .key(objectKey + ".instruction") .build()); } - assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); + // assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); assertFalse(ptInstFile.asUtf8String().isEmpty()); // Read should be enabled by default GetObjectOutput output = decClient.getObject(GetObjectInput.builder() diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 90995dde..351f2c8c 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -548,23 +548,39 @@ public static void Decrypt( EncryptionAlgorithm expectedEncryptionAlgorithm, List expectedPlaintexts ) { + List failures = new ArrayList<>(); for (int i = 0; i < crossLanguageObjects.size(); i++) { - String objectKey = crossLanguageObjects.get(i); - String expectedPlaintext = expectedPlaintexts.get(i); - - GetObjectOutput output = client.getObject(GetObjectInput.builder() - .clientID(S3ECId) - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); + try { + String objectKey = crossLanguageObjects.get(i); + String expectedPlaintext = expectedPlaintexts.get(i); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); - // Then: Pass - assertEquals(expectedPlaintext, new String(output.getBody().array())); - assertEquals( - expectedEncryptionAlgorithm, - GetEncryptionAlgorithm(objectKey), - "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm - ); + // Then: Pass + assertEquals(expectedPlaintext, new String(output.getBody().array())); + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + } catch (Exception e) { + failures.add(String.format( + "Failed to decrypt object '%s' (index %d): %s - %s", + crossLanguageObjects.get(i), i, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Decryption failed for %d out of %d objects:\n%s", + failures.size(), crossLanguageObjects.size(), + String.join("\n", failures) + )); } } From 4d73663712f286a788c6efe398e90969328ba78d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Corella?= <39066999+josecorella@users.noreply.github.com> Date: Tue, 18 Nov 2025 11:25:46 -0800 Subject: [PATCH 12/40] bump php sdk to fixed instruction file (#106) Update PHP v3 to support failure on bad instruction files --- test-server/php-v2-transition-server/src/get_object.php | 4 ++++ test-server/php-v3-server/local-php-sdk | 2 +- test-server/php-v3-server/src/get_object.php | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test-server/php-v2-transition-server/src/get_object.php b/test-server/php-v2-transition-server/src/get_object.php index 5800e850..dcf683b6 100644 --- a/test-server/php-v2-transition-server/src/get_object.php +++ b/test-server/php-v2-transition-server/src/get_object.php @@ -80,6 +80,10 @@ function handleGetObject($params) } if (strpos($e->getMessage(), "@SecurityProfile=V2") !== false) { return S3EncryptionClientError($e->getMessage() . " " . "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"); + } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { + return S3EncryptionClientError($e->getMessage()); } else { error_log("This is the error: " . $e->getMessage()); return GenericServerError("Server error: " . $e->getMessage(), 500); diff --git a/test-server/php-v3-server/local-php-sdk b/test-server/php-v3-server/local-php-sdk index e32c9f2b..88ee9515 160000 --- a/test-server/php-v3-server/local-php-sdk +++ b/test-server/php-v3-server/local-php-sdk @@ -1 +1 @@ -Subproject commit e32c9f2b009a43cf88f2ab35e1e532114c8390c9 +Subproject commit 88ee95156f2884767b72f9219736e976d98a9c96 diff --git a/test-server/php-v3-server/src/get_object.php b/test-server/php-v3-server/src/get_object.php index 3de7f779..6fb28551 100644 --- a/test-server/php-v3-server/src/get_object.php +++ b/test-server/php-v3-server/src/get_object.php @@ -84,6 +84,10 @@ function handleGetObject($params) return S3EncryptionClientError($e->getMessage()); } elseif (strpos($e->getMessage(), "Message is encrypted with a non commiting algorithm but commitment policy is set to REQUIRE_ENCRYPT_REQUIRE_DECRYPT. Select a valid commitment policy to decrypt this object.") !== false) { return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { + return S3EncryptionClientError($e->getMessage()); } else { error_log("This is the error: " . $e->getMessage()); return GenericServerError("Server argument: " . $e->getMessage(), 500); From 63e0ed188f14c3b11b01bedea557622c5db31191 Mon Sep 17 00:00:00 2001 From: seebees Date: Wed, 19 Nov 2025 14:59:58 -0800 Subject: [PATCH 13/40] faster parallel tests (#107) --- .github/workflows/test.yml | 31 +- test-server/Makefile | 18 +- test-server/cpp-v2-server/Makefile | 2 +- test-server/cpp-v2-server/main.cpp | 44 +- test-server/cpp-v2-transition-server/Makefile | 2 +- test-server/cpp-v2-transition-server/main.cpp | 111 ++++- test-server/cpp-v3-server/CMakeLists.txt | 16 +- test-server/cpp-v3-server/Makefile | 2 +- test-server/cpp-v3-server/main.cpp | 124 +++++- test-server/go-v3-server/main.go | 14 +- test-server/go-v3-transition-server/main.go | 14 +- test-server/go-v4-server/main.go | 14 +- test-server/java-tests/build.gradle.kts | 14 + .../amazon/encryption/s3/GCMTestSuite.java | 255 ++++++++++++ .../amazon/encryption/s3/GCMTests.java | 203 --------- .../amazon/encryption/s3/KC_GCMTestSuite.java | 389 ++++++++++++++++++ .../amazon/encryption/s3/KC_GCMTests.java | 264 ------------ .../amazon/encryption/s3/RoundTripTests.java | 14 +- .../amazon/encryption/s3/TestUtils.java | 2 + test-server/java-v3-server/gradle.properties | 19 +- .../s3/CreateClientOperationImpl.java | 16 + .../gradle.properties | 19 +- .../java-v3-transition-server/s3ec-staging | 2 +- .../s3/CreateClientOperationImpl.java | 16 + test-server/java-v4-server/gradle.properties | 19 +- test-server/java-v4-server/s3ec-staging | 2 +- .../s3/CreateClientOperationImpl.java | 16 + .../Controllers/ClientController.cs | 18 +- .../Controllers/ClientController.cs | 18 +- .../s3ec-v3-transition-branch | 2 +- .../Controllers/ClientController.cs | 17 +- test-server/net-v4-server/Makefile | 2 +- .../net-v4-server/s3ec-net-v4-improved | 2 +- .../ruby-v2-server/lib/client_manager.rb | 8 +- .../ruby-v3-server/lib/client_manager.rb | 8 +- 35 files changed, 1121 insertions(+), 596 deletions(-) create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java delete mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java delete mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef47fb14..210e5e7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,15 +41,15 @@ jobs: git config --global credential.helper store echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials - - name: Cache git submodules - uses: actions/cache@v4 - with: - path: | - .git/modules - test-server/*/.git - key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }} - restore-keys: | - ${{ runner.os }}-submodules- + # - name: Cache git submodules + # uses: actions/cache@v4 + # with: + # path: | + # .git/modules + # test-server/*/.git + # key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }} + # restore-keys: | + # ${{ runner.os }}-submodules- - name: Optimize git for performance run: | @@ -59,13 +59,12 @@ jobs: - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} - name: Update cpp submodules recursively with --jobs run: | git submodule update --init --recursive \ - --depth 1 \ - --filter=blob:none \ + --depth 1 --single-branch \ --jobs ${{ steps.cpu-count.outputs.count }} \ --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ @@ -156,25 +155,25 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers + run: cd test-server && make build-all-servers FILTER=ruby,java,php,net,go env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers + run: cd test-server && make start-all-servers FILTER=ruby,java,php,net,go env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers + run: cd test-server && make wait-all-servers FILTER=ruby,java,php,net,go env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} - name: Run run-tests - run: cd test-server && make run-tests + run: cd test-server && make run-tests FILTER=ruby,java,php,net,go env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} diff --git a/test-server/Makefile b/test-server/Makefile index 255323a1..28f6e1be 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -2,9 +2,6 @@ .PHONY: all start-servers run-tests stop-servers clean ci check-env help -# Default target -all: start-all-servers wait-all-servers run-tests - # CI target for GitHub Actions ci: $(MAKE) build-all-servers @@ -20,13 +17,19 @@ START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) # Build all servers in parallel -build-all-servers: export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1) -build-all-servers: $(BUILD_SERVER_TARGETS) +build-all-servers: + @echo "[`date +%H:%M:%S`] Building all servers..." + @$(MAKE) $(BUILD_SERVER_TARGETS) + @echo "[`date +%H:%M:%S`] All servers built." + @echo "Stopping dotnet servers... this is here to speed up CI because if this target is run with -j it will stay open until it shutdown." + @dotnet build-server shutdown + @echo "[`date +%H:%M:%S`] Dotnet build servers stopped" $(BUILD_SERVER_TARGETS): build-%: @if [ -f $*/Makefile ]; then \ - echo "Building server in $*..."; \ + echo "[`date +%H:%M:%S`] Building server in $*..."; \ $(MAKE) -C $* build-server; \ + echo "[`date +%H:%M:%S`] Server $* built successfully"; \ else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ @@ -44,9 +47,8 @@ start-servers: $(MAKE) -C $$dir wait-for-server; \ done -# Start servers sequentially (no parallel execution) start-all-servers: - @$(MAKE) MAKEFLAGS= $(START_SERVER_TARGETS) + @$(MAKE) $(START_SERVER_TARGETS) $(START_SERVER_TARGETS): start-%: @if [ -f $*/Makefile ]; then \ diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 77357c37..e4d2f954 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -11,7 +11,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp V2 server..." - cd build && make + cd build && $(MAKE) start-server: @echo "Starting Cpp V2 server..." diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index a2b05810..1b2de659 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -16,8 +16,8 @@ using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> - client_cache; +std::unordered_map> client_cache_secret; +std::mutex client_mutex; std::string generate_uuid() { uuid_t uuid; @@ -27,6 +27,23 @@ std::string generate_uuid() { return std::string(uuid_str); } +std::shared_ptr get_client(const std::string &client_id) +{ + std::lock_guard lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + return it->second; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + std::lock_guard lock(client_mutex); + client_cache_secret[client_id] = client; +} + std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -74,7 +91,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); - client_cache[client_id] = encryption_client; + set_client(client_id, encryption_client); json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); @@ -144,8 +161,8 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + auto client = get_client(client_id); + if (!client) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -154,18 +171,9 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - // S3EncryptionGetObjectOutcome outcome ; - // if (metadata.empty()) { - // outcome = it->second->GetObject(request); - // } else { - // Aws::Map kmsContextMap; - // fill_context(kmsContextMap, metadata); - // outcome = it->second->GetObject(request, kmsContextMap); - // } - Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - auto outcome = it->second->GetObject(request, kmsContextMap); + auto outcome = client->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { auto &stream = outcome.GetResult().GetBody(); @@ -187,8 +195,8 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, const std::string &client_id, const std::string &body, const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + auto client = get_client(client_id); + if (!client) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -203,7 +211,7 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, auto stream = std::make_shared(body); request.SetBody(stream); - auto outcome = it->second->PutObject(request, kmsContextMap); + auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 16b70796..2ca6ee5f 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp transition server..." - cd build && make + cd build && $(MAKE) start-server: @echo "Starting Cpp transition server..." diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index 1fcedc3c..3514b594 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -16,8 +18,8 @@ using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> - client_cache; +std::unordered_map> client_cache_secret; +std::mutex client_mutex; std::string generate_uuid() { uuid_t uuid; @@ -27,6 +29,23 @@ std::string generate_uuid() { return std::string(uuid_str); } +std::shared_ptr get_client(const std::string &client_id) +{ + std::lock_guard lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + return it->second; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + std::lock_guard lock(client_mutex); + client_cache_secret[client_id] = client; +} + std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -79,7 +98,41 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, return MHD_YES; } - std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + // Extract all key material types + std::string kms_key_id; + std::string rsa_key_blob; + std::string aes_key_blob; + + if (request["config"]["keyMaterial"].contains("kmsKeyId") && + !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { + kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + } + if (request["config"]["keyMaterial"].contains("rsaKey") && + !request["config"]["keyMaterial"]["rsaKey"].is_null()) { + rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; + } + if (request["config"]["keyMaterial"].contains("aesKey") && + !request["config"]["keyMaterial"]["aesKey"].is_null()) { + aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; + } + + // Validate that only one key type is provided + int key_count = 0; + if (!kms_key_id.empty()) key_count++; + if (!rsa_key_blob.empty()) key_count++; + if (!aes_key_blob.empty()) key_count++; + + if (key_count != 1) { + return send_response(connection, 400, + "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); + } + + // RSA is not supported by C++ SDK + if (!rsa_key_blob.empty()) { + return send_response(connection, 501, + "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); + } + bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; bool inst_put = false; @@ -88,8 +141,33 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - auto materials = - std::make_shared(kms_key_id); + // Create appropriate encryption materials based on key type + std::shared_ptr materials; + + if (!aes_key_blob.empty()) { + // Base64 decode the AES key + auto decoded = Aws::Utils::Base64::Decode(aes_key_blob); + if (!decoded.IsSuccess()) { + return send_response(connection, 400, + "{\"error\":\"Failed to decode AES key\"}"); + } + + Aws::Utils::CryptoBuffer key_buffer( + decoded.GetResult().GetUnderlyingData(), + decoded.GetResult().GetLength() + ); + + materials = std::make_shared< + Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( + key_buffer + ); + } else if (!kms_key_id.empty()) { + materials = std::make_shared(kms_key_id); + } else { + return send_response(connection, 400, + "{\"error\":\"No valid key material provided\"}"); + } + CryptoConfigurationV2 config(materials); if (legacy1 || legacy2) config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); @@ -99,7 +177,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); - client_cache[client_id] = encryption_client; + set_client(client_id, encryption_client); json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); @@ -169,8 +247,8 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + auto client = get_client(client_id); + if (!client) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -179,18 +257,9 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - // S3EncryptionGetObjectOutcome outcome ; - // if (metadata.empty()) { - // outcome = it->second->GetObject(request); - // } else { - // Aws::Map kmsContextMap; - // fill_context(kmsContextMap, metadata); - // outcome = it->second->GetObject(request, kmsContextMap); - // } - Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - auto outcome = it->second->GetObject(request, kmsContextMap); + auto outcome = client->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { auto &stream = outcome.GetResult().GetBody(); @@ -212,8 +281,8 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, const std::string &client_id, const std::string &body, const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + auto client = get_client(client_id); + if (!client) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -228,7 +297,7 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, auto stream = std::make_shared(body); request.SetBody(stream); - auto outcome = it->second->PutObject(request, kmsContextMap); + auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); diff --git a/test-server/cpp-v3-server/CMakeLists.txt b/test-server/cpp-v3-server/CMakeLists.txt index b282dbc4..0faac5f0 100644 --- a/test-server/cpp-v3-server/CMakeLists.txt +++ b/test-server/cpp-v3-server/CMakeLists.txt @@ -7,6 +7,7 @@ set(CMAKE_CXX_STANDARD 17) set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") +set(ENABLE_ADDRESS_SANITIZER ON CACHE BOOL "Enable Address Sanitizer") # Add AWS SDK as subdirectory add_subdirectory(aws-sdk-cpp) @@ -18,12 +19,21 @@ find_package(nlohmann_json REQUIRED) add_executable(s3ec-server main.cpp) -target_include_directories(s3ec-server PRIVATE +# Enable Address Sanitizer for the executable +target_compile_options(s3ec-server PRIVATE -fsanitize=address -fno-omit-frame-pointer) +target_link_options(s3ec-server PRIVATE -fsanitize=address) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_include_directories(s3ec-server PRIVATE ${LIBMICROHTTPD_INCLUDE_DIRS} /opt/homebrew/include ) -target_link_directories(s3ec-server PRIVATE +target_link_directories(s3ec-server PRIVATE ${LIBMICROHTTPD_LIBRARY_DIRS} /opt/homebrew/lib ) @@ -36,4 +46,4 @@ target_link_libraries(s3ec-server aws-cpp-sdk-s3-encryption nlohmann_json::nlohmann_json uuid -) \ No newline at end of file +) diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 46f0c9db..05a286f0 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp V3 server..." - cd build && make + cd build && $(MAKE) start-server: @echo "Starting Cpp V3 server..." diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 1f74974c..21360b90 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include #include @@ -12,12 +14,13 @@ #include #include #include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> - client_cache; +std::unordered_map> client_cache_secret; +std::mutex client_mutex; std::string generate_uuid() { uuid_t uuid; @@ -27,6 +30,23 @@ std::string generate_uuid() { return std::string(uuid_str); } +std::shared_ptr get_client(const std::string &client_id) +{ + std::lock_guard lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + return it->second; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + std::lock_guard lock(client_mutex); + client_cache_secret[client_id] = client; +} + std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -69,7 +89,42 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { try { json request = json::parse(body); - std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + + // Extract all key material types + std::string kms_key_id; + std::string rsa_key_blob; + std::string aes_key_blob; + + if (request["config"]["keyMaterial"].contains("kmsKeyId") && + !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { + kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + } + if (request["config"]["keyMaterial"].contains("rsaKey") && + !request["config"]["keyMaterial"]["rsaKey"].is_null()) { + rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; + } + if (request["config"]["keyMaterial"].contains("aesKey") && + !request["config"]["keyMaterial"]["aesKey"].is_null()) { + aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; + } + + // Validate that only one key type is provided + int key_count = 0; + if (!kms_key_id.empty()) key_count++; + if (!rsa_key_blob.empty()) key_count++; + if (!aes_key_blob.empty()) key_count++; + + if (key_count != 1) { + return send_response(connection, 400, + "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); + } + + // RSA is not supported by C++ SDK + if (!rsa_key_blob.empty()) { + return send_response(connection, 501, + "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); + } + bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; bool inst_put = false; @@ -78,9 +133,43 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - auto materials = - std::make_shared(kms_key_id); - CryptoConfigurationV3 config(materials); + // Create appropriate encryption materials based on key type + std::shared_ptr materials; + + if (!aes_key_blob.empty()) { + // Base64 decode the AES key + auto decoded = Aws::Utils::Base64::Decode(aes_key_blob); + if (!decoded.IsSuccess()) { + return send_response(connection, 400, + "{\"error\":\"Failed to decode AES key\"}"); + } + + Aws::Utils::CryptoBuffer key_buffer( + decoded.GetResult().GetUnderlyingData(), + decoded.GetResult().GetLength() + ); + + materials = std::make_shared< + Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( + key_buffer + ); + } else if (!kms_key_id.empty()) { + materials = std::make_shared(kms_key_id); + } else { + return send_response(connection, 400, + "{\"error\":\"No valid key material provided\"}"); + } + + // Configure ClientConfiguration with retry strategy for throttling + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 25; + clientConfig.retryStrategy = Aws::MakeShared( + "S3EncryptionClient", + 5 // maxRetries - will use exponential backoff for throttling + ); + + CryptoConfigurationV3 config(materials); + config.SetClientConfiguration(clientConfig); if (legacy1 || legacy2) config.AllowLegacy(); if (inst_put) @@ -104,7 +193,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); - client_cache[client_id] = encryption_client; + set_client(client_id, encryption_client); json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); @@ -175,8 +264,8 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + auto client = get_client(client_id); + if (!client) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -185,18 +274,9 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - // S3EncryptionGetObjectOutcome outcome ; - // if (metadata.empty()) { - // outcome = it->second->GetObject(request); - // } else { - // Aws::Map kmsContextMap; - // fill_context(kmsContextMap, metadata); - // outcome = it->second->GetObject(request, kmsContextMap); - // } - Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - auto outcome = it->second->GetObject(request, kmsContextMap); + auto outcome = client->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { auto &stream = outcome.GetResult().GetBody(); @@ -220,8 +300,8 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, const std::string &client_id, const std::string &body, const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + auto client = get_client(client_id); + if (!client) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -236,7 +316,7 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, auto stream = std::make_shared(body); request.SetBody(stream); - auto outcome = it->second->PutObject(request, kmsContextMap); + auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); diff --git a/test-server/go-v3-server/main.go b/test-server/go-v3-server/main.go index d201ffe2..a79f007e 100644 --- a/test-server/go-v3-server/main.go +++ b/test-server/go-v3-server/main.go @@ -66,7 +66,12 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -141,7 +146,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index 799a9668..3c92582e 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -68,7 +68,12 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -143,7 +148,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return diff --git a/test-server/go-v4-server/main.go b/test-server/go-v4-server/main.go index 672236ac..d9d897aa 100644 --- a/test-server/go-v4-server/main.go +++ b/test-server/go-v4-server/main.go @@ -68,7 +68,12 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -143,7 +148,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 106f82ef..14c3eec1 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -16,6 +16,9 @@ dependencies { // Test dependencies testImplementation("org.junit.jupiter:junit-jupiter:5.13.0") testRuntimeOnly("org.junit.platform:junit-platform-launcher") + // JUnit Suite support for test ordering + testImplementation("org.junit.platform:junit-platform-suite-api:1.10.0") + testRuntimeOnly("org.junit.platform:junit-platform-suite-engine:1.10.0") testImplementation("com.amazonaws:aws-java-sdk:1.12.788") testImplementation("software.amazon.awssdk:s3:2.37.1") testImplementation("org.bouncycastle:bcprov-jdk15on:1.70") @@ -49,6 +52,17 @@ tasks { classpath = sourceSets["it"].runtimeClasspath outputs.upToDateWhen { false } outputs.cacheIf { false } + + // Enable parallel test execution + systemProperty("junit.jupiter.execution.parallel.enabled", "true") + systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") + systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") + // Configure thread pool size - adjust based on I/O-bound nature of tests + systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") + maxParallelForks = 1 // One JVM + systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", + Math.max(1, Runtime.getRuntime().availableProcessors() - 2).toString()) // Scale with CPU, reserve 2 cores + // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) // For debugging diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java new file mode 100644 index 00000000..4be4d434 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java @@ -0,0 +1,255 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * GCM Test Suite + * + * This suite enforces execution order between GCM encrypt and decrypt phases: + * 1. EncryptTests - All encrypt tests run in parallel (within this phase) + * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel + * + * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion + * and DecryptTests awaits before proceeding. + */ +public class GCMTestSuite { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * GCM Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using GCM (without key commitment) encryption algorithm. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + class EncryptTests { + private static final String sharedObjectKeyBase = "test-gcm-kms"; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe list for storing encrypted object keys + private static final List crossLanguageObjects = + Collections.synchronizedList(new ArrayList<>()); + + /** + * Public accessor for decrypt tests to retrieve encrypted object keys + */ + static List getCrossLanguageObjects() { + return new ArrayList<>(crossLanguageObjects); // Return defensive copy + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @AfterAll + static void signalEncryptionComplete() { + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * GCM Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first (enforced by @Order). + */ + @Nested + class DecryptTests { + private static List crossLanguageObjects; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects from the encrypt phase + crossLanguageObjects = EncryptTests.getCrossLanguageObjects(); + + // Verify we have objects to decrypt + if (crossLanguageObjects.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java deleted file mode 100644 index 6eef0b5f..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java +++ /dev/null @@ -1,203 +0,0 @@ -/* -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0 -*/ - -package software.amazon.encryption.s3; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static software.amazon.encryption.s3.TestUtils.*; - -import java.lang.annotation.ElementType; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import com.amazonaws.services.s3.model.KMSEncryptionMaterials; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.GetObjectInput; -import software.amazon.encryption.s3.model.GetObjectOutput; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.PutObjectInput; -import software.amazon.encryption.s3.model.S3ECConfig; -import software.amazon.encryption.s3.model.S3EncryptionClientError; - -import com.amazonaws.services.s3.AmazonS3Encryption; -import com.amazonaws.services.s3.AmazonS3EncryptionClient; -import com.amazonaws.services.s3.model.CryptoConfiguration; -import com.amazonaws.services.s3.model.CryptoMode; -import com.amazonaws.services.s3.model.CryptoStorageMode; -import software.amazon.encryption.s3.TestUtils.*; -import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; -import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; - -/** -* Exhaustive tests for S3 Encryption Client round-trip operations. -* These tests cover various combinations of client versions, commitment policies, and encryption modes. -* -* Tests are based on the exhaustive test matrix defined at: -* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 -* -*/ - -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class GCMTests { - private static String sharedObjectKeyBase = "test-gcm-kms"; - private static KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - private static List crossLanguageObjects = new ArrayList<>(); - - @Order(1) - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(3) - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(10) - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(11) - @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(12) - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(13) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(14) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java new file mode 100644 index 00000000..c367315f --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java @@ -0,0 +1,389 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.TestAbortedException; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * KC-GCM Test Suite + * + * This suite enforces execution order between KC-GCM encrypt and decrypt phases: + * 1. EncryptTests - All encrypt tests run in parallel (within this phase) + * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel + * + * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion + * and DecryptTests awaits before proceeding. + */ +public class KC_GCMTestSuite { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * KC-GCM Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using Key Commitment GCM encryption algorithm. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + class EncryptTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; + private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-rsa-instruction-file"; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List crossLanguageObjectsMetaDataMode = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsInstructionFiles = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyPair RSA_KEY_PAIR_1; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + } + + /** + * Public accessors for decrypt tests to retrieve encrypted object keys and RSA key + */ + static List getCrossLanguageObjectsMetaDataMode() { + return new ArrayList<>(crossLanguageObjectsMetaDataMode); + } + + static List getCrossLanguageObjectsInstructionFiles() { + return new ArrayList<>(crossLanguageObjectsInstructionFiles); + } + + static KeyPair getRsaKeyPair() { + return RSA_KEY_PAIR_1; + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file( + TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), + crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_encrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @AfterAll + static void signalEncryptionComplete() { + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * KC-GCM Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first (enforced by @Order). + */ + @Nested + class DecryptTests { + private static List crossLanguageObjectsMetaDataMode; + private static List crossLanguageObjectsInstructionFiles; + private static KeyPair RSA_KEY_PAIR_1; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects and RSA key from the encrypt phase + crossLanguageObjectsMetaDataMode = EncryptTests.getCrossLanguageObjectsMetaDataMode(); + crossLanguageObjectsInstructionFiles = EncryptTests.getCrossLanguageObjectsInstructionFiles(); + RSA_KEY_PAIR_1 = EncryptTests.getRsaKeyPair(); + + // Verify we have objects to decrypt + if (crossLanguageObjectsMetaDataMode.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file( + final TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Transition configured with default should decrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file( + final TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java deleted file mode 100644 index ee4279d6..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java +++ /dev/null @@ -1,264 +0,0 @@ -/* -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0 -*/ - -package software.amazon.encryption.s3; - -import static software.amazon.encryption.s3.TestUtils.*; - -import java.nio.ByteBuffer; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.opentest4j.TestAbortedException; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.InstructionFileConfig; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.S3ECConfig; - -/** -* Exhaustive tests for S3 Encryption Client round-trip operations. -* These tests cover various combinations of client versions, commitment policies, and encryption modes. -* -* Tests are based on the exhaustive test matrix defined at: -* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 -* -*/ - -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class KC_GCMTests { - private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; - private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-kms-instruction-file"; - private static KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - private static final List crossLanguageObjectsMetaDataMode = new ArrayList<>(); - private static final List crossLanguageObjectsInstructionFiles = new ArrayList<>(); - private static KeyPair RSA_KEY_PAIR_1; - - @BeforeAll - static void setupKeys() throws Exception { - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); - } - - @Order(1) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file(TestUtils.LanguageServerTarget language) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), crossLanguageObjectsInstructionFiles, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_the_default_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(10) - @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(11) - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(12) - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(13) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(14) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(15) - @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_the_default_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(16) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file(final TestUtils.LanguageServerTarget language) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 347c44e5..2e946030 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -214,9 +214,9 @@ public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLa fail("Expected exception!"); } catch (S3EncryptionClientError e) { if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_CURRENT) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { - assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); } else { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); } } } @@ -278,9 +278,9 @@ public void crossLanguageTestKmsWithIncorrectEncCtxFails(LanguageServerTarget en fail("Expected exception!"); } catch (S3EncryptionClientError e) { if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_CURRENT) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { - assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); } else { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); } } } @@ -427,15 +427,15 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(TestUtils.LanguageServerTarget la || language.getLanguageName().equals(CPP_V2_CURRENT) || language.getLanguageName().equals(CPP_V2_TRANSITION) || language.getLanguageName().equals(CPP_V3)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" - )); + ), "Actual error:" + e.getMessage()); } else if (language.getLanguageName().equals(RUBY_V3) || language.getLanguageName().equals(RUBY_V2_CURRENT) || language.getLanguageName().equals(RUBY_V2_TRANSITION)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object." ), "Actual error:" + e.getMessage()); } else if (language.getLanguageName().equals(PHP_V3)) { - assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."));; + assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."), "Actual error: " + e.getMessage()); } else { - assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); + assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"), "Actual error: " + e.getMessage()); } } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 351f2c8c..a7a86c1d 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -541,6 +541,8 @@ public static void Decrypt( Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, crossLanguageObjects); } + + public static void Decrypt( S3ECTestServerClient client, String S3ECId, diff --git a/test-server/java-v3-server/gradle.properties b/test-server/java-v3-server/gradle.properties index 08afce82..483cd315 100644 --- a/test-server/java-v3-server/gradle.properties +++ b/test-server/java-v3-server/gradle.properties @@ -4,8 +4,21 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index 1d198590..bdb0b30b 100644 --- a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,5 +1,8 @@ package software.amazon.encryption.s3; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.core.traits.Trait; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.S3EncryptionClient; @@ -106,12 +109,25 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + // Client Creation boolean instFilePut = false; if (input.getConfig().getInstructionFileConfig() != null) { instFilePut = input.getConfig().getInstructionFileConfig().isEnableInstructionFilePutObject(); } S3Client s3Client = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) .instructionFileConfig(InstructionFileConfig.builder() .instructionFileClient(S3Client.create()) .enableInstructionFilePutObject(instFilePut) diff --git a/test-server/java-v3-transition-server/gradle.properties b/test-server/java-v3-transition-server/gradle.properties index 08afce82..483cd315 100644 --- a/test-server/java-v3-transition-server/gradle.properties +++ b/test-server/java-v3-transition-server/gradle.properties @@ -4,8 +4,21 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v3-transition-server/s3ec-staging b/test-server/java-v3-transition-server/s3ec-staging index 6413811b..597d5b49 160000 --- a/test-server/java-v3-transition-server/s3ec-staging +++ b/test-server/java-v3-transition-server/s3ec-staging @@ -1 +1 @@ -Subproject commit 6413811bb81037999b8238e02047e0e403f78c1f +Subproject commit 597d5b491ac578f5d03d3fa757201eb48690cd00 diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index 425c0334..e107401e 100644 --- a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,5 +1,8 @@ package software.amazon.encryption.s3; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.algorithms.AlgorithmSuite; @@ -106,9 +109,22 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + // V3 Transition server configuration // Existing Builder defaults to FORBID_ENCRYPT and ALG_AES_256_GCM_IV12_TAG16_NO_KDF S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) .keyring(keyring) .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); diff --git a/test-server/java-v4-server/gradle.properties b/test-server/java-v4-server/gradle.properties index 08afce82..483cd315 100644 --- a/test-server/java-v4-server/gradle.properties +++ b/test-server/java-v4-server/gradle.properties @@ -4,8 +4,21 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v4-server/s3ec-staging b/test-server/java-v4-server/s3ec-staging index db0c743e..2626ed6e 160000 --- a/test-server/java-v4-server/s3ec-staging +++ b/test-server/java-v4-server/s3ec-staging @@ -1 +1 @@ -Subproject commit db0c743eec335d16e6dceaf2b09d84becb0f74f8 +Subproject commit 2626ed6e312c1c5e01abea2f30727ea0f2af299d diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index cb20d5ac..3fdb4b55 100644 --- a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,5 +1,8 @@ package software.amazon.encryption.s3; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.algorithms.AlgorithmSuite; @@ -106,8 +109,21 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + // V4-Improved server configuration S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builderV4() + .wrappedClient(wrappedClient) .keyring(keyring) .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); diff --git a/test-server/net-v2-v3-server/Controllers/ClientController.cs b/test-server/net-v2-v3-server/Controllers/ClientController.cs index e33a58e6..437233a8 100644 --- a/test-server/net-v2-v3-server/Controllers/ClientController.cs +++ b/test-server/net-v2-v3-server/Controllers/ClientController.cs @@ -21,8 +21,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "[NET-current] EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "[NET-current] SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "[NET-current] AesKey not supported" }); try { @@ -47,6 +45,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "Created EncryptionMaterialsV2: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-current] Created EncryptionMaterialsV2: AES"); } else { return StatusCode(501, new GenericServerError { Message = "[NET-current] Unknown or missing key material!" }); @@ -62,6 +69,11 @@ public IActionResult CreateClient([FromBody] ClientRequest request) logger.LogInformation("[NET-current] Created securityProfile= {securityProfile}", securityProfile.ToString()); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile); + + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -91,4 +103,4 @@ public IActionResult CreateClient([FromBody] ClientRequest request) }); } } -} \ No newline at end of file +} diff --git a/test-server/net-v3-transition-server/Controllers/ClientController.cs b/test-server/net-v3-transition-server/Controllers/ClientController.cs index a66fb342..3deeff61 100644 --- a/test-server/net-v3-transition-server/Controllers/ClientController.cs +++ b/test-server/net-v3-transition-server/Controllers/ClientController.cs @@ -21,8 +21,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "AesKey not supported" }); try { @@ -47,6 +45,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "Created EncryptionMaterialsV2: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V3-Transitional] Created EncryptionMaterialsV2: AES"); } else { return StatusCode(501, new GenericServerError { Message = "Unknown or missing key material!" }); @@ -67,6 +74,11 @@ public IActionResult CreateClient([FromBody] ClientRequest request) logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm); + + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -118,4 +130,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti _ => ContentEncryptionAlgorithm.AesGcm }; } -} \ No newline at end of file +} diff --git a/test-server/net-v3-transition-server/s3ec-v3-transition-branch b/test-server/net-v3-transition-server/s3ec-v3-transition-branch index ad825917..d099cfd1 160000 --- a/test-server/net-v3-transition-server/s3ec-v3-transition-branch +++ b/test-server/net-v3-transition-server/s3ec-v3-transition-branch @@ -1 +1 @@ -Subproject commit ad8259173de365a13e8b3932ee02493f599f597f +Subproject commit d099cfd151e2c61fb97dcd417828fb1dd5468b0c diff --git a/test-server/net-v4-server/Controllers/ClientController.cs b/test-server/net-v4-server/Controllers/ClientController.cs index b9fbe3f9..2ef8b921 100644 --- a/test-server/net-v4-server/Controllers/ClientController.cs +++ b/test-server/net-v4-server/Controllers/ClientController.cs @@ -20,8 +20,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "[NET-V4] EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "[NET-V4] SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "[NET-V4] AesKey not supported" }); try { @@ -46,6 +44,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV4(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "[NET-V4] Created EncryptionMaterialsV4: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV4(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V4] Created EncryptionMaterialsV4: AES"); } else { return StatusCode(501, new GenericServerError { Message = "[NET-V4] Unknown or missing key material!" }); @@ -79,6 +86,10 @@ public IActionResult CreateClient([FromBody] ClientRequest request) ? new AmazonS3CryptoConfigurationV4() : new AmazonS3CryptoConfigurationV4(securityProfile, commitmentPolicy, encryptionAlgorithm); + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -130,4 +141,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti _ => ContentEncryptionAlgorithm.AesGcmWithCommitment }; } -} \ No newline at end of file +} diff --git a/test-server/net-v4-server/Makefile b/test-server/net-v4-server/Makefile index e2df658a..b52bbd49 100644 --- a/test-server/net-v4-server/Makefile +++ b/test-server/net-v4-server/Makefile @@ -32,7 +32,7 @@ start-net-V4-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run --no-build & echo $! > $(PID_FILE_NET_V4) + dotnet run --no-build > server.log 2>&1 & echo $! > $(PID_FILE_NET_V4) @echo ".NET V4 server starting..." wait-for-server: diff --git a/test-server/net-v4-server/s3ec-net-v4-improved b/test-server/net-v4-server/s3ec-net-v4-improved index 1c0a458c..8ce8983b 160000 --- a/test-server/net-v4-server/s3ec-net-v4-improved +++ b/test-server/net-v4-server/s3ec-net-v4-improved @@ -1 +1 @@ -Subproject commit 1c0a458c19b351c266199c72072de746362c5326 +Subproject commit 8ce8983bd0edf973651aee0c29894df9091cf97a diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb index 85a91038..3da62b45 100644 --- a/test-server/ruby-v2-server/lib/client_manager.rb +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -60,7 +60,13 @@ def create_client(config) end # Create the S3 encryption client - s3_client = Aws::S3::Client.new(region: 'us-west-2') + # Create the S3 encryption client with retry configuration for throttling + s3_client = Aws::S3::Client.new( + region: 'us-west-2', + retry_mode: 'adaptive', + retry_limit: 5, + retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } + ) encryption_client = Aws::S3::EncryptionV2::Client.new( client: s3_client, **encryption_config diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb index 115183b2..5ee3f1ec 100644 --- a/test-server/ruby-v3-server/lib/client_manager.rb +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -85,7 +85,13 @@ def create_client(config) end # Create the S3 encryption client - s3_client = Aws::S3::Client.new(region: 'us-west-2') + # Create the S3 encryption client with retry configuration for throttling + s3_client = Aws::S3::Client.new( + region: 'us-west-2', + retry_mode: 'adaptive', + retry_limit: 5, + retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } + ) encryption_client = Aws::S3::EncryptionV3::Client.new( client: s3_client, **encryption_config From 91933b7f7694c07c07c1649ef44089a229ca39e9 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 19 Nov 2025 15:03:28 -0800 Subject: [PATCH 14/40] enable cpp --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 210e5e7c..d6b7a9d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,25 +155,25 @@ jobs: aws-region: us-west-2 - name: Build the servers - run: cd test-server && make build-all-servers FILTER=ruby,java,php,net,go + run: cd test-server && make build-all-servers env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} AWS_REGION: us-west-2 - name: Start the servers - run: cd test-server && make start-all-servers FILTER=ruby,java,php,net,go + run: cd test-server && make start-all-servers env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Wait for servers to start - run: cd test-server && make wait-all-servers FILTER=ruby,java,php,net,go + run: cd test-server && make wait-all-servers env: MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} - name: Run run-tests - run: cd test-server && make run-tests FILTER=ruby,java,php,net,go + run: cd test-server && make run-tests env: AWS_REGION: us-west-2 TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} From 490ffb264385b29d139e4180e373b498041948a2 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 19 Nov 2025 15:58:23 -0800 Subject: [PATCH 15/40] fix c++? --- test-server/cpp-v2-transition-server/main.cpp | 106 ++++++++---- test-server/cpp-v3-server/main.cpp | 157 ++++++++++++------ 2 files changed, 177 insertions(+), 86 deletions(-) diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index 3514b594..535a3e0a 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -88,8 +88,11 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { + // Make a copy of body so we own the data even if request_completed fires + std::string body_copy(body); + try { - json request = json::parse(body); + json request = json::parse(body_copy); std::string commitmentPolicy = get_config(request, "commitmentPolicy"); std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); @@ -141,40 +144,58 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - // Create appropriate encryption materials based on key type - std::shared_ptr materials; + // Create CryptoConfigurationV2 and S3EncryptionClientV2 based on key type + std::shared_ptr encryption_client; if (!aes_key_blob.empty()) { // Base64 decode the AES key - auto decoded = Aws::Utils::Base64::Decode(aes_key_blob); - if (!decoded.IsSuccess()) { + Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); + if (decoded.GetLength() == 0) { return send_response(connection, 400, "{\"error\":\"Failed to decode AES key\"}"); } Aws::Utils::CryptoBuffer key_buffer( - decoded.GetResult().GetUnderlyingData(), - decoded.GetResult().GetLength() + decoded.GetUnderlyingData(), + decoded.GetLength() ); - materials = std::make_shared< + auto materials = std::make_shared< Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( key_buffer ); + CryptoConfigurationV2 config(materials); + + if (legacy1 || legacy2) + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (inst_put) + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Configure ClientConfiguration with retry strategy for throttling + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 25; + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + encryption_client = std::make_shared(config, clientConfig); } else if (!kms_key_id.empty()) { - materials = std::make_shared(kms_key_id); + auto materials = std::make_shared(kms_key_id); + CryptoConfigurationV2 config(materials); + + if (legacy1 || legacy2) + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (inst_put) + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Configure ClientConfiguration with retry strategy for throttling + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 25; + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + encryption_client = std::make_shared(config, clientConfig); } else { return send_response(connection, 400, "{\"error\":\"No valid key material provided\"}"); } - - CryptoConfigurationV2 config(materials); - if (legacy1 || legacy2) - config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); set_client(client_id, encryption_client); @@ -287,6 +308,9 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, } try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -294,9 +318,12 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - auto stream = std::make_shared(body); + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); request.SetBody(stream); + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { json response = {{"bucket", bucket}, {"key", key}}; @@ -311,21 +338,31 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, } } +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; + *con_cls = nullptr; + } +} + MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { std::string method_str(method); bool is_push = method_str == "POST" || method_str == "PUT"; - static int dummy; + + // Initialize request context on first call if (*con_cls == nullptr) { - if (is_push) { - *con_cls = new std::string(); - } else { - *con_cls = &dummy; - } + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); return MHD_YES; } + + // Accumulate request body data for POST/PUT requests if (is_push && *upload_data_size) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); @@ -335,11 +372,13 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string url_str(url); + // Handle client creation endpoint if (is_push && url_str == "/client") { - std::unique_ptr body(static_cast(*con_cls)); + std::string *body = static_cast(*con_cls); return handle_create_client(connection, *body); } + // Handle object operations if (url_str.find("/object/") == 0) { std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); @@ -347,18 +386,20 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); - std::string metadata = get_header_value(connection, "content-metadata"); + if (method_str == "GET") { return handle_get_object(connection, bucket, key, client_id, metadata); } else if (method_str == "PUT") { - std::unique_ptr body(static_cast(*con_cls)); - *upload_data_size = 0; + std::string *body = static_cast(*con_cls); return handle_put_object(connection, bucket, key, client_id, *body, metadata); + } else { + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } + // Return error for unrecognized endpoints return send_response(connection, 404, "{\"error\":\"Not idea what is happening\"}"); } @@ -369,8 +410,11 @@ int main() { int port = 8097; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, - &request_handler, NULL, MHD_OPTION_END); + MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_CONNECTION_LIMIT, 100, + MHD_OPTION_END); if (!daemon) { fprintf(stderr, "Failed to start server on port %d\n", port); diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 21360b90..8a795098 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -87,8 +87,11 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { + // Make a copy of body so we own the data even if request_completed fires + std::string body_copy(body); + try { - json request = json::parse(body); + json request = json::parse(body_copy); // Extract all key material types std::string kms_key_id; @@ -133,64 +136,85 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - // Create appropriate encryption materials based on key type - std::shared_ptr materials; + std::string commitmentPolicy = get_config(request, "commitmentPolicy"); + std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); + + // Create CryptoConfigurationV3 and S3EncryptionClientV3 based on key type + std::shared_ptr encryption_client; if (!aes_key_blob.empty()) { // Base64 decode the AES key - auto decoded = Aws::Utils::Base64::Decode(aes_key_blob); - if (!decoded.IsSuccess()) { + Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); + if (decoded.GetLength() == 0) { return send_response(connection, 400, "{\"error\":\"Failed to decode AES key\"}"); } Aws::Utils::CryptoBuffer key_buffer( - decoded.GetResult().GetUnderlyingData(), - decoded.GetResult().GetLength() + decoded.GetUnderlyingData(), + decoded.GetLength() ); - materials = std::make_shared< + auto materials = std::make_shared< Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( key_buffer ); + CryptoConfigurationV3 config(materials); + + if (legacy1 || legacy2) + config.AllowLegacy(); + if (inst_put) + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + } + + // Configure ClientConfiguration with retry strategy for throttling + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 25; + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + encryption_client = std::make_shared(config, clientConfig); } else if (!kms_key_id.empty()) { - materials = std::make_shared(kms_key_id); + auto materials = std::make_shared(kms_key_id); + CryptoConfigurationV3 config(materials); + + if (legacy1 || legacy2) + config.AllowLegacy(); + if (inst_put) + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + } + + // Configure ClientConfiguration with retry strategy for throttling + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 25; + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + encryption_client = std::make_shared(config, clientConfig); } else { return send_response(connection, 400, "{\"error\":\"No valid key material provided\"}"); } - - // Configure ClientConfiguration with retry strategy for throttling - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 25; - clientConfig.retryStrategy = Aws::MakeShared( - "S3EncryptionClient", - 5 // maxRetries - will use exponential backoff for throttling - ); - - CryptoConfigurationV3 config(materials); - config.SetClientConfiguration(clientConfig); - if (legacy1 || legacy2) - config.AllowLegacy(); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - std::string commitmentPolicy = get_config(request, "commitmentPolicy"); - std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); - - if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); - } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); - } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); - } - - auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); set_client(client_id, encryption_client); @@ -306,6 +330,9 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, } try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -313,9 +340,12 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - auto stream = std::make_shared(body); + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); request.SetBody(stream); + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { json response = {{"bucket", bucket}, {"key", key}}; @@ -332,21 +362,31 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, } } +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; + *con_cls = nullptr; + } +} + MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { std::string method_str(method); bool is_push = method_str == "POST" || method_str == "PUT"; - static int dummy; + + // Initialize request context on first call if (*con_cls == nullptr) { - if (is_push) { - *con_cls = new std::string(); - } else { - *con_cls = &dummy; - } + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); return MHD_YES; } + + // Accumulate request body data for POST/PUT requests if (is_push && *upload_data_size) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); @@ -356,11 +396,13 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string url_str(url); + // Handle client creation endpoint if (is_push && url_str == "/client") { - std::unique_ptr body(static_cast(*con_cls)); + std::string *body = static_cast(*con_cls); return handle_create_client(connection, *body); } + // Handle object operations if (url_str.find("/object/") == 0) { std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); @@ -368,18 +410,20 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); - std::string metadata = get_header_value(connection, "content-metadata"); + if (method_str == "GET") { return handle_get_object(connection, bucket, key, client_id, metadata); } else if (method_str == "PUT") { - std::unique_ptr body(static_cast(*con_cls)); - *upload_data_size = 0; + std::string *body = static_cast(*con_cls); return handle_put_object(connection, bucket, key, client_id, *body, metadata); + } else { + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } + // Return error for unrecognized endpoints return send_response(connection, 404, "{\"error\":\"Not idea what is happening\"}"); } @@ -390,8 +434,11 @@ int main() { int port = 8091; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, - &request_handler, NULL, MHD_OPTION_END); + MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_CONNECTION_LIMIT, 100, + MHD_OPTION_END); if (!daemon) { fprintf(stderr, "Failed to start server on port %d\n", port); From d1366a9d02bffac1ae4b7d3b021169d2a73f03dc Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 19 Nov 2025 16:20:44 -0800 Subject: [PATCH 16/40] Make the instruction file tests parallel --- .../s3/InstructionFileFailures.java | 1305 +++++++++-------- .../amazon/encryption/s3/TestUtils.java | 42 +- 2 files changed, 734 insertions(+), 613 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java index 4f86a90b..6096d59e 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -11,23 +11,23 @@ import java.security.KeyPair; import java.security.KeyPairGenerator; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; import org.opentest4j.TestAbortedException; import software.amazon.awssdk.core.ResponseBytes; @@ -47,649 +47,736 @@ import software.amazon.encryption.s3.model.S3ECConfig; /** -* Exhaustive tests for S3 Encryption Client round-trip operations. -* These tests cover various combinations of client versions, commitment policies, and encryption modes. +* Instruction File Failures Test Suite +* +* This suite enforces execution order between encrypt and decrypt phases: +* 1. EncryptTests - Encrypts objects with various key materials and creates test copies +* 2. DecryptTests - Waits for encrypt phase to complete, then tests decryption scenarios +* +* Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion +* and DecryptTests awaits before proceeding. * * Tests are based on the exhaustive test matrix defined at: * https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 -* */ - -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class InstructionFileFailures { - private static final String sharedObjectKeyBaseMetaDataMode = "test-instruction-files-cases"; - private static KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - private static final List crossLanguageObjectsKms = new ArrayList<>(); - private static final List crossLanguageObjectsRsa = new ArrayList<>(); - private static final List crossLanguageObjectsAes = new ArrayList<>(); - - private static KeyMaterial RSA_KEY; - private static KeyMaterial AES_KEY; - - @BeforeAll - static void setupKeys() throws Exception { - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - KeyPair keyPair = keyPairGen.generateKeyPair(); - - RSA_KEY = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) - .build(); - - KeyGenerator keyGen = KeyGenerator.getInstance("AES"); - keyGen.init(256); - SecretKey aesSecretKey = keyGen.generateKey(); - - AES_KEY = KeyMaterial.builder() - .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) +public class InstructionFileFailures { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using various key materials (KMS, RSA, AES) with instruction files. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + class EncryptTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-instruction-files-cases"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) .build(); - } + + // Thread-safe lists for storing encrypted object keys + private static final List crossLanguageObjectsKms = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsRsa = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsAes = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + RSA_KEY = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) + .build(); + + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesSecretKey = keyGen.generateKey(); + + AES_KEY = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) + .build(); + } - public static Stream improvedClientsCanPutKMSWithInstructionFile() { - return improvedClientsForTest() - .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) - .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - } + /** + * Public accessors for decrypt tests to retrieve encrypted object keys and key materials + */ + static List getCrossLanguageObjectsKms() { + return new ArrayList<>(crossLanguageObjectsKms); + } - public static Stream improvedClientsCanPutRawWithInstructionFile() { - return improvedClientsForTest() - .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) - .filter(target -> RAW_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - } + static List getCrossLanguageObjectsRsa() { + return new ArrayList<>(crossLanguageObjectsRsa); + } - public static Stream clientsCanGetKMSWithInstructionFile() { - Stream improved = improvedClientsForTest() - .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - Stream transition = transitionClientsForTest() - .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - return Stream.concat(improved, transition); - } + static List getCrossLanguageObjectsAes() { + return new ArrayList<>(crossLanguageObjectsAes); + } - public static Stream clientsCanGetRawWithInstructionFile() { - Stream improved = improvedClientsForTest() - .filter(target -> RAW_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - Stream transition = transitionClientsForTest() - .filter(target -> RAW_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - return Stream.concat(improved, transition); - } + static KeyMaterial getRsaKey() { + return RSA_KEY; + } - @Order(1) - @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutKMSWithInstructionFile") - void encrypt_with_instruction_files_kms_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .instructionFileConfig( - InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build() - ) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt( - client, - S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-kms" + language.getLanguageName()), - crossLanguageObjectsKms, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + static KeyMaterial getAesKey() { + return AES_KEY; + } - @Order(2) - @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction files") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutRawWithInstructionFile") - void encrypt_with_instruction_files_rsa_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) + static KeyMaterial getKmsKeyArn() { + return kmsKeyArn; + } + + public static Stream improvedClientsCanPutKMSWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawRSAWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawAESWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encrypt_with_instruction_files_kms_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) .instructionFileConfig( - InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build() + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() ) .build()) - .build()); + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-kms" + language.getLanguageName()), + crossLanguageObjectsKms, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - String S3ECId = clientOutput.getClientId(); + @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encrypt_with_instruction_files_rsa_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-rsa" + language.getLanguageName()), + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt AES KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encrypt_with_instruction_files_aes_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + static void make_copies_to_verify_things() throws Exception { + // Create a plaintext S3 client to copy objects with instruction files + try (S3Client ptS3Client = S3Client.create()) { + List allCrossLanguageObjects = Stream.of( + crossLanguageObjectsKms.stream(), + crossLanguageObjectsRsa.stream(), + crossLanguageObjectsAes.stream() + ).flatMap(s -> s).collect(Collectors.toList()); + for (String objectKey : allCrossLanguageObjects) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Get the instruction file + String instructionFileKey = objectKey + ".instruction"; + ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + + String instructionFileJson = instructionFile.asUtf8String(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Put a strict copy, to verify that we know how to do this + putObjectWithInstructionFile( + ptS3Client, + objectKey + "-good-copy", + encryptedObject.asByteArray(), + objectMetadata, + instructionFileJson + ); + + ObjectMapper mapper = new ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); + instructionFileMap.put("x-amz-d", objectMetadata.get("x-amz-d")); + instructionFileMap.put("x-amz-i", objectMetadata.get("x-amz-i")); + + String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); + + // Put instruction files that should fail: + putObjectWithInstructionFile( + ptS3Client, + objectKey + "-bad-both-meta-and-instruction", + encryptedObject.asByteArray(), + objectMetadata, + instructionFileWithCommitmentValues + ); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + "-bad-only-instruction", + encryptedObject.asByteArray(), + Map.of(), + instructionFileWithCommitmentValues + ); + + } + } + } + + static void putObjectWithInstructionFile( + S3Client ptS3Client, + String newObjectKey, + byte[] objectData, + Map objectMetadata, + String instructionFileJson + ) { + + // Put the encrypted object copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + + // Put the instruction file copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey + ".instruction") + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } - TestUtils.Encrypt( - client, - S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-rsa" + language.getLanguageName()), - crossLanguageObjectsRsa, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); + @AfterAll + static void signalEncryptionComplete() throws Exception { + make_copies_to_verify_things(); + + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } } - @Order(3) - @ParameterizedTest(name = "{0}: Encrypt AES KC-GCM with instruction files") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#improvedClientsCanPutRawWithInstructionFile") - void encrypt_with_instruction_files_aes_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .instructionFileConfig( - InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build() - ) + /** + * Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first. + */ + @Nested + class DecryptTests { + private static List crossLanguageObjectsKms; + private static List crossLanguageObjectsRsa; + private static List crossLanguageObjectsAes; + private static KeyMaterial kmsKeyArn; + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects and key materials from the encrypt phase + crossLanguageObjectsKms = EncryptTests.getCrossLanguageObjectsKms(); + crossLanguageObjectsRsa = EncryptTests.getCrossLanguageObjectsRsa(); + crossLanguageObjectsAes = EncryptTests.getCrossLanguageObjectsAes(); + kmsKeyArn = EncryptTests.getKmsKeyArn(); + RSA_KEY = EncryptTests.getRsaKey(); + AES_KEY = EncryptTests.getAesKey(); + + // Verify we have objects to decrypt + if (crossLanguageObjectsKms.isEmpty() && crossLanguageObjectsRsa.isEmpty() && crossLanguageObjectsAes.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + public static Stream clientsCanGetKMSWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream clientsCanGetRawRSAWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream clientsCanGetRawAESWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + // KMS instruction files decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt KMS encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decrypt_kms_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) .build()) - .build()); + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsKms, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsKms + ); + } - String S3ECId = clientOutput.getClientId(); + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decrypt_kms_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - TestUtils.Encrypt( - client, - S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), - crossLanguageObjectsAes, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - @Order(9) - @Test - void make_copies_to_verify_things() throws Exception { - // Create a plaintext S3 client to copy objects with instruction files - try (S3Client ptS3Client = S3Client.create()) { - List allCrossLanguageObjects = Stream.of( - crossLanguageObjectsKms.stream(), - crossLanguageObjectsRsa.stream(), - crossLanguageObjectsAes.stream() - ).flatMap(s -> s).collect(Collectors.toList()); - for (String objectKey : allCrossLanguageObjects) { - // Get the encrypted object - ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); - - // Get the instruction file - String instructionFileKey = objectKey + ".instruction"; - ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder - .bucket(TestUtils.BUCKET) - .key(instructionFileKey) - .build()); - - String instructionFileJson = instructionFile.asUtf8String(); - Map objectMetadata = encryptedObject.response().metadata(); - - // Put a strict copy, to verify that we know how to do this - putObjectWithInstructionFile( - ptS3Client, - objectKey + "-good-copy", - encryptedObject.asByteArray(), - objectMetadata, - instructionFileJson - ); - - ObjectMapper mapper = new ObjectMapper(); - Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); - - instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); - instructionFileMap.put("x-amz-d", objectMetadata.get("x-amz-d")); - instructionFileMap.put("x-amz-i", objectMetadata.get("x-amz-i")); - - String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); - - // Put instruction files that should fail: - putObjectWithInstructionFile( - ptS3Client, - objectKey + "-bad-both-meta-and-instruction", - encryptedObject.asByteArray(), - objectMetadata, - instructionFileWithCommitmentValues - ); - - putObjectWithInstructionFile( - ptS3Client, - objectKey + "-bad-only-instruction", - encryptedObject.asByteArray(), - Map.of(), - instructionFileWithCommitmentValues - ); + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decrypt_kms_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); } - } - void putObjectWithInstructionFile( - S3Client ptS3Client, - String newObjectKey, - byte[] objectData, - Map objectMetadata, - String instructionFileJson - ) { - - // Put the encrypted object copy - ptS3Client.putObject(builder -> builder - .bucket(TestUtils.BUCKET) - .key(newObjectKey) - .metadata(objectMetadata) - .build(), - software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); - - // Put the instruction file copy - ptS3Client.putObject(builder -> builder - .bucket(TestUtils.BUCKET) - .key(newObjectKey + ".instruction") - .build(), - software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); - } + @ParameterizedTest(name = "{0}: Fail to decrypt KMS duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decrypt_kms_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - // KMS instruction files decrypt - - @Order(10) - @ParameterizedTest(name = "{0}: Successfully decrypt KMS encrypted original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsKms, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-good-copy") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjectsKms - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - @Order(11) - @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + @ParameterizedTest(name = "{0}: Fail to decrypt KMS instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decrypt_kms_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - @Order(12) - @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - @Order(13) - @ParameterizedTest(name = "{0}: Fail to decrypt KMS duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + // RSA instruction file decrypt - @Order(14) - @ParameterizedTest(name = "{0}: Fail to decrypt KMS instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + @ParameterizedTest(name = "{0}: Successfully decrypt RSA encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decrypt_rsa_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - // RSA instruction file decrypt - - @Order(20) - @ParameterizedTest(name = "{0}: Successfully decrypt RSA encrypted original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_rsa_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsRsa, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-good-copy") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjectsRsa - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsRsa + ); + } - @Order(21) - @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_rsa_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decrypt_rsa_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - @Order(22) - @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_rsa_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - @Order(23) - @ParameterizedTest(name = "{0}: Fail to decrypt RSA duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_rsa_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decrypt_rsa_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - @Order(24) - @ParameterizedTest(name = "{0}: Fail to decrypt RSA instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_rsa_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - // AES instruction file decrypt - - @Order(30) - @ParameterizedTest(name = "{0}: Successfully decrypt AES encrypted original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_aes_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsAes, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-good-copy") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjectsAes - ); - } + @ParameterizedTest(name = "{0}: Fail to decrypt RSA duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decrypt_rsa_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - @Order(31) - @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_aes_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - @Order(32) - @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_aes_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + @ParameterizedTest(name = "{0}: Fail to decrypt RSA instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decrypt_rsa_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - @Order(33) - @ParameterizedTest(name = "{0}: Fail to decrypt AES duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_aes_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } - @Order(34) - @ParameterizedTest(name = "{0}: Fail to decrypt AES instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures#clientsCanGetRawWithInstructionFile") - void decrypt_aes_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - + // AES instruction file decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt AES encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decrypt_aes_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsAes + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decrypt_aes_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decrypt_aes_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decrypt_aes_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-both-meta-and-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decrypt_aes_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + "-bad-only-instruction") + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index a7a86c1d..6f274747 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -99,13 +99,27 @@ public class TestUtils { public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = Set.of(NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4); - // For now, only .NET and Java have RSA support - public static final Set RAW_SUPPORTED = + + // Cpp only supports Raw AES + public static final Set RAW_AES_SUPPORTED = + Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 + , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 + , RUBY_V2_TRANSITION, RUBY_V3 + , CPP_V2_CURRENT, CPP_V2_TRANSITION, CPP_V3 + ); + + public static final Set RAW_RSA_SUPPORTED = Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 , RUBY_V2_TRANSITION, RUBY_V3 ); + // Intersection of RAW_AES_SUPPORTED and RAW_RSA_SUPPORTED + public static final Set RAW_SUPPORTED = + RAW_AES_SUPPORTED.stream() + .filter(RAW_RSA_SUPPORTED::contains) + .collect(Collectors.toSet()); + // .NET only supports decrypting instruction files using AES and RSA. // Python MUST support decrypting KMS instruction files, but does not yet. public static final Set KMS_INSTRUCTION_FILE_UNSUPPORTED = @@ -366,6 +380,28 @@ public static Stream improvedClientsForTest() { .map(Arguments::of); } + /** + * Get stream of arguments for clients that support RAW AES (includes CPP). + */ + public static Stream clientsRawAesForTest() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + Stream transition = transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + return Stream.concat(improved, transition); + } + + /** + * Get stream of arguments for clients that support RAW RSA (excludes CPP). + */ + public static Stream clientsRawRsaForTest() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + Stream transition = transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + return Stream.concat(improved, transition); + } + /** * These functions provide a stream of arguments for parameterized tests. * @return Stream of Arguments containing pairs of LanguageServerTarget for encryption and decryption @@ -541,8 +577,6 @@ public static void Decrypt( Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, crossLanguageObjects); } - - public static void Decrypt( S3ECTestServerClient client, String S3ECId, From c7f19ec3824ee4011ae67107d6106cae1856e189 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Wed, 19 Nov 2025 21:48:30 -0800 Subject: [PATCH 17/40] fix cpp --- test-server/cpp-v3-server/aws-sdk-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/cpp-v3-server/aws-sdk-cpp b/test-server/cpp-v3-server/aws-sdk-cpp index 87402c99..4039810c 160000 --- a/test-server/cpp-v3-server/aws-sdk-cpp +++ b/test-server/cpp-v3-server/aws-sdk-cpp @@ -1 +1 @@ -Subproject commit 87402c99fd3c9107c6ccc6edf545fd4b05b2b551 +Subproject commit 4039810cd5d32429a64e70733175940d4a73f13c From ab1ba3a28824525da0256e49e9e3579b11e473e4 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 10:19:52 -0800 Subject: [PATCH 18/40] go concurency --- test-server/go-v3-server/main.go | 14 +++++++++++--- test-server/go-v3-transition-server/main.go | 14 +++++++++++--- test-server/go-v4-server/main.go | 14 +++++++++++--- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/test-server/go-v3-server/main.go b/test-server/go-v3-server/main.go index a79f007e..0384c5ff 100644 --- a/test-server/go-v3-server/main.go +++ b/test-server/go-v3-server/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" @@ -23,6 +24,7 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client + mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -182,8 +184,10 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache + // Store client in cache (protected by mutex) + s.mu.Lock() s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -204,8 +208,10 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -274,8 +280,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index 3c92582e..64556f12 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" @@ -24,6 +25,7 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client + mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -199,8 +201,10 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache + // Store client in cache (protected by mutex) + s.mu.Lock() s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -221,8 +225,10 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -291,8 +297,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/go-v4-server/main.go b/test-server/go-v4-server/main.go index d9d897aa..50999e95 100644 --- a/test-server/go-v4-server/main.go +++ b/test-server/go-v4-server/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "github.com/aws/amazon-s3-encryption-client-go/v4/client" "github.com/aws/amazon-s3-encryption-client-go/v4/materials" @@ -24,6 +25,7 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV4 kmsClient *kms.Client + mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -199,8 +201,10 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache + // Store client in cache (protected by mutex) + s.mu.Lock() s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -221,8 +225,10 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -291,8 +297,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) From 99f05fe6dbc8c7f0d8c3aab24459ef4d90269722 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 12:09:14 -0800 Subject: [PATCH 19/40] cpp is hard --- test-server/cpp-v2-server/Makefile | 4 +- test-server/cpp-v2-server/main.cpp | 5 ++- test-server/cpp-v2-transition-server/Makefile | 4 +- test-server/cpp-v2-transition-server/main.cpp | 3 +- test-server/cpp-v3-server/Makefile | 4 +- test-server/cpp-v3-server/main.cpp | 38 ++++++++++++++----- 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index e4d2f954..2d0a4b55 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -20,7 +20,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V2 server starting..." stop-server: @@ -31,7 +31,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f build/server.log + @rm -f server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index 1b2de659..8ba5a49a 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -283,7 +283,10 @@ int main() { int port = 8085; struct MHD_Daemon *daemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, - &request_handler, NULL, MHD_OPTION_END); + &request_handler, NULL, + MHD_OPTION_CONNECTION_LIMIT, 250, + MHD_OPTION_CONNECTION_TIMEOUT, 30, + MHD_OPTION_END); if (!daemon) { fprintf(stderr, "Failed to start server on port %d\n", port); diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 2ca6ee5f..0383b4d8 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -19,7 +19,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp transition server starting..." stop-server: @@ -30,7 +30,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f build/server.log + @rm -f server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index 535a3e0a..f7dbef11 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -413,7 +413,8 @@ int main() { MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, &request_handler, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_CONNECTION_LIMIT, 100, + MHD_OPTION_CONNECTION_LIMIT, 250, + MHD_OPTION_CONNECTION_TIMEOUT, 30, MHD_OPTION_END); if (!daemon) { diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 05a286f0..e90c8d73 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -19,7 +19,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V3 server starting..." stop-server: @@ -30,7 +30,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f build/server.log + @rm -f server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 8a795098..bbc3fbf1 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -288,8 +288,12 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, const std::string &metadata) { + fprintf(stderr, "[CPP-V3] GetObject request: bucket=%s, key=%s, client_id=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str()); + auto client = get_client(client_id); if (!client) { + fprintf(stderr, "[CPP-V3] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -306,15 +310,17 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, auto &stream = outcome.GetResult().GetBody(); std::string content((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); + fprintf(stderr, "[CPP-V3] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); return send_response(connection, 200, content); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "handle_get_object error %s\n", msg.c_str()); + fprintf(stderr, "[CPP-V3] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "handle_get_object exception %s\n", e.what()); - auto msg = make_error("An exception was thrown", 500); + fprintf(stderr, "[CPP-V3] GetObject exception: %s\n", e.what()); + auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } @@ -324,8 +330,12 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, const std::string &client_id, const std::string &body, const std::string &metadata) { + fprintf(stderr, "[CPP-V3] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + auto client = get_client(client_id); if (!client) { + fprintf(stderr, "[CPP-V3] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -348,15 +358,16 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, // body_ptr keeps the data alive through this entire operation auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V3] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "handle_put_object error %s\n", msg.c_str()); + fprintf(stderr, "[CPP-V3] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "handle_put_object exception %s\n", e.what()); + fprintf(stderr, "[CPP-V3] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } @@ -387,19 +398,23 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, } // Accumulate request body data for POST/PUT requests - if (is_push && *upload_data_size) { + if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request std::string url_str(url); // Handle client creation endpoint if (is_push && url_str == "/client") { std::string *body = static_cast(*con_cls); - return handle_create_client(connection, *body); + MHD_Result result = handle_create_client(connection, *body); + return result; } // Handle object operations @@ -413,10 +428,12 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string metadata = get_header_value(connection, "content-metadata"); if (method_str == "GET") { - return handle_get_object(connection, bucket, key, client_id, metadata); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); + return result; } else if (method_str == "PUT") { std::string *body = static_cast(*con_cls); - return handle_put_object(connection, bucket, key, client_id, *body, metadata); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + return result; } else { return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } @@ -437,7 +454,8 @@ int main() { MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, &request_handler, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_CONNECTION_LIMIT, 100, + MHD_OPTION_CONNECTION_LIMIT, 250, + MHD_OPTION_CONNECTION_TIMEOUT, 30, MHD_OPTION_END); if (!daemon) { From 35e87728233d8503c8f9156e21842d8361643a25 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 12:34:33 -0800 Subject: [PATCH 20/40] try this --- test-server/cpp-v3-server/main.cpp | 35 ++++++++++++++++++++++--- test-server/java-tests/build.gradle.kts | 2 +- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index bbc3fbf1..64f2c9ef 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -304,15 +304,40 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); + + // Keep outcome alive to ensure stream remains valid auto outcome = client->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V3] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + fprintf(stderr, "[CPP-V3] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", bucket.c_str(), key.c_str(), content.length()); - return send_response(connection, 200, content); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); fprintf(stderr, "[CPP-V3] GetObject AWS error: %s\n", msg.c_str()); @@ -322,6 +347,10 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, fprintf(stderr, "[CPP-V3] GetObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V3] GetObject unknown exception\n"); + auto msg = make_error("Unknown error in GetObject", 500); + return send_response(connection, 500, msg); } } diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 14c3eec1..1c0d4308 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -61,7 +61,7 @@ tasks { systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") maxParallelForks = 1 // One JVM systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", - Math.max(1, Runtime.getRuntime().availableProcessors() - 2).toString()) // Scale with CPU, reserve 2 cores + Math.max(1, Runtime.getRuntime().availableProcessors() - 6).toString()) // Scale with CPU, reserve 2 cores // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) From 954b55921ceadd8dfcf1ac8366a5eb55e664d06b Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 14:05:42 -0800 Subject: [PATCH 21/40] next cut --- test-server/cpp-v2-server/main.cpp | 2 +- test-server/cpp-v2-transition-server/main.cpp | 2 +- test-server/cpp-v3-server/main.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index 8ba5a49a..33e60bfb 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -284,7 +284,7 @@ int main() { struct MHD_Daemon *daemon = MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, &request_handler, NULL, - MHD_OPTION_CONNECTION_LIMIT, 250, + MHD_OPTION_CONNECTION_LIMIT, 100, MHD_OPTION_CONNECTION_TIMEOUT, 30, MHD_OPTION_END); diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index f7dbef11..8163376f 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -413,7 +413,7 @@ int main() { MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, &request_handler, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_CONNECTION_LIMIT, 250, + MHD_OPTION_CONNECTION_LIMIT, 100, MHD_OPTION_CONNECTION_TIMEOUT, 30, MHD_OPTION_END); diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 64f2c9ef..c9cac063 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -483,7 +483,7 @@ int main() { MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, &request_handler, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_CONNECTION_LIMIT, 250, + MHD_OPTION_CONNECTION_LIMIT, 100, MHD_OPTION_CONNECTION_TIMEOUT, 30, MHD_OPTION_END); From ff1a84a262947ab0fd8462889ce3e829d09d994a Mon Sep 17 00:00:00 2001 From: Andrew Jewell <107044381+ajewellamz@users.noreply.github.com> Date: Thu, 20 Nov 2025 17:19:48 -0500 Subject: [PATCH 22/40] Enable address sanitizer during tests (#105) Fail loud! From 0858f67d21e06a8c1ab002acf34854d05fe3f5ff Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 14:29:56 -0800 Subject: [PATCH 23/40] try not deleting the body --- test-server/cpp-v3-server/main.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index c9cac063..ac1a8dd7 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -407,7 +407,9 @@ void request_completed(void *cls, struct MHD_Connection *connection, // Clean up the request-specific context when request is truly complete if (*con_cls != nullptr) { std::string *body = static_cast(*con_cls); - delete body; + // This tests that we never can delete a string that someone is using. + // this seems safe to do for a test server because these strings are small. + // delete body; *con_cls = nullptr; } } From 5bfc9af38f0834b87e091d93ef73c643680a9950 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 14:35:18 -0800 Subject: [PATCH 24/40] simplify? --- test-server/cpp-v3-server/main.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index ac1a8dd7..c287c781 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -285,9 +285,10 @@ void fill_context(Aws::Map &map, } MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { + std::string bucket, + std::string key, + std::string client_id, + std::string metadata) { fprintf(stderr, "[CPP-V3] GetObject request: bucket=%s, key=%s, client_id=%s\n", bucket.c_str(), key.c_str(), client_id.c_str()); @@ -355,10 +356,11 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, } MHD_Result handle_put_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &body, - const std::string &metadata) { + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { fprintf(stderr, "[CPP-V3] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); From 8d9ff3052860ba9f31d0e5cfd3479fe5acccc269 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 14:44:17 -0800 Subject: [PATCH 25/40] try less work --- test-server/java-tests/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 1c0d4308..d30888a3 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -61,7 +61,7 @@ tasks { systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") maxParallelForks = 1 // One JVM systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", - Math.max(1, Runtime.getRuntime().availableProcessors() - 6).toString()) // Scale with CPU, reserve 2 cores + Math.max(1, Runtime.getRuntime().availableProcessors() - 9).toString()) // Scale with CPU, reserve 2 cores // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) From 03d315179e827072701ebc2b408dd781122231cd Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 15:17:31 -0800 Subject: [PATCH 26/40] try a different direction --- test-server/cpp-v3-server/main.cpp | 96 +++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index c287c781..95b5b523 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -1,3 +1,44 @@ +/* + * S3 Encryption Test Server - C++ V3 + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_SELECT_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (select/poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + #include #include #include @@ -15,12 +56,16 @@ #include #include #include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; std::unordered_map> client_cache_secret; -std::mutex client_mutex; +std::shared_mutex client_mutex; // Using shared_mutex for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() std::string generate_uuid() { uuid_t uuid; @@ -32,7 +77,8 @@ std::string generate_uuid() { std::shared_ptr get_client(const std::string &client_id) { - std::lock_guard lock(client_mutex); + // Use shared_lock for concurrent reads - multiple threads can read simultaneously + std::shared_lock lock(client_mutex); auto it = client_cache_secret.find(client_id); if (it == client_cache_secret.end()) { return std::shared_ptr(); @@ -43,8 +89,10 @@ std::shared_ptr get_client(const std::string &client_id) void set_client(const std::string &client_id, std::shared_ptr client) { - std::lock_guard lock(client_mutex); - client_cache_secret[client_id] = client; + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + client_cache_secret.emplace(client_id, client); } std::string get_header_value(struct MHD_Connection *connection, @@ -87,11 +135,11 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { - // Make a copy of body so we own the data even if request_completed fires - std::string body_copy(body); + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller try { - json request = json::parse(body_copy); + json request = json::parse(body); // Extract all key material types std::string kms_key_id; @@ -179,8 +227,9 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, } // Configure ClientConfiguration with retry strategy for throttling + // Match S3 connection pool size to thread pool size for optimal resource usage Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 25; + clientConfig.maxConnections = g_thread_pool_size; clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); encryption_client = std::make_shared(config, clientConfig); @@ -206,8 +255,9 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, } // Configure ClientConfiguration with retry strategy for throttling + // Match S3 connection pool size to thread pool size for optimal resource usage Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 25; + clientConfig.maxConnections = g_thread_pool_size; clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); encryption_client = std::make_shared(config, clientConfig); @@ -407,11 +457,10 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, void request_completed(void *cls, struct MHD_Connection *connection, void **con_cls, enum MHD_RequestTerminationCode toe) { // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent if (*con_cls != nullptr) { std::string *body = static_cast(*con_cls); - // This tests that we never can delete a string that someone is using. - // this seems safe to do for a test server because these strings are small. - // delete body; + delete body; // Safe to delete now - all synchronous operations are complete *con_cls = nullptr; } } @@ -481,13 +530,32 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, int main() { Aws::SDKOptions options; Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; // Fallback if detection fails + fprintf(stderr, "[WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + // Thread pool size = num_cores * 2 (allows for I/O blocking without starving throughput) + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] S3 client maxConnections: %u\n", g_thread_pool_size); + int port = 8091; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, &request_handler, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_CONNECTION_LIMIT, 100, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, MHD_OPTION_CONNECTION_TIMEOUT, 30, MHD_OPTION_END); From b4416dc16d0db8f5d4b26a30c0c526c2e9d32593 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 15:36:37 -0800 Subject: [PATCH 27/40] cpp build update --- test-server/cpp-v3-server/main.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 95b5b523..4663b4ec 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -56,13 +56,14 @@ #include #include #include +#include #include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; std::unordered_map> client_cache_secret; -std::shared_mutex client_mutex; // Using shared_mutex for concurrent reads +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads // Threading configuration - set at startup based on CPU cores unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() @@ -78,7 +79,7 @@ std::string generate_uuid() { std::shared_ptr get_client(const std::string &client_id) { // Use shared_lock for concurrent reads - multiple threads can read simultaneously - std::shared_lock lock(client_mutex); + std::shared_lock lock(client_mutex); auto it = client_cache_secret.find(client_id); if (it == client_cache_secret.end()) { return std::shared_ptr(); @@ -91,7 +92,7 @@ void set_client(const std::string &client_id, std::shared_ptr lock(client_mutex); + std::unique_lock lock(client_mutex); client_cache_secret.emplace(client_id, client); } From 30d797e11145b624397a885233fcf17f1e704d32 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Thu, 20 Nov 2025 19:36:33 -0800 Subject: [PATCH 28/40] Making C test servers concurent Make the servers cuoncurent. Needed to add a LRU, so there is a subtle difference with Cpp. Also added logging, including SDK logging. --- test-server/cpp-v2-server/main.cpp | 435 ++++++++++++++++-- test-server/cpp-v2-transition-server/main.cpp | 433 ++++++++++++++--- test-server/cpp-v3-server/main.cpp | 326 ++++++++++--- test-server/java-tests/build.gradle.kts | 2 +- test-server/php-v2-server/Makefile | 2 +- test-server/php-v2-transition-server/Makefile | 2 +- test-server/php-v3-server/Makefile | 2 +- 7 files changed, 1015 insertions(+), 187 deletions(-) diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index 33e60bfb..f5c4e406 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -1,8 +1,53 @@ +/* + * S3 Encryption Test Server - C++ V2 + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + #include +#include #include #include #include #include +#include +#include #include #include #include @@ -10,14 +55,33 @@ #include #include +#include #include #include +#include +#include +#include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> client_cache_secret; -std::mutex client_mutex; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() std::string generate_uuid() { uuid_t uuid; @@ -29,19 +93,51 @@ std::string generate_uuid() { std::shared_ptr get_client(const std::string &client_id) { - std::lock_guard lock(client_mutex); + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); auto it = client_cache_secret.find(client_id); if (it == client_cache_secret.end()) { return std::shared_ptr(); } else { - return it->second; + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; } } void set_client(const std::string &client_id, std::shared_ptr client) { - std::lock_guard lock(client_mutex); - client_cache_secret[client_id] = client; + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V2] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V2] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); } std::string get_header_value(struct MHD_Connection *connection, @@ -69,6 +165,9 @@ std::string make_error(const std::string &message, int status_code) { MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller + try { json request = json::parse(body); std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; @@ -88,7 +187,12 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, if (inst_put) config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - auto encryption_client = std::make_shared(config); + // Each client gets a large connection pool since we cannot share HTTP clients + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + auto encryption_client = std::make_shared(config, clientConfig); std::string client_id = generate_uuid(); set_client(client_id, encryption_client); @@ -96,6 +200,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2] handle_create_client exception: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"An exception was thrown.\"}"); } catch (...) { @@ -106,13 +211,18 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: metadata is empty\n"); return; } + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; + int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -145,6 +255,9 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + // Add to map map.emplace(key, value); @@ -155,14 +268,24 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } + + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { + std::string bucket, + std::string key, + std::string client_id, + std::string metadata) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length()); + auto client = get_client(client_id); if (!client) { + fprintf(stderr, "[CPP-V2] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -173,34 +296,98 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid auto outcome = client->GetObject(request, kmsContextMap); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); - return send_response(connection, 200, content); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V2] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V2] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V2] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V2] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V2] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V2] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - auto msg = make_error("An exception was thrown", 500); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &body, - const std::string &metadata) { + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V2] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + auto client = get_client(client_id); if (!client) { + fprintf(stderr, "[CPP-V2] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -208,88 +395,248 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - auto stream = std::make_shared(body); + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); request.SetBody(stream); + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V2] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V2] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; // Safe to delete now - all synchronous operations are complete + *con_cls = nullptr; + } +} + MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - std::string method_str(method); - bool is_push = method_str == "POST" || method_str == "PUT"; - static int dummy; - if (*con_cls == nullptr) { - if (is_push) { - *con_cls = new std::string(); - } else { - *con_cls = &dummy; + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V2] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); } + + // Initialize request context on first call + if (*con_cls == nullptr) { + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); + fprintf(stderr, "[CPP-V2] REQUEST INIT: allocated new request context for %s %s\n", method, url); return MHD_YES; } - if (is_push && *upload_data_size) { + + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V2] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V2] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + + // Accumulate request body data for POST/PUT requests + if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - std::string url_str(url); - + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V2] PROCESSING: %s %s\n", method, url); + + // Handle client creation endpoint if (is_push && url_str == "/client") { - std::unique_ptr body(static_cast(*con_cls)); - return handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2] Handling /client endpoint\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2] /client handler returned: %d\n", result); + return result; } + // Handle object operations if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V2] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); - std::string metadata = get_header_value(connection, "content-metadata"); + + fprintf(stderr, "[CPP-V2] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + if (method_str == "GET") { - return handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V2] Dispatching to handle_get_object\n"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V2] handle_get_object returned: %d\n", result); + return result; } else if (method_str == "PUT") { - std::unique_ptr body(static_cast(*con_cls)); - *upload_data_size = 0; - return handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2] Dispatching to handle_put_object\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2] handle_put_object returned: %d\n", result); + return result; + } else { + fprintf(stderr, "[CPP-V2] Method not allowed: %s\n", method); + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V2] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V2] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V2] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V2] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V2] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; } int main() { Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; + fprintf(stderr, "[CPP-V2] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + int port = 8085; + struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, &request_handler, NULL, - MHD_OPTION_CONNECTION_LIMIT, 100, - MHD_OPTION_CONNECTION_TIMEOUT, 30, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "Failed to start server on port %d\n", port); + fprintf(stderr, "[CPP-V2] Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index 8163376f..dc863723 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -1,10 +1,55 @@ +/* + * S3 Encryption Test Server - C++ V2 Transition + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + #include +#include #include #include #include #include #include #include +#include +#include #include #include #include @@ -12,14 +57,33 @@ #include #include +#include #include #include +#include +#include +#include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> client_cache_secret; -std::mutex client_mutex; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() std::string generate_uuid() { uuid_t uuid; @@ -31,19 +95,51 @@ std::string generate_uuid() { std::shared_ptr get_client(const std::string &client_id) { - std::lock_guard lock(client_mutex); + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); auto it = client_cache_secret.find(client_id); if (it == client_cache_secret.end()) { return std::shared_ptr(); } else { - return it->second; + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; } } void set_client(const std::string &client_id, std::shared_ptr client) { - std::lock_guard lock(client_mutex); - client_cache_secret[client_id] = client; + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); } std::string get_header_value(struct MHD_Connection *connection, @@ -88,11 +184,11 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { - // Make a copy of body so we own the data even if request_completed fires - std::string body_copy(body); + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller try { - json request = json::parse(body_copy); + json request = json::parse(body); std::string commitmentPolicy = get_config(request, "commitmentPolicy"); std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); @@ -144,8 +240,8 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - // Create CryptoConfigurationV2 and S3EncryptionClientV2 based on key type - std::shared_ptr encryption_client; + // Create CryptoConfigurationV2 based on key type + std::shared_ptr config; if (!aes_key_blob.empty()) { // Base64 decode the AES key @@ -164,38 +260,26 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( key_buffer ); - CryptoConfigurationV2 config(materials); - - if (legacy1 || legacy2) - config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - // Configure ClientConfiguration with retry strategy for throttling - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 25; - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - - encryption_client = std::make_shared(config, clientConfig); + config = std::make_shared(materials); } else if (!kms_key_id.empty()) { auto materials = std::make_shared(kms_key_id); - CryptoConfigurationV2 config(materials); - - if (legacy1 || legacy2) - config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - // Configure ClientConfiguration with retry strategy for throttling - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 25; - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - - encryption_client = std::make_shared(config, clientConfig); + config = std::make_shared(materials); } else { return send_response(connection, 400, "{\"error\":\"No valid key material provided\"}"); } + + // Apply common configuration settings (applies to both AES and KMS) + if (legacy1 || legacy2) + config->SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (inst_put) + config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Create S3EncryptionClientV2 with standard configuration + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + auto encryption_client = std::make_shared(*config, clientConfig); std::string client_id = generate_uuid(); set_client(client_id, encryption_client); @@ -203,6 +287,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] handle_create_client exception: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"An exception was thrown.\"}"); } catch (...) { @@ -213,13 +298,18 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: metadata is empty\n"); return; } + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; + int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -252,6 +342,9 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + // Add to map map.emplace(key, value); @@ -262,14 +355,24 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { + std::string bucket, + std::string key, + std::string client_id, + std::string metadata) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length()); + auto client = get_client(client_id); if (!client) { + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -280,30 +383,91 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid auto outcome = client->GetObject(request, kmsContextMap); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); - return send_response(connection, 200, content); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - auto msg = make_error("An exception was thrown", 500); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &body, - const std::string &metadata) { + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + auto client = get_client(client_id); if (!client) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -326,13 +490,16 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, // body_ptr keeps the data alive through this entire operation auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } @@ -341,9 +508,36 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, void request_completed(void *cls, struct MHD_Connection *connection, void **con_cls, enum MHD_RequestTerminationCode toe) { // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V2-TRANSITION] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + if (*con_cls != nullptr) { std::string *body = static_cast(*con_cls); - delete body; + delete body; // Safe to delete now - all synchronous operations are complete *con_cls = nullptr; } } @@ -352,34 +546,58 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - std::string method_str(method); - bool is_push = method_str == "POST" || method_str == "PUT"; + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); + } // Initialize request context on first call if (*con_cls == nullptr) { // Allocate unique state for each request to avoid race conditions *con_cls = new std::string(); + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST INIT: allocated new request context for %s %s\n", method, url); return MHD_YES; } + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + // Accumulate request body data for POST/PUT requests - if (is_push && *upload_data_size) { + if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - std::string url_str(url); + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V2-TRANSITION] PROCESSING: %s %s\n", method, url); // Handle client creation endpoint if (is_push && url_str == "/client") { + fprintf(stderr, "[CPP-V2-TRANSITION] Handling /client endpoint\n"); std::string *body = static_cast(*con_cls); - return handle_create_client(connection, *body); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2-TRANSITION] /client handler returned: %d\n", result); + return result; } // Handle object operations if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { @@ -388,37 +606,124 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string client_id = get_header_value(connection, "clientid"); std::string metadata = get_header_value(connection, "content-metadata"); + fprintf(stderr, "[CPP-V2-TRANSITION] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + if (method_str == "GET") { - return handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_get_object\n"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] handle_get_object returned: %d\n", result); + return result; } else if (method_str == "PUT") { + fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_put_object\n"); std::string *body = static_cast(*con_cls); - return handle_put_object(connection, bucket, key, client_id, *body, metadata); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] handle_put_object returned: %d\n", result); + return result; } else { + fprintf(stderr, "[CPP-V2-TRANSITION] Method not allowed: %s\n", method); return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } - // Return error for unrecognized endpoints - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V2-TRANSITION] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; } int main() { Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; + fprintf(stderr, "[CPP-V2-TRANSITION] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + int port = 8097; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_CONNECTION_LIMIT, 100, - MHD_OPTION_CONNECTION_TIMEOUT, 30, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "Failed to start server on port %d\n", port); + fprintf(stderr, "[CPP-V2-TRANSITION] Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 4663b4ec..a407d1e4 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -4,11 +4,12 @@ * CONCURRENCY AND SYNCHRONIZATION DESIGN: * * 1. Threading Model: - * - Uses MHD_USE_SELECT_INTERNALLY with fixed thread pool + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool * - Thread pool size = CPU cores * 2 (auto-detected at startup) * - Threads are reused across connections for efficiency - * - I/O multiplexing (select/poll) distributes connections across thread pool + * - I/O multiplexing (poll) distributes connections across thread pool * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() * * 2. Resource Scaling: * - All limits automatically scale with detected CPU count: @@ -40,12 +41,16 @@ */ #include +#include #include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -58,11 +63,23 @@ #include #include #include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> client_cache_secret; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads // Threading configuration - set at startup based on CPU cores @@ -78,13 +95,18 @@ std::string generate_uuid() { std::shared_ptr get_client(const std::string &client_id) { - // Use shared_lock for concurrent reads - multiple threads can read simultaneously - std::shared_lock lock(client_mutex); + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); auto it = client_cache_secret.find(client_id); if (it == client_cache_secret.end()) { return std::shared_ptr(); } else { - return it->second; + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; } } @@ -93,7 +115,31 @@ void set_client(const std::string &client_id, std::shared_ptr lock(client_mutex); - client_cache_secret.emplace(client_id, client); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V3] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V3] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); } std::string get_header_value(struct MHD_Connection *connection, @@ -188,8 +234,8 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, std::string commitmentPolicy = get_config(request, "commitmentPolicy"); std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); - // Create CryptoConfigurationV3 and S3EncryptionClientV3 based on key type - std::shared_ptr encryption_client; + // Create CryptoConfigurationV3 based on key type + CryptoConfigurationV3 config; if (!aes_key_blob.empty()) { // Base64 decode the AES key @@ -208,64 +254,45 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( key_buffer ); - CryptoConfigurationV3 config(materials); - - if (legacy1 || legacy2) - config.AllowLegacy(); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); - } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); - } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); - } - - // Configure ClientConfiguration with retry strategy for throttling - // Match S3 connection pool size to thread pool size for optimal resource usage - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = g_thread_pool_size; - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - - encryption_client = std::make_shared(config, clientConfig); + config = CryptoConfigurationV3(materials); } else if (!kms_key_id.empty()) { auto materials = std::make_shared(kms_key_id); - CryptoConfigurationV3 config(materials); - - if (legacy1 || legacy2) - config.AllowLegacy(); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); - } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); - } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); - } - - // Configure ClientConfiguration with retry strategy for throttling - // Match S3 connection pool size to thread pool size for optimal resource usage - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = g_thread_pool_size; - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - - encryption_client = std::make_shared(config, clientConfig); + config = CryptoConfigurationV3(materials); } else { return send_response(connection, 400, "{\"error\":\"No valid key material provided\"}"); } + + // Apply common configuration settings (applies to both AES and KMS) + if (legacy1 || legacy2) + config.AllowLegacy(); + if (inst_put) + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Configure commitment policy (applies to both AES and KMS) + if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF" || + encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + } + + // Create S3EncryptionClientV3 with standard configuration + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + auto encryption_client = std::make_shared(config, clientConfig); std::string client_id = generate_uuid(); set_client(client_id, encryption_client); @@ -284,13 +311,18 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: metadata is empty\n"); return; } + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; + int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -323,6 +355,9 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + // Add to map map.emplace(key, value); @@ -333,6 +368,8 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } + + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, @@ -340,8 +377,11 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, std::string key, std::string client_id, std::string metadata) { - fprintf(stderr, "[CPP-V3] GetObject request: bucket=%s, key=%s, client_id=%s\n", - bucket.c_str(), key.c_str(), client_id.c_str()); + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length()); auto client = get_client(client_id); if (!client) { @@ -357,9 +397,20 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + // Keep outcome alive to ensure stream remains valid auto outcome = client->GetObject(request, kmsContextMap); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + if (outcome.IsSuccess()) { // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); @@ -391,16 +442,28 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, return ret; } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V3] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + auto msg = make_error(outcome.GetError().GetMessage(), 500); fprintf(stderr, "[CPP-V3] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V3] GetObject exception: %s\n", e.what()); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } catch (...) { - fprintf(stderr, "[CPP-V3] GetObject unknown exception\n"); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); auto msg = make_error("Unknown error in GetObject", 500); return send_response(connection, 500, msg); } @@ -459,6 +522,32 @@ void request_completed(void *cls, struct MHD_Connection *connection, void **con_cls, enum MHD_RequestTerminationCode toe) { // Clean up the request-specific context when request is truly complete // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V3] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + if (*con_cls != nullptr) { std::string *body = static_cast(*con_cls); delete body; // Safe to delete now - all synchronous operations are complete @@ -470,16 +559,32 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - std::string method_str(method); - bool is_push = method_str == "POST" || method_str == "PUT"; + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V3] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); + } // Initialize request context on first call if (*con_cls == nullptr) { // Allocate unique state for each request to avoid race conditions *con_cls = new std::string(); + fprintf(stderr, "[CPP-V3] REQUEST INIT: allocated new request context for %s %s\n", method, url); return MHD_YES; } + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V3] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V3] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + // Accumulate request body data for POST/PUT requests if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); @@ -490,18 +595,22 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, // At this point, *upload_data_size == 0, meaning we have all the data // Now we can safely process the request - - std::string url_str(url); + + // LOG: About to process request + fprintf(stderr, "[CPP-V3] PROCESSING: %s %s\n", method, url); // Handle client creation endpoint if (is_push && url_str == "/client") { + fprintf(stderr, "[CPP-V3] Handling /client endpoint\n"); std::string *body = static_cast(*con_cls); MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V3] /client handler returned: %d\n", result); return result; } // Handle object operations if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V3] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { @@ -510,26 +619,90 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string client_id = get_header_value(connection, "clientid"); std::string metadata = get_header_value(connection, "content-metadata"); + fprintf(stderr, "[CPP-V3] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + if (method_str == "GET") { + fprintf(stderr, "[CPP-V3] Dispatching to handle_get_object\n"); MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V3] handle_get_object returned: %d\n", result); return result; } else if (method_str == "PUT") { + fprintf(stderr, "[CPP-V3] Dispatching to handle_put_object\n"); std::string *body = static_cast(*con_cls); MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V3] handle_put_object returned: %d\n", result); return result; } else { + fprintf(stderr, "[CPP-V3] Method not allowed: %s\n", method); return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } - // Return error for unrecognized endpoints - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V3] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V3] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V3] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V3] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V3] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; } int main() { Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + Aws::InitAPI(options); // Detect CPU core count and configure threading @@ -547,17 +720,20 @@ int main() { fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); - fprintf(stderr, "[CONFIG] S3 client maxConnections: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); int port = 8091; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD, port, NULL, NULL, + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, MHD_OPTION_CONNECTION_LIMIT, connection_limit, - MHD_OPTION_CONNECTION_TIMEOUT, 30, + MHD_OPTION_CONNECTION_TIMEOUT, 10, MHD_OPTION_END); if (!daemon) { diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index d30888a3..1c0d4308 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -61,7 +61,7 @@ tasks { systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") maxParallelForks = 1 // One JVM systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", - Math.max(1, Runtime.getRuntime().availableProcessors() - 9).toString()) // Scale with CPU, reserve 2 cores + Math.max(1, Runtime.getRuntime().availableProcessors() - 6).toString()) // Scale with CPU, reserve 2 cores // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) diff --git a/test-server/php-v2-server/Makefile b/test-server/php-v2-server/Makefile index a9d04134..719ea238 100644 --- a/test-server/php-v2-server/Makefile +++ b/test-server/php-v2-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 server starting..." stop-server: diff --git a/test-server/php-v2-transition-server/Makefile b/test-server/php-v2-transition-server/Makefile index 61eb3a84..a3d038de 100644 --- a/test-server/php-v2-transition-server/Makefile +++ b/test-server/php-v2-transition-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 Transition server starting..." stop-server: diff --git a/test-server/php-v3-server/Makefile b/test-server/php-v3-server/Makefile index 2b9661f2..9460d4ed 100644 --- a/test-server/php-v3-server/Makefile +++ b/test-server/php-v3-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V3 server starting..." stop-server: From d0936710ac0d2258e512bcba4b9fbc3a324ccc5d Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 21 Nov 2025 21:39:35 -0800 Subject: [PATCH 29/40] update the test --- test-server/java-tests/build.gradle.kts | 2 +- .../amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 1c0d4308..14c3eec1 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -61,7 +61,7 @@ tasks { systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") maxParallelForks = 1 // One JVM systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", - Math.max(1, Runtime.getRuntime().availableProcessors() - 6).toString()) // Scale with CPU, reserve 2 cores + Math.max(1, Runtime.getRuntime().availableProcessors() - 2).toString()) // Scale with CPU, reserve 2 cores // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java index 100925a9..9565639b 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java @@ -180,7 +180,7 @@ public void GIVEN_KCGCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncry ) { S3ECTestServerClient encClient = TestUtils.testServerClientFor(encLang); - final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang; + final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang + "-" + decLang; final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(TestUtils.KMS_KEY_ARN) From 2e315c707205f799b59c7c12658511eb31c566c5 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Sat, 22 Nov 2025 16:44:01 -0800 Subject: [PATCH 30/40] fix c build issue --- test-server/cpp-v3-server/main.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index a407d1e4..fc0d0452 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -64,6 +64,7 @@ #include #include #include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; @@ -235,7 +236,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); // Create CryptoConfigurationV3 based on key type - CryptoConfigurationV3 config; + std::optional config; if (!aes_key_blob.empty()) { // Base64 decode the AES key @@ -254,10 +255,10 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( key_buffer ); - config = CryptoConfigurationV3(materials); + config.emplace(materials); } else if (!kms_key_id.empty()) { auto materials = std::make_shared(kms_key_id); - config = CryptoConfigurationV3(materials); + config.emplace(materials); } else { return send_response(connection, 400, "{\"error\":\"No valid key material provided\"}"); @@ -265,9 +266,9 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, // Apply common configuration settings (applies to both AES and KMS) if (legacy1 || legacy2) - config.AllowLegacy(); + config->AllowLegacy(); if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); // Configure commitment policy (applies to both AES and KMS) if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { @@ -275,24 +276,24 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") { return unsupported(connection, commitmentPolicy, encryptionAlgorithm); } - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") { return unsupported(connection, commitmentPolicy, encryptionAlgorithm); } - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") { return unsupported(connection, commitmentPolicy, encryptionAlgorithm); } - config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + config->SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); } // Create S3EncryptionClientV3 with standard configuration Aws::Client::ClientConfiguration clientConfig; clientConfig.maxConnections = 512; // Large pool per client clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - auto encryption_client = std::make_shared(config, clientConfig); + auto encryption_client = std::make_shared(*config, clientConfig); std::string client_id = generate_uuid(); set_client(client_id, encryption_client); From b3c4997e3a7454755f2ddfddc013e2e69fb21499 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 24 Nov 2025 16:26:00 -0800 Subject: [PATCH 31/40] net v2 transition --- .gitmodules | 4 + .../amazon/encryption/s3/TestUtils.java | 12 +- .../.duvet/.gitignore | 3 + .../.duvet/config.toml | 27 ++++ .../net-v2-transition-server/.gitignore | 44 +++++++ .../Controllers/ClientController.cs | 121 ++++++++++++++++++ .../Controllers/ObjectController.cs | 105 +++++++++++++++ test-server/net-v2-transition-server/Makefile | 43 +++++++ .../Models/ClientRequest.cs | 55 ++++++++ .../Models/ClientResponse.cs | 8 ++ .../Models/ErrorModels.cs | 17 +++ .../NetV2TransitionServer.csproj | 27 ++++ .../net-v2-transition-server/Program.cs | 17 +++ .../net-v2-transition-server/README.md | 66 ++++++++++ .../Services/ClientCacheService.cs | 28 ++++ .../s3ec-v2-transition-branch | 1 + 16 files changed, 572 insertions(+), 6 deletions(-) create mode 100644 test-server/net-v2-transition-server/.duvet/.gitignore create mode 100644 test-server/net-v2-transition-server/.duvet/config.toml create mode 100644 test-server/net-v2-transition-server/.gitignore create mode 100644 test-server/net-v2-transition-server/Controllers/ClientController.cs create mode 100644 test-server/net-v2-transition-server/Controllers/ObjectController.cs create mode 100644 test-server/net-v2-transition-server/Makefile create mode 100644 test-server/net-v2-transition-server/Models/ClientRequest.cs create mode 100644 test-server/net-v2-transition-server/Models/ClientResponse.cs create mode 100644 test-server/net-v2-transition-server/Models/ErrorModels.cs create mode 100644 test-server/net-v2-transition-server/NetV2TransitionServer.csproj create mode 100644 test-server/net-v2-transition-server/Program.cs create mode 100644 test-server/net-v2-transition-server/README.md create mode 100644 test-server/net-v2-transition-server/Services/ClientCacheService.cs create mode 160000 test-server/net-v2-transition-server/s3ec-v2-transition-branch diff --git a/.gitmodules b/.gitmodules index a88a83f9..27d4960b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,3 +58,7 @@ [submodule "test-server/cpp-v2-server/aws-sdk-cpp"] path = test-server/cpp-v2-server/aws-sdk-cpp url = git@github.com:awslabs/aws-sdk-cpp-staging.git +[submodule "test-server/net-v2-transition-server/s3ec-v2-transition-branch"] + path = test-server/net-v2-transition-server/s3ec-v2-transition-branch + url = https://github.com/aws/private-amazon-s3-encryption-client-dotnet-staging.git + branch = s3ec-v2-transition-integ diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index d3d25d58..a008d3f9 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -90,21 +90,21 @@ public class TestUtils { // Sets of unsupported features by language public static final Set ENCRYPTION_CONTEXT_ON_DECRYPT_UNSUPPORTED = - Set.of(GO_V3_CURRENT, PHP_V2_CURRENT, PHP_V2_TRANSITION, PHP_V3, NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4); + Set.of(GO_V3_CURRENT, PHP_V2_CURRENT, PHP_V2_TRANSITION, PHP_V3, NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = - Set.of(NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4); + Set.of(NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); // For now, only .NET and Java have RSA support public static final Set RAW_SUPPORTED = Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 - , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 + , NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4 ); // .NET only supports decrypting instruction files using AES and RSA. // Python MUST support decrypting KMS instruction files, but does not yet. public static final Set KMS_INSTRUCTION_FILE_UNSUPPORTED = - Set.of(NET_V2_CURRENT, NET_V2_TRANSITION, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4); + Set.of(NET_V2_CURRENT, NET_V2_TRANSITION, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); // Go does not write with instruction files public static final Set INSTRUCTION_FILE_PUT_UNSUPPORTED = @@ -131,7 +131,7 @@ public class TestUtils { Set.of( JAVA_V3_TRANSITION, GO_V3_TRANSITION, - // NET_V2_TRANSITION, + NET_V2_TRANSITION, NET_V3_TRANSITION, CPP_V2_TRANSITION, PHP_V2_TRANSITION, @@ -169,7 +169,7 @@ public class TestUtils { servers.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); servers.put(JAVA_V3_TRANSITION, new LanguageServerTarget(JAVA_V3_TRANSITION, "8094")); servers.put(GO_V3_TRANSITION, new LanguageServerTarget(GO_V3_TRANSITION, "8095")); - // servers.put(NET_V2_TRANSITION, new LanguageServerTarget(NET_V2_TRANSITION, "8096")); + servers.put(NET_V2_TRANSITION, new LanguageServerTarget(NET_V2_TRANSITION, "8096")); servers.put(RUBY_V2_TRANSITION, new LanguageServerTarget(RUBY_V2_TRANSITION, "8098")); servers.put(PHP_V2_TRANSITION, new LanguageServerTarget(PHP_V2_TRANSITION, "8099")); servers.put(JAVA_V4, new LanguageServerTarget(JAVA_V4, "8088")); diff --git a/test-server/net-v2-transition-server/.duvet/.gitignore b/test-server/net-v2-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..93956e36 --- /dev/null +++ b/test-server/net-v2-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ \ No newline at end of file diff --git a/test-server/net-v2-transition-server/.duvet/config.toml b/test-server/net-v2-transition-server/.duvet/config.toml new file mode 100644 index 00000000..416dcfb9 --- /dev/null +++ b/test-server/net-v2-transition-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "**/*.cs" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/net-v2-transition-server/.gitignore b/test-server/net-v2-transition-server/.gitignore new file mode 100644 index 00000000..4c20cbc8 --- /dev/null +++ b/test-server/net-v2-transition-server/.gitignore @@ -0,0 +1,44 @@ +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# NuGet Packages +*.nupkg +*.snupkg +packages/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# VS Code +.vscode/ + +# macOS +.DS_Store + +# Temporary files +*.tmp +*.temp diff --git a/test-server/net-v2-transition-server/Controllers/ClientController.cs b/test-server/net-v2-transition-server/Controllers/ClientController.cs new file mode 100644 index 00000000..a66fb342 --- /dev/null +++ b/test-server/net-v2-transition-server/Controllers/ClientController.cs @@ -0,0 +1,121 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text.Json; +using Amazon.Extensions.S3.Encryption; +using Amazon.Extensions.S3.Encryption.Primitives; +using Microsoft.AspNetCore.Mvc; +using NetV3TransitionServer.Models; +using NetV3TransitionServer.Services; + +namespace NetV3TransitionServer.Controllers; + +[ApiController] +[Route("[controller]")] +public class ClientController(IClientCacheService clientCacheService, ILogger logger) : ControllerBase +{ + [HttpPost] + public IActionResult CreateClient([FromBody] ClientRequest request) + { + // Return 501 for not implemented features by the server + if (request.Config.EnableDelayedAuthenticationMode) + return StatusCode(501, new GenericServerError { Message = "EnableDelayedAuthenticationMode not supported" }); + if (request.Config.SetBufferSize.HasValue) + return StatusCode(501, new GenericServerError { Message = "SetBufferSize not supported" }); + if (request.Config.KeyMaterial.AesKey != null) + return StatusCode(501, new GenericServerError { Message = "AesKey not supported" }); + + try + { + EncryptionMaterialsV2 encryptionMaterial; + if (request.Config.KeyMaterial.KmsKeyId != null) + { + // The POST request does not contain encryption context. + // However, encryption context is a required field when using KMS. + // So, we are passing empty dictionary. + var encryptionContext = new Dictionary(); + var kmsKeyId = request.Config.KeyMaterial.KmsKeyId; + encryptionMaterial = new EncryptionMaterialsV2(kmsKeyId, KmsType.KmsContext, encryptionContext); + logger.LogInformation( + "[NET-V3-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}", + kmsKeyId); + } + else if (request.Config.KeyMaterial.RsaKey != null) + { + var rsaKeyBytes = request.Config.KeyMaterial.RsaKey; + var rsaKey = RSA.Create(); + rsaKey.ImportPkcs8PrivateKey(new ReadOnlySpan(rsaKeyBytes), out _); + encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); + logger.LogInformation( + "Created EncryptionMaterialsV2: RSA"); + } else + { + return StatusCode(501, new GenericServerError { Message = "Unknown or missing key material!" }); + } + + var enableLegacyUnauthenticatedModes = request.Config.EnableLegacyUnauthenticatedModes; + var enableLegacyWrappingAlgorithms = request.Config.EnableLegacyWrappingAlgorithms; + var commitmentPolicy = MapCommitmentPolicy(request.Config.CommitmentPolicy); + + // SecurityProfile V2AndLegacy can decrypt from legacy S3EC but V2 cannot + var enableLegacyMode = enableLegacyUnauthenticatedModes || enableLegacyWrappingAlgorithms; + var securityProfile = enableLegacyMode ? SecurityProfile.V2AndLegacy : SecurityProfile.V2; + logger.LogInformation("[NET-V3-Transitional] Created securityProfile= {securityProfile}", securityProfile.ToString()); + + var encryptionAlgorithm = MapEncryptionAlgorithm(request.Config.EncryptionAlgorithm); + // var encryptionAlgorithm = commitmentPolicy == Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt ? ContentEncryptionAlgorithm.AesGcm : ContentEncryptionAlgorithm.AesGcmWithCommitment; + logger.LogInformation("[NET-V3-Transitional] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy); + logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); + + var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm); + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) + { + configuration.StorageMode = CryptoStorageMode.InstructionFile; + logger.LogInformation("[NET-V3-Transitional] Created StorageMode= InstructionFile"); + } + // Create S3 encryption client + var encryptionClient = new AmazonS3EncryptionClientV2(configuration, encryptionMaterial); + // Add to cache and return client ID + var clientId = clientCacheService.AddClient(encryptionClient); + var response = new ClientResponse { ClientId = clientId }; + + logger.LogInformation("[NET-V3-Transitional] Created S3EC client with ID: {clientId}", clientId); + + return new ContentResult + { + Content = JsonSerializer.Serialize(response), + ContentType = "application/json", + StatusCode = 200 + }; + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V3-Transitional] Failed to create S3EC client"); + return StatusCode(500, new S3EncryptionClientError + { + Message = $"Failed to create client: {ex.Message}" + }); + } + } + + private static Amazon.Extensions.S3.Encryption.CommitmentPolicy MapCommitmentPolicy(Models.CommitmentPolicy? policy) + { + return policy switch + { + Models.CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptRequireDecrypt, + Models.CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.RequireEncryptAllowDecrypt, + Models.CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT => Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt, + _ => Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt + }; + } + + // This is redundant but useful when tests starts sending EncryptionAlgorithm + private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.EncryptionAlgorithm? algorithm) + { + return algorithm switch + { + Models.EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF => ContentEncryptionAlgorithm.AesGcm, + Models.EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY => ContentEncryptionAlgorithm.AesGcmWithCommitment, + _ => ContentEncryptionAlgorithm.AesGcm + }; + } +} \ No newline at end of file diff --git a/test-server/net-v2-transition-server/Controllers/ObjectController.cs b/test-server/net-v2-transition-server/Controllers/ObjectController.cs new file mode 100644 index 00000000..76548815 --- /dev/null +++ b/test-server/net-v2-transition-server/Controllers/ObjectController.cs @@ -0,0 +1,105 @@ +using System.Text.Json; +using Amazon.S3.Model; +using Microsoft.AspNetCore.Mvc; +using NetV3TransitionServer.Models; +using NetV3TransitionServer.Services; + +namespace NetV3TransitionServer.Controllers; + +[ApiController] +[Route("[controller]")] +public class ObjectController(IClientCacheService clientCacheService, ILogger logger) : ControllerBase +{ + [HttpPut("{bucket}/{key}")] + public async Task PutObject(string bucket, string key) + { + logger.LogInformation("Starting PutObject"); + var clientId = Request.Headers["clientId"].FirstOrDefault(); + if (string.IsNullOrEmpty(clientId)) + return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" }); + + var client = clientCacheService.GetClient(clientId); + if (client == null) + return NotFound(new GenericServerError { Message = $"[NET-V3-Transitional] No client found for ClientID: {clientId}" }); + + try + { + // Read raw body data + using var memoryStream = new MemoryStream(); + // Request is the HTTP request this method is currently handling + await Request.Body.CopyToAsync(memoryStream); + var bodyBytes = memoryStream.ToArray(); + + // Create put request + var putRequest = new PutObjectRequest + { + BucketName = bucket, + Key = key, + InputStream = new MemoryStream(bodyBytes) + }; + + await client.PutObjectAsync(putRequest); + + var response = new { bucket, key }; + + logger.LogInformation( + "[NET-V3-Transitional] Put object succeeded for bucket={bucket}, key={key} and clientId = {clientId}", + bucket, key, clientId); + return new ContentResult + { + Content = JsonSerializer.Serialize(response), + ContentType = "application/json", + StatusCode = 200 + }; + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V3-Transitional] Failed to put object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = $"[NET-V3-Transitional] Failed to put object: {ex.Message}" }); + } + } + + [HttpGet("{bucket}/{key}")] + public async Task GetObject(string bucket, string key) + { + logger.LogInformation("Starting GetObject"); + var clientId = Request.Headers["clientId"].FirstOrDefault(); + if (string.IsNullOrEmpty(clientId)) + return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" }); + + var client = clientCacheService.GetClient(clientId); + if (client == null) + return NotFound(new GenericServerError { Message = $"[NET-V3-Transitional] No client found for ClientID: {clientId}" }); + + try + { + var getRequest = new GetObjectRequest + { + BucketName = bucket, + Key = key + }; + var response = await client.GetObjectAsync(getRequest); + logger.LogInformation("[NET-V3-Transitional] Got object from S3 for bucket={bucket}, key={key}", bucket, key); + // Read response body + using var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + var bodyBytes = memoryStream.ToArray(); + + // Convert metadata to content-metadata header format + var metadataList = response.Metadata.Keys + .Select(metaDataKey => $"{metaDataKey}={response.Metadata[metaDataKey]}") + .ToList(); + var metadataStr = string.Join(",", metadataList); + + // Set response headers + Response.Headers["Content-Metadata"] = metadataStr; + + return File(bodyBytes, "application/octet-stream"); + } + catch (Exception ex) + { + logger.LogError(ex, "[NET-V3-Transitional] Failed to get object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = ex.Message }); + } + } +} \ No newline at end of file diff --git a/test-server/net-v2-transition-server/Makefile b/test-server/net-v2-transition-server/Makefile new file mode 100644 index 00000000..eba78e1c --- /dev/null +++ b/test-server/net-v2-transition-server/Makefile @@ -0,0 +1,43 @@ +# Makefile for S3 Encryption Client .NET Testing + +.PHONY: build-server start-server stop-server wait-for-server + +PID_FILE_NET_V3_TRANSITION := net-v3-transition-server.pid +PORT_NET_V3_TRANSITION := 8100 + +build-server: + @echo "Building .NET V3 transition server..." + dotnet build + +start-server: + $(MAKE) start-net-v3-transition-server + +stop-server: + @echo "Stopping .NET V3 Transition server on port $(PORT_NET_V3_TRANSITION)..." + @lsof -ti:$(PORT_NET_V3_TRANSITION) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE_NET_V3_TRANSITION) ]; then \ + pkill -P $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V3_TRANSITION); \ + fi + @rm -f server.log + @echo "Server stopped" + +# Start .NET V3 transition server in background +start-net-v3-transition-server: + @echo "Starting .NET V3 transition server..." + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + dotnet run > server.log 2>&1 & echo $$! > $(PID_FILE_NET_V3_TRANSITION) + @echo ".NET V3 transition server starting..." + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT_NET_V3_TRANSITION) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/net-v2-transition-server/Models/ClientRequest.cs b/test-server/net-v2-transition-server/Models/ClientRequest.cs new file mode 100644 index 00000000..07fe8520 --- /dev/null +++ b/test-server/net-v2-transition-server/Models/ClientRequest.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; + +namespace NetV3TransitionServer.Models; + +public class ClientRequest +{ + [Required] + public ClientConfig Config { get; set; } = new(); +} + +public class ClientConfig +{ + public bool EnableLegacyUnauthenticatedModes { get; set; } = false; + public bool EnableLegacyWrappingAlgorithms { get; set; } = false; + public bool EnableDelayedAuthenticationMode { get; set; } = false; + public long? SetBufferSize { get; set; } + [Required] + public KeyMaterial KeyMaterial { get; set; } = new(); + [JsonPropertyName("commitmentPolicy")] + public CommitmentPolicy? CommitmentPolicy { get; set; } + [JsonPropertyName("encryptionAlgorithm")] + public EncryptionAlgorithm? EncryptionAlgorithm { get; set; } + public InstructionFileConfig? InstructionFileConfig { get; set; } +} + +public class KeyMaterial +{ + public byte[]? RsaKey { get; set; } + public byte[]? AesKey { get; set; } + public string? KmsKeyId { get; set; } +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum CommitmentPolicy +{ + REQUIRE_ENCRYPT_REQUIRE_DECRYPT, + REQUIRE_ENCRYPT_ALLOW_DECRYPT, + FORBID_ENCRYPT_ALLOW_DECRYPT +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum EncryptionAlgorithm +{ + ALG_AES_256_CBC_IV16_NO_KDF, + ALG_AES_256_GCM_IV12_TAG16_NO_KDF, + ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY +} + +public class InstructionFileConfig +{ + public string? ClientId { get; set; } + public bool EnableInstructionFilePutObject { get; set; } = false; + public bool DisableInstructionFile { get; set; } = false; +} \ No newline at end of file diff --git a/test-server/net-v2-transition-server/Models/ClientResponse.cs b/test-server/net-v2-transition-server/Models/ClientResponse.cs new file mode 100644 index 00000000..43c94a3e --- /dev/null +++ b/test-server/net-v2-transition-server/Models/ClientResponse.cs @@ -0,0 +1,8 @@ +using System.Text.Json.Serialization; + +namespace NetV3TransitionServer.Models; + +public class ClientResponse +{ + [JsonPropertyName("clientId")] public string ClientId { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/test-server/net-v2-transition-server/Models/ErrorModels.cs b/test-server/net-v2-transition-server/Models/ErrorModels.cs new file mode 100644 index 00000000..7fbf6680 --- /dev/null +++ b/test-server/net-v2-transition-server/Models/ErrorModels.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace NetV3TransitionServer.Models; + +public class GenericServerError +{ + [JsonPropertyName("__type")] + public string Type { get; set; } = "software.amazon.encryption.s3#GenericServerError"; + public string Message { get; set; } = string.Empty; +} + +public class S3EncryptionClientError +{ + [JsonPropertyName("__type")] + public string Type { get; set; } = "software.amazon.encryption.s3#S3EncryptionClientError"; + public string Message { get; set; } = string.Empty; +} diff --git a/test-server/net-v2-transition-server/NetV2TransitionServer.csproj b/test-server/net-v2-transition-server/NetV2TransitionServer.csproj new file mode 100644 index 00000000..1f91ea9a --- /dev/null +++ b/test-server/net-v2-transition-server/NetV2TransitionServer.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + enable + enable + false + + + + false + + + + + + + + + + + + + + + + diff --git a/test-server/net-v2-transition-server/Program.cs b/test-server/net-v2-transition-server/Program.cs new file mode 100644 index 00000000..138743c9 --- /dev/null +++ b/test-server/net-v2-transition-server/Program.cs @@ -0,0 +1,17 @@ +using NetV3TransitionServer.Services; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddControllers(); +builder.Services.AddSingleton(); + +const int port = 8100; + +builder.WebHost.UseUrls($"http://localhost:{port}"); + +var app = builder.Build(); + +app.MapControllers(); + +Console.WriteLine($"Starting server on port {port}"); +app.Run(); diff --git a/test-server/net-v2-transition-server/README.md b/test-server/net-v2-transition-server/README.md new file mode 100644 index 00000000..7634a4e7 --- /dev/null +++ b/test-server/net-v2-transition-server/README.md @@ -0,0 +1,66 @@ +# Net-V2-V3-Server + +A .NET test server for Amazon S3 encryption client .NET v3 transition. + +## Project Structure + +``` +net-v2-v3-server/ +├── Controllers/ # API controllers +├── Models/ # Data models +├── Services/ # Business logic services +├── Program.cs # Application entry point +├── NetV3TransitionServer.csproj # Project file +└── README.md # This file +``` + +## Running the Server + +For S3 Encryption Client v3 transition (runs on port 8100): + +```bash +dotnet run +``` + +## API Endpoints + +### Client Management + +- `POST /Client` - Create a new S3 encryption client + +### Object Operations + +- `PUT /{bucket}/{key}` - Upload an encrypted object to S3 +- `GET /{bucket}/{key}` - Download and decrypt an object from S3 + +All object operations require a `clientId` header to specify which client to use. + +## Example Usage + +### Create a Client + +```bash +curl -i -X POST \ + -H "Content-Type: application/json" \ + -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ + -d '{"config":{"enableLegacyUnauthenticatedModes":true,"enableLegacyWrappingAlgorithms":true,"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}, "encryptionContext": {"abc": "b"}, "CommitmentPolicy":"REQUIRE_ENCRYPT_REQUIRE_DECRYPT"}}' \ + http://localhost:8100/client +``` + +### Upload an Object + +```bash +curl -X PUT \ + -H "clientid: 7978763a-a02b-4dea-a5d4-78ef11d13d12" \ + -H "content-type: application/octet-stream" \ + -d "simple-test-input-net" \ + http://localhost:8083/object/s3ec-test-server-github-bucket/cross-lang-test-key-kms-ec-dotnet +``` + +### Download an Object + +```bash +curl -X GET \ + -H "clientid: 7978763a-a02b-4dea-a5d4-78ef11d13d12" \ + http://localhost:8083/object/s3ec-test-server-github-bucket/cross-lang-test-key-kms-ec-dotnet +``` diff --git a/test-server/net-v2-transition-server/Services/ClientCacheService.cs b/test-server/net-v2-transition-server/Services/ClientCacheService.cs new file mode 100644 index 00000000..0e7332ca --- /dev/null +++ b/test-server/net-v2-transition-server/Services/ClientCacheService.cs @@ -0,0 +1,28 @@ +using Amazon.Extensions.S3.Encryption; +using System.Collections.Concurrent; + +namespace NetV3TransitionServer.Services; + +public interface IClientCacheService +{ + string AddClient(AmazonS3EncryptionClientV2 client); + AmazonS3EncryptionClientV2? GetClient(string clientId); +} + +public class ClientCacheService : IClientCacheService +{ + private readonly ConcurrentDictionary _clients = new(); + + public string AddClient(AmazonS3EncryptionClientV2 client) + { + var clientId = Guid.NewGuid().ToString(); + _clients[clientId] = client; + return clientId; + } + + public AmazonS3EncryptionClientV2? GetClient(string clientId) + { + _clients.TryGetValue(clientId, out var client); + return client; + } +} diff --git a/test-server/net-v2-transition-server/s3ec-v2-transition-branch b/test-server/net-v2-transition-server/s3ec-v2-transition-branch new file mode 160000 index 00000000..034a2686 --- /dev/null +++ b/test-server/net-v2-transition-server/s3ec-v2-transition-branch @@ -0,0 +1 @@ +Subproject commit 034a2686c8d08cdd8079cc3f95fe1aaf8e3f1a77 From cb6e80f10dda66c596be65ac543fd0e2511aa023 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 24 Nov 2025 16:31:32 -0800 Subject: [PATCH 32/40] m --- test-server/net-v2-transition-server/Makefile | 2 +- test-server/net-v2-transition-server/Program.cs | 2 +- test-server/net-v2-transition-server/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-server/net-v2-transition-server/Makefile b/test-server/net-v2-transition-server/Makefile index eba78e1c..646d2e54 100644 --- a/test-server/net-v2-transition-server/Makefile +++ b/test-server/net-v2-transition-server/Makefile @@ -3,7 +3,7 @@ .PHONY: build-server start-server stop-server wait-for-server PID_FILE_NET_V3_TRANSITION := net-v3-transition-server.pid -PORT_NET_V3_TRANSITION := 8100 +PORT_NET_V3_TRANSITION := 8096 build-server: @echo "Building .NET V3 transition server..." diff --git a/test-server/net-v2-transition-server/Program.cs b/test-server/net-v2-transition-server/Program.cs index 138743c9..d7af9fd3 100644 --- a/test-server/net-v2-transition-server/Program.cs +++ b/test-server/net-v2-transition-server/Program.cs @@ -5,7 +5,7 @@ builder.Services.AddControllers(); builder.Services.AddSingleton(); -const int port = 8100; +const int port = 8096; builder.WebHost.UseUrls($"http://localhost:{port}"); diff --git a/test-server/net-v2-transition-server/README.md b/test-server/net-v2-transition-server/README.md index 7634a4e7..8e352436 100644 --- a/test-server/net-v2-transition-server/README.md +++ b/test-server/net-v2-transition-server/README.md @@ -16,7 +16,7 @@ net-v2-v3-server/ ## Running the Server -For S3 Encryption Client v3 transition (runs on port 8100): +For S3 Encryption Client v3 transition (runs on port 8096): ```bash dotnet run @@ -44,7 +44,7 @@ curl -i -X POST \ -H "Content-Type: application/json" \ -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ -d '{"config":{"enableLegacyUnauthenticatedModes":true,"enableLegacyWrappingAlgorithms":true,"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}, "encryptionContext": {"abc": "b"}, "CommitmentPolicy":"REQUIRE_ENCRYPT_REQUIRE_DECRYPT"}}' \ - http://localhost:8100/client + http://localhost:8096/client ``` ### Upload an Object From 305be032e72de8786b1b19e9c9b597acce23d412 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 24 Nov 2025 16:32:58 -0800 Subject: [PATCH 33/40] m --- test-server/net-v2-transition-server/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test-server/net-v2-transition-server/README.md b/test-server/net-v2-transition-server/README.md index 8e352436..aeb6e8d5 100644 --- a/test-server/net-v2-transition-server/README.md +++ b/test-server/net-v2-transition-server/README.md @@ -1,22 +1,21 @@ -# Net-V2-V3-Server +# Net-V2-transition-Server -A .NET test server for Amazon S3 encryption client .NET v3 transition. +A .NET test server for Amazon S3 encryption client .NET v2 transition. ## Project Structure ``` -net-v2-v3-server/ ├── Controllers/ # API controllers ├── Models/ # Data models ├── Services/ # Business logic services ├── Program.cs # Application entry point -├── NetV3TransitionServer.csproj # Project file +├── NetV2TransitionServer.csproj # Project file └── README.md # This file ``` ## Running the Server -For S3 Encryption Client v3 transition (runs on port 8096): +For S3 Encryption Client v2 transition (runs on port 8096): ```bash dotnet run From 03db2bece3e0df1ec78145e3e25b9fc5d5094b90 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 24 Nov 2025 16:36:03 -0800 Subject: [PATCH 34/40] m --- .../src/it/java/software/amazon/encryption/s3/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index a008d3f9..93947357 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -104,7 +104,7 @@ public class TestUtils { // .NET only supports decrypting instruction files using AES and RSA. // Python MUST support decrypting KMS instruction files, but does not yet. public static final Set KMS_INSTRUCTION_FILE_UNSUPPORTED = - Set.of(NET_V2_CURRENT, NET_V2_TRANSITION, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); + Set.of(NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); // Go does not write with instruction files public static final Set INSTRUCTION_FILE_PUT_UNSUPPORTED = From 666bb314342d19c1e4d4ef495c6cfd91e1e2ed97 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 24 Nov 2025 16:39:16 -0800 Subject: [PATCH 35/40] m --- test-server/net-v2-transition-server/Makefile | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test-server/net-v2-transition-server/Makefile b/test-server/net-v2-transition-server/Makefile index 646d2e54..65a4339d 100644 --- a/test-server/net-v2-transition-server/Makefile +++ b/test-server/net-v2-transition-server/Makefile @@ -2,39 +2,39 @@ .PHONY: build-server start-server stop-server wait-for-server -PID_FILE_NET_V3_TRANSITION := net-v3-transition-server.pid -PORT_NET_V3_TRANSITION := 8096 +PID_FILE_NET_V2_TRANSITION := net-v2-transition-server.pid +PORT_NET_V2_TRANSITION := 8096 build-server: - @echo "Building .NET V3 transition server..." + @echo "Building .NET V2 transition server..." dotnet build start-server: - $(MAKE) start-net-v3-transition-server + $(MAKE) start-net-v2-transition-server stop-server: - @echo "Stopping .NET V3 Transition server on port $(PORT_NET_V3_TRANSITION)..." - @lsof -ti:$(PORT_NET_V3_TRANSITION) | xargs kill -9 2>/dev/null || true - @if [ -f $(PID_FILE_NET_V3_TRANSITION) ]; then \ - pkill -P $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ - kill -9 $$(cat $(PID_FILE_NET_V3_TRANSITION)) 2>/dev/null || true; \ - rm -f $(PID_FILE_NET_V3_TRANSITION); \ + @echo "Stopping .NET V2 Transition server on port $(PORT_NET_V2_TRANSITION)..." + @lsof -ti:$(PORT_NET_V2_TRANSITION) | xargs kill -9 2>/dev/null || true + @if [ -f $(PID_FILE_NET_V2_TRANSITION) ]; then \ + pkill -P $$(cat $(PID_FILE_NET_V2_TRANSITION)) 2>/dev/null || true; \ + kill -9 $$(cat $(PID_FILE_NET_V2_TRANSITION)) 2>/dev/null || true; \ + rm -f $(PID_FILE_NET_V2_TRANSITION); \ fi @rm -f server.log @echo "Server stopped" -# Start .NET V3 transition server in background -start-net-v3-transition-server: - @echo "Starting .NET V3 transition server..." +# Start .NET V2 transition server in background +start-net-v2-transition-server: + @echo "Starting .NET V2 transition server..." AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run > server.log 2>&1 & echo $$! > $(PID_FILE_NET_V3_TRANSITION) - @echo ".NET V3 transition server starting..." + dotnet run > server.log 2>&1 & echo $$! > $(PID_FILE_NET_V2_TRANSITION) + @echo ".NET V2 transition server starting..." wait-for-server: - $(MAKE) -C .. wait-for-port PORT=$(PORT_NET_V3_TRANSITION) + $(MAKE) -C .. wait-for-port PORT=$(PORT_NET_V2_TRANSITION) duvet: duvet report From c1c0445186427c4c6faa57da49952e01df73ff73 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 24 Nov 2025 17:52:37 -0800 Subject: [PATCH 36/40] m --- .../it/java/software/amazon/encryption/s3/RoundTripTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 468fc708..e6fab412 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -423,7 +423,7 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(TestUtils.LanguageServerTarget la .build()); fail("Expected Exception"); } catch (S3EncryptionClientError e) { - if (language.getLanguageName().equals(NET_V3_CURRENT) || language.getLanguageName().equals(NET_V2_CURRENT) || language.getLanguageName().equals(NET_V3_TRANSITION) || language.getLanguageName().equals(NET_V4) + if (language.getLanguageName().equals(NET_V3_CURRENT) || language.getLanguageName().equals(NET_V2_CURRENT) || language.getLanguageName().equals(NET_V2_TRANSITION) || language.getLanguageName().equals(NET_V3_TRANSITION) || language.getLanguageName().equals(NET_V4) || language.getLanguageName().equals(CPP_V2_CURRENT) || language.getLanguageName().equals(CPP_V2_TRANSITION) || language.getLanguageName().equals(CPP_V3)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" From 9d317ec1d558f4443eb0dfdb57d329e9a1f896c6 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Tue, 25 Nov 2025 09:28:12 -0800 Subject: [PATCH 37/40] m --- .../Controllers/ClientController.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test-server/net-v2-transition-server/Controllers/ClientController.cs b/test-server/net-v2-transition-server/Controllers/ClientController.cs index a66fb342..b41a28d6 100644 --- a/test-server/net-v2-transition-server/Controllers/ClientController.cs +++ b/test-server/net-v2-transition-server/Controllers/ClientController.cs @@ -21,8 +21,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "AesKey not supported" }); try { @@ -39,6 +37,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) "[NET-V3-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}", kmsKeyId); } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V3-Transitional] Created EncryptionMaterialsV2: AES"); + } else if (request.Config.KeyMaterial.RsaKey != null) { var rsaKeyBytes = request.Config.KeyMaterial.RsaKey; From d4502d5b675e200d89ca9765a6462cac3d4ac34f Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Tue, 25 Nov 2025 14:26:17 -0800 Subject: [PATCH 38/40] revert Ryan's branch merge --- .github/workflows/test.yml | 4 +- test-server/Makefile | 2 +- .../go-v3-transition-server/local-go-s3ec | 2 +- test-server/go-v4-server/local-go-s3ec | 2 +- .../s3/InstructionFileFailures.java | 782 ------------------ .../amazon/encryption/s3/RoundTripTests.java | 2 +- .../amazon/encryption/s3/TestUtils.java | 122 +-- test-server/java-v3-server/Makefile | 2 +- .../java-v3-transition-server/Makefile | 2 +- test-server/java-v4-server/Makefile | 2 +- .../src/get_object.php | 4 - test-server/php-v3-server/local-php-sdk | 2 +- test-server/php-v3-server/src/get_object.php | 4 - test-server/ruby-v2-server/app.rb | 4 +- .../ruby-v2-server/lib/client_manager.rb | 36 +- .../ruby-v3-server/lib/client_manager.rb | 38 +- 16 files changed, 51 insertions(+), 959 deletions(-) delete mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6b7a9d0..57163fad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -170,7 +170,9 @@ jobs: - name: Wait for servers to start run: cd test-server && make wait-all-servers env: - MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} + AWS_REGION: us-west-2 + TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} + TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} - name: Run run-tests run: cd test-server && make run-tests diff --git a/test-server/Makefile b/test-server/Makefile index 28f6e1be..cabb09b1 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -84,7 +84,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --info --parallel --no-daemon integ -Dtest.filter.servers="$(FILTER)" + ./gradlew --build-cache --info --parallel integ -Dtest.filter.servers="$(FILTER)" @echo "Tests completed successfully" # Stop the servers diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index e59a38ca..f51a4402 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit e59a38caeddfcfbf41e064e125b5783cdfce3878 +Subproject commit f51a4402c741cd989c7984336de560e9c54baf17 diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec index e59a38ca..f51a4402 160000 --- a/test-server/go-v4-server/local-go-s3ec +++ b/test-server/go-v4-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit e59a38caeddfcfbf41e064e125b5783cdfce3878 +Subproject commit f51a4402c741cd989c7984336de560e9c54baf17 diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java deleted file mode 100644 index 6096d59e..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java +++ /dev/null @@ -1,782 +0,0 @@ -/* -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0 -*/ - -package software.amazon.encryption.s3; - -import static software.amazon.encryption.s3.TestUtils.*; - -import java.nio.ByteBuffer; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.crypto.KeyGenerator; -import javax.crypto.SecretKey; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.opentest4j.TestAbortedException; - -import software.amazon.awssdk.core.ResponseBytes; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.InstructionFileConfig; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.S3ECConfig; - -/** -* Instruction File Failures Test Suite -* -* This suite enforces execution order between encrypt and decrypt phases: -* 1. EncryptTests - Encrypts objects with various key materials and creates test copies -* 2. DecryptTests - Waits for encrypt phase to complete, then tests decryption scenarios -* -* Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion -* and DecryptTests awaits before proceeding. -* -* Tests are based on the exhaustive test matrix defined at: -* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 -*/ -public class InstructionFileFailures { - // Synchronization latch - released when encrypt phase completes - private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); - - /** - * Encryption Tests - Encrypt Phase - * - * These tests encrypt objects using various key materials (KMS, RSA, AES) with instruction files. - * All tests in this class can run in parallel with each other. - * The encrypted objects are stored in thread-safe lists for use by DecryptTests. - */ - @Nested - class EncryptTests { - private static final String sharedObjectKeyBaseMetaDataMode = "test-instruction-files-cases"; - private static KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - - // Thread-safe lists for storing encrypted object keys - private static final List crossLanguageObjectsKms = - Collections.synchronizedList(new ArrayList<>()); - private static final List crossLanguageObjectsRsa = - Collections.synchronizedList(new ArrayList<>()); - private static final List crossLanguageObjectsAes = - Collections.synchronizedList(new ArrayList<>()); - - private static KeyMaterial RSA_KEY; - private static KeyMaterial AES_KEY; - - @BeforeAll - static void setupKeys() throws Exception { - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - KeyPair keyPair = keyPairGen.generateKeyPair(); - - RSA_KEY = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) - .build(); - - KeyGenerator keyGen = KeyGenerator.getInstance("AES"); - keyGen.init(256); - SecretKey aesSecretKey = keyGen.generateKey(); - - AES_KEY = KeyMaterial.builder() - .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) - .build(); - } - - /** - * Public accessors for decrypt tests to retrieve encrypted object keys and key materials - */ - static List getCrossLanguageObjectsKms() { - return new ArrayList<>(crossLanguageObjectsKms); - } - - static List getCrossLanguageObjectsRsa() { - return new ArrayList<>(crossLanguageObjectsRsa); - } - - static List getCrossLanguageObjectsAes() { - return new ArrayList<>(crossLanguageObjectsAes); - } - - static KeyMaterial getRsaKey() { - return RSA_KEY; - } - - static KeyMaterial getAesKey() { - return AES_KEY; - } - - static KeyMaterial getKmsKeyArn() { - return kmsKeyArn; - } - - public static Stream improvedClientsCanPutKMSWithInstructionFile() { - return improvedClientsForTest() - .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) - .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - } - - public static Stream improvedClientsCanPutRawRSAWithInstructionFile() { - return improvedClientsForTest() - .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) - .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - } - - public static Stream improvedClientsCanPutRawAESWithInstructionFile() { - return improvedClientsForTest() - .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) - .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - } - - @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") - void encrypt_with_instruction_files_kms_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .instructionFileConfig( - InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build() - ) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt( - client, - S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-kms" + language.getLanguageName()), - crossLanguageObjectsKms, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction files") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") - void encrypt_with_instruction_files_rsa_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .instructionFileConfig( - InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build() - ) - .build()) - .build()); - - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt( - client, - S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-rsa" + language.getLanguageName()), - crossLanguageObjectsRsa, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Encrypt AES KC-GCM with instruction files") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") - void encrypt_with_instruction_files_aes_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .instructionFileConfig( - InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build() - ) - .build()) - .build()); - - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt( - client, - S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), - crossLanguageObjectsAes, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - static void make_copies_to_verify_things() throws Exception { - // Create a plaintext S3 client to copy objects with instruction files - try (S3Client ptS3Client = S3Client.create()) { - List allCrossLanguageObjects = Stream.of( - crossLanguageObjectsKms.stream(), - crossLanguageObjectsRsa.stream(), - crossLanguageObjectsAes.stream() - ).flatMap(s -> s).collect(Collectors.toList()); - for (String objectKey : allCrossLanguageObjects) { - // Get the encrypted object - ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); - - // Get the instruction file - String instructionFileKey = objectKey + ".instruction"; - ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder - .bucket(TestUtils.BUCKET) - .key(instructionFileKey) - .build()); - - String instructionFileJson = instructionFile.asUtf8String(); - Map objectMetadata = encryptedObject.response().metadata(); - - // Put a strict copy, to verify that we know how to do this - putObjectWithInstructionFile( - ptS3Client, - objectKey + "-good-copy", - encryptedObject.asByteArray(), - objectMetadata, - instructionFileJson - ); - - ObjectMapper mapper = new ObjectMapper(); - Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); - - instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); - instructionFileMap.put("x-amz-d", objectMetadata.get("x-amz-d")); - instructionFileMap.put("x-amz-i", objectMetadata.get("x-amz-i")); - - String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); - - // Put instruction files that should fail: - putObjectWithInstructionFile( - ptS3Client, - objectKey + "-bad-both-meta-and-instruction", - encryptedObject.asByteArray(), - objectMetadata, - instructionFileWithCommitmentValues - ); - - putObjectWithInstructionFile( - ptS3Client, - objectKey + "-bad-only-instruction", - encryptedObject.asByteArray(), - Map.of(), - instructionFileWithCommitmentValues - ); - - } - } - } - - static void putObjectWithInstructionFile( - S3Client ptS3Client, - String newObjectKey, - byte[] objectData, - Map objectMetadata, - String instructionFileJson - ) { - - // Put the encrypted object copy - ptS3Client.putObject(builder -> builder - .bucket(TestUtils.BUCKET) - .key(newObjectKey) - .metadata(objectMetadata) - .build(), - software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); - - // Put the instruction file copy - ptS3Client.putObject(builder -> builder - .bucket(TestUtils.BUCKET) - .key(newObjectKey + ".instruction") - .build(), - software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); - } - - @AfterAll - static void signalEncryptionComplete() throws Exception { - make_copies_to_verify_things(); - - // Signal that all encryption tests have completed - encryptPhaseComplete.countDown(); - } - } - - /** - * Decryption Tests - Decrypt Phase - * - * These tests decrypt objects that were encrypted by EncryptTests. - * All tests in this class can run fully in parallel with each other. - * They depend on EncryptTests completing first. - */ - @Nested - class DecryptTests { - private static List crossLanguageObjectsKms; - private static List crossLanguageObjectsRsa; - private static List crossLanguageObjectsAes; - private static KeyMaterial kmsKeyArn; - private static KeyMaterial RSA_KEY; - private static KeyMaterial AES_KEY; - - @BeforeAll - static void setup() throws InterruptedException { - // Wait for all encryption tests to complete - encryptPhaseComplete.await(); - - // Import encrypted objects and key materials from the encrypt phase - crossLanguageObjectsKms = EncryptTests.getCrossLanguageObjectsKms(); - crossLanguageObjectsRsa = EncryptTests.getCrossLanguageObjectsRsa(); - crossLanguageObjectsAes = EncryptTests.getCrossLanguageObjectsAes(); - kmsKeyArn = EncryptTests.getKmsKeyArn(); - RSA_KEY = EncryptTests.getRsaKey(); - AES_KEY = EncryptTests.getAesKey(); - - // Verify we have objects to decrypt - if (crossLanguageObjectsKms.isEmpty() && crossLanguageObjectsRsa.isEmpty() && crossLanguageObjectsAes.isEmpty()) { - throw new IllegalStateException( - "No encrypted objects found. Ensure EncryptTests runs first."); - } - } - - public static Stream clientsCanGetKMSWithInstructionFile() { - Stream improved = improvedClientsForTest() - .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - Stream transition = transitionClientsForTest() - .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - return Stream.concat(improved, transition); - } - - public static Stream clientsCanGetRawRSAWithInstructionFile() { - Stream improved = improvedClientsForTest() - .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - Stream transition = transitionClientsForTest() - .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - return Stream.concat(improved, transition); - } - - public static Stream clientsCanGetRawAESWithInstructionFile() { - Stream improved = improvedClientsForTest() - .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - Stream transition = transitionClientsForTest() - .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - - return Stream.concat(improved, transition); - } - - // KMS instruction files decrypt - - @ParameterizedTest(name = "{0}: Successfully decrypt KMS encrypted original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsKms, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-good-copy") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjectsKms - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt KMS duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt KMS instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") - void decrypt_kms_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsKms - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - // RSA instruction file decrypt - - @ParameterizedTest(name = "{0}: Successfully decrypt RSA encrypted original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") - void decrypt_rsa_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsRsa, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-good-copy") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjectsRsa - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") - void decrypt_rsa_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") - void decrypt_rsa_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt RSA duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") - void decrypt_rsa_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt RSA instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") - void decrypt_rsa_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(RSA_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsRsa - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - // AES instruction file decrypt - - @ParameterizedTest(name = "{0}: Successfully decrypt AES encrypted original and good-copy objects") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") - void decrypt_aes_original_and_good_copy_objects_succeeds(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsAes, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - - TestUtils.Decrypt( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-good-copy") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, - crossLanguageObjectsAes - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is duplicated in metadata and instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") - void decrypt_aes_with_duplicate_commitment_in_metadata_and_instruction_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is only in instruction file") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") - void decrypt_aes_with_commitment_only_in_instruction_file_fails(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt AES duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") - void decrypt_aes_with_duplicate_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-both-meta-and-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - - @ParameterizedTest(name = "{0}: Fail to decrypt AES instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") - @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") - void decrypt_aes_with_instruction_file_commitment_fails_with_forbid_policy(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(AES_KEY) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails( - client, - S3ECId, - crossLanguageObjectsAes - .stream() - .map(key -> key + "-bad-only-instruction") - .collect(Collectors.toList()), - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY - ); - } - } -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 326a6433..add811eb 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -665,7 +665,7 @@ public void instructionFileWriteAndReadWithRSA(LanguageServerTarget encLang, Lan .key(objectKey + ".instruction") .build()); } - // assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); assertFalse(ptInstFile.asUtf8String().isEmpty()); // Read should be enabled by default GetObjectOutput output = decClient.getObject(GetObjectInput.builder() diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 4e2878f4..759c26c5 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -17,14 +17,10 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.amazonaws.services.s3.model.S3Object; -import com.fasterxml.jackson.databind.ObjectMapper; - import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -111,7 +107,6 @@ public class TestUtils { public static final Set RAW_RSA_SUPPORTED = Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 , NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4 - , RUBY_V2_TRANSITION, RUBY_V3 ); // Intersection of RAW_AES_SUPPORTED and RAW_RSA_SUPPORTED @@ -490,58 +485,22 @@ public static String appendTestSuffix(final String s) { private static AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); public static EncryptionAlgorithm GetEncryptionAlgorithm(String objectKey) { - // Lambda to determine encryption algorithm from a metadata map - java.util.function.Function, Optional> getAlgorithmFromMap = (map) -> { - if (map.containsKey("x-amz-c")) { - return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } else if (map.containsKey("x-amz-cek-alg")) { - String cek = (String) map.get("x-amz-cek-alg"); - if (cek.contains("CBC")) { - return Optional.of(EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); - } else if (cek.contains("GCM")) { - return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - } - return Optional.empty(); - }; - ObjectMetadata metadata = s3Client.getObjectMetadata(TestUtils.BUCKET, objectKey); Map userMetadata = metadata.getUserMetadata(); - // Try to get algorithm from object metadata - Optional algorithm = getAlgorithmFromMap.apply(userMetadata); - if (algorithm.isPresent()) { - return algorithm.get(); - } - - // Check instruction file - try { - String instructionFileKey = objectKey + ".instruction"; - com.amazonaws.services.s3.model.S3Object instructionFileObject = - s3Client.getObject(TestUtils.BUCKET, instructionFileKey); - - // Read instruction file content - java.io.InputStream inputStream = instructionFileObject.getObjectContent(); - String instructionFileJson = new String( - inputStream.readAllBytes(), - java.nio.charset.StandardCharsets.UTF_8 - ); - inputStream.close(); - - // Parse JSON to get metadata - com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); - Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); - - // Try to get algorithm from instruction file - algorithm = getAlgorithmFromMap.apply(instructionFileMap); - if (algorithm.isPresent()) { - return algorithm.get(); + // This is optimized to not need to go to the instruction files for commit_key + if (userMetadata.containsKey("x-amz-c")) { + return EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY; + } else if (userMetadata.containsKey("x-amz-cek-alg")) { + String cek = userMetadata.get("x-amz-cek-alg"); + if (cek.contains("CBC")) { + return EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF; + } else if (cek.contains("GCM")) { + return EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF; } - } catch (Exception e) { - // Instruction file doesn't exist or couldn't be read } - throw new RuntimeException("Could not determine encryption algorithm from object metadata or instruction file!"); + throw new RuntimeException("Need to support instruction files!"); } public static void Encrypt( @@ -569,54 +528,23 @@ public static void Encrypt( public static void Decrypt( S3ECTestServerClient client, - String S3ECId, - List crossLanguageObjects, + String S3ECId, List crossLanguageObjects, EncryptionAlgorithm expectedEncryptionAlgorithm ) { - // Call 5-parameter version with crossLanguageObjects as expectedPlaintexts - Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, crossLanguageObjects); - } - - public static void Decrypt( - S3ECTestServerClient client, - String S3ECId, - List crossLanguageObjects, - EncryptionAlgorithm expectedEncryptionAlgorithm, - List expectedPlaintexts - ) { - List failures = new ArrayList<>(); - for (int i = 0; i < crossLanguageObjects.size(); i++) { - try { - String objectKey = crossLanguageObjects.get(i); - String expectedPlaintext = expectedPlaintexts.get(i); - - GetObjectOutput output = client.getObject(GetObjectInput.builder() - .clientID(S3ECId) - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); - - // Then: Pass - assertEquals(expectedPlaintext, new String(output.getBody().array())); - assertEquals( - expectedEncryptionAlgorithm, - GetEncryptionAlgorithm(objectKey), - "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm - ); - } catch (Exception e) { - failures.add(String.format( - "Failed to decrypt object '%s' (index %d): %s - %s", - crossLanguageObjects.get(i), i, e.getClass().getSimpleName(), e.getMessage() - )); - } - } - - if (!failures.isEmpty()) { - throw new AssertionError(String.format( - "Decryption failed for %d out of %d objects:\n%s", - failures.size(), crossLanguageObjects.size(), - String.join("\n", failures) - )); + for (String objectKey : crossLanguageObjects) { + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Then: Pass + assertEquals(objectKey, new String(output.getBody().array())); + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); } } diff --git a/test-server/java-v3-server/Makefile b/test-server/java-v3-server/Makefile index 59dcdff5..692e80b3 100644 --- a/test-server/java-v3-server/Makefile +++ b/test-server/java-v3-server/Makefile @@ -7,7 +7,7 @@ PORT := 8080 build-server: @echo "Building Java V3 server..." - ./gradlew --build-cache --parallel --no-daemon build + ./gradlew --build-cache --parallel build start-server: @echo "Starting Java V3 server..." diff --git a/test-server/java-v3-transition-server/Makefile b/test-server/java-v3-transition-server/Makefile index 81726b59..5a25a8aa 100644 --- a/test-server/java-v3-transition-server/Makefile +++ b/test-server/java-v3-transition-server/Makefile @@ -10,7 +10,7 @@ build-server: cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." @echo "Building Java V3 Transition server..." - ./gradlew --build-cache --parallel --no-daemon build + ./gradlew --build-cache --parallel build start-server: @echo "Starting Java V3 Transition server..." diff --git a/test-server/java-v4-server/Makefile b/test-server/java-v4-server/Makefile index 3d1aae2a..418e0127 100644 --- a/test-server/java-v4-server/Makefile +++ b/test-server/java-v4-server/Makefile @@ -10,7 +10,7 @@ build-server: cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." @echo "Building Java V4 server..." - ./gradlew --build-cache --parallel --no-daemon build + ./gradlew --build-cache --parallel build start-server: @echo "Starting Java V4 server..." diff --git a/test-server/php-v2-transition-server/src/get_object.php b/test-server/php-v2-transition-server/src/get_object.php index dcf683b6..5800e850 100644 --- a/test-server/php-v2-transition-server/src/get_object.php +++ b/test-server/php-v2-transition-server/src/get_object.php @@ -80,10 +80,6 @@ function handleGetObject($params) } if (strpos($e->getMessage(), "@SecurityProfile=V2") !== false) { return S3EncryptionClientError($e->getMessage() . " " . "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"); - } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { - return S3EncryptionClientError($e->getMessage()); - } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { - return S3EncryptionClientError($e->getMessage()); } else { error_log("This is the error: " . $e->getMessage()); return GenericServerError("Server error: " . $e->getMessage(), 500); diff --git a/test-server/php-v3-server/local-php-sdk b/test-server/php-v3-server/local-php-sdk index 88ee9515..e32c9f2b 160000 --- a/test-server/php-v3-server/local-php-sdk +++ b/test-server/php-v3-server/local-php-sdk @@ -1 +1 @@ -Subproject commit 88ee95156f2884767b72f9219736e976d98a9c96 +Subproject commit e32c9f2b009a43cf88f2ab35e1e532114c8390c9 diff --git a/test-server/php-v3-server/src/get_object.php b/test-server/php-v3-server/src/get_object.php index 6fb28551..3de7f779 100644 --- a/test-server/php-v3-server/src/get_object.php +++ b/test-server/php-v3-server/src/get_object.php @@ -84,10 +84,6 @@ function handleGetObject($params) return S3EncryptionClientError($e->getMessage()); } elseif (strpos($e->getMessage(), "Message is encrypted with a non commiting algorithm but commitment policy is set to REQUIRE_ENCRYPT_REQUIRE_DECRYPT. Select a valid commitment policy to decrypt this object.") !== false) { return S3EncryptionClientError($e->getMessage()); - } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { - return S3EncryptionClientError($e->getMessage()); - } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { - return S3EncryptionClientError($e->getMessage()); } else { error_log("This is the error: " . $e->getMessage()); return GenericServerError("Server argument: " . $e->getMessage(), 500); diff --git a/test-server/ruby-v2-server/app.rb b/test-server/ruby-v2-server/app.rb index 5a39e2ea..cde757a3 100644 --- a/test-server/ruby-v2-server/app.rb +++ b/test-server/ruby-v2-server/app.rb @@ -132,7 +132,7 @@ def initialize metadata: response_metadata }.to_json - rescue Aws::S3::EncryptionV2::Errors::EncryptionError, Aws::S3::EncryptionV3::Errors::EncryptionError => e + rescue Aws::S3::EncryptionV2::Errors::EncryptionError => e S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) ErrorHandlers.send_s3_encryption_client_error(self, e.message) rescue StandardError => e @@ -201,7 +201,7 @@ def initialize content_type 'application/octet-stream' body - rescue Aws::S3::EncryptionV2::Errors::DecryptionError, Aws::S3::EncryptionV3::Errors::DecryptionError => e + rescue Aws::S3::EncryptionV2::Errors::DecryptionError => e S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) ErrorHandlers.send_s3_encryption_client_error(self, e.message) rescue StandardError => e diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb index 3da62b45..533ff67c 100644 --- a/test-server/ruby-v2-server/lib/client_manager.rb +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -2,8 +2,6 @@ require 'securerandom' require 'aws-sdk-s3' require 'aws-sdk-kms' -require 'openssl' -require 'base64' require_relative 'logger' # Manages S3 Encryption Client instances @@ -16,42 +14,20 @@ def initialize # Create a new S3 encryption client and return its ID def create_client(config) - # Extract all key material types + # Extract configuration kms_key_id = config.dig('keyMaterial', 'kmsKeyId') - rsa_key_blob = config.dig('keyMaterial', 'rsaKey') - aes_key_blob = config.dig('keyMaterial', 'aesKey') inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') - # Validate that only one key type is provided - key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count - raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 + raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? # Create S3 encryption client configuration encryption_config = { + kms_key_id: kms_key_id, + kms_client: @kms_client, + key_wrap_schema: :kms_context, content_encryption_schema: :aes_gcm_no_padding, envelope_location: inst_file_put ? :instruction_file : :metadata - } - - # Configure based on key type - if kms_key_id - encryption_config[:kms_key_id] = kms_key_id - encryption_config[:kms_client] = @kms_client - encryption_config[:key_wrap_schema] = :kms_context - elsif rsa_key_blob - # Parse RSA private key from PKCS8 format - key_bytes = Base64.decode64(rsa_key_blob) - rsa_key = OpenSSL::PKey::RSA.new(key_bytes) - encryption_config[:encryption_key] = rsa_key - encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 - elsif aes_key_blob - # Extract AES key bytes - key_bytes = Base64.decode64(aes_key_blob) - encryption_config[:encryption_key] = key_bytes - encryption_config[:key_wrap_schema] = :aes_gcm - end - - # Apply legacy settings - encryption_config.tap do |hash| + }.tap do |hash| if !config['enableLegacyWrappingAlgorithms'].nil? || !config['enableLegacyUnauthenticatedModes'].nil? legacy_modes = config['enableLegacyWrappingAlgorithms'] || config['enableLegacyUnauthenticatedModes'] # Set security profile based on legacy wrapping algorithms setting diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb index 5ee3f1ec..0f48d563 100644 --- a/test-server/ruby-v3-server/lib/client_manager.rb +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -2,8 +2,6 @@ require 'securerandom' require 'aws-sdk-s3' require 'aws-sdk-kms' -require 'openssl' -require 'base64' require_relative 'logger' # Manages S3 Encryption Client instances @@ -16,17 +14,11 @@ def initialize # Create a new S3 encryption client and return its ID def create_client(config) - # Extract all key material types + # Extract configuration kms_key_id = config.dig('keyMaterial', 'kmsKeyId') - rsa_key_blob = config.dig('keyMaterial', 'rsaKey') - aes_key_blob = config.dig('keyMaterial', 'aesKey') inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') content_alg = config.dig('encryptionAlgorithm') - # Validate that only one key type is provided - key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count - raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 - # translate between canonical AlgSuite and Ruby symbols if content_alg.nil? || content_alg == 'ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY' content_alg = :alg_aes_256_gcm_hkdf_sha512_commit_key @@ -36,32 +28,16 @@ def create_client(config) raise 'Unknown content encryption algorithm provided: ' + content_alg end + raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? + # Create S3 encryption client configuration encryption_config = { + kms_key_id: kms_key_id, + kms_client: @kms_client, + key_wrap_schema: :kms_context, envelope_location: inst_file_put ? :instruction_file : :metadata, content_encryption_schema: content_alg - } - - # Configure based on key type - if kms_key_id - encryption_config[:kms_key_id] = kms_key_id - encryption_config[:kms_client] = @kms_client - encryption_config[:key_wrap_schema] = :kms_context - elsif rsa_key_blob - # Parse RSA private key from PKCS8 format - key_bytes = Base64.decode64(rsa_key_blob) - rsa_key = OpenSSL::PKey::RSA.new(key_bytes) - encryption_config[:encryption_key] = rsa_key - encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 - elsif aes_key_blob - # Extract AES key bytes - key_bytes = Base64.decode64(aes_key_blob) - encryption_config[:encryption_key] = key_bytes - encryption_config[:key_wrap_schema] = :aes_gcm - end - - # Apply additional configuration - encryption_config.tap do |hash| + }.tap do |hash| if !config['commitmentPolicy'].nil? hash[:commitment_policy] = case config['commitmentPolicy'] when 'FORBID_ENCRYPT_ALLOW_DECRYPT' From 22dc07cf2cdc0a646a138c5e3d4366c9e4d2e639 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Tue, 25 Nov 2025 14:27:10 -0800 Subject: [PATCH 39/40] revert Ryan's branch merge --- .github/workflows/test.yml | 23 +- test-server/Makefile | 18 +- test-server/cpp-v2-server/Makefile | 6 +- test-server/cpp-v2-server/main.cpp | 468 ++------------ test-server/cpp-v2-transition-server/Makefile | 6 +- test-server/cpp-v2-transition-server/main.cpp | 543 ++--------------- test-server/cpp-v3-server/CMakeLists.txt | 16 +- test-server/cpp-v3-server/Makefile | 6 +- test-server/cpp-v3-server/aws-sdk-cpp | 2 +- test-server/cpp-v3-server/main.cpp | 574 +++--------------- test-server/go-v3-server/main.go | 28 +- test-server/go-v3-transition-server/main.go | 28 +- test-server/go-v4-server/main.go | 28 +- test-server/java-tests/build.gradle.kts | 14 - .../s3/ExhaustiveRoundTripTests1_25.java | 2 +- .../amazon/encryption/s3/GCMTestSuite.java | 255 -------- .../amazon/encryption/s3/GCMTests.java | 203 +++++++ .../amazon/encryption/s3/KC_GCMTestSuite.java | 389 ------------ .../amazon/encryption/s3/KC_GCMTests.java | 264 ++++++++ .../amazon/encryption/s3/RoundTripTests.java | 14 +- .../amazon/encryption/s3/TestUtils.java | 40 +- test-server/java-v3-server/gradle.properties | 19 +- .../s3/CreateClientOperationImpl.java | 16 - .../gradle.properties | 19 +- .../java-v3-transition-server/s3ec-staging | 2 +- .../s3/CreateClientOperationImpl.java | 16 - test-server/java-v4-server/gradle.properties | 19 +- test-server/java-v4-server/s3ec-staging | 2 +- .../s3/CreateClientOperationImpl.java | 16 - .../Controllers/ClientController.cs | 18 +- .../Controllers/ClientController.cs | 18 +- .../s3ec-v3-transition-branch | 2 +- .../Controllers/ClientController.cs | 17 +- test-server/net-v4-server/Makefile | 2 +- .../net-v4-server/s3ec-net-v4-improved | 2 +- test-server/php-v2-server/Makefile | 2 +- test-server/php-v2-transition-server/Makefile | 2 +- test-server/php-v3-server/Makefile | 2 +- .../ruby-v2-server/lib/client_manager.rb | 8 +- .../ruby-v3-server/lib/client_manager.rb | 8 +- 40 files changed, 745 insertions(+), 2372 deletions(-) delete mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java delete mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java create mode 100644 test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57163fad..880ca3f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,15 +41,15 @@ jobs: git config --global credential.helper store echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials - # - name: Cache git submodules - # uses: actions/cache@v4 - # with: - # path: | - # .git/modules - # test-server/*/.git - # key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }} - # restore-keys: | - # ${{ runner.os }}-submodules- + - name: Cache git submodules + uses: actions/cache@v4 + with: + path: | + .git/modules + test-server/*/.git + key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }} + restore-keys: | + ${{ runner.os }}-submodules- - name: Optimize git for performance run: | @@ -59,12 +59,13 @@ jobs: - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} - name: Update cpp submodules recursively with --jobs run: | git submodule update --init --recursive \ - --depth 1 --single-branch \ + --depth 1 \ + --filter=blob:none \ --jobs ${{ steps.cpu-count.outputs.count }} \ --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ diff --git a/test-server/Makefile b/test-server/Makefile index cabb09b1..9b18b857 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -2,6 +2,9 @@ .PHONY: all start-servers run-tests stop-servers clean ci check-env help +# Default target +all: start-all-servers wait-all-servers run-tests + # CI target for GitHub Actions ci: $(MAKE) build-all-servers @@ -17,19 +20,13 @@ START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) # Build all servers in parallel -build-all-servers: - @echo "[`date +%H:%M:%S`] Building all servers..." - @$(MAKE) $(BUILD_SERVER_TARGETS) - @echo "[`date +%H:%M:%S`] All servers built." - @echo "Stopping dotnet servers... this is here to speed up CI because if this target is run with -j it will stay open until it shutdown." - @dotnet build-server shutdown - @echo "[`date +%H:%M:%S`] Dotnet build servers stopped" +build-all-servers: export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1) +build-all-servers: $(BUILD_SERVER_TARGETS) $(BUILD_SERVER_TARGETS): build-%: @if [ -f $*/Makefile ]; then \ - echo "[`date +%H:%M:%S`] Building server in $*..."; \ + echo "Building server in $*..."; \ $(MAKE) -C $* build-server; \ - echo "[`date +%H:%M:%S`] Server $* built successfully"; \ else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ @@ -47,8 +44,9 @@ start-servers: $(MAKE) -C $$dir wait-for-server; \ done +# Start servers sequentially (no parallel execution) start-all-servers: - @$(MAKE) $(START_SERVER_TARGETS) + @$(MAKE) MAKEFLAGS= $(START_SERVER_TARGETS) $(START_SERVER_TARGETS): start-%: @if [ -f $*/Makefile ]; then \ diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 2d0a4b55..77357c37 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -11,7 +11,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp V2 server..." - cd build && $(MAKE) + cd build && make start-server: @echo "Starting Cpp V2 server..." @@ -20,7 +20,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V2 server starting..." stop-server: @@ -31,7 +31,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f server.log + @rm -f build/server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index f5c4e406..a2b05810 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -1,53 +1,8 @@ -/* - * S3 Encryption Test Server - C++ V2 - * - * CONCURRENCY AND SYNCHRONIZATION DESIGN: - * - * 1. Threading Model: - * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool - * - Thread pool size = CPU cores * 2 (auto-detected at startup) - * - Threads are reused across connections for efficiency - * - I/O multiplexing (poll) distributes connections across thread pool - * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding - * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() - * - * 2. Resource Scaling: - * - All limits automatically scale with detected CPU count: - * * Thread pool size = num_cores * 2 - * * Connection limit = num_cores * 2 - * * S3 client maxConnections = num_cores * 2 - * - Multiplier of 2 accounts for I/O blocking without starving throughput - * - Ensures optimal resource usage on any hardware configuration - * - * 3. Client Cache (client_cache_secret): - * - Protected by std::shared_mutex for efficient concurrent access - * - get_client() uses shared_lock (multiple threads can read simultaneously) - * - set_client() uses unique_lock (exclusive write access) - * - This allows concurrent GET/PUT operations without serialization - * - UUID-based keys guarantee uniqueness (always insert, never update) - * - * 4. Memory Management: - * - Request body allocated in request_handler (*con_cls = new std::string()) - * - Body lifetime managed by libmicrohttpd - valid until request_completed() - * - All handler functions complete synchronously before returning - * - request_completed() safely deletes body after response sent - * - No memory leaks under sustained concurrent load - * - * 5. Synchronous Operation Guarantees: - * - GetObject: Waits for S3, reads full response stream, then returns - * - PutObject: Waits for S3 operation to complete, then returns - * - No async callbacks or background operations - * - Client receives response only after S3 operation completes - */ - #include -#include #include #include #include #include -#include -#include #include #include #include @@ -55,33 +10,14 @@ #include #include -#include #include #include -#include -#include -#include -#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; - -// LRU cache for S3 encryption clients -// Limits memory and connection pool growth by evicting least recently used clients -const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations - -struct ClientCacheEntry { - std::shared_ptr client; - std::list::iterator lru_iter; -}; - -std::unordered_map client_cache_secret; -std::list lru_order; // Most recently used at front -std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads - -// Threading configuration - set at startup based on CPU cores -unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() +std::unordered_map> + client_cache; std::string generate_uuid() { uuid_t uuid; @@ -91,55 +27,6 @@ std::string generate_uuid() { return std::string(uuid_str); } -std::shared_ptr get_client(const std::string &client_id) -{ - // Need unique_lock to update LRU order even on reads - std::unique_lock lock(client_mutex); - auto it = client_cache_secret.find(client_id); - if (it == client_cache_secret.end()) { - return std::shared_ptr(); - } else { - // Move to front of LRU list (mark as most recently used) - lru_order.erase(it->second.lru_iter); - lru_order.push_front(client_id); - it->second.lru_iter = lru_order.begin(); - - return it->second.client; - } -} - -void set_client(const std::string &client_id, std::shared_ptr client) -{ - // UUID guarantees unique keys - always insert, never update - // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts - std::unique_lock lock(client_mutex); - - // Add to front of LRU list (most recently used) - lru_order.push_front(client_id); - - ClientCacheEntry entry; - entry.client = client; - entry.lru_iter = lru_order.begin(); - - client_cache_secret.emplace(client_id, entry); - - // Evict least recently used clients if we exceed the limit - while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { - std::string lru_client_id = lru_order.back(); - lru_order.pop_back(); - - auto evict_it = client_cache_secret.find(lru_client_id); - if (evict_it != client_cache_secret.end()) { - fprintf(stderr, "[CPP-V2] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", - lru_client_id.c_str(), client_cache_secret.size()); - client_cache_secret.erase(evict_it); - } - } - - fprintf(stderr, "[CPP-V2] [CACHE-ADD] Added client %s (cache size now %zu)\n", - client_id.c_str(), client_cache_secret.size()); -} - std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -165,9 +52,6 @@ std::string make_error(const std::string &message, int status_code) { MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { - // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly - // All operations here are synchronous and complete before returning to caller - try { json request = json::parse(body); std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; @@ -187,20 +71,14 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, if (inst_put) config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - // Each client gets a large connection pool since we cannot share HTTP clients - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 512; // Large pool per client - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - - auto encryption_client = std::make_shared(config, clientConfig); + auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); - set_client(client_id, encryption_client); + client_cache[client_id] = encryption_client; json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2] handle_create_client exception: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"An exception was thrown.\"}"); } catch (...) { @@ -211,18 +89,13 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { - fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: metadata is empty\n"); return; } - fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", - metadata.c_str(), metadata.length()); - // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; - int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -255,9 +128,6 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); - fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", - ++pair_count, key.c_str(), value.c_str()); - // Add to map map.emplace(key, value); @@ -268,24 +138,14 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } - - fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - std::string bucket, - std::string key, - std::string client_id, - std::string metadata) { - // Get thread ID for debugging concurrent operations - std::thread::id thread_id = std::this_thread::get_id(); - - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu\n", - (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length()); - - auto client = get_client(client_id); - if (!client) { - fprintf(stderr, "[CPP-V2] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -294,100 +154,45 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); + // S3EncryptionGetObjectOutcome outcome ; + // if (metadata.empty()) { + // outcome = it->second->GetObject(request); + // } else { + // Aws::Map kmsContextMap; + // fill_context(kmsContextMap, metadata); + // outcome = it->second->GetObject(request, kmsContextMap); + // } + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - - // Log the encryption context map size and contents - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); - for (const auto& pair : kmsContextMap) { - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: context['%s']='%s'\n", - pair.first.c_str(), pair.second.c_str()); - } - - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); - - // Keep outcome alive to ensure stream remains valid - auto outcome = client->GetObject(request, kmsContextMap); - - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + auto outcome = it->second->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { - // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::stringstream buffer; - buffer << stream.rdbuf(); - std::string content = buffer.str(); - - // Validate we read something - if (content.empty() && stream.fail()) { - fprintf(stderr, "[CPP-V2] GetObject error: Failed to read stream for bucket=%s, key=%s\n", - bucket.c_str(), key.c_str()); - auto msg = make_error("Failed to read response stream", 500); - return send_response(connection, 500, msg); - } - - fprintf(stderr, "[CPP-V2] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", - bucket.c_str(), key.c_str(), content.length()); - - // Create and send response - struct MHD_Response *response = MHD_create_response_from_buffer( - content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); - - // Add keep-alive header - MHD_add_response_header(response, "Connection", "keep-alive"); - MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); - - MHD_Result ret = MHD_queue_response(connection, 200, response); - MHD_destroy_response(response); - - return ret; + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); } else { - // Enhanced error logging with thread info - auto error = outcome.GetError(); - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str()); - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject error details:\n"); - fprintf(stderr, "[CPP-V2] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); - fprintf(stderr, "[CPP-V2] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); - fprintf(stderr, "[CPP-V2] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); - fprintf(stderr, "[CPP-V2] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); - auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "[CPP-V2] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); - auto msg = make_error(e.what(), 500); - return send_response(connection, 500, msg); - } catch (...) { - fprintf(stderr, "[CPP-V2] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str()); - auto msg = make_error("Unknown error in GetObject", 500); + auto msg = make_error("An exception was thrown", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - std::string bucket, - std::string key, - std::string client_id, - std::string body, - std::string metadata) { - fprintf(stderr, "[CPP-V2] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", - bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); - - auto client = get_client(client_id); - if (!client) { - fprintf(stderr, "[CPP-V2] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &body, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { - // Create owned copy of body data to ensure it lives through the S3 operation - auto body_ptr = std::make_shared(body); - Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -395,248 +200,85 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - // Create stream from owned body data - auto stream = std::make_shared(*body_ptr); + auto stream = std::make_shared(body); request.SetBody(stream); - // Synchronous call - waits for S3 operation to complete - // body_ptr keeps the data alive through this entire operation - auto outcome = client->PutObject(request, kmsContextMap); + auto outcome = it->second->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { - fprintf(stderr, "[CPP-V2] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "[CPP-V2] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } -void request_completed(void *cls, struct MHD_Connection *connection, - void **con_cls, enum MHD_RequestTerminationCode toe) { - // Clean up the request-specific context when request is truly complete - // This is called AFTER all handlers have returned and the response has been sent - - // Log why the request was terminated - const char* reason = "UNKNOWN"; - switch (toe) { - case MHD_REQUEST_TERMINATED_COMPLETED_OK: - reason = "COMPLETED_OK"; - break; - case MHD_REQUEST_TERMINATED_WITH_ERROR: - reason = "WITH_ERROR"; - break; - case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: - reason = "TIMEOUT_REACHED"; - break; - case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: - reason = "DAEMON_SHUTDOWN"; - break; - case MHD_REQUEST_TERMINATED_READ_ERROR: - reason = "READ_ERROR"; - break; - case MHD_REQUEST_TERMINATED_CLIENT_ABORT: - reason = "CLIENT_ABORT"; - break; - } - fprintf(stderr, "[CPP-V2] request_completed called, reason=%s, con_cls=%p\n", - reason, *con_cls); - - if (*con_cls != nullptr) { - std::string *body = static_cast(*con_cls); - delete body; // Safe to delete now - all synchronous operations are complete - *con_cls = nullptr; - } -} - MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - try { - std::string method_str(method); - std::string url_str(url); - bool is_push = method_str == "POST" || method_str == "PUT"; - - // LOG: Every request entry (even first-time calls) - if (*con_cls == nullptr) { - fprintf(stderr, "[CPP-V2] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", - method, url, version, *upload_data_size); - } - - // Initialize request context on first call + std::string method_str(method); + bool is_push = method_str == "POST" || method_str == "PUT"; + static int dummy; if (*con_cls == nullptr) { - // Allocate unique state for each request to avoid race conditions - *con_cls = new std::string(); - fprintf(stderr, "[CPP-V2] REQUEST INIT: allocated new request context for %s %s\n", method, url); + if (is_push) { + *con_cls = new std::string(); + } else { + *con_cls = &dummy; + } return MHD_YES; } - - // LOG: Subsequent calls - if (is_push && *upload_data_size > 0) { - fprintf(stderr, "[CPP-V2] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); - } else if (*upload_data_size == 0) { - fprintf(stderr, "[CPP-V2] REQUEST COMPLETE: %s %s ready for processing\n", method, url); - } - - // Accumulate request body data for POST/PUT requests - if (is_push && *upload_data_size > 0) { + if (is_push && *upload_data_size) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - // At this point, *upload_data_size == 0, meaning we have all the data - // Now we can safely process the request - - // LOG: About to process request - fprintf(stderr, "[CPP-V2] PROCESSING: %s %s\n", method, url); - - // Handle client creation endpoint + + std::string url_str(url); + if (is_push && url_str == "/client") { - fprintf(stderr, "[CPP-V2] Handling /client endpoint\n"); - std::string *body = static_cast(*con_cls); - MHD_Result result = handle_create_client(connection, *body); - fprintf(stderr, "[CPP-V2] /client handler returned: %d\n", result); - return result; + std::unique_ptr body(static_cast(*con_cls)); + return handle_create_client(connection, *body); } - // Handle object operations if (url_str.find("/object/") == 0) { - fprintf(stderr, "[CPP-V2] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); + std::string metadata = get_header_value(connection, "content-metadata"); - - fprintf(stderr, "[CPP-V2] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", - bucket.c_str(), key.c_str(), client_id.c_str(), method); - if (method_str == "GET") { - fprintf(stderr, "[CPP-V2] Dispatching to handle_get_object\n"); - MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); - fprintf(stderr, "[CPP-V2] handle_get_object returned: %d\n", result); - return result; + return handle_get_object(connection, bucket, key, client_id, metadata); } else if (method_str == "PUT") { - fprintf(stderr, "[CPP-V2] Dispatching to handle_put_object\n"); - std::string *body = static_cast(*con_cls); - MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); - fprintf(stderr, "[CPP-V2] handle_put_object returned: %d\n", result); - return result; - } else { - fprintf(stderr, "[CPP-V2] Method not allowed: %s\n", method); - return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); + std::unique_ptr body(static_cast(*con_cls)); + *upload_data_size = 0; + return handle_put_object(connection, bucket, key, client_id, *body, metadata); } } } - // Return error for unrecognized endpoints - fprintf(stderr, "[CPP-V2] ERROR: Unrecognized endpoint: %s %s\n", method, url); - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); - } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", - e.what(), method, url); - // Try to send error response, but connection might already be broken - try { - return send_response(connection, 500, - "{\"error\":\"Internal server error: unhandled exception\"}"); - } catch (...) { - fprintf(stderr, "[CPP-V2] FATAL: Failed to send error response\n"); - return MHD_NO; - } - } catch (...) { - fprintf(stderr, "[CPP-V2] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", - method, url); - // Try to send error response, but connection might already be broken - try { - return send_response(connection, 500, - "{\"error\":\"Internal server error: unknown exception\"}"); - } catch (...) { - fprintf(stderr, "[CPP-V2] FATAL: Failed to send error response\n"); - return MHD_NO; - } - } -} - -// Error log callback for libmicrohttpd -void log_mhd_error(void* cls, const char* fmt, va_list ap) { - fprintf(stderr, "[CPP-V2] [MHD-ERROR] "); - vfprintf(stderr, fmt, ap); - fprintf(stderr, "\n"); -} - -// Connection notification callback - called when a client connects -MHD_Result notify_connection(void *cls, - struct MHD_Connection *connection, - void **socket_context, - enum MHD_ConnectionNotificationCode toe) { - if (toe == MHD_CONNECTION_NOTIFY_STARTED) { - fprintf(stderr, "[CPP-V2] [MHD-CONNECT] New connection started\n"); - } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { - fprintf(stderr, "[CPP-V2] [MHD-DISCONNECT] Connection closed\n"); - } - return MHD_YES; + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); } int main() { Aws::SDKOptions options; - - // Configure AWS SDK logging to output to stderr (which goes to server.log) - // Using Debug level to capture all SDK activity including CryptoModule errors - options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; - options.loggingOptions.logger_create_fn = []() { - return std::make_shared( - Aws::Utils::Logging::LogLevel::Debug - ); - }; - - fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); - Aws::InitAPI(options); - - // Detect CPU core count and configure threading - unsigned int num_cores = std::thread::hardware_concurrency(); - if (num_cores == 0) { - num_cores = 4; - fprintf(stderr, "[CPP-V2] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); - } - - g_thread_pool_size = num_cores * 2; - unsigned int connection_limit = g_thread_pool_size; - - // Log configuration - fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); - fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); - fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); - fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); - int port = 8085; - struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, - port, NULL, NULL, - &request_handler, NULL, - MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, - MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, - MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, - MHD_OPTION_CONNECTION_LIMIT, connection_limit, - MHD_OPTION_CONNECTION_TIMEOUT, 10, - MHD_OPTION_END); + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, + &request_handler, NULL, MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "[CPP-V2] Failed to start server on port %d\n", port); + fprintf(stderr, "Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 0383b4d8..16b70796 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp transition server..." - cd build && $(MAKE) + cd build && make start-server: @echo "Starting Cpp transition server..." @@ -19,7 +19,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp transition server starting..." stop-server: @@ -30,7 +30,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f server.log + @rm -f build/server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index dc863723..1fcedc3c 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -1,55 +1,8 @@ -/* - * S3 Encryption Test Server - C++ V2 Transition - * - * CONCURRENCY AND SYNCHRONIZATION DESIGN: - * - * 1. Threading Model: - * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool - * - Thread pool size = CPU cores * 2 (auto-detected at startup) - * - Threads are reused across connections for efficiency - * - I/O multiplexing (poll) distributes connections across thread pool - * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding - * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() - * - * 2. Resource Scaling: - * - All limits automatically scale with detected CPU count: - * * Thread pool size = num_cores * 2 - * * Connection limit = num_cores * 2 - * * S3 client maxConnections = num_cores * 2 - * - Multiplier of 2 accounts for I/O blocking without starving throughput - * - Ensures optimal resource usage on any hardware configuration - * - * 3. Client Cache (client_cache_secret): - * - Protected by std::shared_mutex for efficient concurrent access - * - get_client() uses shared_lock (multiple threads can read simultaneously) - * - set_client() uses unique_lock (exclusive write access) - * - This allows concurrent GET/PUT operations without serialization - * - UUID-based keys guarantee uniqueness (always insert, never update) - * - * 4. Memory Management: - * - Request body allocated in request_handler (*con_cls = new std::string()) - * - Body lifetime managed by libmicrohttpd - valid until request_completed() - * - All handler functions complete synchronously before returning - * - request_completed() safely deletes body after response sent - * - No memory leaks under sustained concurrent load - * - * 5. Synchronous Operation Guarantees: - * - GetObject: Waits for S3, reads full response stream, then returns - * - PutObject: Waits for S3 operation to complete, then returns - * - No async callbacks or background operations - * - Client receives response only after S3 operation completes - */ - #include -#include #include #include #include #include -#include -#include -#include -#include #include #include #include @@ -57,33 +10,14 @@ #include #include -#include #include #include -#include -#include -#include -#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; - -// LRU cache for S3 encryption clients -// Limits memory and connection pool growth by evicting least recently used clients -const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations - -struct ClientCacheEntry { - std::shared_ptr client; - std::list::iterator lru_iter; -}; - -std::unordered_map client_cache_secret; -std::list lru_order; // Most recently used at front -std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads - -// Threading configuration - set at startup based on CPU cores -unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() +std::unordered_map> + client_cache; std::string generate_uuid() { uuid_t uuid; @@ -93,55 +27,6 @@ std::string generate_uuid() { return std::string(uuid_str); } -std::shared_ptr get_client(const std::string &client_id) -{ - // Need unique_lock to update LRU order even on reads - std::unique_lock lock(client_mutex); - auto it = client_cache_secret.find(client_id); - if (it == client_cache_secret.end()) { - return std::shared_ptr(); - } else { - // Move to front of LRU list (mark as most recently used) - lru_order.erase(it->second.lru_iter); - lru_order.push_front(client_id); - it->second.lru_iter = lru_order.begin(); - - return it->second.client; - } -} - -void set_client(const std::string &client_id, std::shared_ptr client) -{ - // UUID guarantees unique keys - always insert, never update - // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts - std::unique_lock lock(client_mutex); - - // Add to front of LRU list (most recently used) - lru_order.push_front(client_id); - - ClientCacheEntry entry; - entry.client = client; - entry.lru_iter = lru_order.begin(); - - client_cache_secret.emplace(client_id, entry); - - // Evict least recently used clients if we exceed the limit - while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { - std::string lru_client_id = lru_order.back(); - lru_order.pop_back(); - - auto evict_it = client_cache_secret.find(lru_client_id); - if (evict_it != client_cache_secret.end()) { - fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", - lru_client_id.c_str(), client_cache_secret.size()); - client_cache_secret.erase(evict_it); - } - } - - fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-ADD] Added client %s (cache size now %zu)\n", - client_id.c_str(), client_cache_secret.size()); -} - std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -184,9 +69,6 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { - // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly - // All operations here are synchronous and complete before returning to caller - try { json request = json::parse(body); std::string commitmentPolicy = get_config(request, "commitmentPolicy"); @@ -197,41 +79,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, return MHD_YES; } - // Extract all key material types - std::string kms_key_id; - std::string rsa_key_blob; - std::string aes_key_blob; - - if (request["config"]["keyMaterial"].contains("kmsKeyId") && - !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { - kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; - } - if (request["config"]["keyMaterial"].contains("rsaKey") && - !request["config"]["keyMaterial"]["rsaKey"].is_null()) { - rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; - } - if (request["config"]["keyMaterial"].contains("aesKey") && - !request["config"]["keyMaterial"]["aesKey"].is_null()) { - aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; - } - - // Validate that only one key type is provided - int key_count = 0; - if (!kms_key_id.empty()) key_count++; - if (!rsa_key_blob.empty()) key_count++; - if (!aes_key_blob.empty()) key_count++; - - if (key_count != 1) { - return send_response(connection, 400, - "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); - } - - // RSA is not supported by C++ SDK - if (!rsa_key_blob.empty()) { - return send_response(connection, 501, - "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); - } - + std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; bool inst_put = false; @@ -240,54 +88,22 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - // Create CryptoConfigurationV2 based on key type - std::shared_ptr config; - - if (!aes_key_blob.empty()) { - // Base64 decode the AES key - Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); - if (decoded.GetLength() == 0) { - return send_response(connection, 400, - "{\"error\":\"Failed to decode AES key\"}"); - } - - Aws::Utils::CryptoBuffer key_buffer( - decoded.GetUnderlyingData(), - decoded.GetLength() - ); - - auto materials = std::make_shared< - Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( - key_buffer - ); - config = std::make_shared(materials); - } else if (!kms_key_id.empty()) { - auto materials = std::make_shared(kms_key_id); - config = std::make_shared(materials); - } else { - return send_response(connection, 400, - "{\"error\":\"No valid key material provided\"}"); - } - - // Apply common configuration settings (applies to both AES and KMS) + auto materials = + std::make_shared(kms_key_id); + CryptoConfigurationV2 config(materials); if (legacy1 || legacy2) - config->SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); if (inst_put) - config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - // Create S3EncryptionClientV2 with standard configuration - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 512; // Large pool per client - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - auto encryption_client = std::make_shared(*config, clientConfig); + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); - set_client(client_id, encryption_client); + client_cache[client_id] = encryption_client; json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2-TRANSITION] handle_create_client exception: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"An exception was thrown.\"}"); } catch (...) { @@ -298,18 +114,13 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: metadata is empty\n"); return; } - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", - metadata.c_str(), metadata.length()); - // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; - int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -342,9 +153,6 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", - ++pair_count, key.c_str(), value.c_str()); - // Add to map map.emplace(key, value); @@ -355,24 +163,14 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } - - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - std::string bucket, - std::string key, - std::string client_id, - std::string metadata) { - // Get thread ID for debugging concurrent operations - std::thread::id thread_id = std::this_thread::get_id(); - - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu\n", - (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length()); - - auto client = get_client(client_id); - if (!client) { - fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -381,100 +179,45 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); + // S3EncryptionGetObjectOutcome outcome ; + // if (metadata.empty()) { + // outcome = it->second->GetObject(request); + // } else { + // Aws::Map kmsContextMap; + // fill_context(kmsContextMap, metadata); + // outcome = it->second->GetObject(request, kmsContextMap); + // } + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - - // Log the encryption context map size and contents - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); - for (const auto& pair : kmsContextMap) { - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: context['%s']='%s'\n", - pair.first.c_str(), pair.second.c_str()); - } - - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); - - // Keep outcome alive to ensure stream remains valid - auto outcome = client->GetObject(request, kmsContextMap); - - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + auto outcome = it->second->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { - // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::stringstream buffer; - buffer << stream.rdbuf(); - std::string content = buffer.str(); - - // Validate we read something - if (content.empty() && stream.fail()) { - fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Failed to read stream for bucket=%s, key=%s\n", - bucket.c_str(), key.c_str()); - auto msg = make_error("Failed to read response stream", 500); - return send_response(connection, 500, msg); - } - - fprintf(stderr, "[CPP-V2-TRANSITION] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", - bucket.c_str(), key.c_str(), content.length()); - - // Create and send response - struct MHD_Response *response = MHD_create_response_from_buffer( - content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); - - // Add keep-alive header - MHD_add_response_header(response, "Connection", "keep-alive"); - MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); - - MHD_Result ret = MHD_queue_response(connection, 200, response); - MHD_destroy_response(response); - - return ret; + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); } else { - // Enhanced error logging with thread info - auto error = outcome.GetError(); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str()); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject error details:\n"); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); - auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "[CPP-V2-TRANSITION] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); - auto msg = make_error(e.what(), 500); - return send_response(connection, 500, msg); - } catch (...) { - fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str()); - auto msg = make_error("Unknown error in GetObject", 500); + auto msg = make_error("An exception was thrown", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - std::string bucket, - std::string key, - std::string client_id, - std::string body, - std::string metadata) { - fprintf(stderr, "[CPP-V2-TRANSITION] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", - bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); - - auto client = get_client(client_id); - if (!client) { - fprintf(stderr, "[CPP-V2-TRANSITION] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &body, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { - // Create owned copy of body data to ensure it lives through the S3 operation - auto body_ptr = std::make_shared(body); - Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -482,248 +225,86 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - // Create stream from owned body data - auto stream = std::make_shared(*body_ptr); + auto stream = std::make_shared(body); request.SetBody(stream); - // Synchronous call - waits for S3 operation to complete - // body_ptr keeps the data alive through this entire operation - auto outcome = client->PutObject(request, kmsContextMap); + auto outcome = it->second->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { - fprintf(stderr, "[CPP-V2-TRANSITION] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "[CPP-V2-TRANSITION] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2-TRANSITION] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } -void request_completed(void *cls, struct MHD_Connection *connection, - void **con_cls, enum MHD_RequestTerminationCode toe) { - // Clean up the request-specific context when request is truly complete - // This is called AFTER all handlers have returned and the response has been sent - - // Log why the request was terminated - const char* reason = "UNKNOWN"; - switch (toe) { - case MHD_REQUEST_TERMINATED_COMPLETED_OK: - reason = "COMPLETED_OK"; - break; - case MHD_REQUEST_TERMINATED_WITH_ERROR: - reason = "WITH_ERROR"; - break; - case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: - reason = "TIMEOUT_REACHED"; - break; - case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: - reason = "DAEMON_SHUTDOWN"; - break; - case MHD_REQUEST_TERMINATED_READ_ERROR: - reason = "READ_ERROR"; - break; - case MHD_REQUEST_TERMINATED_CLIENT_ABORT: - reason = "CLIENT_ABORT"; - break; - } - fprintf(stderr, "[CPP-V2-TRANSITION] request_completed called, reason=%s, con_cls=%p\n", - reason, *con_cls); - - if (*con_cls != nullptr) { - std::string *body = static_cast(*con_cls); - delete body; // Safe to delete now - all synchronous operations are complete - *con_cls = nullptr; - } -} - MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - try { - std::string method_str(method); - std::string url_str(url); - bool is_push = method_str == "POST" || method_str == "PUT"; - - // LOG: Every request entry (even first-time calls) - if (*con_cls == nullptr) { - fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", - method, url, version, *upload_data_size); - } - - // Initialize request context on first call + std::string method_str(method); + bool is_push = method_str == "POST" || method_str == "PUT"; + static int dummy; if (*con_cls == nullptr) { - // Allocate unique state for each request to avoid race conditions - *con_cls = new std::string(); - fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST INIT: allocated new request context for %s %s\n", method, url); + if (is_push) { + *con_cls = new std::string(); + } else { + *con_cls = &dummy; + } return MHD_YES; } - - // LOG: Subsequent calls - if (is_push && *upload_data_size > 0) { - fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); - } else if (*upload_data_size == 0) { - fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST COMPLETE: %s %s ready for processing\n", method, url); - } - - // Accumulate request body data for POST/PUT requests - if (is_push && *upload_data_size > 0) { + if (is_push && *upload_data_size) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - // At this point, *upload_data_size == 0, meaning we have all the data - // Now we can safely process the request - - // LOG: About to process request - fprintf(stderr, "[CPP-V2-TRANSITION] PROCESSING: %s %s\n", method, url); - - // Handle client creation endpoint + + std::string url_str(url); + if (is_push && url_str == "/client") { - fprintf(stderr, "[CPP-V2-TRANSITION] Handling /client endpoint\n"); - std::string *body = static_cast(*con_cls); - MHD_Result result = handle_create_client(connection, *body); - fprintf(stderr, "[CPP-V2-TRANSITION] /client handler returned: %d\n", result); - return result; + std::unique_ptr body(static_cast(*con_cls)); + return handle_create_client(connection, *body); } - // Handle object operations if (url_str.find("/object/") == 0) { - fprintf(stderr, "[CPP-V2-TRANSITION] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); + std::string metadata = get_header_value(connection, "content-metadata"); - - fprintf(stderr, "[CPP-V2-TRANSITION] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", - bucket.c_str(), key.c_str(), client_id.c_str(), method); - if (method_str == "GET") { - fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_get_object\n"); - MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); - fprintf(stderr, "[CPP-V2-TRANSITION] handle_get_object returned: %d\n", result); - return result; + return handle_get_object(connection, bucket, key, client_id, metadata); } else if (method_str == "PUT") { - fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_put_object\n"); - std::string *body = static_cast(*con_cls); - MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); - fprintf(stderr, "[CPP-V2-TRANSITION] handle_put_object returned: %d\n", result); - return result; - } else { - fprintf(stderr, "[CPP-V2-TRANSITION] Method not allowed: %s\n", method); - return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); + std::unique_ptr body(static_cast(*con_cls)); + *upload_data_size = 0; + return handle_put_object(connection, bucket, key, client_id, *body, metadata); } } } - // Return error for unrecognized endpoints - fprintf(stderr, "[CPP-V2-TRANSITION] ERROR: Unrecognized endpoint: %s %s\n", method, url); - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); - } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", - e.what(), method, url); - // Try to send error response, but connection might already be broken - try { - return send_response(connection, 500, - "{\"error\":\"Internal server error: unhandled exception\"}"); - } catch (...) { - fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); - return MHD_NO; - } - } catch (...) { - fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", - method, url); - // Try to send error response, but connection might already be broken - try { - return send_response(connection, 500, - "{\"error\":\"Internal server error: unknown exception\"}"); - } catch (...) { - fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); - return MHD_NO; - } - } -} - -// Error log callback for libmicrohttpd -void log_mhd_error(void* cls, const char* fmt, va_list ap) { - fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-ERROR] "); - vfprintf(stderr, fmt, ap); - fprintf(stderr, "\n"); -} - -// Connection notification callback - called when a client connects -MHD_Result notify_connection(void *cls, - struct MHD_Connection *connection, - void **socket_context, - enum MHD_ConnectionNotificationCode toe) { - if (toe == MHD_CONNECTION_NOTIFY_STARTED) { - fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-CONNECT] New connection started\n"); - } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { - fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-DISCONNECT] Connection closed\n"); - } - return MHD_YES; + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); } int main() { Aws::SDKOptions options; - - // Configure AWS SDK logging to output to stderr (which goes to server.log) - // Using Debug level to capture all SDK activity including CryptoModule errors - options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; - options.loggingOptions.logger_create_fn = []() { - return std::make_shared( - Aws::Utils::Logging::LogLevel::Debug - ); - }; - - fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); - Aws::InitAPI(options); - - // Detect CPU core count and configure threading - unsigned int num_cores = std::thread::hardware_concurrency(); - if (num_cores == 0) { - num_cores = 4; - fprintf(stderr, "[CPP-V2-TRANSITION] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); - } - - g_thread_pool_size = num_cores * 2; - unsigned int connection_limit = g_thread_pool_size; - - // Log configuration - fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); - fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); - fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); - fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); - int port = 8097; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, - port, NULL, NULL, - &request_handler, NULL, - MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, - MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, - MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, - MHD_OPTION_CONNECTION_LIMIT, connection_limit, - MHD_OPTION_CONNECTION_TIMEOUT, 10, - MHD_OPTION_END); + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, + &request_handler, NULL, MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "[CPP-V2-TRANSITION] Failed to start server on port %d\n", port); + fprintf(stderr, "Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } diff --git a/test-server/cpp-v3-server/CMakeLists.txt b/test-server/cpp-v3-server/CMakeLists.txt index 0faac5f0..b282dbc4 100644 --- a/test-server/cpp-v3-server/CMakeLists.txt +++ b/test-server/cpp-v3-server/CMakeLists.txt @@ -7,7 +7,6 @@ set(CMAKE_CXX_STANDARD 17) set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") -set(ENABLE_ADDRESS_SANITIZER ON CACHE BOOL "Enable Address Sanitizer") # Add AWS SDK as subdirectory add_subdirectory(aws-sdk-cpp) @@ -19,21 +18,12 @@ find_package(nlohmann_json REQUIRED) add_executable(s3ec-server main.cpp) -# Enable Address Sanitizer for the executable -target_compile_options(s3ec-server PRIVATE -fsanitize=address -fno-omit-frame-pointer) -target_link_options(s3ec-server PRIVATE -fsanitize=address) - -target_include_directories(s3ec-server PRIVATE - ${LIBMICROHTTPD_INCLUDE_DIRS} - /opt/homebrew/include -) - -target_include_directories(s3ec-server PRIVATE +target_include_directories(s3ec-server PRIVATE ${LIBMICROHTTPD_INCLUDE_DIRS} /opt/homebrew/include ) -target_link_directories(s3ec-server PRIVATE +target_link_directories(s3ec-server PRIVATE ${LIBMICROHTTPD_LIBRARY_DIRS} /opt/homebrew/lib ) @@ -46,4 +36,4 @@ target_link_libraries(s3ec-server aws-cpp-sdk-s3-encryption nlohmann_json::nlohmann_json uuid -) +) \ No newline at end of file diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index e90c8d73..46f0c9db 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp V3 server..." - cd build && $(MAKE) + cd build && make start-server: @echo "Starting Cpp V3 server..." @@ -19,7 +19,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V3 server starting..." stop-server: @@ -30,7 +30,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f server.log + @rm -f build/server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v3-server/aws-sdk-cpp b/test-server/cpp-v3-server/aws-sdk-cpp index 4039810c..87402c99 160000 --- a/test-server/cpp-v3-server/aws-sdk-cpp +++ b/test-server/cpp-v3-server/aws-sdk-cpp @@ -1 +1 @@ -Subproject commit 4039810cd5d32429a64e70733175940d4a73f13c +Subproject commit 87402c99fd3c9107c6ccc6edf545fd4b05b2b551 diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index fc0d0452..1f74974c 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -1,56 +1,8 @@ -/* - * S3 Encryption Test Server - C++ V3 - * - * CONCURRENCY AND SYNCHRONIZATION DESIGN: - * - * 1. Threading Model: - * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool - * - Thread pool size = CPU cores * 2 (auto-detected at startup) - * - Threads are reused across connections for efficiency - * - I/O multiplexing (poll) distributes connections across thread pool - * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding - * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() - * - * 2. Resource Scaling: - * - All limits automatically scale with detected CPU count: - * * Thread pool size = num_cores * 2 - * * Connection limit = num_cores * 2 - * * S3 client maxConnections = num_cores * 2 - * - Multiplier of 2 accounts for I/O blocking without starving throughput - * - Ensures optimal resource usage on any hardware configuration - * - * 3. Client Cache (client_cache_secret): - * - Protected by std::shared_mutex for efficient concurrent access - * - get_client() uses shared_lock (multiple threads can read simultaneously) - * - set_client() uses unique_lock (exclusive write access) - * - This allows concurrent GET/PUT operations without serialization - * - UUID-based keys guarantee uniqueness (always insert, never update) - * - * 4. Memory Management: - * - Request body allocated in request_handler (*con_cls = new std::string()) - * - Body lifetime managed by libmicrohttpd - valid until request_completed() - * - All handler functions complete synchronously before returning - * - request_completed() safely deletes body after response sent - * - No memory leaks under sustained concurrent load - * - * 5. Synchronous Operation Guarantees: - * - GetObject: Waits for S3, reads full response stream, then returns - * - PutObject: Waits for S3 operation to complete, then returns - * - No async callbacks or background operations - * - Client receives response only after S3 operation completes - */ - #include -#include #include -#include #include #include #include -#include -#include -#include -#include #include #include #include @@ -60,31 +12,12 @@ #include #include #include -#include -#include -#include -#include -#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; - -// LRU cache for S3 encryption clients -// Limits memory and connection pool growth by evicting least recently used clients -const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations - -struct ClientCacheEntry { - std::shared_ptr client; - std::list::iterator lru_iter; -}; - -std::unordered_map client_cache_secret; -std::list lru_order; // Most recently used at front -std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads - -// Threading configuration - set at startup based on CPU cores -unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() +std::unordered_map> + client_cache; std::string generate_uuid() { uuid_t uuid; @@ -94,55 +27,6 @@ std::string generate_uuid() { return std::string(uuid_str); } -std::shared_ptr get_client(const std::string &client_id) -{ - // Need unique_lock to update LRU order even on reads - std::unique_lock lock(client_mutex); - auto it = client_cache_secret.find(client_id); - if (it == client_cache_secret.end()) { - return std::shared_ptr(); - } else { - // Move to front of LRU list (mark as most recently used) - lru_order.erase(it->second.lru_iter); - lru_order.push_front(client_id); - it->second.lru_iter = lru_order.begin(); - - return it->second.client; - } -} - -void set_client(const std::string &client_id, std::shared_ptr client) -{ - // UUID guarantees unique keys - always insert, never update - // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts - std::unique_lock lock(client_mutex); - - // Add to front of LRU list (most recently used) - lru_order.push_front(client_id); - - ClientCacheEntry entry; - entry.client = client; - entry.lru_iter = lru_order.begin(); - - client_cache_secret.emplace(client_id, entry); - - // Evict least recently used clients if we exceed the limit - while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { - std::string lru_client_id = lru_order.back(); - lru_order.pop_back(); - - auto evict_it = client_cache_secret.find(lru_client_id); - if (evict_it != client_cache_secret.end()) { - fprintf(stderr, "[CPP-V3] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", - lru_client_id.c_str(), client_cache_secret.size()); - client_cache_secret.erase(evict_it); - } - } - - fprintf(stderr, "[CPP-V3] [CACHE-ADD] Added client %s (cache size now %zu)\n", - client_id.c_str(), client_cache_secret.size()); -} - std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -183,47 +67,9 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { - // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly - // All operations here are synchronous and complete before returning to caller - try { json request = json::parse(body); - - // Extract all key material types - std::string kms_key_id; - std::string rsa_key_blob; - std::string aes_key_blob; - - if (request["config"]["keyMaterial"].contains("kmsKeyId") && - !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { - kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; - } - if (request["config"]["keyMaterial"].contains("rsaKey") && - !request["config"]["keyMaterial"]["rsaKey"].is_null()) { - rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; - } - if (request["config"]["keyMaterial"].contains("aesKey") && - !request["config"]["keyMaterial"]["aesKey"].is_null()) { - aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; - } - - // Validate that only one key type is provided - int key_count = 0; - if (!kms_key_id.empty()) key_count++; - if (!rsa_key_blob.empty()) key_count++; - if (!aes_key_blob.empty()) key_count++; - - if (key_count != 1) { - return send_response(connection, 400, - "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); - } - - // RSA is not supported by C++ SDK - if (!rsa_key_blob.empty()) { - return send_response(connection, 501, - "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); - } - + std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; bool inst_put = false; @@ -232,71 +78,33 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - std::string commitmentPolicy = get_config(request, "commitmentPolicy"); - std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); - - // Create CryptoConfigurationV3 based on key type - std::optional config; - - if (!aes_key_blob.empty()) { - // Base64 decode the AES key - Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); - if (decoded.GetLength() == 0) { - return send_response(connection, 400, - "{\"error\":\"Failed to decode AES key\"}"); - } - - Aws::Utils::CryptoBuffer key_buffer( - decoded.GetUnderlyingData(), - decoded.GetLength() - ); - - auto materials = std::make_shared< - Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( - key_buffer - ); - config.emplace(materials); - } else if (!kms_key_id.empty()) { - auto materials = std::make_shared(kms_key_id); - config.emplace(materials); - } else { - return send_response(connection, 400, - "{\"error\":\"No valid key material provided\"}"); - } - - // Apply common configuration settings (applies to both AES and KMS) + auto materials = + std::make_shared(kms_key_id); + CryptoConfigurationV3 config(materials); if (legacy1 || legacy2) - config->AllowLegacy(); + config.AllowLegacy(); if (inst_put) - config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - // Configure commitment policy (applies to both AES and KMS) + config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + std::string commitmentPolicy = get_config(request, "commitmentPolicy"); + std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); + if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF" || - encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") { - return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - } - config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") { - return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - } - config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") { - return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - } - config->SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); } - - // Create S3EncryptionClientV3 with standard configuration - Aws::Client::ClientConfiguration clientConfig; - clientConfig.maxConnections = 512; // Large pool per client - clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); - auto encryption_client = std::make_shared(*config, clientConfig); + + auto encryption_client = std::make_shared(config); std::string client_id = generate_uuid(); - set_client(client_id, encryption_client); + client_cache[client_id] = encryption_client; json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); @@ -312,18 +120,13 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { - fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: metadata is empty\n"); return; } - fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", - metadata.c_str(), metadata.length()); - // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; - int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -356,9 +159,6 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); - fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", - ++pair_count, key.c_str(), value.c_str()); - // Add to map map.emplace(key, value); @@ -369,24 +169,14 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } - - fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - std::string bucket, - std::string key, - std::string client_id, - std::string metadata) { - // Get thread ID for debugging concurrent operations - std::thread::id thread_id = std::this_thread::get_id(); - - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu\n", - (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length()); - - auto client = get_client(client_id); - if (!client) { - fprintf(stderr, "[CPP-V3] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -395,100 +185,47 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); + // S3EncryptionGetObjectOutcome outcome ; + // if (metadata.empty()) { + // outcome = it->second->GetObject(request); + // } else { + // Aws::Map kmsContextMap; + // fill_context(kmsContextMap, metadata); + // outcome = it->second->GetObject(request, kmsContextMap); + // } + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - - // Log the encryption context map size and contents - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); - for (const auto& pair : kmsContextMap) { - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: context['%s']='%s'\n", - pair.first.c_str(), pair.second.c_str()); - } - - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); - - // Keep outcome alive to ensure stream remains valid - auto outcome = client->GetObject(request, kmsContextMap); - - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); + auto outcome = it->second->GetObject(request, kmsContextMap); if (outcome.IsSuccess()) { - // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::stringstream buffer; - buffer << stream.rdbuf(); - std::string content = buffer.str(); - - // Validate we read something - if (content.empty() && stream.fail()) { - fprintf(stderr, "[CPP-V3] GetObject error: Failed to read stream for bucket=%s, key=%s\n", - bucket.c_str(), key.c_str()); - auto msg = make_error("Failed to read response stream", 500); - return send_response(connection, 500, msg); - } - - fprintf(stderr, "[CPP-V3] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", - bucket.c_str(), key.c_str(), content.length()); - - // Create and send response - struct MHD_Response *response = MHD_create_response_from_buffer( - content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); - - // Add keep-alive header - MHD_add_response_header(response, "Connection", "keep-alive"); - MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); - - MHD_Result ret = MHD_queue_response(connection, 200, response); - MHD_destroy_response(response); - - return ret; + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); } else { - // Enhanced error logging with thread info - auto error = outcome.GetError(); - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str()); - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject error details:\n"); - fprintf(stderr, "[CPP-V3] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); - fprintf(stderr, "[CPP-V3] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); - fprintf(stderr, "[CPP-V3] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); - fprintf(stderr, "[CPP-V3] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); - auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "[CPP-V3] GetObject AWS error: %s\n", msg.c_str()); + fprintf(stderr, "handle_get_object error %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); - auto msg = make_error(e.what(), 500); - return send_response(connection, 500, msg); - } catch (...) { - fprintf(stderr, "[CPP-V3] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", - (unsigned long)std::hash{}(thread_id), key.c_str()); - auto msg = make_error("Unknown error in GetObject", 500); + fprintf(stderr, "handle_get_object exception %s\n", e.what()); + auto msg = make_error("An exception was thrown", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - std::string bucket, - std::string key, - std::string client_id, - std::string body, - std::string metadata) { - fprintf(stderr, "[CPP-V3] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", - bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); - - auto client = get_client(client_id); - if (!client) { - fprintf(stderr, "[CPP-V3] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &body, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { - // Create owned copy of body data to ensure it lives through the S3 operation - auto body_ptr = std::make_shared(body); - Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -496,246 +233,85 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - // Create stream from owned body data - auto stream = std::make_shared(*body_ptr); + auto stream = std::make_shared(body); request.SetBody(stream); - // Synchronous call - waits for S3 operation to complete - // body_ptr keeps the data alive through this entire operation - auto outcome = client->PutObject(request, kmsContextMap); + auto outcome = it->second->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { - fprintf(stderr, "[CPP-V3] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "[CPP-V3] PutObject AWS error: %s\n", msg.c_str()); + fprintf(stderr, "handle_put_object error %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V3] PutObject exception: %s\n", e.what()); + fprintf(stderr, "handle_put_object exception %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } -void request_completed(void *cls, struct MHD_Connection *connection, - void **con_cls, enum MHD_RequestTerminationCode toe) { - // Clean up the request-specific context when request is truly complete - // This is called AFTER all handlers have returned and the response has been sent - - // Log why the request was terminated - const char* reason = "UNKNOWN"; - switch (toe) { - case MHD_REQUEST_TERMINATED_COMPLETED_OK: - reason = "COMPLETED_OK"; - break; - case MHD_REQUEST_TERMINATED_WITH_ERROR: - reason = "WITH_ERROR"; - break; - case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: - reason = "TIMEOUT_REACHED"; - break; - case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: - reason = "DAEMON_SHUTDOWN"; - break; - case MHD_REQUEST_TERMINATED_READ_ERROR: - reason = "READ_ERROR"; - break; - case MHD_REQUEST_TERMINATED_CLIENT_ABORT: - reason = "CLIENT_ABORT"; - break; - } - fprintf(stderr, "[CPP-V3] request_completed called, reason=%s, con_cls=%p\n", - reason, *con_cls); - - if (*con_cls != nullptr) { - std::string *body = static_cast(*con_cls); - delete body; // Safe to delete now - all synchronous operations are complete - *con_cls = nullptr; - } -} - MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - try { - std::string method_str(method); - std::string url_str(url); - bool is_push = method_str == "POST" || method_str == "PUT"; - - // LOG: Every request entry (even first-time calls) - if (*con_cls == nullptr) { - fprintf(stderr, "[CPP-V3] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", - method, url, version, *upload_data_size); - } - - // Initialize request context on first call + std::string method_str(method); + bool is_push = method_str == "POST" || method_str == "PUT"; + static int dummy; if (*con_cls == nullptr) { - // Allocate unique state for each request to avoid race conditions - *con_cls = new std::string(); - fprintf(stderr, "[CPP-V3] REQUEST INIT: allocated new request context for %s %s\n", method, url); + if (is_push) { + *con_cls = new std::string(); + } else { + *con_cls = &dummy; + } return MHD_YES; } - - // LOG: Subsequent calls - if (is_push && *upload_data_size > 0) { - fprintf(stderr, "[CPP-V3] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); - } else if (*upload_data_size == 0) { - fprintf(stderr, "[CPP-V3] REQUEST COMPLETE: %s %s ready for processing\n", method, url); - } - - // Accumulate request body data for POST/PUT requests - if (is_push && *upload_data_size > 0) { + if (is_push && *upload_data_size) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - // At this point, *upload_data_size == 0, meaning we have all the data - // Now we can safely process the request - - // LOG: About to process request - fprintf(stderr, "[CPP-V3] PROCESSING: %s %s\n", method, url); - // Handle client creation endpoint + std::string url_str(url); + if (is_push && url_str == "/client") { - fprintf(stderr, "[CPP-V3] Handling /client endpoint\n"); - std::string *body = static_cast(*con_cls); - MHD_Result result = handle_create_client(connection, *body); - fprintf(stderr, "[CPP-V3] /client handler returned: %d\n", result); - return result; + std::unique_ptr body(static_cast(*con_cls)); + return handle_create_client(connection, *body); } - // Handle object operations if (url_str.find("/object/") == 0) { - fprintf(stderr, "[CPP-V3] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); + std::string metadata = get_header_value(connection, "content-metadata"); - - fprintf(stderr, "[CPP-V3] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", - bucket.c_str(), key.c_str(), client_id.c_str(), method); - if (method_str == "GET") { - fprintf(stderr, "[CPP-V3] Dispatching to handle_get_object\n"); - MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata); - fprintf(stderr, "[CPP-V3] handle_get_object returned: %d\n", result); - return result; + return handle_get_object(connection, bucket, key, client_id, metadata); } else if (method_str == "PUT") { - fprintf(stderr, "[CPP-V3] Dispatching to handle_put_object\n"); - std::string *body = static_cast(*con_cls); - MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); - fprintf(stderr, "[CPP-V3] handle_put_object returned: %d\n", result); - return result; - } else { - fprintf(stderr, "[CPP-V3] Method not allowed: %s\n", method); - return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); + std::unique_ptr body(static_cast(*con_cls)); + *upload_data_size = 0; + return handle_put_object(connection, bucket, key, client_id, *body, metadata); } } } - // Return error for unrecognized endpoints - fprintf(stderr, "[CPP-V3] ERROR: Unrecognized endpoint: %s %s\n", method, url); - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); - } catch (const std::exception &e) { - fprintf(stderr, "[CPP-V3] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", - e.what(), method, url); - // Try to send error response, but connection might already be broken - try { - return send_response(connection, 500, - "{\"error\":\"Internal server error: unhandled exception\"}"); - } catch (...) { - fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); - return MHD_NO; - } - } catch (...) { - fprintf(stderr, "[CPP-V3] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", - method, url); - // Try to send error response, but connection might already be broken - try { - return send_response(connection, 500, - "{\"error\":\"Internal server error: unknown exception\"}"); - } catch (...) { - fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); - return MHD_NO; - } - } -} - -// Error log callback for libmicrohttpd -void log_mhd_error(void* cls, const char* fmt, va_list ap) { - fprintf(stderr, "[CPP-V3] [MHD-ERROR] "); - vfprintf(stderr, fmt, ap); - fprintf(stderr, "\n"); -} - -// Connection notification callback - called when a client connects -MHD_Result notify_connection(void *cls, - struct MHD_Connection *connection, - void **socket_context, - enum MHD_ConnectionNotificationCode toe) { - if (toe == MHD_CONNECTION_NOTIFY_STARTED) { - fprintf(stderr, "[CPP-V3] [MHD-CONNECT] New connection started\n"); - } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { - fprintf(stderr, "[CPP-V3] [MHD-DISCONNECT] Connection closed\n"); - } - return MHD_YES; + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); } int main() { Aws::SDKOptions options; - - // Configure AWS SDK logging to output to stderr (which goes to server.log) - // Using Debug level to capture all SDK activity including CryptoModule errors - options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; - options.loggingOptions.logger_create_fn = []() { - return std::make_shared( - Aws::Utils::Logging::LogLevel::Debug - ); - }; - - fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); - Aws::InitAPI(options); - - // Detect CPU core count and configure threading - unsigned int num_cores = std::thread::hardware_concurrency(); - if (num_cores == 0) { - num_cores = 4; // Fallback if detection fails - fprintf(stderr, "[WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); - } - - // Thread pool size = num_cores * 2 (allows for I/O blocking without starving throughput) - g_thread_pool_size = num_cores * 2; - unsigned int connection_limit = g_thread_pool_size; - - // Log configuration - fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); - fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); - fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); - fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); - int port = 8091; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, - port, NULL, NULL, - &request_handler, NULL, - MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, - MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, - MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, - MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, - MHD_OPTION_CONNECTION_LIMIT, connection_limit, - MHD_OPTION_CONNECTION_TIMEOUT, 10, - MHD_OPTION_END); + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, + &request_handler, NULL, MHD_OPTION_END); if (!daemon) { fprintf(stderr, "Failed to start server on port %d\n", port); diff --git a/test-server/go-v3-server/main.go b/test-server/go-v3-server/main.go index 0384c5ff..d201ffe2 100644 --- a/test-server/go-v3-server/main.go +++ b/test-server/go-v3-server/main.go @@ -8,7 +8,6 @@ import ( "log" "net/http" "strings" - "sync" "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" @@ -24,7 +23,6 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client - mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -68,12 +66,7 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion("us-west-2"), - config.WithRetryMaxAttempts(5), - config.WithRetryMode(aws.RetryModeAdaptive), - ) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -148,12 +141,7 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion("us-west-2"), - config.WithRetryMaxAttempts(5), - config.WithRetryMode(aws.RetryModeAdaptive), - ) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return @@ -184,10 +172,8 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache (protected by mutex) - s.mu.Lock() + // Store client in cache s.clientCache[clientID] = s3EncryptionClient - s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -208,10 +194,8 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache (protected by mutex) - s.mu.RLock() + // Get client from cache client, exists := s.clientCache[clientID] - s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -280,10 +264,8 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache (protected by mutex) - s.mu.RLock() + // Get client from cache client, exists := s.clientCache[clientID] - s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index 64556f12..799a9668 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -8,7 +8,6 @@ import ( "log" "net/http" "strings" - "sync" "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" @@ -25,7 +24,6 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client - mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -70,12 +68,7 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion("us-west-2"), - config.WithRetryMaxAttempts(5), - config.WithRetryMode(aws.RetryModeAdaptive), - ) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -150,12 +143,7 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion("us-west-2"), - config.WithRetryMaxAttempts(5), - config.WithRetryMode(aws.RetryModeAdaptive), - ) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return @@ -201,10 +189,8 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache (protected by mutex) - s.mu.Lock() + // Store client in cache s.clientCache[clientID] = s3EncryptionClient - s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -225,10 +211,8 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache (protected by mutex) - s.mu.RLock() + // Get client from cache client, exists := s.clientCache[clientID] - s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -297,10 +281,8 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache (protected by mutex) - s.mu.RLock() + // Get client from cache client, exists := s.clientCache[clientID] - s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/go-v4-server/main.go b/test-server/go-v4-server/main.go index 50999e95..672236ac 100644 --- a/test-server/go-v4-server/main.go +++ b/test-server/go-v4-server/main.go @@ -8,7 +8,6 @@ import ( "log" "net/http" "strings" - "sync" "github.com/aws/amazon-s3-encryption-client-go/v4/client" "github.com/aws/amazon-s3-encryption-client-go/v4/materials" @@ -25,7 +24,6 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV4 kmsClient *kms.Client - mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -70,12 +68,7 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion("us-west-2"), - config.WithRetryMaxAttempts(5), - config.WithRetryMode(aws.RetryModeAdaptive), - ) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -150,12 +143,7 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithRegion("us-west-2"), - config.WithRetryMaxAttempts(5), - config.WithRetryMode(aws.RetryModeAdaptive), - ) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return @@ -201,10 +189,8 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache (protected by mutex) - s.mu.Lock() + // Store client in cache s.clientCache[clientID] = s3EncryptionClient - s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -225,10 +211,8 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache (protected by mutex) - s.mu.RLock() + // Get client from cache client, exists := s.clientCache[clientID] - s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -297,10 +281,8 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache (protected by mutex) - s.mu.RLock() + // Get client from cache client, exists := s.clientCache[clientID] - s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 14c3eec1..106f82ef 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -16,9 +16,6 @@ dependencies { // Test dependencies testImplementation("org.junit.jupiter:junit-jupiter:5.13.0") testRuntimeOnly("org.junit.platform:junit-platform-launcher") - // JUnit Suite support for test ordering - testImplementation("org.junit.platform:junit-platform-suite-api:1.10.0") - testRuntimeOnly("org.junit.platform:junit-platform-suite-engine:1.10.0") testImplementation("com.amazonaws:aws-java-sdk:1.12.788") testImplementation("software.amazon.awssdk:s3:2.37.1") testImplementation("org.bouncycastle:bcprov-jdk15on:1.70") @@ -52,17 +49,6 @@ tasks { classpath = sourceSets["it"].runtimeClasspath outputs.upToDateWhen { false } outputs.cacheIf { false } - - // Enable parallel test execution - systemProperty("junit.jupiter.execution.parallel.enabled", "true") - systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") - systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") - // Configure thread pool size - adjust based on I/O-bound nature of tests - systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") - maxParallelForks = 1 // One JVM - systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", - Math.max(1, Runtime.getRuntime().availableProcessors() - 2).toString()) // Scale with CPU, reserve 2 cores - // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) // For debugging diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java index 9565639b..100925a9 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java @@ -180,7 +180,7 @@ public void GIVEN_KCGCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncry ) { S3ECTestServerClient encClient = TestUtils.testServerClientFor(encLang); - final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang + "-" + decLang; + final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang; final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(TestUtils.KMS_KEY_ARN) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java deleted file mode 100644 index 4be4d434..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.encryption.s3; - -import static software.amazon.encryption.s3.TestUtils.*; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.S3ECConfig; - -/** - * GCM Test Suite - * - * This suite enforces execution order between GCM encrypt and decrypt phases: - * 1. EncryptTests - All encrypt tests run in parallel (within this phase) - * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel - * - * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion - * and DecryptTests awaits before proceeding. - */ -public class GCMTestSuite { - // Synchronization latch - released when encrypt phase completes - private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); - - /** - * GCM Encryption Tests - Encrypt Phase - * - * These tests encrypt objects using GCM (without key commitment) encryption algorithm. - * All tests in this class can run in parallel with each other. - * The encrypted objects are stored in thread-safe lists for use by DecryptTests. - */ - @Nested - class EncryptTests { - private static final String sharedObjectKeyBase = "test-gcm-kms"; - private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - - // Thread-safe list for storing encrypted object keys - private static final List crossLanguageObjects = - Collections.synchronizedList(new ArrayList<>()); - - /** - * Public accessor for decrypt tests to retrieve encrypted object keys - */ - static List getCrossLanguageObjects() { - return new ArrayList<>(crossLanguageObjects); // Return defensive copy - } - - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), - crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_encrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), - crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), - crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @AfterAll - static void signalEncryptionComplete() { - // Signal that all encryption tests have completed - encryptPhaseComplete.countDown(); - } - } - - /** - * GCM Decryption Tests - Decrypt Phase - * - * These tests decrypt objects that were encrypted by EncryptTests. - * All tests in this class can run fully in parallel with each other. - * They depend on EncryptTests completing first (enforced by @Order). - */ - @Nested - class DecryptTests { - private static List crossLanguageObjects; - private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - - @BeforeAll - static void setup() throws InterruptedException { - // Wait for all encryption tests to complete - encryptPhaseComplete.await(); - - // Import encrypted objects from the encrypt phase - crossLanguageObjects = EncryptTests.getCrossLanguageObjects(); - - // Verify we have objects to decrypt - if (crossLanguageObjects.isEmpty()) { - throw new IllegalStateException( - "No encrypted objects found. Ensure EncryptTests runs first."); - } - } - - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_decrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, - EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - } -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java new file mode 100644 index 00000000..6eef0b5f --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java @@ -0,0 +1,203 @@ +/* +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.lang.annotation.ElementType; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import com.amazonaws.services.s3.model.KMSEncryptionMaterials; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.PutObjectInput; +import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3EncryptionClientError; + +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import software.amazon.encryption.s3.TestUtils.*; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +/** +* Exhaustive tests for S3 Encryption Client round-trip operations. +* These tests cover various combinations of client versions, commitment policies, and encryption modes. +* +* Tests are based on the exhaustive test matrix defined at: +* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 +* +*/ + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class GCMTests { + private static String sharedObjectKeyBase = "test-gcm-kms"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + private static List crossLanguageObjects = new ArrayList<>(); + + @Order(1) + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(2) + @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(3) + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(10) + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(11) + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(12) + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(13) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @Order(14) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java deleted file mode 100644 index c367315f..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -package software.amazon.encryption.s3; - -import static software.amazon.encryption.s3.TestUtils.*; - -import java.nio.ByteBuffer; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.opentest4j.TestAbortedException; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.InstructionFileConfig; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.S3ECConfig; - -/** - * KC-GCM Test Suite - * - * This suite enforces execution order between KC-GCM encrypt and decrypt phases: - * 1. EncryptTests - All encrypt tests run in parallel (within this phase) - * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel - * - * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion - * and DecryptTests awaits before proceeding. - */ -public class KC_GCMTestSuite { - // Synchronization latch - released when encrypt phase completes - private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); - - /** - * KC-GCM Encryption Tests - Encrypt Phase - * - * These tests encrypt objects using Key Commitment GCM encryption algorithm. - * All tests in this class can run in parallel with each other. - * The encrypted objects are stored in thread-safe lists for use by DecryptTests. - */ - @Nested - class EncryptTests { - private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; - private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-rsa-instruction-file"; - private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - - // Thread-safe lists for storing encrypted object keys - private static final List crossLanguageObjectsMetaDataMode = - Collections.synchronizedList(new ArrayList<>()); - private static final List crossLanguageObjectsInstructionFiles = - Collections.synchronizedList(new ArrayList<>()); - - private static KeyPair RSA_KEY_PAIR_1; - - @BeforeAll - static void setupKeys() throws Exception { - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); - } - - /** - * Public accessors for decrypt tests to retrieve encrypted object keys and RSA key - */ - static List getCrossLanguageObjectsMetaDataMode() { - return new ArrayList<>(crossLanguageObjectsMetaDataMode); - } - - static List getCrossLanguageObjectsInstructionFiles() { - return new ArrayList<>(crossLanguageObjectsInstructionFiles); - } - - static KeyPair getRsaKeyPair() { - return RSA_KEY_PAIR_1; - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), - crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM (instruction file)") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file( - TestUtils.LanguageServerTarget language - ) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), - crossLanguageObjectsInstructionFiles, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), - crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_the_default_should_encrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, - appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), - crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @AfterAll - static void signalEncryptionComplete() { - // Signal that all encryption tests have completed - encryptPhaseComplete.countDown(); - } - } - - /** - * KC-GCM Decryption Tests - Decrypt Phase - * - * These tests decrypt objects that were encrypted by EncryptTests. - * All tests in this class can run fully in parallel with each other. - * They depend on EncryptTests completing first (enforced by @Order). - */ - @Nested - class DecryptTests { - private static List crossLanguageObjectsMetaDataMode; - private static List crossLanguageObjectsInstructionFiles; - private static KeyPair RSA_KEY_PAIR_1; - private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - - @BeforeAll - static void setup() throws InterruptedException { - // Wait for all encryption tests to complete - encryptPhaseComplete.await(); - - // Import encrypted objects and RSA key from the encrypt phase - crossLanguageObjectsMetaDataMode = EncryptTests.getCrossLanguageObjectsMetaDataMode(); - crossLanguageObjectsInstructionFiles = EncryptTests.getCrossLanguageObjectsInstructionFiles(); - RSA_KEY_PAIR_1 = EncryptTests.getRsaKeyPair(); - - // Verify we have objects to decrypt - if (crossLanguageObjectsMetaDataMode.isEmpty()) { - throw new IllegalStateException( - "No encrypted objects found. Ensure EncryptTests runs first."); - } - } - - @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_decrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_the_default_should_decrypt_kc_gcm( - TestUtils.LanguageServerTarget language - ) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM (instruction file)") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file( - final TestUtils.LanguageServerTarget language - ) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @ParameterizedTest(name = "{0}: Transition configured with default should decrypt KC-GCM (instruction file)") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file( - final TestUtils.LanguageServerTarget language - ) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, - EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - } -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java new file mode 100644 index 00000000..ee4279d6 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java @@ -0,0 +1,264 @@ +/* +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.opentest4j.TestAbortedException; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** +* Exhaustive tests for S3 Encryption Client round-trip operations. +* These tests cover various combinations of client versions, commitment policies, and encryption modes. +* +* Tests are based on the exhaustive test matrix defined at: +* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 +* +*/ + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class KC_GCMTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; + private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-kms-instruction-file"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + private static final List crossLanguageObjectsMetaDataMode = new ArrayList<>(); + private static final List crossLanguageObjectsInstructionFiles = new ArrayList<>(); + private static KeyPair RSA_KEY_PAIR_1; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + } + + @Order(1) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(2) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file(TestUtils.LanguageServerTarget language) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), crossLanguageObjectsInstructionFiles, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(2) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(2) + @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(10) + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(11) + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(12) + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(13) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(14) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(15) + @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @Order(16) + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file(final TestUtils.LanguageServerTarget language) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index add811eb..e6fab412 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -214,9 +214,9 @@ public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLa fail("Expected exception!"); } catch (S3EncryptionClientError e) { if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_CURRENT) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { - assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); } else { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); } } } @@ -278,9 +278,9 @@ public void crossLanguageTestKmsWithIncorrectEncCtxFails(LanguageServerTarget en fail("Expected exception!"); } catch (S3EncryptionClientError e) { if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_CURRENT) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { - assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); } else { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); } } } @@ -427,15 +427,15 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(TestUtils.LanguageServerTarget la || language.getLanguageName().equals(CPP_V2_CURRENT) || language.getLanguageName().equals(CPP_V2_TRANSITION) || language.getLanguageName().equals(CPP_V3)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" - ), "Actual error:" + e.getMessage()); + )); } else if (language.getLanguageName().equals(RUBY_V3) || language.getLanguageName().equals(RUBY_V2_CURRENT) || language.getLanguageName().equals(RUBY_V2_TRANSITION)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object." ), "Actual error:" + e.getMessage()); } else if (language.getLanguageName().equals(PHP_V3)) { - assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."), "Actual error: " + e.getMessage()); + assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."));; } else { - assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"), "Actual error: " + e.getMessage()); + assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); } } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 759c26c5..93947357 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -95,26 +95,12 @@ public class TestUtils { public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = Set.of(NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4); - - // Cpp only supports Raw AES - public static final Set RAW_AES_SUPPORTED = - Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 - , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 - , RUBY_V2_TRANSITION, RUBY_V3 - , CPP_V2_CURRENT, CPP_V2_TRANSITION, CPP_V3 - ); - - public static final Set RAW_RSA_SUPPORTED = + // For now, only .NET and Java have RSA support + public static final Set RAW_SUPPORTED = Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 , NET_V2_CURRENT, NET_V3_CURRENT, NET_V2_TRANSITION, NET_V3_TRANSITION, NET_V4 ); - // Intersection of RAW_AES_SUPPORTED and RAW_RSA_SUPPORTED - public static final Set RAW_SUPPORTED = - RAW_AES_SUPPORTED.stream() - .filter(RAW_RSA_SUPPORTED::contains) - .collect(Collectors.toSet()); - // .NET only supports decrypting instruction files using AES and RSA. // Python MUST support decrypting KMS instruction files, but does not yet. public static final Set KMS_INSTRUCTION_FILE_UNSUPPORTED = @@ -375,28 +361,6 @@ public static Stream improvedClientsForTest() { .map(Arguments::of); } - /** - * Get stream of arguments for clients that support RAW AES (includes CPP). - */ - public static Stream clientsRawAesForTest() { - Stream improved = improvedClientsForTest() - .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - Stream transition = transitionClientsForTest() - .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - return Stream.concat(improved, transition); - } - - /** - * Get stream of arguments for clients that support RAW RSA (excludes CPP). - */ - public static Stream clientsRawRsaForTest() { - Stream improved = improvedClientsForTest() - .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - Stream transition = transitionClientsForTest() - .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); - return Stream.concat(improved, transition); - } - /** * These functions provide a stream of arguments for parameterized tests. * @return Stream of Arguments containing pairs of LanguageServerTarget for encryption and decryption diff --git a/test-server/java-v3-server/gradle.properties b/test-server/java-v3-server/gradle.properties index 483cd315..08afce82 100644 --- a/test-server/java-v3-server/gradle.properties +++ b/test-server/java-v3-server/gradle.properties @@ -4,21 +4,8 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings - -# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive -org.gradle.daemon=false - -# Set minimal idle timeout for any daemon-like behavior (1 second) -org.gradle.daemon.idletimeout=1000 - -# JVM arguments to prevent forking a separate JVM process -# By matching the JVM args here with what Gradle expects, we avoid the -# "single-use Daemon process will be forked" behavior -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC - -# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true - -# Configure on demand to reduce startup time -org.gradle.configureondemand=true +org.gradle.daemon=true +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError +org.gradle.workers.max=4 diff --git a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index bdb0b30b..1d198590 100644 --- a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,8 +1,5 @@ package software.amazon.encryption.s3; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.retry.RetryPolicy; -import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.core.traits.Trait; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.S3EncryptionClient; @@ -109,25 +106,12 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } - // Configure S3 client with adaptive retry for throttling - RetryPolicy retryPolicy = RetryPolicy.builder() - .numRetries(5) - .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) - .build(); - - S3Client wrappedClient = S3Client.builder() - .overrideConfiguration(ClientOverrideConfiguration.builder() - .retryPolicy(retryPolicy) - .build()) - .build(); - // Client Creation boolean instFilePut = false; if (input.getConfig().getInstructionFileConfig() != null) { instFilePut = input.getConfig().getInstructionFileConfig().isEnableInstructionFilePutObject(); } S3Client s3Client = S3EncryptionClient.builder() - .wrappedClient(wrappedClient) .instructionFileConfig(InstructionFileConfig.builder() .instructionFileClient(S3Client.create()) .enableInstructionFilePutObject(instFilePut) diff --git a/test-server/java-v3-transition-server/gradle.properties b/test-server/java-v3-transition-server/gradle.properties index 483cd315..08afce82 100644 --- a/test-server/java-v3-transition-server/gradle.properties +++ b/test-server/java-v3-transition-server/gradle.properties @@ -4,21 +4,8 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings - -# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive -org.gradle.daemon=false - -# Set minimal idle timeout for any daemon-like behavior (1 second) -org.gradle.daemon.idletimeout=1000 - -# JVM arguments to prevent forking a separate JVM process -# By matching the JVM args here with what Gradle expects, we avoid the -# "single-use Daemon process will be forked" behavior -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC - -# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true - -# Configure on demand to reduce startup time -org.gradle.configureondemand=true +org.gradle.daemon=true +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError +org.gradle.workers.max=4 diff --git a/test-server/java-v3-transition-server/s3ec-staging b/test-server/java-v3-transition-server/s3ec-staging index 597d5b49..6413811b 160000 --- a/test-server/java-v3-transition-server/s3ec-staging +++ b/test-server/java-v3-transition-server/s3ec-staging @@ -1 +1 @@ -Subproject commit 597d5b491ac578f5d03d3fa757201eb48690cd00 +Subproject commit 6413811bb81037999b8238e02047e0e403f78c1f diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index e107401e..425c0334 100644 --- a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,8 +1,5 @@ package software.amazon.encryption.s3; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.retry.RetryPolicy; -import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.algorithms.AlgorithmSuite; @@ -109,22 +106,9 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } - // Configure S3 client with adaptive retry for throttling - RetryPolicy retryPolicy = RetryPolicy.builder() - .numRetries(5) - .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) - .build(); - - S3Client wrappedClient = S3Client.builder() - .overrideConfiguration(ClientOverrideConfiguration.builder() - .retryPolicy(retryPolicy) - .build()) - .build(); - // V3 Transition server configuration // Existing Builder defaults to FORBID_ENCRYPT and ALG_AES_256_GCM_IV12_TAG16_NO_KDF S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builder() - .wrappedClient(wrappedClient) .keyring(keyring) .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); diff --git a/test-server/java-v4-server/gradle.properties b/test-server/java-v4-server/gradle.properties index 483cd315..08afce82 100644 --- a/test-server/java-v4-server/gradle.properties +++ b/test-server/java-v4-server/gradle.properties @@ -4,21 +4,8 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings - -# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive -org.gradle.daemon=false - -# Set minimal idle timeout for any daemon-like behavior (1 second) -org.gradle.daemon.idletimeout=1000 - -# JVM arguments to prevent forking a separate JVM process -# By matching the JVM args here with what Gradle expects, we avoid the -# "single-use Daemon process will be forked" behavior -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC - -# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true - -# Configure on demand to reduce startup time -org.gradle.configureondemand=true +org.gradle.daemon=true +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError +org.gradle.workers.max=4 diff --git a/test-server/java-v4-server/s3ec-staging b/test-server/java-v4-server/s3ec-staging index 2626ed6e..db0c743e 160000 --- a/test-server/java-v4-server/s3ec-staging +++ b/test-server/java-v4-server/s3ec-staging @@ -1 +1 @@ -Subproject commit 2626ed6e312c1c5e01abea2f30727ea0f2af299d +Subproject commit db0c743eec335d16e6dceaf2b09d84becb0f74f8 diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index 3fdb4b55..cb20d5ac 100644 --- a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,8 +1,5 @@ package software.amazon.encryption.s3; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.retry.RetryPolicy; -import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.algorithms.AlgorithmSuite; @@ -109,21 +106,8 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } - // Configure S3 client with adaptive retry for throttling - RetryPolicy retryPolicy = RetryPolicy.builder() - .numRetries(5) - .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) - .build(); - - S3Client wrappedClient = S3Client.builder() - .overrideConfiguration(ClientOverrideConfiguration.builder() - .retryPolicy(retryPolicy) - .build()) - .build(); - // V4-Improved server configuration S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builderV4() - .wrappedClient(wrappedClient) .keyring(keyring) .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); diff --git a/test-server/net-v2-v3-server/Controllers/ClientController.cs b/test-server/net-v2-v3-server/Controllers/ClientController.cs index 437233a8..e33a58e6 100644 --- a/test-server/net-v2-v3-server/Controllers/ClientController.cs +++ b/test-server/net-v2-v3-server/Controllers/ClientController.cs @@ -21,6 +21,8 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "[NET-current] EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "[NET-current] SetBufferSize not supported" }); + if (request.Config.KeyMaterial.AesKey != null) + return StatusCode(501, new GenericServerError { Message = "[NET-current] AesKey not supported" }); try { @@ -45,15 +47,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "Created EncryptionMaterialsV2: RSA"); - } - else if (request.Config.KeyMaterial.AesKey != null) - { - var aesKeyBytes = request.Config.KeyMaterial.AesKey; - var aes = Aes.Create(); - aes.Key = aesKeyBytes; - encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); - logger.LogInformation( - "[NET-current] Created EncryptionMaterialsV2: AES"); } else { return StatusCode(501, new GenericServerError { Message = "[NET-current] Unknown or missing key material!" }); @@ -69,11 +62,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) logger.LogInformation("[NET-current] Created securityProfile= {securityProfile}", securityProfile.ToString()); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile); - - // Add retry configuration for throttling - configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; - configuration.MaxErrorRetry = 5; - if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -103,4 +91,4 @@ public IActionResult CreateClient([FromBody] ClientRequest request) }); } } -} +} \ No newline at end of file diff --git a/test-server/net-v3-transition-server/Controllers/ClientController.cs b/test-server/net-v3-transition-server/Controllers/ClientController.cs index 3deeff61..a66fb342 100644 --- a/test-server/net-v3-transition-server/Controllers/ClientController.cs +++ b/test-server/net-v3-transition-server/Controllers/ClientController.cs @@ -21,6 +21,8 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "SetBufferSize not supported" }); + if (request.Config.KeyMaterial.AesKey != null) + return StatusCode(501, new GenericServerError { Message = "AesKey not supported" }); try { @@ -45,15 +47,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "Created EncryptionMaterialsV2: RSA"); - } - else if (request.Config.KeyMaterial.AesKey != null) - { - var aesKeyBytes = request.Config.KeyMaterial.AesKey; - var aes = Aes.Create(); - aes.Key = aesKeyBytes; - encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); - logger.LogInformation( - "[NET-V3-Transitional] Created EncryptionMaterialsV2: AES"); } else { return StatusCode(501, new GenericServerError { Message = "Unknown or missing key material!" }); @@ -74,11 +67,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm); - - // Add retry configuration for throttling - configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; - configuration.MaxErrorRetry = 5; - if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -130,4 +118,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti _ => ContentEncryptionAlgorithm.AesGcm }; } -} +} \ No newline at end of file diff --git a/test-server/net-v3-transition-server/s3ec-v3-transition-branch b/test-server/net-v3-transition-server/s3ec-v3-transition-branch index d099cfd1..ad825917 160000 --- a/test-server/net-v3-transition-server/s3ec-v3-transition-branch +++ b/test-server/net-v3-transition-server/s3ec-v3-transition-branch @@ -1 +1 @@ -Subproject commit d099cfd151e2c61fb97dcd417828fb1dd5468b0c +Subproject commit ad8259173de365a13e8b3932ee02493f599f597f diff --git a/test-server/net-v4-server/Controllers/ClientController.cs b/test-server/net-v4-server/Controllers/ClientController.cs index 2ef8b921..b9fbe3f9 100644 --- a/test-server/net-v4-server/Controllers/ClientController.cs +++ b/test-server/net-v4-server/Controllers/ClientController.cs @@ -20,6 +20,8 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "[NET-V4] EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "[NET-V4] SetBufferSize not supported" }); + if (request.Config.KeyMaterial.AesKey != null) + return StatusCode(501, new GenericServerError { Message = "[NET-V4] AesKey not supported" }); try { @@ -44,15 +46,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV4(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "[NET-V4] Created EncryptionMaterialsV4: RSA"); - } - else if (request.Config.KeyMaterial.AesKey != null) - { - var aesKeyBytes = request.Config.KeyMaterial.AesKey; - var aes = Aes.Create(); - aes.Key = aesKeyBytes; - encryptionMaterial = new EncryptionMaterialsV4(aes, SymmetricAlgorithmType.AesGcm); - logger.LogInformation( - "[NET-V4] Created EncryptionMaterialsV4: AES"); } else { return StatusCode(501, new GenericServerError { Message = "[NET-V4] Unknown or missing key material!" }); @@ -86,10 +79,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) ? new AmazonS3CryptoConfigurationV4() : new AmazonS3CryptoConfigurationV4(securityProfile, commitmentPolicy, encryptionAlgorithm); - // Add retry configuration for throttling - configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; - configuration.MaxErrorRetry = 5; - if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -141,4 +130,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti _ => ContentEncryptionAlgorithm.AesGcmWithCommitment }; } -} +} \ No newline at end of file diff --git a/test-server/net-v4-server/Makefile b/test-server/net-v4-server/Makefile index b52bbd49..e2df658a 100644 --- a/test-server/net-v4-server/Makefile +++ b/test-server/net-v4-server/Makefile @@ -32,7 +32,7 @@ start-net-V4-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run --no-build > server.log 2>&1 & echo $! > $(PID_FILE_NET_V4) + dotnet run --no-build & echo $! > $(PID_FILE_NET_V4) @echo ".NET V4 server starting..." wait-for-server: diff --git a/test-server/net-v4-server/s3ec-net-v4-improved b/test-server/net-v4-server/s3ec-net-v4-improved index 8ce8983b..1c0a458c 160000 --- a/test-server/net-v4-server/s3ec-net-v4-improved +++ b/test-server/net-v4-server/s3ec-net-v4-improved @@ -1 +1 @@ -Subproject commit 8ce8983bd0edf973651aee0c29894df9091cf97a +Subproject commit 1c0a458c19b351c266199c72072de746362c5326 diff --git a/test-server/php-v2-server/Makefile b/test-server/php-v2-server/Makefile index 719ea238..a9d04134 100644 --- a/test-server/php-v2-server/Makefile +++ b/test-server/php-v2-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 server starting..." stop-server: diff --git a/test-server/php-v2-transition-server/Makefile b/test-server/php-v2-transition-server/Makefile index a3d038de..61eb3a84 100644 --- a/test-server/php-v2-transition-server/Makefile +++ b/test-server/php-v2-transition-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 Transition server starting..." stop-server: diff --git a/test-server/php-v3-server/Makefile b/test-server/php-v3-server/Makefile index 9460d4ed..2b9661f2 100644 --- a/test-server/php-v3-server/Makefile +++ b/test-server/php-v3-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V3 server starting..." stop-server: diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb index 533ff67c..717003bf 100644 --- a/test-server/ruby-v2-server/lib/client_manager.rb +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -36,13 +36,7 @@ def create_client(config) end # Create the S3 encryption client - # Create the S3 encryption client with retry configuration for throttling - s3_client = Aws::S3::Client.new( - region: 'us-west-2', - retry_mode: 'adaptive', - retry_limit: 5, - retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } - ) + s3_client = Aws::S3::Client.new(region: 'us-west-2') encryption_client = Aws::S3::EncryptionV2::Client.new( client: s3_client, **encryption_config diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb index 0f48d563..a6fb551f 100644 --- a/test-server/ruby-v3-server/lib/client_manager.rb +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -61,13 +61,7 @@ def create_client(config) end # Create the S3 encryption client - # Create the S3 encryption client with retry configuration for throttling - s3_client = Aws::S3::Client.new( - region: 'us-west-2', - retry_mode: 'adaptive', - retry_limit: 5, - retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } - ) + s3_client = Aws::S3::Client.new(region: 'us-west-2') encryption_client = Aws::S3::EncryptionV3::Client.new( client: s3_client, **encryption_config From 6e103b84dd4a0271307d91ee3335a17a9047f648 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Tue, 25 Nov 2025 15:09:10 -0800 Subject: [PATCH 40/40] change V3 to V2 in logs and in class names --- .../Controllers/ClientController.cs | 22 ++++++++--------- .../Controllers/ObjectController.cs | 24 +++++++++---------- .../Models/ClientRequest.cs | 2 +- .../Models/ClientResponse.cs | 2 +- .../Models/ErrorModels.cs | 2 +- .../net-v2-transition-server/Program.cs | 2 +- .../Services/ClientCacheService.cs | 2 +- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test-server/net-v2-transition-server/Controllers/ClientController.cs b/test-server/net-v2-transition-server/Controllers/ClientController.cs index b41a28d6..135a7db9 100644 --- a/test-server/net-v2-transition-server/Controllers/ClientController.cs +++ b/test-server/net-v2-transition-server/Controllers/ClientController.cs @@ -4,10 +4,10 @@ using Amazon.Extensions.S3.Encryption; using Amazon.Extensions.S3.Encryption.Primitives; using Microsoft.AspNetCore.Mvc; -using NetV3TransitionServer.Models; -using NetV3TransitionServer.Services; +using NetV2TransitionServer.Models; +using NetV2TransitionServer.Services; -namespace NetV3TransitionServer.Controllers; +namespace NetV2TransitionServer.Controllers; [ApiController] [Route("[controller]")] @@ -34,7 +34,7 @@ public IActionResult CreateClient([FromBody] ClientRequest request) var kmsKeyId = request.Config.KeyMaterial.KmsKeyId; encryptionMaterial = new EncryptionMaterialsV2(kmsKeyId, KmsType.KmsContext, encryptionContext); logger.LogInformation( - "[NET-V3-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}", + "[NET-V2-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}", kmsKeyId); } else if (request.Config.KeyMaterial.AesKey != null) @@ -44,7 +44,7 @@ public IActionResult CreateClient([FromBody] ClientRequest request) aes.Key = aesKeyBytes; encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); logger.LogInformation( - "[NET-V3-Transitional] Created EncryptionMaterialsV2: AES"); + "[NET-V2-Transitional] Created EncryptionMaterialsV2: AES"); } else if (request.Config.KeyMaterial.RsaKey != null) { @@ -66,18 +66,18 @@ public IActionResult CreateClient([FromBody] ClientRequest request) // SecurityProfile V2AndLegacy can decrypt from legacy S3EC but V2 cannot var enableLegacyMode = enableLegacyUnauthenticatedModes || enableLegacyWrappingAlgorithms; var securityProfile = enableLegacyMode ? SecurityProfile.V2AndLegacy : SecurityProfile.V2; - logger.LogInformation("[NET-V3-Transitional] Created securityProfile= {securityProfile}", securityProfile.ToString()); + logger.LogInformation("[NET-V2-Transitional] Created securityProfile= {securityProfile}", securityProfile.ToString()); var encryptionAlgorithm = MapEncryptionAlgorithm(request.Config.EncryptionAlgorithm); // var encryptionAlgorithm = commitmentPolicy == Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt ? ContentEncryptionAlgorithm.AesGcm : ContentEncryptionAlgorithm.AesGcmWithCommitment; - logger.LogInformation("[NET-V3-Transitional] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy); - logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); + logger.LogInformation("[NET-V2-Transitional] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy); + logger.LogInformation("[NET-V2-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm); if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; - logger.LogInformation("[NET-V3-Transitional] Created StorageMode= InstructionFile"); + logger.LogInformation("[NET-V2-Transitional] Created StorageMode= InstructionFile"); } // Create S3 encryption client var encryptionClient = new AmazonS3EncryptionClientV2(configuration, encryptionMaterial); @@ -85,7 +85,7 @@ public IActionResult CreateClient([FromBody] ClientRequest request) var clientId = clientCacheService.AddClient(encryptionClient); var response = new ClientResponse { ClientId = clientId }; - logger.LogInformation("[NET-V3-Transitional] Created S3EC client with ID: {clientId}", clientId); + logger.LogInformation("[NET-V2-Transitional] Created S3EC client with ID: {clientId}", clientId); return new ContentResult { @@ -96,7 +96,7 @@ public IActionResult CreateClient([FromBody] ClientRequest request) } catch (Exception ex) { - logger.LogError(ex, "[NET-V3-Transitional] Failed to create S3EC client"); + logger.LogError(ex, "[NET-V2-Transitional] Failed to create S3EC client"); return StatusCode(500, new S3EncryptionClientError { Message = $"Failed to create client: {ex.Message}" diff --git a/test-server/net-v2-transition-server/Controllers/ObjectController.cs b/test-server/net-v2-transition-server/Controllers/ObjectController.cs index 76548815..d9eda828 100644 --- a/test-server/net-v2-transition-server/Controllers/ObjectController.cs +++ b/test-server/net-v2-transition-server/Controllers/ObjectController.cs @@ -1,10 +1,10 @@ using System.Text.Json; using Amazon.S3.Model; using Microsoft.AspNetCore.Mvc; -using NetV3TransitionServer.Models; -using NetV3TransitionServer.Services; +using NetV2TransitionServer.Models; +using NetV2TransitionServer.Services; -namespace NetV3TransitionServer.Controllers; +namespace NetV2TransitionServer.Controllers; [ApiController] [Route("[controller]")] @@ -16,11 +16,11 @@ public async Task PutObject(string bucket, string key) logger.LogInformation("Starting PutObject"); var clientId = Request.Headers["clientId"].FirstOrDefault(); if (string.IsNullOrEmpty(clientId)) - return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" }); + return BadRequest(new GenericServerError { Message = "[NET-V2-Transitional] ClientID header is required" }); var client = clientCacheService.GetClient(clientId); if (client == null) - return NotFound(new GenericServerError { Message = $"[NET-V3-Transitional] No client found for ClientID: {clientId}" }); + return NotFound(new GenericServerError { Message = $"[NET-V2-Transitional] No client found for ClientID: {clientId}" }); try { @@ -43,7 +43,7 @@ public async Task PutObject(string bucket, string key) var response = new { bucket, key }; logger.LogInformation( - "[NET-V3-Transitional] Put object succeeded for bucket={bucket}, key={key} and clientId = {clientId}", + "[NET-V2-Transitional] Put object succeeded for bucket={bucket}, key={key} and clientId = {clientId}", bucket, key, clientId); return new ContentResult { @@ -54,8 +54,8 @@ public async Task PutObject(string bucket, string key) } catch (Exception ex) { - logger.LogError(ex, "[NET-V3-Transitional] Failed to put object from S3 for bucket={bucket}, key={key}", bucket, key); - return StatusCode(500, new S3EncryptionClientError { Message = $"[NET-V3-Transitional] Failed to put object: {ex.Message}" }); + logger.LogError(ex, "[NET-V2-Transitional] Failed to put object from S3 for bucket={bucket}, key={key}", bucket, key); + return StatusCode(500, new S3EncryptionClientError { Message = $"[NET-V2-Transitional] Failed to put object: {ex.Message}" }); } } @@ -65,11 +65,11 @@ public async Task GetObject(string bucket, string key) logger.LogInformation("Starting GetObject"); var clientId = Request.Headers["clientId"].FirstOrDefault(); if (string.IsNullOrEmpty(clientId)) - return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" }); + return BadRequest(new GenericServerError { Message = "[NET-V2-Transitional] ClientID header is required" }); var client = clientCacheService.GetClient(clientId); if (client == null) - return NotFound(new GenericServerError { Message = $"[NET-V3-Transitional] No client found for ClientID: {clientId}" }); + return NotFound(new GenericServerError { Message = $"[NET-V2-Transitional] No client found for ClientID: {clientId}" }); try { @@ -79,7 +79,7 @@ public async Task GetObject(string bucket, string key) Key = key }; var response = await client.GetObjectAsync(getRequest); - logger.LogInformation("[NET-V3-Transitional] Got object from S3 for bucket={bucket}, key={key}", bucket, key); + logger.LogInformation("[NET-V2-Transitional] Got object from S3 for bucket={bucket}, key={key}", bucket, key); // Read response body using var memoryStream = new MemoryStream(); await response.ResponseStream.CopyToAsync(memoryStream); @@ -98,7 +98,7 @@ public async Task GetObject(string bucket, string key) } catch (Exception ex) { - logger.LogError(ex, "[NET-V3-Transitional] Failed to get object from S3 for bucket={bucket}, key={key}", bucket, key); + logger.LogError(ex, "[NET-V2-Transitional] Failed to get object from S3 for bucket={bucket}, key={key}", bucket, key); return StatusCode(500, new S3EncryptionClientError { Message = ex.Message }); } } diff --git a/test-server/net-v2-transition-server/Models/ClientRequest.cs b/test-server/net-v2-transition-server/Models/ClientRequest.cs index 07fe8520..7cdde249 100644 --- a/test-server/net-v2-transition-server/Models/ClientRequest.cs +++ b/test-server/net-v2-transition-server/Models/ClientRequest.cs @@ -1,7 +1,7 @@ using System.ComponentModel.DataAnnotations; using System.Text.Json.Serialization; -namespace NetV3TransitionServer.Models; +namespace NetV2TransitionServer.Models; public class ClientRequest { diff --git a/test-server/net-v2-transition-server/Models/ClientResponse.cs b/test-server/net-v2-transition-server/Models/ClientResponse.cs index 43c94a3e..cb91c1f5 100644 --- a/test-server/net-v2-transition-server/Models/ClientResponse.cs +++ b/test-server/net-v2-transition-server/Models/ClientResponse.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace NetV3TransitionServer.Models; +namespace NetV2TransitionServer.Models; public class ClientResponse { diff --git a/test-server/net-v2-transition-server/Models/ErrorModels.cs b/test-server/net-v2-transition-server/Models/ErrorModels.cs index 7fbf6680..11b45599 100644 --- a/test-server/net-v2-transition-server/Models/ErrorModels.cs +++ b/test-server/net-v2-transition-server/Models/ErrorModels.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace NetV3TransitionServer.Models; +namespace NetV2TransitionServer.Models; public class GenericServerError { diff --git a/test-server/net-v2-transition-server/Program.cs b/test-server/net-v2-transition-server/Program.cs index d7af9fd3..d905e470 100644 --- a/test-server/net-v2-transition-server/Program.cs +++ b/test-server/net-v2-transition-server/Program.cs @@ -1,4 +1,4 @@ -using NetV3TransitionServer.Services; +using NetV2TransitionServer.Services; var builder = WebApplication.CreateBuilder(args); diff --git a/test-server/net-v2-transition-server/Services/ClientCacheService.cs b/test-server/net-v2-transition-server/Services/ClientCacheService.cs index 0e7332ca..e8f09b89 100644 --- a/test-server/net-v2-transition-server/Services/ClientCacheService.cs +++ b/test-server/net-v2-transition-server/Services/ClientCacheService.cs @@ -1,7 +1,7 @@ using Amazon.Extensions.S3.Encryption; using System.Collections.Concurrent; -namespace NetV3TransitionServer.Services; +namespace NetV2TransitionServer.Services; public interface IClientCacheService {