chore: add instruction file support to TestServer

.github/workflows/lint.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   lint:
-    runs-on: macos-13
+    runs-on: macos-15
 
     steps:
       - name: Checkout code

test-server/cpp-v2-server/main.cpp

@@ -57,12 +57,19 @@ MHD_Result handle_create_client(struct MHD_Connection *connection,
     std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"];
     bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"];
     bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"];
+    bool inst_put = false;
+    if (request["config"].contains("instructionFileConfig") &&
+        request["config"]["instructionFileConfig"].contains("enableInstructionFilePutObject")) {
+        inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"];
+    }
 
     auto materials =
         std::make_shared<KMSWithContextEncryptionMaterials>(kms_key_id);
     CryptoConfigurationV2 config(materials);
     if (legacy1 || legacy2)
       config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY);
+    if (inst_put)
+      config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE);
 
     auto encryption_client = std::make_shared<S3EncryptionClientV2>(config);
 

test-server/cpp-v2-transition-server/main.cpp

@@ -82,12 +82,19 @@ MHD_Result handle_create_client(struct MHD_Connection *connection,
     std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"];
     bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"];
     bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"];
+    bool inst_put = false;
+    if (request["config"].contains("instructionFileConfig") &&
+        request["config"]["instructionFileConfig"].contains("enableInstructionFilePutObject")) {
+        inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"];
+    }
 
     auto materials =
         std::make_shared<KMSWithContextEncryptionMaterials>(kms_key_id);
     CryptoConfigurationV2 config(materials);
     if (legacy1 || legacy2)
       config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY);
+    if (inst_put)
+      config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE);
 
     auto encryption_client = std::make_shared<S3EncryptionClientV2>(config);
 

test-server/cpp-v3-server/main.cpp

@@ -72,12 +72,19 @@ MHD_Result handle_create_client(struct MHD_Connection *connection,
     std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"];
     bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"];
     bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"];
+    bool inst_put = false;
+    if (request["config"].contains("instructionFileConfig") &&
+        request["config"]["instructionFileConfig"].contains("enableInstructionFilePutObject")) {
+        inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"];
+    }
 
     auto materials =
         std::make_shared<KMSWithContextEncryptionMaterials>(kms_key_id);
     CryptoConfigurationV3 config(materials);
     if (legacy1 || legacy2)
       config.AllowLegacy();
+    if (inst_put)
+      config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE);
 
     std::string commitmentPolicy = get_config(request, "commitmentPolicy");
     std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm");

test-server/java-tests/build.gradle.kts

@@ -17,6 +17,7 @@ dependencies {
     testImplementation("org.junit.jupiter:junit-jupiter:5.13.0")
     testRuntimeOnly("org.junit.platform:junit-platform-launcher")
     testImplementation("com.amazonaws:aws-java-sdk:1.12.788")
+    testImplementation("software.amazon.awssdk:s3:2.37.1")
     testImplementation("org.bouncycastle:bcprov-jdk15on:1.70")
 }
 

test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java

@@ -6,6 +6,7 @@
 package software.amazon.encryption.s3;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 import static software.amazon.encryption.s3.TestUtils.*;
@@ -15,19 +16,26 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Stream;
 
+import com.amazonaws.services.s3.AmazonS3EncryptionClientV2;
+import com.amazonaws.services.s3.AmazonS3EncryptionV2;
+import com.amazonaws.services.s3.model.CryptoConfigurationV2;
 import com.amazonaws.services.s3.model.KMSEncryptionMaterials;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.opentest4j.TestAbortedException;
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.encryption.s3.client.S3ECTestServerClient;
 import software.amazon.encryption.s3.model.CommitmentPolicy;
 import software.amazon.encryption.s3.model.CreateClientInput;
 import software.amazon.encryption.s3.model.CreateClientOutput;
+import software.amazon.encryption.s3.model.EncryptionAlgorithm;
 import software.amazon.encryption.s3.model.GetObjectInput;
 import software.amazon.encryption.s3.model.GetObjectOutput;
+import software.amazon.encryption.s3.model.InstructionFileConfig;
 import software.amazon.encryption.s3.model.KeyMaterial;
 import software.amazon.encryption.s3.model.PutObjectInput;
 import software.amazon.encryption.s3.model.S3ECConfig;
@@ -38,7 +46,6 @@
 import com.amazonaws.services.s3.model.CryptoConfiguration;
 import com.amazonaws.services.s3.model.CryptoMode;
 import com.amazonaws.services.s3.model.CryptoStorageMode;
-import software.amazon.encryption.s3.TestUtils.*;
 import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
 import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
 
@@ -416,4 +423,125 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(TestUtils.LanguageServerTarget la
             }
         }
     }
+
+    @ParameterizedTest(name = "{displayName} for Encrypt: Java, Decrypt: {0}")
+    @MethodSource("software.amazon.encryption.s3.TestUtils#clientsForTest")
+    public void instructionFileReadV2Format(TestUtils.LanguageServerTarget language) {
+        if (KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(language.getLanguageName())) {
+            throw new TestAbortedException(String.format("%s does not support KMS instruction files", language.getLanguageName()));
+        }
+        if (INSTRUCTION_FILE_GET_UNSUPPORTED.contains(language.getLanguageName())) {
+            throw new TestAbortedException(String.format("%s does not support instruction file Gets", language.getLanguageName()));
+        }
+        S3ECTestServerClient client = testServerClientFor(language);
+        final String objectKey = appendTestSuffix("read-instruction-file-v2-" + language);
+        final String input = "simple-test-input";
+        KeyMaterial kmsKeyArn = KeyMaterial.builder()
+          .kmsKeyId(KMS_KEY_ARN)
+          .build();
+        CreateClientOutput output1 = client.createClient(CreateClientInput.builder()
+          .config(S3ECConfig.builder()
+            .enableLegacyWrappingAlgorithms(true)
+            .keyMaterial(kmsKeyArn)
+            .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT)
+            .build())
+          .build());
+        String s3ECId = output1.getClientId();
+
+        // Write with instruction file using V2 client
+        EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(KMS_KEY_ARN);
+        CryptoConfigurationV2 cryptoConfigurationV2 = new CryptoConfigurationV2();
+        cryptoConfigurationV2.setStorageMode(CryptoStorageMode.InstructionFile);
+        AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder()
+          .withEncryptionMaterialsProvider(materialsProvider)
+          .withCryptoConfiguration(cryptoConfigurationV2)
+          .build();
+        v2Client.putObject(BUCKET, objectKey, input);
+
+        // Read should be enabled by default
+        GetObjectOutput output = client.getObject(GetObjectInput.builder()
+          .clientID(s3ECId)
+          .bucket(BUCKET)
+          .key(objectKey)
+          .build());
+
+        assertEquals(input, new String(output.getBody().array()));
+    }
+
+    @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}")
+    @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients")
+    public void instructionFileWriteAndRead(LanguageServerTarget encLang, LanguageServerTarget decLang) {
+        if (INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(encLang.getLanguageName())) {
+            throw new TestAbortedException("not testing " + encLang.getLanguageName());
+        }
+        if (INSTRUCTION_FILE_GET_UNSUPPORTED.contains(decLang.getLanguageName())) {
+            throw new TestAbortedException("not testing " + encLang.getLanguageName());
+        }
+        if (KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(encLang.getLanguageName())) {
+            throw new TestAbortedException("not testing " + encLang.getLanguageName());
+        }
+        if (KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(decLang.getLanguageName())) {
+            throw new TestAbortedException("not testing " + encLang.getLanguageName());
+        }
+        if (INSTRUCTION_FILE_ROUNDTRIP_TEMP_UNSUPPORTED.contains(encLang.getLanguageName())) {
+            throw new TestAbortedException("not testing " + encLang.getLanguageName());
+        }
+        S3ECTestServerClient encClient = testServerClientFor(encLang);
+        S3ECTestServerClient decClient = testServerClientFor(decLang);
+        final String objectKey = appendTestSuffix(String.format("write-%s-read-%s-instruction-file", encLang.getLanguageName(), decLang.getLanguageName()));
+        final String input = "simple-test-input";
+        KeyMaterial kmsKeyArn = KeyMaterial.builder()
+          .kmsKeyId(KMS_KEY_ARN)
+          .build();
+        CreateClientOutput encOutput = encClient.createClient(CreateClientInput.builder()
+          .config(S3ECConfig.builder()
+            .keyMaterial(kmsKeyArn)
+            .instructionFileConfig(InstructionFileConfig.builder()
+              .enableInstructionFilePutObject(true)
+              .build())
+            .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT)
+            .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF)
+            .build())
+          .build());
+        String encS3ECId = encOutput.getClientId();
+        CreateClientOutput decOutput = decClient.createClient(CreateClientInput.builder()
+          .config(S3ECConfig.builder()
+            .keyMaterial(kmsKeyArn)
+            .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT)
+            .build())
+          .build());
+        String decS3ECId = decOutput.getClientId();
+
+        // Write with instruction file
+        encClient.putObject(PutObjectInput.builder()
+          .clientID(encS3ECId)
+          .bucket(BUCKET)
+          .key(objectKey)
+          .body(ByteBuffer.wrap(input.getBytes(StandardCharsets.UTF_8)))
+          .build());
+
+        // Assert using Java plaintext client that an instruction file exists
+        ResponseBytes<GetObjectResponse> ptInstFile;
+        try (S3Client ptS3Client = S3Client.create()) {
+            ptInstFile = ptS3Client.getObjectAsBytes(builder -> builder
+              .bucket(BUCKET)
+              .key(objectKey + ".instruction")
+              .build());
+        }
+        // Check for inst file key
+        if (!encLang.getLanguageName().contains("Ruby")) {
+            // Ruby doesn't include it :(
+            assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file"));
+        }
+        assertFalse(ptInstFile.asUtf8String().isEmpty());
+
+        // Read should be enabled by default
+        GetObjectOutput output = decClient.getObject(GetObjectInput.builder()
+          .clientID(decS3ECId)
+          .bucket(BUCKET)
+          .key(objectKey)
+          .build());
+
+        assertEquals(input, new String(output.getBody().array()));
+    }
 }

test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java

@@ -22,32 +22,29 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import org.joda.time.DateTime;
 import org.joda.time.format.DateTimeFormat;
 import org.junit.jupiter.params.provider.Arguments;
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.Regions;
-import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol;
-import software.amazon.smithy.java.client.core.ClientConfig;
-import software.amazon.smithy.java.client.core.ClientProtocol;
-import software.amazon.smithy.java.client.core.endpoint.EndpointResolver;
-import software.amazon.encryption.s3.client.S3ECTestServerClient;
 import software.amazon.encryption.s3.model.EncryptionAlgorithm;
 import software.amazon.encryption.s3.model.GetObjectInput;
 import software.amazon.encryption.s3.model.GetObjectOutput;
 import software.amazon.encryption.s3.model.PutObjectInput;
 import software.amazon.encryption.s3.model.PutObjectOutput;
-import software.amazon.encryption.s3.model.S3ECConfig;
-import software.amazon.encryption.s3.model.S3ECTestServerApiService;
 import software.amazon.encryption.s3.model.S3EncryptionClientError;
+import software.amazon.smithy.java.aws.client.restjson.RestJsonClientProtocol;
+import software.amazon.smithy.java.client.core.ClientConfig;
+import software.amazon.smithy.java.client.core.ClientProtocol;
+import software.amazon.smithy.java.client.core.endpoint.EndpointResolver;
+import software.amazon.encryption.s3.client.S3ECTestServerClient;
+import software.amazon.encryption.s3.model.S3ECTestServerApiService;
 import software.amazon.smithy.java.http.api.HttpRequest;
 import software.amazon.smithy.java.http.api.HttpResponse;
 
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-
 public class TestUtils {
 
     // Version name constants
@@ -97,6 +94,25 @@ public class TestUtils {
     public static final Set<String> ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED =
         Set.of(NET_V2_CURRENT, NET_V3);
 
+    // .NET only supports decrypting instruction files using AES and RSA.
+    // Python MUST support decrypting KMS instruction files, but does not yet.
+    public static final Set<String> KMS_INSTRUCTION_FILE_UNSUPPORTED =
+      Set.of(NET_V2_CURRENT, NET_V2_TRANSITION, NET_V3);
+
+    // Go does not write with instruction files
+    public static final Set<String> INSTRUCTION_FILE_PUT_UNSUPPORTED =
+      Set.of(GO_V3_CURRENT, GO_V3_TRANSITION, GO_V4, PYTHON_V3
+        // Apparently C++ V2 Current does not work, even though it should
+      , CPP_V2_CURRENT);
+
+    // Not implemented yet in Python.
+    public static final Set<String> INSTRUCTION_FILE_GET_UNSUPPORTED =
+      Set.of(PYTHON_V3);
+
+    // PHP doesn't work but it should, temporarily disable
+    public static final Set<String> INSTRUCTION_FILE_ROUNDTRIP_TEMP_UNSUPPORTED =
+      Set.of(PHP_V2_CURRENT, PHP_V2_TRANSITION, PHP_V3);
+
     public static final Set<String> CURRENT_VERSIONS =
         Set.of(
             JAVA_V3_CURRENT,
@@ -286,7 +302,7 @@ public static List<String> metadataMapToList(Map<String, String> md) {
     public static void validateServersRunning() {
         for (LanguageServerTarget server : serverMap.values()) {
             if (!serverListening(server.getServerURI())) {
-                throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", 
+                throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s",
                     server.getLanguageName(), server.getServerURI()));
             }
         }
@@ -430,7 +446,7 @@ public static EncryptionAlgorithm GetEncryptionAlgorithm(String objectKey)
                 return EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF;
             }
         }
-        
+
         throw new RuntimeException("Need to support instruction files!");
     }
 
@@ -456,7 +472,7 @@ public static void Encrypt(
 
         crossLanguageObjects.add(objectKey);
     }
-    
+
     public static void Decrypt(
         S3ECTestServerClient client,
         String S3ECId, List<String> crossLanguageObjects,
@@ -468,7 +484,7 @@ public static void Decrypt(
             .bucket(TestUtils.BUCKET)
             .key(objectKey)
             .build());
-            
+
             // Then: Pass
             assertEquals(objectKey, new String(output.getBody().array()));
             assertEquals(
@@ -478,7 +494,7 @@ public static void Decrypt(
             );
         }
     }
-    
+
     public static void Decrypt_fails(
         S3ECTestServerClient client,
         String S3ECId, List<String> crossLanguageObjects,

test-server/java-v3-server/build.gradle.kts

@@ -14,7 +14,8 @@ dependencies {
     implementation("software.amazon.smithy.java:aws-server-restjson:$smithyJavaVersion")
 
     compileOnly("software.amazon.awssdk:aws-sdk-java:2.31.66")
-    implementation("software.amazon.encryption.s3:amazon-s3-encryption-client-java:3.3.5")
+    // This MUST stay at 3.5.0
+    implementation("software.amazon.encryption.s3:amazon-s3-encryption-client-java:3.5.0")
 }
 
 // Use that application plugin to start the service via the `run` task.