From 7a43842b31b2e3a0afb0f93a04a22af74a8ae90d Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 5 Nov 2025 15:15:34 -0800 Subject: [PATCH 1/8] m --- .gitmodules | 4 + .../go-v3-transition-server/.duvet/.gitignore | 3 + .../.duvet/config.toml | 27 ++++ .../.duvet/snapshot.txt | 128 ++++++++++++++++++ .../go-v3-transition-server/local-go-s3ec | 1 + .../amazon/encryption/s3/TestUtils.java | 2 +- 6 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 test-server/go-v3-transition-server/.duvet/.gitignore create mode 100644 test-server/go-v3-transition-server/.duvet/config.toml create mode 100644 test-server/go-v3-transition-server/.duvet/snapshot.txt create mode 160000 test-server/go-v3-transition-server/local-go-s3ec diff --git a/.gitmodules b/.gitmodules index 0bf186eb..c103cf8f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -39,3 +39,7 @@ path = test-server/net-v2-v3-server/s3ec-net-v3 url = https://github.com/aws/private-amazon-s3-encryption-client-dotnet-staging.git branch = s3ec-v3 +[submodule "test-server/go-v3-transition-server/local-go-s3ec"] + path = test-server/go-v3-transition-server/local-go-s3ec + url = https://github.com/aws/private-amazon-s3-encryption-client-go-staging + branch = v3-strip diff --git a/test-server/go-v3-transition-server/.duvet/.gitignore b/test-server/go-v3-transition-server/.duvet/.gitignore new file mode 100644 index 00000000..32ad579b --- /dev/null +++ b/test-server/go-v3-transition-server/.duvet/.gitignore @@ -0,0 +1,3 @@ +reports/ +requirements/ +specification/ diff --git a/test-server/go-v3-transition-server/.duvet/config.toml b/test-server/go-v3-transition-server/.duvet/config.toml new file mode 100644 index 00000000..713e72d3 --- /dev/null +++ b/test-server/go-v3-transition-server/.duvet/config.toml @@ -0,0 +1,27 @@ +'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" + +[[source]] +pattern = "local-go-s3ec/v4/**/*.go" + +# Include required specifications here +[[specification]] +source = "../specification/s3-encryption/client.md" +[[specification]] +source = "../specification/s3-encryption/decryption.md" +[[specification]] +source = "../specification/s3-encryption/encryption.md" +[[specification]] +source = "../specification/s3-encryption/key-commitment.md" +[[specification]] +source = "../specification/s3-encryption/key-derivation.md" +[[specification]] +source = "../specification/s3-encryption/data-format/content-metadata.md" +[[specification]] +source = "../specification/s3-encryption/data-format/metadata-strategy.md" + +[report.html] +enabled = true + +# Enable snapshots to prevent requirement coverage regressions +[report.snapshot] +enabled = false diff --git a/test-server/go-v3-transition-server/.duvet/snapshot.txt b/test-server/go-v3-transition-server/.duvet/snapshot.txt new file mode 100644 index 00000000..e432d768 --- /dev/null +++ b/test-server/go-v3-transition-server/.duvet/snapshot.txt @@ -0,0 +1,128 @@ +SPECIFICATION: [Content Metadata](../specification/s3-encryption/data-format/content-metadata.md) + SECTION: [Content Metadata MapKeys](#content-metadata-mapkeys) + TEXT[!MUST]: The "x-amz-meta-" prefix is automatically added by the S3 server and MUST NOT be included in implementation code. + TEXT[!MUST]: The "x-amz-" prefix denotes that the metadata is owned by an Amazon product and MUST be prepended to all S3EC metadata mapkeys. + TEXT[!SHOULD]: - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects. + TEXT[!MUST]: - The mapkey "x-amz-key" MUST be present for V1 format objects. + TEXT[!MUST]: - The mapkey "x-amz-matdesc" MUST be present for V1 format objects. + TEXT[!MUST]: - The mapkey "x-amz-iv" MUST be present for V1 format objects. + TEXT[!MUST]: - The mapkey "x-amz-key-v2" MUST be present for V2 format objects. + TEXT[!MUST]: - The mapkey "x-amz-matdesc" MUST be present for V2 format objects. + TEXT[!MUST]: - The mapkey "x-amz-iv" MUST be present for V2 format objects. + TEXT[!MUST]: - The mapkey "x-amz-wrap-alg" MUST be present for V2 format objects. + TEXT[!MUST]: - The mapkey "x-amz-cek-alg" MUST be present for V2 format objects. + TEXT[!MUST]: - The mapkey "x-amz-tag-len" MUST be present for V2 format objects. + TEXT[!MUST]: - The mapkey "x-amz-c" MUST be present for V3 format objects. + TEXT[!SHOULD]: - This mapkey ("x-amz-c") SHOULD be represented by a constant named "CONTENT_CIPHER_V3" or similar in the implementation code. + TEXT[!MUST]: - The mapkey "x-amz-3" MUST be present for V3 format objects. + TEXT[!SHOULD]: - This mapkey ("x-amz-3") SHOULD be represented by a constant named "ENCRYPTED_DATA_KEY_V3" or similar in the implementation code. + TEXT[!SHOULD]: - The mapkey "x-amz-m" SHOULD be present for V3 format objects that use Raw Keyring Material Description. + TEXT[!SHOULD]: - This mapkey ("x-amz-m") SHOULD be represented by a constant named "MAT_DESC_V3" or similar in the implementation code. + TEXT[!SHOULD]: - The mapkey "x-amz-t" SHOULD be present for V3 format objects that use KMS Encryption Context. + TEXT[!SHOULD]: - This mapkey ("x-amz-t") SHOULD be represented by a constant named "ENCRYPTION_CONTEXT_V3" or similar in the implementation code. + TEXT[!MUST]: - The mapkey "x-amz-w" MUST be present for V3 format objects. + TEXT[!SHOULD]: - This mapkey ("x-amz-w") SHOULD be represented by a constant named "ENCRYPTED_DATA_KEY_ALGORITHM_V3" or similar in the implementation code. + TEXT[!MUST]: - The mapkey "x-amz-d" MUST be present for V3 format objects. + TEXT[!SHOULD]: - This mapkey ("x-amz-d") SHOULD be represented by a constant named "KEY_COMMITMENT_V3" or similar in the implementation code. + TEXT[!MUST]: - The mapkey "x-amz-i" MUST be present for V3 format objects. + TEXT[!SHOULD]: - This mapkey ("x-amz-i") SHOULD be represented by a constant named "MESSAGE_ID_V3" or similar in the implementation code. + TEXT[!MUST]: In the V3 format, the mapkeys "x-amz-c", "x-amz-d", and "x-amz-i" MUST be stored exclusively in the Object Metadata. + + SECTION: [Determining S3EC Object Status](#determining-s3ec-object-status) + TEXT[!MUST]: - If the metadata contains "x-amz-iv" and "x-amz-key" then the object MUST be considered as an S3EC-encrypted object using the V1 format. + TEXT[!MUST]: - If the metadata contains "x-amz-iv" and "x-amz-metadata-x-amz-key-v2" then the object MUST be considered as an S3EC-encrypted object using the V2 format. + TEXT[!MUST]: - If the metadata contains "x-amz-3" and "x-amz-d" and "x-amz-i" then the object MUST be considered an S3EC-encrypted object using the V3 format. + TEXT[!MUST]: If the object matches none of the V1/V2/V3 formats, the S3EC MUST attempt to get the instruction file. + TEXT[!SHOULD]: If there are multiple mapkeys which are meant to be exclusive, such as "x-amz-key", "x-amz-key-v2", and "x-amz-3" then the S3EC SHOULD throw an exception. + TEXT[!SHOULD]: In general, if there is any deviation from the above format, with the exception of additional unrelated mapkeys, then the S3EC SHOULD throw an exception. + + SECTION: [V1/V2 Shared](#v1-v2-shared) + TEXT[!MAY]: This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + + SECTION: [V3 Only](#v3-only) + TEXT[!MAY]: This material description string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + TEXT[!MUST]: The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). + TEXT[!MUST]: If the mapkey is not present, the default Material Description value MUST be set to an empty map (`{}`). + TEXT[!MAY]: This encryption context string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. + TEXT[!MUST]: The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. + TEXT[!MUST]: - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. + TEXT[!MUST]: - The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. + TEXT[!MUST]: - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. + + SECTION: [Algorithm Suite and Message Format Version Compatibility](#algorithm-suite-and-message-format-version-compatibility) + TEXT[!MAY]: Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version. + TEXT[!MUST]: Objects encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF MUST use the V2 message format version only. + TEXT[!MUST]: Objects encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY MUST use the V3 message format version only. + +SPECIFICATION: [Content Metadata Strategy](../specification/s3-encryption/data-format/metadata-strategy.md) + SECTION: [Object Metadata](#object-metadata) + TEXT[!MUST]: By default, the S3EC MUST store content metadata in the S3 Object Metadata. + TEXT[!SHOULD]: The S3EC SHOULD support decoding the S3 Server's "double encoding". + TEXT[!MUST]: If the S3EC does not support decoding the S3 Server's "double encoding" then it MUST return the content metadata untouched. + + SECTION: [Instruction File](#instruction-file) + TEXT[!MUST]: The S3EC MUST support writing some or all (depending on format) content metadata to an Instruction File. + TEXT[!MUST]: The content metadata stored in the Instruction File MUST be serialized to a JSON string. + TEXT[!MUST]: The serialized JSON string MUST be the only contents of the Instruction File. + TEXT[!MUST]: Instruction File writes MUST NOT be enabled by default. + TEXT[!MUST]: Instruction File writes MUST be optionally configured during client creation or on each PutObject request. + TEXT[!MAY]: The S3EC MAY support re-encryption/key rotation via Instruction Files. + TEXT[!MUST]: The S3EC MUST NOT support providing a custom Instruction File suffix on ordinary writes; custom suffixes MUST only be used during re-encryption. + TEXT[!SHOULD]: The S3EC SHOULD support providing a custom Instruction File suffix on GetObject requests, regardless of whether or not re-encryption is supported. + + SECTION: [V1/V2 Instruction Files](#v1-v2-instruction-files) + TEXT[!MUST]: In the V1/V2 message format, all of the content metadata MUST be stored in the Instruction File. + + SECTION: [V3 Instruction Files](#v3-instruction-files) + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-c" and its value in the Object Metadata when writing with an Instruction File. + TEXT[!MUST]: - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-d" and its value in the Object Metadata when writing with an Instruction File. + TEXT[!MUST]: - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-i" and its value in the Object Metadata when writing with an Instruction File. + TEXT[!MUST]: - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-3" and its value in the Instruction File. + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-w" and its value in the Instruction File. + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-m" and its value (when present in the content metadata) in the Instruction File. + TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-t" and its value (when present in the content metadata) in the Instruction File. + +SPECIFICATION: [Encryption](../specification/s3-encryption/encryption.md) + SECTION: [Content Encryption](#content-encryption) + TEXT[!MUST]: The S3EC MUST use the encryption algorithm configured during [client](./client.md) initialization. + TEXT[!MUST]: The client MUST validate that the length of the plaintext bytes does not exceed the algorithm suite's cipher's maximum content length in bytes. + TEXT[!MUST]: The client MUST generate an IV or Message ID using the length of the IV or Message ID defined in the algorithm suite. + TEXT[!MUST]: The generated IV or Message ID MUST be set or returned from the encryption process such that it can be included in the content metadata. + + SECTION: [Cipher Initialization](#cipher-initialization) + TEXT[!SHOULD]: The client SHOULD validate that the generated IV or Message ID is not zeros. + + SECTION: [ALG_AES_256_CTR_IV16_TAG16_NO_KDF](#alg-aes-256-ctr-iv16-tag16-no-kdf) + TEXT[!MUST]: Attempts to encrypt using AES-CTR MUST fail. + + SECTION: [ALG_AES_256_CTR_HKDF_SHA512_COMMIT_KEY](#alg-aes-256-ctr-hkdf-sha512-commit-key) + TEXT[!MUST]: Attempts to encrypt using key committing AES-CTR MUST fail. + + SECTION: [ALG_AES_256_GCM_IV12_TAG16_NO_KDF](#alg-aes-256-gcm-iv12-tag16-no-kdf) + TEXT[!MUST]: The client MUST initialize the cipher, or call an AES-GCM encryption API, with the plaintext data key, the generated IV, and the tag length defined in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + TEXT[!MUST]: The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. + TEXT[!MUST]: The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + + SECTION: [ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY](#alg-aes-256-gcm-hkdf-sha512-commit-key) + TEXT[!MUST]: The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). + TEXT[!MUST]: The derived key commitment value MUST be set or returned from the encryption process such that it can be included in the content metadata. + TEXT[!MUST]: The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. + +SPECIFICATION: [Key Derivation](../specification/s3-encryption/key-derivation.md) + SECTION: [HKDF Operation](#hkdf-operation) + TEXT[!MUST]: - The hash function MUST be specified by the algorithm suite commitment settings. + TEXT[!MUST]: - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. + TEXT[!MUST]: - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. + TEXT[!MUST]: - The salt MUST be the Message ID with the length defined in the algorithm suite. + TEXT[!MUST]: - The DEK input pseudorandom key MUST be the output from the extract step. + TEXT[!MUST]: - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. + TEXT[!MUST]: - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. + TEXT[!MUST]: - The CK input pseudorandom key MUST be the output from the extract step. + TEXT[!MUST]: - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. + TEXT[!MUST]: - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. + TEXT[!MUST]: When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, the IV used in the AES-GCM content encryption/decryption MUST contain only zeros of the length defined in the algorithm suite. + TEXT[!MUST]: The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only zeros, and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. + TEXT[!MUST]: The client MUST set the AAD to the Algorithm Suite ID represented as bytes. diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec new file mode 160000 index 00000000..8615988f --- /dev/null +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -0,0 +1 @@ +Subproject commit 8615988fd52f8df6db9b377e1a2dcdd1f866e07a diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 4b0c8e74..c57e8623 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -110,7 +110,7 @@ public class TestUtils { public static final Set TRANSITION_VERSIONS = Set.of( // JAVA_V3_TRANSITION, - // GO_V3_TRANSITION, + GO_V3_TRANSITION, // NET_V2_TRANSITION, CPP_V2_TRANSITION, // PHP_V2_TRANSITION, From df5b4bdb1c6fa411ed23af9c9878958da0c647db Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 5 Nov 2025 15:57:33 -0800 Subject: [PATCH 2/8] m --- test-server/go-v3-transition-server/Makefile | 31 ++ test-server/go-v3-transition-server/README.md | 23 ++ test-server/go-v3-transition-server/go.mod | 35 ++ test-server/go-v3-transition-server/go.sum | 45 +++ test-server/go-v3-transition-server/main.go | 350 ++++++++++++++++++ 5 files changed, 484 insertions(+) create mode 100644 test-server/go-v3-transition-server/Makefile create mode 100644 test-server/go-v3-transition-server/README.md create mode 100644 test-server/go-v3-transition-server/go.mod create mode 100644 test-server/go-v3-transition-server/go.sum create mode 100644 test-server/go-v3-transition-server/main.go diff --git a/test-server/go-v3-transition-server/Makefile b/test-server/go-v3-transition-server/Makefile new file mode 100644 index 00000000..cfaf32fe --- /dev/null +++ b/test-server/go-v3-transition-server/Makefile @@ -0,0 +1,31 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8089 + +start-server: + @echo "Starting Go V4 server..." + go mod tidy + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + go run . & echo $$! > $(PID_FILE) + @echo "Go V4 server starting..." + +stop-server: + @if [ -f $(PID_FILE) ]; then \ + kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm $(PID_FILE); \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) + +duvet: + duvet report + +view-report-mac: + open .duvet/reports/report.html diff --git a/test-server/go-v3-transition-server/README.md b/test-server/go-v3-transition-server/README.md new file mode 100644 index 00000000..d97a37bf --- /dev/null +++ b/test-server/go-v3-transition-server/README.md @@ -0,0 +1,23 @@ +# S3EC Go V4 Test Server + +This is the Go implementation of the S3ECTestServer framework for S3EC Go V4. It provides a server implementation for testing Go S3 Encryption Client V4 functionality. + +## Overview + +The S3EC Go test server implements the S3ECTestServer service defined in the shared Smithy model. It provides endpoints for: + +- Creating S3 Encryption Clients +- Putting objects with encryption +- Getting and decrypting objects + +## Usage + +To run the server: + +```console +go run . +``` + +This will start the server running on port `8089`. + +The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations. diff --git a/test-server/go-v3-transition-server/go.mod b/test-server/go-v3-transition-server/go.mod new file mode 100644 index 00000000..4ab1895c --- /dev/null +++ b/test-server/go-v3-transition-server/go.mod @@ -0,0 +1,35 @@ +module github.com/aws/amazon-s3-encryption-client-python/test-server/go-server + +go 1.21 + +require ( + github.com/aws/amazon-s3-encryption-client-go/v4 v4.0.0 + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 + github.com/google/uuid v1.5.0 + github.com/gorilla/mux v1.8.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect +) + +// S3EC Go V4 is not released to pkg.go.dev as of writing. +// It is included as a submodule and referenced locally. +replace github.com/aws/amazon-s3-encryption-client-go/v4 => ./local-go-s3ec/v4 diff --git a/test-server/go-v3-transition-server/go.sum b/test-server/go-v3-transition-server/go.sum new file mode 100644 index 00000000..1bb969a3 --- /dev/null +++ b/test-server/go-v3-transition-server/go.sum @@ -0,0 +1,45 @@ + +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 h1:c75pHGBV3h6WOsIjbJhLyOnlCPXzap45nbiP2Z5jk5M= +github.com/aws/aws-sdk-go-v2/service/kms v1.27.4/go.mod h1:D9FVDkZjkZnnFHymJ3fPVz0zOUlNSd0xcIIVmmrAac8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go new file mode 100644 index 00000000..75871d5f --- /dev/null +++ b/test-server/go-v3-transition-server/main.go @@ -0,0 +1,350 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + + "github.com/aws/amazon-s3-encryption-client-go/v4/client" + "github.com/aws/amazon-s3-encryption-client-go/v4/materials" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/gorilla/mux" +) + +// Server represents the Go test server +type Server struct { + clientCache map[string]*client.S3EncryptionClientV4 + kmsClient *kms.Client +} + +// CreateClientInput represents the input for creating a client +type CreateClientInput struct { + Config S3ECConfig `json:"config"` +} + +// CreateClientOutput represents the output for creating a client +type CreateClientOutput struct { + ClientID string `json:"clientId"` +} + +// S3ECConfig represents the S3 encryption client configuration +type S3ECConfig struct { + EnableLegacyUnauthenticatedModes bool `json:"enableLegacyUnauthenticatedModes"` + EnableDelayedAuthenticationMode bool `json:"enableDelayedAuthenticationMode"` + EnableLegacyWrappingAlgorithms bool `json:"enableLegacyWrappingAlgorithms"` + SetBufferSize int64 `json:"setBufferSize"` + KeyMaterial KeyMaterial `json:"keyMaterial"` +} + +// KeyMaterial represents the key material for encryption +type KeyMaterial struct { + RSAKey []byte `json:"rsaKey"` + AESKey []byte `json:"aesKey"` + KMSKeyID string `json:"kmsKeyId"` +} + +// PutObjectOutput represents the output for put object operation +type PutObjectOutput struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + Metadata []string `json:"metadata"` +} + +// ErrorResponse represents an error response +type ErrorResponse struct { + Type string `json:"__type"` + Message string `json:"message"` +} + +// NewServer creates a new server instance +func NewServer() (*Server, error) { + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + if err != nil { + return nil, fmt.Errorf("failed to load AWS config: %w", err) + } + + return &Server{ + clientCache: make(map[string]*client.S3EncryptionClientV4), + kmsClient: kms.NewFromConfig(cfg), + }, nil +} + +// createGenericServerError creates a generic server error response +func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("[Go V4] GenericServerError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#GenericServerError", + Message: message, + }) +} + +// createS3EncryptionClientError creates an S3 encryption client error response +func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) { + // Echo error to console + log.Printf("[Go V4] S3EncryptionClientError: %s (Status: %d)", message, statusCode) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + json.NewEncoder(w).Encode(ErrorResponse{ + Type: "software.amazon.encryption.s3#S3EncryptionClientError", + Message: message, + }) +} + +// metadataStringToMap converts metadata string to map +func metadataStringToMap(mdString string) (map[string]string, error) { + md := make(map[string]string) + if mdString == "" { + return md, nil + } + + mdList := strings.Split(mdString, ",") + for _, entry := range mdList { + // Split on "]:[" to separate key and value + parts := strings.Split(entry, "]:[") + if len(parts) == 2 { + // Remove remaining brackets from start and end + key := parts[0][1:] // Remove first character + value := parts[1][:len(parts[1])-1] // Remove last character + md[key] = value + } else { + return nil, fmt.Errorf("malformed metadata list entry: %s", entry) + } + } + return md, nil +} + +// createClient handles POST /client +func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + var input CreateClientInput + if err := json.Unmarshal(body, &input); err != nil { + s.createGenericServerError(w, "Invalid JSON in request body", http.StatusBadRequest) + return + } + + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) + return + } + + // Create KMS keyring + kmsClient := kms.NewFromConfig(cfg) + keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { + options.EnableLegacyWrappingAlgorithms = input.Config.EnableLegacyWrappingAlgorithms + }) + cmm, err := materials.NewCryptographicMaterialsManager(keyring) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create CMM: %v", err), http.StatusInternalServerError) + return + } + + // Create S3 encryption client + var s3EncryptionClient *client.S3EncryptionClientV4 + s3PlaintextClient := s3.NewFromConfig(cfg) + s3EncryptionClient, err = client.New(s3PlaintextClient, cmm) + + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create S3EC: %v", err), http.StatusInternalServerError) + return + } + + // Generate client ID + clientID := uuid.New().String() + + // Store client in cache + s.clientCache[clientID] = s3EncryptionClient + + // Return response + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(CreateClientOutput{ + ClientID: clientID, + }) +} + +// putObject handles PUT /object/{bucket}/{key} +func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + // Get client from cache + client, exists := s.clientCache[clientID] + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Read body + body, err := io.ReadAll(r.Body) + if err != nil { + s.createGenericServerError(w, "Failed to read request body", http.StatusBadRequest) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + // Create context with encryption context + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + // Create put object input + putInput := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: strings.NewReader(string(body)), + } + + // Add metadata if present + if len(encCtx) > 0 { + putInput.Metadata = encCtx + } + + // Make the put object request using the encryption client + _, err = client.PutObject(encryptionContext, putInput) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to put object: %v", err), http.StatusInternalServerError) + return + } + + log.Printf("[Go V4] PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Return response + w.Header().Set("Content-Type", "application/json") + resp := PutObjectOutput{ + Bucket: bucket, + Key: key, + Metadata: []string{}, // TODO: pass metadata back in response + } + json.NewEncoder(w).Encode(resp) +} + +// getObject handles GET /object/{bucket}/{key} +func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + bucket := vars["bucket"] + key := vars["key"] + + clientID := r.Header.Get("ClientID") + if clientID == "" { + s.createGenericServerError(w, "ClientID header is required", http.StatusBadRequest) + return + } + + // Get client from cache + client, exists := s.clientCache[clientID] + + if !exists { + s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) + return + } + + // Get metadata from header + metadataHeader := r.Header.Get("Content-Metadata") + encCtx, err := metadataStringToMap(metadataHeader) + + ctx := context.Background() + encryptionContext := context.WithValue(ctx, "EncryptionContext", encCtx) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to parse metadata: %v", err), http.StatusBadRequest) + return + } + + // Create get object input + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + // Make the get object request using the encryption client + result, err := client.GetObject(encryptionContext, getInput) + if err != nil { + errMsg := err.Error() + // Shim the S3EC error message to the error message expected by the test server. + // We don't want to change the S3EC error message but the test server expects a specific error message; + // This is the appropriate place to rewrite the error message. + if strings.Contains(errMsg, "to decrypt x-amz-cek-alg value `kms` you must enable legacyWrappingAlgorithms on the keyring") { + s.createS3EncryptionClientError(w, "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms", http.StatusInternalServerError) + return + } + + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to get object: %v", err), http.StatusInternalServerError) + return + } + defer result.Body.Close() + + // Read the body + body, err := io.ReadAll(result.Body) + if err != nil { + s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to read object body: %v", err), http.StatusInternalServerError) + return + } + + // Convert metadata to string format + var metadataList []string + if result.Metadata != nil { + for k, v := range result.Metadata { + metadataList = append(metadataList, fmt.Sprintf("%s=%s", k, v)) + } + } + + metadataStr := strings.Join(metadataList, ",") + + log.Printf("[Go V4] GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + + // Set response headers + w.Header().Set("Content-Metadata", metadataStr) + + // Return the body as response + w.Write(body) +} + +func main() { + server, err := NewServer() + if err != nil { + log.Fatalf("[Go V4] Failed to create Go V4 server: %v", err) + } + + r := mux.NewRouter() + + // Register routes + r.HandleFunc("/client", server.createClient).Methods("POST") + r.HandleFunc("/object/{bucket}/{key}", server.putObject).Methods("PUT") + r.HandleFunc("/object/{bucket}/{key}", server.getObject).Methods("GET") + + fmt.Println("[Go V4] Starting Go V4 server on :8089...") + log.Fatal(http.ListenAndServe(":8089", r)) +} From c9d59d8b366a5112759cc0ce61921eab942a7d59 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 6 Nov 2025 10:57:47 -0800 Subject: [PATCH 3/8] m --- test-server/go-v3-transition-server/Makefile | 6 ++--- test-server/go-v3-transition-server/go.mod | 4 ++-- .../go-v3-transition-server/local-go-s3ec | 2 +- test-server/go-v3-transition-server/main.go | 24 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test-server/go-v3-transition-server/Makefile b/test-server/go-v3-transition-server/Makefile index cfaf32fe..b03ea80b 100644 --- a/test-server/go-v3-transition-server/Makefile +++ b/test-server/go-v3-transition-server/Makefile @@ -3,17 +3,17 @@ .PHONY: start-server stop-server wait-for-server PID_FILE := server.pid -PORT := 8089 +PORT := 8095 start-server: - @echo "Starting Go V4 server..." + @echo "Starting Go V3 Transition server..." go mod tidy AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ go run . & echo $$! > $(PID_FILE) - @echo "Go V4 server starting..." + @echo "Go V3 Transition server starting..." stop-server: @if [ -f $(PID_FILE) ]; then \ diff --git a/test-server/go-v3-transition-server/go.mod b/test-server/go-v3-transition-server/go.mod index 4ab1895c..50f1259a 100644 --- a/test-server/go-v3-transition-server/go.mod +++ b/test-server/go-v3-transition-server/go.mod @@ -3,7 +3,7 @@ module github.com/aws/amazon-s3-encryption-client-python/test-server/go-server go 1.21 require ( - github.com/aws/amazon-s3-encryption-client-go/v4 v4.0.0 + github.com/aws/amazon-s3-encryption-client-go/v3 v3.0.0 github.com/aws/aws-sdk-go-v2 v1.24.0 github.com/aws/aws-sdk-go-v2/config v1.26.1 github.com/aws/aws-sdk-go-v2/service/kms v1.27.4 @@ -32,4 +32,4 @@ require ( // S3EC Go V4 is not released to pkg.go.dev as of writing. // It is included as a submodule and referenced locally. -replace github.com/aws/amazon-s3-encryption-client-go/v4 => ./local-go-s3ec/v4 +replace github.com/aws/amazon-s3-encryption-client-go/v3 => ./local-go-s3ec/v3 diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index 8615988f..7a29344c 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit 8615988fd52f8df6db9b377e1a2dcdd1f866e07a +Subproject commit 7a29344cc0c431fd5ac6d0a08ce4db455d75c175 diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index 75871d5f..b3d42956 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -9,8 +9,8 @@ import ( "net/http" "strings" - "github.com/aws/amazon-s3-encryption-client-go/v4/client" - "github.com/aws/amazon-s3-encryption-client-go/v4/materials" + "github.com/aws/amazon-s3-encryption-client-go/v3/client" + "github.com/aws/amazon-s3-encryption-client-go/v3/materials" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/kms" @@ -21,7 +21,7 @@ import ( // Server represents the Go test server type Server struct { - clientCache map[string]*client.S3EncryptionClientV4 + clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client } @@ -72,7 +72,7 @@ func NewServer() (*Server, error) { } return &Server{ - clientCache: make(map[string]*client.S3EncryptionClientV4), + clientCache: make(map[string]*client.S3EncryptionClientV3), kmsClient: kms.NewFromConfig(cfg), }, nil } @@ -80,7 +80,7 @@ func NewServer() (*Server, error) { // createGenericServerError creates a generic server error response func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) { // Echo error to console - log.Printf("[Go V4] GenericServerError: %s (Status: %d)", message, statusCode) + log.Printf("[Go V3-Transition] GenericServerError: %s (Status: %d)", message, statusCode) w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) @@ -93,7 +93,7 @@ func (s *Server) createGenericServerError(w http.ResponseWriter, message string, // createS3EncryptionClientError creates an S3 encryption client error response func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) { // Echo error to console - log.Printf("[Go V4] S3EncryptionClientError: %s (Status: %d)", message, statusCode) + log.Printf("[Go V3-Transition] S3EncryptionClientError: %s (Status: %d)", message, statusCode) w.Header().Set("Content-Type", "application/json") w.WriteHeader(statusCode) @@ -160,7 +160,7 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { } // Create S3 encryption client - var s3EncryptionClient *client.S3EncryptionClientV4 + var s3EncryptionClient *client.S3EncryptionClientV3 s3PlaintextClient := s3.NewFromConfig(cfg) s3EncryptionClient, err = client.New(s3PlaintextClient, cmm) @@ -240,7 +240,7 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - log.Printf("[Go V4] PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + log.Printf("[Go V3-Transition] PutObject SUCCESS: Bucket=%s, Key=%s", bucket, key) // Return response w.Header().Set("Content-Type", "application/json") @@ -323,7 +323,7 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { metadataStr := strings.Join(metadataList, ",") - log.Printf("[Go V4] GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) + log.Printf("[Go V3-Transition] GetObject SUCCESS: Bucket=%s, Key=%s", bucket, key) // Set response headers w.Header().Set("Content-Metadata", metadataStr) @@ -335,7 +335,7 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { func main() { server, err := NewServer() if err != nil { - log.Fatalf("[Go V4] Failed to create Go V4 server: %v", err) + log.Fatalf("[Go V3-Transition] Failed to create Go V3 Transition server: %v", err) } r := mux.NewRouter() @@ -345,6 +345,6 @@ func main() { r.HandleFunc("/object/{bucket}/{key}", server.putObject).Methods("PUT") r.HandleFunc("/object/{bucket}/{key}", server.getObject).Methods("GET") - fmt.Println("[Go V4] Starting Go V4 server on :8089...") - log.Fatal(http.ListenAndServe(":8089", r)) + fmt.Println("[Go V3-Transition] Starting Go V3 Transition server on :8095...") + log.Fatal(http.ListenAndServe(":8095", r)) } From d008f2228d18fc4bd8b088372763ab12edfc2463 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 6 Nov 2025 12:00:43 -0800 Subject: [PATCH 4/8] m --- .../src/it/java/software/amazon/encryption/s3/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index c57e8623..ed5d510b 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -147,7 +147,7 @@ public class TestUtils { servers.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); // TODO: Create and add transition servers servers.put(JAVA_V3_TRANSITION, new LanguageServerTarget(JAVA_V3_TRANSITION, "8094")); - // servers.put(GO_V3_TRANSITION, new LanguageServerTarget(GO_V3_TRANSITION, "8095")); + servers.put(GO_V3_TRANSITION, new LanguageServerTarget(GO_V3_TRANSITION, "8095")); // servers.put(NET_V2_TRANSITION, new LanguageServerTarget(NET_V2_TRANSITION, "8096")); servers.put(RUBY_V2_TRANSITION, new LanguageServerTarget(RUBY_V2_TRANSITION, "8098")); servers.put(PHP_V2_TRANSITION, new LanguageServerTarget(PHP_V2_TRANSITION, "8099")); From 7a01efc94b90f3dfdd420a1326228f00cce503b0 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 6 Nov 2025 13:19:31 -0800 Subject: [PATCH 5/8] m --- test-server/go-v3-transition-server/main.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index b3d42956..edcdaa78 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -42,6 +42,7 @@ type S3ECConfig struct { EnableLegacyWrappingAlgorithms bool `json:"enableLegacyWrappingAlgorithms"` SetBufferSize int64 `json:"setBufferSize"` KeyMaterial KeyMaterial `json:"keyMaterial"` + CommitmentPolicy string `json:"commitmentPolicy"` } // KeyMaterial represents the key material for encryption @@ -147,6 +148,16 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } + var commitmentPolicy commitment.CommitmentPolicy + switch input.Config.CommitmentPolicy { + case "REQUIRE_ENCRYPT_REQUIRE_DECRYPT": + commitmentPolicy = commitment.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + case "REQUIRE_ENCRYPT_ALLOW_DECRYPT": + commitmentPolicy = commitment.REQUIRE_ENCRYPT_ALLOW_DECRYPT + case "FORBID_ENCRYPT_ALLOW_DECRYPT": + commitmentPolicy = commitment.FORBID_ENCRYPT_ALLOW_DECRYPT + } + // Create KMS keyring kmsClient := kms.NewFromConfig(cfg) keyring := materials.NewKmsKeyring(kmsClient, input.Config.KeyMaterial.KMSKeyID, func(options *materials.KeyringOptions) { @@ -162,7 +173,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Create S3 encryption client var s3EncryptionClient *client.S3EncryptionClientV3 s3PlaintextClient := s3.NewFromConfig(cfg) - s3EncryptionClient, err = client.New(s3PlaintextClient, cmm) + s3EncryptionClient, err = client.New(s3PlaintextClient, cmm, func(clientOptions *client.EncryptionClientOptions) { + if input.Config.CommitmentPolicy != "" { + clientOptions.CommitmentPolicy = commitmentPolicy + } + clientOptions.EnableLegacyUnauthenticatedModes = input.Config.EnableLegacyUnauthenticatedModes + }) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to create S3EC: %v", err), http.StatusInternalServerError) From f289b6c6b2b4e56b17c251f1d25ffcedcf5fe9d4 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 6 Nov 2025 13:35:43 -0800 Subject: [PATCH 6/8] m --- test-server/go-v3-server/.duvet/.gitignore | 3 --- test-server/go-v3-server/.duvet/config.toml | 21 --------------------- 2 files changed, 24 deletions(-) delete mode 100644 test-server/go-v3-server/.duvet/.gitignore delete mode 100644 test-server/go-v3-server/.duvet/config.toml diff --git a/test-server/go-v3-server/.duvet/.gitignore b/test-server/go-v3-server/.duvet/.gitignore deleted file mode 100644 index 93956e36..00000000 --- a/test-server/go-v3-server/.duvet/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -reports/ -requirements/ -specification/ \ No newline at end of file diff --git a/test-server/go-v3-server/.duvet/config.toml b/test-server/go-v3-server/.duvet/config.toml deleted file mode 100644 index 4729a668..00000000 --- a/test-server/go-v3-server/.duvet/config.toml +++ /dev/null @@ -1,21 +0,0 @@ -'$schema' = "https://awslabs.github.io/duvet/config/v0.4.0.json" - -[[source]] -pattern = "**/*.go" - -# Include required specifications here -[[specification]] -source = "../specification/s3-encryption/data-format/content-metadata.md" -[[specification]] -source = "../specification/s3-encryption/data-format/metadata-strategy.md" -[[specification]] -source = "../specification/s3-encryption/encryption.md" -[[specification]] -source = "../specification/s3-encryption/key-derivation.md" - -[report.html] -enabled = true - -# Enable snapshots to prevent requirement coverage regressions -[report.snapshot] -enabled = false From 1c88d51b4e7f4133e5cc3037ec57b1c505c4c5fc Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 6 Nov 2025 13:56:30 -0800 Subject: [PATCH 7/8] m --- test-server/go-v3-transition-server/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index edcdaa78..799a9668 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -11,6 +11,7 @@ import ( "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" + "github.com/aws/amazon-s3-encryption-client-go/v3/commitment" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/kms" From 21b9acccd9ffca66923670952f1e6c0613de92b3 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 6 Nov 2025 14:34:25 -0800 Subject: [PATCH 8/8] m --- .../.duvet/snapshot.txt | 128 ------------------ test-server/go-v3-transition-server/README.md | 6 +- 2 files changed, 3 insertions(+), 131 deletions(-) delete mode 100644 test-server/go-v3-transition-server/.duvet/snapshot.txt diff --git a/test-server/go-v3-transition-server/.duvet/snapshot.txt b/test-server/go-v3-transition-server/.duvet/snapshot.txt deleted file mode 100644 index e432d768..00000000 --- a/test-server/go-v3-transition-server/.duvet/snapshot.txt +++ /dev/null @@ -1,128 +0,0 @@ -SPECIFICATION: [Content Metadata](../specification/s3-encryption/data-format/content-metadata.md) - SECTION: [Content Metadata MapKeys](#content-metadata-mapkeys) - TEXT[!MUST]: The "x-amz-meta-" prefix is automatically added by the S3 server and MUST NOT be included in implementation code. - TEXT[!MUST]: The "x-amz-" prefix denotes that the metadata is owned by an Amazon product and MUST be prepended to all S3EC metadata mapkeys. - TEXT[!SHOULD]: - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects. - TEXT[!MUST]: - The mapkey "x-amz-key" MUST be present for V1 format objects. - TEXT[!MUST]: - The mapkey "x-amz-matdesc" MUST be present for V1 format objects. - TEXT[!MUST]: - The mapkey "x-amz-iv" MUST be present for V1 format objects. - TEXT[!MUST]: - The mapkey "x-amz-key-v2" MUST be present for V2 format objects. - TEXT[!MUST]: - The mapkey "x-amz-matdesc" MUST be present for V2 format objects. - TEXT[!MUST]: - The mapkey "x-amz-iv" MUST be present for V2 format objects. - TEXT[!MUST]: - The mapkey "x-amz-wrap-alg" MUST be present for V2 format objects. - TEXT[!MUST]: - The mapkey "x-amz-cek-alg" MUST be present for V2 format objects. - TEXT[!MUST]: - The mapkey "x-amz-tag-len" MUST be present for V2 format objects. - TEXT[!MUST]: - The mapkey "x-amz-c" MUST be present for V3 format objects. - TEXT[!SHOULD]: - This mapkey ("x-amz-c") SHOULD be represented by a constant named "CONTENT_CIPHER_V3" or similar in the implementation code. - TEXT[!MUST]: - The mapkey "x-amz-3" MUST be present for V3 format objects. - TEXT[!SHOULD]: - This mapkey ("x-amz-3") SHOULD be represented by a constant named "ENCRYPTED_DATA_KEY_V3" or similar in the implementation code. - TEXT[!SHOULD]: - The mapkey "x-amz-m" SHOULD be present for V3 format objects that use Raw Keyring Material Description. - TEXT[!SHOULD]: - This mapkey ("x-amz-m") SHOULD be represented by a constant named "MAT_DESC_V3" or similar in the implementation code. - TEXT[!SHOULD]: - The mapkey "x-amz-t" SHOULD be present for V3 format objects that use KMS Encryption Context. - TEXT[!SHOULD]: - This mapkey ("x-amz-t") SHOULD be represented by a constant named "ENCRYPTION_CONTEXT_V3" or similar in the implementation code. - TEXT[!MUST]: - The mapkey "x-amz-w" MUST be present for V3 format objects. - TEXT[!SHOULD]: - This mapkey ("x-amz-w") SHOULD be represented by a constant named "ENCRYPTED_DATA_KEY_ALGORITHM_V3" or similar in the implementation code. - TEXT[!MUST]: - The mapkey "x-amz-d" MUST be present for V3 format objects. - TEXT[!SHOULD]: - This mapkey ("x-amz-d") SHOULD be represented by a constant named "KEY_COMMITMENT_V3" or similar in the implementation code. - TEXT[!MUST]: - The mapkey "x-amz-i" MUST be present for V3 format objects. - TEXT[!SHOULD]: - This mapkey ("x-amz-i") SHOULD be represented by a constant named "MESSAGE_ID_V3" or similar in the implementation code. - TEXT[!MUST]: In the V3 format, the mapkeys "x-amz-c", "x-amz-d", and "x-amz-i" MUST be stored exclusively in the Object Metadata. - - SECTION: [Determining S3EC Object Status](#determining-s3ec-object-status) - TEXT[!MUST]: - If the metadata contains "x-amz-iv" and "x-amz-key" then the object MUST be considered as an S3EC-encrypted object using the V1 format. - TEXT[!MUST]: - If the metadata contains "x-amz-iv" and "x-amz-metadata-x-amz-key-v2" then the object MUST be considered as an S3EC-encrypted object using the V2 format. - TEXT[!MUST]: - If the metadata contains "x-amz-3" and "x-amz-d" and "x-amz-i" then the object MUST be considered an S3EC-encrypted object using the V3 format. - TEXT[!MUST]: If the object matches none of the V1/V2/V3 formats, the S3EC MUST attempt to get the instruction file. - TEXT[!SHOULD]: If there are multiple mapkeys which are meant to be exclusive, such as "x-amz-key", "x-amz-key-v2", and "x-amz-3" then the S3EC SHOULD throw an exception. - TEXT[!SHOULD]: In general, if there is any deviation from the above format, with the exception of additional unrelated mapkeys, then the S3EC SHOULD throw an exception. - - SECTION: [V1/V2 Shared](#v1-v2-shared) - TEXT[!MAY]: This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. - - SECTION: [V3 Only](#v3-only) - TEXT[!MAY]: This material description string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. - TEXT[!MUST]: The Material Description MUST be used for wrapping algorithms `AES/GCM` (`02`) and `RSA-OAEP-SHA1` (`22`). - TEXT[!MUST]: If the mapkey is not present, the default Material Description value MUST be set to an empty map (`{}`). - TEXT[!MAY]: This encryption context string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server. - TEXT[!MUST]: The Encryption Context value MUST be used for wrapping algorithm `kms+context` or `12`. - TEXT[!MUST]: - The wrapping algorithm value "02" MUST be translated to AES/GCM upon retrieval, and vice versa on write. - TEXT[!MUST]: - The wrapping algorithm value "12" MUST be translated to kms+context upon retrieval, and vice versa on write. - TEXT[!MUST]: - The wrapping algorithm value "22" MUST be translated to RSA-OAEP-SHA1 upon retrieval, and vice versa on write. - - SECTION: [Algorithm Suite and Message Format Version Compatibility](#algorithm-suite-and-message-format-version-compatibility) - TEXT[!MAY]: Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version. - TEXT[!MUST]: Objects encrypted with ALG_AES_256_GCM_IV12_TAG16_NO_KDF MUST use the V2 message format version only. - TEXT[!MUST]: Objects encrypted with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY MUST use the V3 message format version only. - -SPECIFICATION: [Content Metadata Strategy](../specification/s3-encryption/data-format/metadata-strategy.md) - SECTION: [Object Metadata](#object-metadata) - TEXT[!MUST]: By default, the S3EC MUST store content metadata in the S3 Object Metadata. - TEXT[!SHOULD]: The S3EC SHOULD support decoding the S3 Server's "double encoding". - TEXT[!MUST]: If the S3EC does not support decoding the S3 Server's "double encoding" then it MUST return the content metadata untouched. - - SECTION: [Instruction File](#instruction-file) - TEXT[!MUST]: The S3EC MUST support writing some or all (depending on format) content metadata to an Instruction File. - TEXT[!MUST]: The content metadata stored in the Instruction File MUST be serialized to a JSON string. - TEXT[!MUST]: The serialized JSON string MUST be the only contents of the Instruction File. - TEXT[!MUST]: Instruction File writes MUST NOT be enabled by default. - TEXT[!MUST]: Instruction File writes MUST be optionally configured during client creation or on each PutObject request. - TEXT[!MAY]: The S3EC MAY support re-encryption/key rotation via Instruction Files. - TEXT[!MUST]: The S3EC MUST NOT support providing a custom Instruction File suffix on ordinary writes; custom suffixes MUST only be used during re-encryption. - TEXT[!SHOULD]: The S3EC SHOULD support providing a custom Instruction File suffix on GetObject requests, regardless of whether or not re-encryption is supported. - - SECTION: [V1/V2 Instruction Files](#v1-v2-instruction-files) - TEXT[!MUST]: In the V1/V2 message format, all of the content metadata MUST be stored in the Instruction File. - - SECTION: [V3 Instruction Files](#v3-instruction-files) - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-c" and its value in the Object Metadata when writing with an Instruction File. - TEXT[!MUST]: - The V3 message format MUST NOT store the mapkey "x-amz-c" and its value in the Instruction File. - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-d" and its value in the Object Metadata when writing with an Instruction File. - TEXT[!MUST]: - The V3 message format MUST NOT store the mapkey "x-amz-d" and its value in the Instruction File. - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-i" and its value in the Object Metadata when writing with an Instruction File. - TEXT[!MUST]: - The V3 message format MUST NOT store the mapkey "x-amz-i" and its value in the Instruction File. - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-3" and its value in the Instruction File. - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-w" and its value in the Instruction File. - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-m" and its value (when present in the content metadata) in the Instruction File. - TEXT[!MUST]: - The V3 message format MUST store the mapkey "x-amz-t" and its value (when present in the content metadata) in the Instruction File. - -SPECIFICATION: [Encryption](../specification/s3-encryption/encryption.md) - SECTION: [Content Encryption](#content-encryption) - TEXT[!MUST]: The S3EC MUST use the encryption algorithm configured during [client](./client.md) initialization. - TEXT[!MUST]: The client MUST validate that the length of the plaintext bytes does not exceed the algorithm suite's cipher's maximum content length in bytes. - TEXT[!MUST]: The client MUST generate an IV or Message ID using the length of the IV or Message ID defined in the algorithm suite. - TEXT[!MUST]: The generated IV or Message ID MUST be set or returned from the encryption process such that it can be included in the content metadata. - - SECTION: [Cipher Initialization](#cipher-initialization) - TEXT[!SHOULD]: The client SHOULD validate that the generated IV or Message ID is not zeros. - - SECTION: [ALG_AES_256_CTR_IV16_TAG16_NO_KDF](#alg-aes-256-ctr-iv16-tag16-no-kdf) - TEXT[!MUST]: Attempts to encrypt using AES-CTR MUST fail. - - SECTION: [ALG_AES_256_CTR_HKDF_SHA512_COMMIT_KEY](#alg-aes-256-ctr-hkdf-sha512-commit-key) - TEXT[!MUST]: Attempts to encrypt using key committing AES-CTR MUST fail. - - SECTION: [ALG_AES_256_GCM_IV12_TAG16_NO_KDF](#alg-aes-256-gcm-iv12-tag16-no-kdf) - TEXT[!MUST]: The client MUST initialize the cipher, or call an AES-GCM encryption API, with the plaintext data key, the generated IV, and the tag length defined in the Algorithm Suite when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. - TEXT[!MUST]: The client MUST NOT provide any AAD when encrypting with ALG_AES_256_GCM_IV12_TAG16_NO_KDF. - TEXT[!MUST]: The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. - - SECTION: [ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY](#alg-aes-256-gcm-hkdf-sha512-commit-key) - TEXT[!MUST]: The client MUST use HKDF to derive the key commitment value and the derived encrypting key as described in [Key Derivation](key-derivation.md). - TEXT[!MUST]: The derived key commitment value MUST be set or returned from the encryption process such that it can be included in the content metadata. - TEXT[!MUST]: The client MUST append the GCM auth tag to the ciphertext if the underlying crypto provider does not do so automatically. - -SPECIFICATION: [Key Derivation](../specification/s3-encryption/key-derivation.md) - SECTION: [HKDF Operation](#hkdf-operation) - TEXT[!MUST]: - The hash function MUST be specified by the algorithm suite commitment settings. - TEXT[!MUST]: - The input keying material MUST be the plaintext data key (PDK) generated by the key provider. - TEXT[!MUST]: - The length of the input keying material MUST equal the key derivation input length specified by the algorithm suite commit key derivation setting. - TEXT[!MUST]: - The salt MUST be the Message ID with the length defined in the algorithm suite. - TEXT[!MUST]: - The DEK input pseudorandom key MUST be the output from the extract step. - TEXT[!MUST]: - The length of the output keying material MUST equal the encryption key length specified by the algorithm suite encryption settings. - TEXT[!MUST]: - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string DERIVEKEY as UTF8 encoded bytes. - TEXT[!MUST]: - The CK input pseudorandom key MUST be the output from the extract step. - TEXT[!MUST]: - The length of the output keying material MUST equal the commit key length specified by the supported algorithm suites. - TEXT[!MUST]: - The input info MUST be a concatenation of the algorithm suite ID as bytes followed by the string COMMITKEY as UTF8 encoded bytes. - TEXT[!MUST]: When encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, the IV used in the AES-GCM content encryption/decryption MUST contain only zeros of the length defined in the algorithm suite. - TEXT[!MUST]: The client MUST initialize the cipher, or call an AES-GCM encryption API, with the derived encryption key, an IV containing only zeros, and the tag length defined in the Algorithm Suite when encrypting or decrypting with ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY. - TEXT[!MUST]: The client MUST set the AAD to the Algorithm Suite ID represented as bytes. diff --git a/test-server/go-v3-transition-server/README.md b/test-server/go-v3-transition-server/README.md index d97a37bf..e7e226f7 100644 --- a/test-server/go-v3-transition-server/README.md +++ b/test-server/go-v3-transition-server/README.md @@ -1,6 +1,6 @@ -# S3EC Go V4 Test Server +# S3EC Go V3 Transition Test Server -This is the Go implementation of the S3ECTestServer framework for S3EC Go V4. It provides a server implementation for testing Go S3 Encryption Client V4 functionality. +This is the Go implementation of the S3ECTestServer framework for S3EC Go V3 Transition. It provides a server implementation for testing Go S3 Encryption Client V3 Transition functionality. ## Overview @@ -18,6 +18,6 @@ To run the server: go run . ``` -This will start the server running on port `8089`. +This will start the server running on port `8095`. The server is used as part of the testing framework to verify cross-language compatibility of the S3 Encryption Client implementations.