From 11c92bc28de4c1f8c6c4f70427a4dcbfc290446d Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Tue, 16 Sep 2025 12:49:05 -0700 Subject: [PATCH 1/5] Ruby test servers V2 and v3 pending changes --- .github/workflows/test.yml | 10 +- .gitmodules | 6 + test-server/Makefile | 27 +- test-server/java-tests/build.gradle.kts | 9 + .../amazon/encryption/s3/RoundTripTests.java | 24 +- test-server/ruby-v2-server/.gitignore | 2 + test-server/ruby-v2-server/Gemfile | 15 ++ test-server/ruby-v2-server/Gemfile.lock | 102 ++++++++ test-server/ruby-v2-server/Makefile | 29 +++ test-server/ruby-v2-server/README.md | 73 ++++++ test-server/ruby-v2-server/app.rb | 231 ++++++++++++++++++ test-server/ruby-v2-server/config.ru | 3 + .../ruby-v2-server/lib/client_manager.rb | 74 ++++++ .../ruby-v2-server/lib/error_handlers.rb | 42 ++++ test-server/ruby-v2-server/lib/logger.rb | 105 ++++++++ .../ruby-v2-server/lib/metadata_utils.rb | 50 ++++ test-server/ruby-v2-server/local-ruby-sdk | 1 + test-server/ruby-v3-server/.bundle/config | 2 + test-server/ruby-v3-server/.gitignore | 2 + test-server/ruby-v3-server/Gemfile | 15 ++ test-server/ruby-v3-server/Gemfile.lock | 102 ++++++++ test-server/ruby-v3-server/Makefile | 29 +++ test-server/ruby-v3-server/README.md | 74 ++++++ test-server/ruby-v3-server/app.rb | 231 ++++++++++++++++++ test-server/ruby-v3-server/config.ru | 3 + .../ruby-v3-server/lib/client_manager.rb | 74 ++++++ .../ruby-v3-server/lib/error_handlers.rb | 42 ++++ test-server/ruby-v3-server/lib/logger.rb | 105 ++++++++ .../ruby-v3-server/lib/metadata_utils.rb | 50 ++++ test-server/ruby-v3-server/local-ruby-sdk | 1 + 30 files changed, 1521 insertions(+), 12 deletions(-) create mode 100644 .gitmodules create mode 100644 test-server/ruby-v2-server/.gitignore create mode 100644 test-server/ruby-v2-server/Gemfile create mode 100644 test-server/ruby-v2-server/Gemfile.lock create mode 100644 test-server/ruby-v2-server/Makefile create mode 100644 test-server/ruby-v2-server/README.md create mode 100644 test-server/ruby-v2-server/app.rb create mode 100644 test-server/ruby-v2-server/config.ru create mode 100644 test-server/ruby-v2-server/lib/client_manager.rb create mode 100644 test-server/ruby-v2-server/lib/error_handlers.rb create mode 100644 test-server/ruby-v2-server/lib/logger.rb create mode 100644 test-server/ruby-v2-server/lib/metadata_utils.rb create mode 160000 test-server/ruby-v2-server/local-ruby-sdk create mode 100644 test-server/ruby-v3-server/.bundle/config create mode 100644 test-server/ruby-v3-server/.gitignore create mode 100644 test-server/ruby-v3-server/Gemfile create mode 100644 test-server/ruby-v3-server/Gemfile.lock create mode 100644 test-server/ruby-v3-server/Makefile create mode 100644 test-server/ruby-v3-server/README.md create mode 100644 test-server/ruby-v3-server/app.rb create mode 100644 test-server/ruby-v3-server/config.ru create mode 100644 test-server/ruby-v3-server/lib/client_manager.rb create mode 100644 test-server/ruby-v3-server/lib/error_handlers.rb create mode 100644 test-server/ruby-v3-server/lib/logger.rb create mode 100644 test-server/ruby-v3-server/lib/metadata_utils.rb create mode 160000 test-server/ruby-v3-server/local-ruby-sdk diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0639f45a..180ed6de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,11 +20,19 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: true + token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version || '3.11' }} + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' - name: Set up PHP with Composer uses: shivammathur/setup-php@v2 @@ -61,7 +69,7 @@ jobs: path: | ~/.gradle/caches ~/.gradle/wrapper - test-server/java-server/.gradle + test-server/java-v3-server/.gradle test-server/java-tests/.gradle key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: | diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..b2b112a0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "test-server/ruby-v2-server/local-ruby-sdk"] + path = test-server/ruby-v2-server/local-ruby-sdk + url = git@github.com:aws/aws-sdk-ruby-staging.git +[submodule "test-server/ruby-v3-server/local-ruby-sdk"] + path = test-server/ruby-v3-server/local-ruby-sdk + url = git@github.com:aws/aws-sdk-ruby-staging.git diff --git a/test-server/Makefile b/test-server/Makefile index 90d15648..7008565c 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -10,26 +10,35 @@ ci: start-servers run-tests stop-servers SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | sort) -SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) +START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) +WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) # Start all servers in parallel start-servers: @echo "Starting all servers..." $(MAKE) start-all-servers @echo "Waiting for servers to start..." - @for dir in $(SERVER_DIRS); do \ - echo "Waiting for server in $$dir..."; \ - $(MAKE) -C $$dir wait-for-server; \ - done + $(MAKE) wait-all-servers -start-all-servers: $(SERVER_TARGETS) +start-all-servers: $(START_SERVER_TARGETS) -$(SERVER_TARGETS): start-%: +$(START_SERVER_TARGETS): start-%: @if [ -f $*/Makefile ]; then \ echo "Starting server in $*..."; \ $(MAKE) -C $* start-server; \ else \ - echo "❌ Error: no Makefile found in $$dir"; \ + echo "❌ Error: no Makefile found in $*"; \ + exit 1; \ + fi; \ + +wait-all-servers: $(WAIT_SERVER_TARGETS) + +$(WAIT_SERVER_TARGETS): wait-%: + @if [ -f $*/Makefile ]; then \ + echo "Waiting server in $*..."; \ + $(MAKE) -C $* wait-for-server; \ + else \ + echo "❌ Error: no Makefile found in $*"; \ exit 1; \ fi; \ @@ -44,7 +53,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --rerun-tasks --parallel integ + ./gradlew --build-cache --parallel integ @echo "Tests completed successfully" # Stop the servers diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 813e8369..3a00e348 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -48,6 +48,15 @@ tasks { classpath = sourceSets["it"].runtimeClasspath outputs.upToDateWhen { false } outputs.cacheIf { false } + // For debugging + // // Enable System.out output + // testLogging { + // events("passed", "skipped", "failed", "standardOut", "standardError") + // showStandardStreams = true + // } + + // // Disable AWS SDK v1 deprecation warnings + // systemProperty("aws.java.v1.disableDeprecationAnnouncement", "true") } } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index b6ecc6a6..23219611 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -61,6 +61,10 @@ public class RoundTripTests { private static final String NET_V3 = "NET-V3"; private static final String PHP_V2 = "PHP-V2"; private static final String PHP_V3 = "PHP-V3"; + private static final String RUBY_V2 = "Ruby-V2"; + private static final String RUBY_V3 = "Ruby-V3"; + + private static final List serverList; private static final Map serverMap; @@ -80,6 +84,8 @@ public class RoundTripTests { serverList.add(new LanguageServerTarget(NET_V3, "8084")); serverList.add(new LanguageServerTarget(PHP_V2, "8087")); serverList.add(new LanguageServerTarget(PHP_V3, "8093")); + serverList.add(new LanguageServerTarget(RUBY_V2, "8086")); + serverList.add(new LanguageServerTarget(RUBY_V3, "8092")); serverMap = new HashMap<>(14); serverMap.put(JAVA_V3, new LanguageServerTarget(JAVA_V3, "8080")); @@ -89,6 +95,8 @@ public class RoundTripTests { serverMap.put(NET_V3, new LanguageServerTarget(NET_V3, "8084")); serverMap.put(PHP_V2, new LanguageServerTarget(PHP_V2, "8087")); serverMap.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); + serverMap.put(RUBY_V2, new LanguageServerTarget(RUBY_V2, "8086")); + serverMap.put(RUBY_V3, new LanguageServerTarget(RUBY_V3, "8092")); } // Encryption context validation behavior varies by implementation: @@ -337,7 +345,13 @@ public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLa .build()); fail("Expected exception!"); } catch (S3EncryptionClientError e) { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + assertTrue( + e.getMessage().contains("Provided encryption context does not match information retrieved from S3") || + // Ruby error message + (e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context") + && decLang.languageName.startsWith("ruby-v") + ) + ); } } @@ -389,7 +403,11 @@ public void crossLanguageTestKmsWithIncorrectEncCtxFails(LanguageServerTarget en .build()); fail("Expected exception!"); } catch (S3EncryptionClientError e) { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + if (decLang.languageName.equals(RUBY_V3) || decLang.languageName.equals(RUBY_V2)) { + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); + } else { + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + } } } @@ -529,6 +547,8 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration V2." )); + } else if (language.equals(RUBY_V3) || language.equals(RUBY_V2)) { + assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object.")); } else { assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); } diff --git a/test-server/ruby-v2-server/.gitignore b/test-server/ruby-v2-server/.gitignore new file mode 100644 index 00000000..d20a29c0 --- /dev/null +++ b/test-server/ruby-v2-server/.gitignore @@ -0,0 +1,2 @@ +vendor +server.pid \ No newline at end of file diff --git a/test-server/ruby-v2-server/Gemfile b/test-server/ruby-v2-server/Gemfile new file mode 100644 index 00000000..2759a87b --- /dev/null +++ b/test-server/ruby-v2-server/Gemfile @@ -0,0 +1,15 @@ +source 'https://rubygems.org' + +ruby '~> 3.0' + +gem 'sinatra', '~> 3.0' +gem 'puma', '~> 6.0' +gem 'aws-sdk-s3', path: 'local-ruby-sdk/gems/aws-sdk-s3' +gem 'aws-sdk-kms', path: 'local-ruby-sdk/gems/aws-sdk-kms' +gem 'json', '~> 2.0' +gem 'concurrent-ruby', '~> 1.0' +gem 'nokogiri', '~> 1.13' + +group :development do + gem 'rubocop', '~> 1.0' +end diff --git a/test-server/ruby-v2-server/Gemfile.lock b/test-server/ruby-v2-server/Gemfile.lock new file mode 100644 index 00000000..660aadd5 --- /dev/null +++ b/test-server/ruby-v2-server/Gemfile.lock @@ -0,0 +1,102 @@ +PATH + remote: local-ruby-sdk/gems/aws-sdk-kms + specs: + aws-sdk-kms (1.112.0) + aws-sdk-core (~> 3, >= 3.231.0) + aws-sigv4 (~> 1.5) + +PATH + remote: local-ruby-sdk/gems/aws-sdk-s3 + specs: + aws-sdk-s3 (1.199.0) + aws-sdk-core (~> 3, >= 3.231.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + aws-eventstream (1.4.0) + aws-partitions (1.1160.0) + aws-sdk-core (3.232.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + base64 + bigdecimal + jmespath (~> 1, >= 1.6.1) + logger + aws-sigv4 (1.12.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.3.0) + bigdecimal (3.2.3) + concurrent-ruby (1.3.5) + jmespath (1.6.2) + json (2.13.2) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + nio4r (2.7.4) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + parallel (1.27.0) + parser (3.3.9.0) + ast (~> 2.4.1) + racc + prism (1.5.1) + puma (6.6.1) + nio4r (~> 2.0) + racc (1.8.1) + rack (2.2.17) + rack-protection (3.2.0) + base64 (>= 0.1.0) + rack (~> 2.2, >= 2.2.4) + rainbow (3.1.1) + regexp_parser (2.11.3) + rubocop (1.80.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.46.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.46.0) + parser (>= 3.3.7.2) + prism (~> 1.4) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + sinatra (3.2.0) + mustermann (~> 3.0) + rack (~> 2.2, >= 2.2.4) + rack-protection (= 3.2.0) + tilt (~> 2.0) + tilt (2.6.1) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.1.0) + +PLATFORMS + arm64-darwin-24 + +DEPENDENCIES + aws-sdk-kms! + aws-sdk-s3! + concurrent-ruby (~> 1.0) + json (~> 2.0) + nokogiri (~> 1.13) + puma (~> 6.0) + rubocop (~> 1.0) + sinatra (~> 3.0) + +RUBY VERSION + ruby 3.4.5p51 + +BUNDLED WITH + 2.6.9 diff --git a/test-server/ruby-v2-server/Makefile b/test-server/ruby-v2-server/Makefile new file mode 100644 index 00000000..5d552aac --- /dev/null +++ b/test-server/ruby-v2-server/Makefile @@ -0,0 +1,29 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8086 + +start-server: + @if [ -f $(PID_FILE) ]; then \ + echo "❌ Error: Server already running. Stop before starting."; \ + exit 1; \ + fi; + @echo "Starting Ruby V2 server..." + bundle install + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + bundle exec ruby app.rb & echo $$! > server.pid + @echo "Ruby V2 server starting..." + +stop-server: + @if [ -f server.pid ]; then \ + kill $$(cat server.pid) 2>/dev/null || true; \ + rm server.pid; \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=8086 \ No newline at end of file diff --git a/test-server/ruby-v2-server/README.md b/test-server/ruby-v2-server/README.md new file mode 100644 index 00000000..4b3e5209 --- /dev/null +++ b/test-server/ruby-v2-server/README.md @@ -0,0 +1,73 @@ +# Ruby S3 Encryption Client Test Server + +This is a Ruby implementation of the S3 Encryption Client test server +that provides an invariant interface around the S3 Encryption Client v2. +It's designed to work alongside other implementations of test servers for cross-language compatibility testing. + +## Overview + +The server provides a REST API that wraps the AWS S3 Encryption Client v2, +allowing tests to verify that all language implementations behave consistently. + +## Endpoints + +- `POST /client` - Create a new S3 encryption client instance +- `PUT /object/{bucket}/{key}` - Encrypt and store an object +- `GET /object/{bucket}/{key}` - Retrieve and decrypt an object +- `GET /health` - Health check endpoint + +## Configuration + +The server runs on port **8086** by default. + +## Setup + +1. Install Ruby 3.x +2. Install dependencies: + + ```bash + cd test-server/ruby-v2-server + bundle install + ``` + +3. Set up AWS credentials (via AWS CLI, environment variables, or IAM roles) + +4. Start the server: + + ```bash + ruby app.rb + # or using Rack + bundle exec rackup -p 8086 + ``` + +## Usage + +The server is designed to be used by the Java test suite in `test-server/java-tests/`. +The tests will automatically discover and use this server for cross-language compatibility testing. + +### Environment Variables + +- `TEST_SERVER_KMS_KEY_ARN` - KMS key ARN for encryption (defaults to test key) +- `TEST_SERVER_S3_BUCKET` - S3 bucket for testing (defaults to test bucket) + +## Architecture + +- `app.rb` - Main Sinatra application +- `lib/client_manager.rb` - Manages S3 encryption client instances +- `lib/metadata_utils.rb` - Handles metadata serialization/deserialization +- `lib/error_handlers.rb` - Smithy-compliant error responses + +## Error Handling + +The server returns errors in the format expected by the Smithy model: + +- `GenericServerError` - Internal server errors +- `S3EncryptionClientError` - Errors from the S3 Encryption Client + +## Compatibility + +This server is compatible with: + +- S3 Encryption Client v2 +- Legacy v1 clients (when `enableLegacyWrappingAlgorithms` is true) +- Cross-language testing with other implementations diff --git a/test-server/ruby-v2-server/app.rb b/test-server/ruby-v2-server/app.rb new file mode 100644 index 00000000..755379d1 --- /dev/null +++ b/test-server/ruby-v2-server/app.rb @@ -0,0 +1,231 @@ +require 'sinatra' +require 'json' +require_relative 'lib/client_manager' +require_relative 'lib/metadata_utils' +require_relative 'lib/error_handlers' +require_relative 'lib/logger' + +class S3ECRubyServer < Sinatra::Base + configure do + set :port, 8086 + set :bind, '0.0.0.0' + set :show_exceptions, false + set :raise_errors, false + end + + def initialize + super + @client_manager = ClientManager.new + S3ECLogger.info("S3EC_SERVER: Ruby server initialized on port #{settings.port}") + end + + # Request logging middleware + before do + @request_id = S3ECLogger.generate_request_id + S3ECLogger.log_request(request.request_method, request.path_info, request.env, @request_id) + end + + # Response logging middleware + after do + S3ECLogger.log_response(response.status, @request_id) + end + + # Health check endpoint + get '/health' do + content_type :json + { status: 'OK', server: 'Ruby S3EC Test Server', port: settings.port.to_i }.to_json + end + + # POST /client - Create S3 encryption client + post '/client' do + begin + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Processing client creation request") + + # Parse request body + request_body = request.body.read + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Request body size: #{request_body.length} bytes") + + parsed_data = JSON.parse(request_body) + config = parsed_data['config'] || {} + + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Parsed config: #{config.inspect}") + + # Create client using client manager + client_id = @client_manager.create_client(config) + + S3ECLogger.info("CLIENT_ENDPOINT [#{@request_id}]: Successfully created client #{client_id}") + + # Return client ID + content_type :json + { clientId: client_id }.to_json + + rescue JSON::ParserError => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'JSON parsing' }, @request_id) + ErrorHandlers.send_generic_server_error(self, "Invalid JSON in request body", 400) + rescue => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'client creation', config: config }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, "Failed to create client: #{e.message}") + end + end + + # PUT /object/{bucket}/{key} - Encrypt and put object + put '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Processing PUT request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Get request body + body = request.body.read + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Request body size: #{body.length} bytes") + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 put_object parameters + put_params = { + bucket: bucket, + key: key, + body: body + } + + # Add encryption context if present + put_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + + # Log S3 operation + S3ECLogger.log_s3_operation('put', bucket, key, encryption_context, "ClientID: #{client_id}, BodySize: #{body.length}") + + # Make the put_object request + response = client.put_object(put_params) + + S3ECLogger.info("PUT_ENDPOINT [#{@request_id}]: Successfully put object s3://#{bucket}/#{key}") + + # Prepare response metadata + response_metadata = MetadataUtils.map_to_array(encryption_context) + S3ECLogger.log_metadata_processing('response', encryption_context, response_metadata) + + # Return response matching Smithy model + content_type :json + { + bucket: bucket, + key: key, + metadata: response_metadata + }.to_json + + rescue Aws::S3::EncryptionV2::Errors::EncryptionError => e + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # GET /object/{bucket}/{key} - Get and decrypt object + get '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Processing GET request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 get_object parameters + get_params = { + bucket: bucket, + key: key + } + + # Add encryption context if present + get_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + + # Log S3 operation + S3ECLogger.log_s3_operation('get', bucket, key, encryption_context, "ClientID: #{client_id}") + + # Make the get_object request + response = client.get_object(get_params) + + # Extract body and metadata + body = response.body.read + metadata = response.metadata || {} + + S3ECLogger.info("GET_ENDPOINT [#{@request_id}]: Successfully got object s3://#{bucket}/#{key}, BodySize: #{body.length}") + + # Set Content-Metadata header in response + metadata_str = MetadataUtils.map_to_string(metadata) + S3ECLogger.log_metadata_processing('response', metadata, metadata_str) + + headers['Content-Metadata'] = metadata_str unless metadata_str.empty? + + # Return the body as response + content_type 'application/octet-stream' + body + + rescue Aws::S3::EncryptionV2::Errors::DecryptionError => e + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # Global error handler + error do + error = env['sinatra.error'] + context = { + endpoint: request.path_info, + method: request.request_method, + params: params, + error_type: 'global_error_handler' + } + + S3ECLogger.log_error(error, context, @request_id) + ErrorHandlers.send_generic_server_error(self, "Internal server error: #{error.message}") + end + + # Start server when run directly + if __FILE__ == $0 + S3ECLogger.info("S3EC_SERVER: Starting Ruby server on port #{settings.port}...") + run! + end +end diff --git a/test-server/ruby-v2-server/config.ru b/test-server/ruby-v2-server/config.ru new file mode 100644 index 00000000..99d3a689 --- /dev/null +++ b/test-server/ruby-v2-server/config.ru @@ -0,0 +1,3 @@ +require_relative 'app' + +run S3ECRubyServer diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb new file mode 100644 index 00000000..d3b12b23 --- /dev/null +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -0,0 +1,74 @@ +require 'concurrent-ruby' +require 'securerandom' +require 'aws-sdk-s3' +require 'aws-sdk-kms' +require_relative 'logger' + +# Manages S3 Encryption Client instances +class ClientManager + def initialize + @client_cache = Concurrent::Hash.new + @kms_client = Aws::KMS::Client.new(region: 'us-west-2') + S3ECLogger.info("CLIENT_MANAGER: Initialized with KMS client for us-west-2") + end + + # Create a new S3 encryption client and return its ID + def create_client(config) + # Extract configuration + kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + enable_legacy_wrapping = config['enableLegacyWrappingAlgorithms'] || false + + raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? + + # Create S3 encryption client configuration + encryption_config = { + kms_key_id: kms_key_id, + kms_client: @kms_client, + key_wrap_schema: :kms_context, + content_encryption_schema: :aes_gcm_no_padding, + # Set security profile based on legacy wrapping algorithms setting + security_profile: enable_legacy_wrapping ? :v2_and_legacy : :v2 + } + + # Create the S3 encryption client + s3_client = Aws::S3::Client.new(region: 'us-west-2') + encryption_client = Aws::S3::EncryptionV2::Client.new( + client: s3_client, + **encryption_config + ) + + # Generate client ID and store in cache + client_id = SecureRandom.uuid + @client_cache[client_id] = encryption_client + + # Log client creation + S3ECLogger.log_client_creation(config, client_id) + S3ECLogger.log_cache_stats(@client_cache.size) + + client_id + end + + # Get a client by ID + def get_client(client_id) + client = @client_cache[client_id] + if client + S3ECLogger.log_client_cache_hit(client_id) + else + S3ECLogger.log_client_cache_miss(client_id) + end + client + end + + # Remove a client from cache (optional cleanup) + def remove_client(client_id) + removed = @client_cache.delete(client_id) + S3ECLogger.info("CLIENT_CACHE: Removed client #{client_id} from cache") if removed + S3ECLogger.log_cache_stats(@client_cache.size) + removed + end + + # Get cache size (for debugging) + def cache_size + @client_cache.size + end +end diff --git a/test-server/ruby-v2-server/lib/error_handlers.rb b/test-server/ruby-v2-server/lib/error_handlers.rb new file mode 100644 index 00000000..234a9a55 --- /dev/null +++ b/test-server/ruby-v2-server/lib/error_handlers.rb @@ -0,0 +1,42 @@ +# Error handling utilities to match Smithy error types +require 'json' + +class ErrorHandlers + # Create a response that matches the GenericServerError type from the Smithy model + # Used for internal server errors + def self.create_generic_server_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#GenericServerError', + 'message' => message + }.to_json + } + end + + # Create a response that matches the S3EncryptionClientError type from the Smithy model + # Used for errors thrown by the S3 Encryption Client + def self.create_s3_encryption_client_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#S3EncryptionClientError', + 'message' => message + }.to_json + } + end + + # Helper method to send error response in Sinatra + def self.send_generic_server_error(app, message, status_code = 500) + error_response = create_generic_server_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end + + # Helper method to send S3EC error response in Sinatra + def self.send_s3_encryption_client_error(app, message, status_code = 500) + error_response = create_s3_encryption_client_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end +end diff --git a/test-server/ruby-v2-server/lib/logger.rb b/test-server/ruby-v2-server/lib/logger.rb new file mode 100644 index 00000000..2febcbab --- /dev/null +++ b/test-server/ruby-v2-server/lib/logger.rb @@ -0,0 +1,105 @@ +require 'logger' +require 'securerandom' + +# Centralized logging utility for the S3EC Ruby server +class S3ECLogger + def self.instance + @instance ||= new + end + + def initialize + @logger = Logger.new(STDOUT) + @logger.level = ENV['LOG_LEVEL'] ? Logger.const_get(ENV['LOG_LEVEL'].upcase) : Logger::INFO + @logger.formatter = proc do |severity, datetime, progname, msg| + "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity}: #{msg}\n" + end + end + + # Generate a unique request ID for correlation + def self.generate_request_id + SecureRandom.hex(8) + end + + # Request/Response logging + def self.log_request(method, path, headers = {}, request_id = nil) + client_id = headers['HTTP_CLIENTID'] || headers['ClientID'] || 'none' + content_metadata = headers['HTTP_CONTENT_METADATA'] || headers['Content-Metadata'] || 'none' + + instance.logger.info("REQUEST [#{request_id}] #{method} #{path} | ClientID: #{client_id} | Metadata: #{content_metadata}") + end + + def self.log_response(status, request_id = nil, additional_info = "") + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("RESPONSE [#{request_id}] Status: #{status}#{info_str}") + end + + # Operation-level logging + def self.log_client_creation(config, client_id) + kms_key = config.dig('keyMaterial', 'kmsKeyId') || 'unknown' + legacy_enabled = config['enableLegacyWrappingAlgorithms'] || false + instance.logger.info("CLIENT_CREATION: Created S3EC client #{client_id} | KMS Key: #{kms_key} | Legacy: #{legacy_enabled}") + end + + def self.log_client_cache_hit(client_id) + instance.logger.debug("CACHE_HIT: Found client #{client_id} in cache") + end + + def self.log_client_cache_miss(client_id) + instance.logger.warn("CACHE_MISS: Client #{client_id} not found in cache") + end + + def self.log_cache_stats(cache_size) + instance.logger.debug("CACHE_STATS: Current client cache size: #{cache_size}") + end + + def self.log_s3_operation(operation, bucket, key, encryption_context = {}, additional_info = "") + enc_ctx_str = encryption_context.empty? ? "none" : encryption_context.inspect + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("S3_OPERATION: #{operation.upcase} s3://#{bucket}/#{key} | EncCtx: #{enc_ctx_str}#{info_str}") + end + + def self.log_metadata_processing(operation, input, output) + instance.logger.debug("METADATA_#{operation.upcase}: Input: #{input.inspect} | Output: #{output.inspect}") + end + + # Enhanced error logging + def self.log_error(error, context = {}, request_id = nil) + error_context = context.empty? ? "" : " | Context: #{context.inspect}" + instance.logger.error("ERROR [#{request_id}] #{error.class}: #{error.message}#{error_context}") + + if error.backtrace && instance.debug? + instance.logger.debug("ERROR_BACKTRACE [#{request_id}]:\n#{error.backtrace.join("\n")}") + end + end + + def self.log_validation_error(field, value, request_id = nil) + instance.logger.warn("VALIDATION_ERROR [#{request_id}] Invalid #{field}: #{value}") + end + + def self.log_aws_error(error, operation, request_id = nil) + instance.logger.error("AWS_ERROR [#{request_id}] #{operation} failed: #{error.class} - #{error.message}") + end + + # Standard logging methods + def self.debug(message) + instance.logger.debug(message) + end + + def self.info(message) + instance.logger.info(message) + end + + def self.warn(message) + instance.logger.warn(message) + end + + def self.error(message) + instance.logger.error(message) + end + + attr_reader :logger + + def debug? + @logger.debug? + end +end diff --git a/test-server/ruby-v2-server/lib/metadata_utils.rb b/test-server/ruby-v2-server/lib/metadata_utils.rb new file mode 100644 index 00000000..72015fcc --- /dev/null +++ b/test-server/ruby-v2-server/lib/metadata_utils.rb @@ -0,0 +1,50 @@ +# Utility class for handling metadata serialization/deserialization +# Matches the format used by Java and Python servers: [key]:[value],[key2]:[value2] +class MetadataUtils + # Convert metadata string to hash + # Input: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + # Output: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + def self.string_to_map(metadata_string) + return {} if metadata_string.nil? || metadata_string.empty? + + metadata = {} + entries = metadata_string.split(',') + + entries.each do |entry| + # Split on "]:[" to separate key and value + parts = entry.split(']:[') + if parts.length == 2 + # Remove remaining brackets from start and end + key = parts[0].delete_prefix("[") # Remove first character '[' + value = parts[1].delete_suffix("]") # Remove last character ']' + metadata[key] = value + else + raise "Malformed metadata list entry: #{entry}" + end + end + + metadata + end + + # Convert hash to metadata string + # Input: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + # Output: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + def self.map_to_string(metadata_hash) + return '' if metadata_hash.nil? || metadata_hash.empty? + + entries = metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + + entries.join(',') + end + + # Convert metadata hash to array format (for JSON responses) + def self.map_to_array(metadata_hash) + return [] if metadata_hash.nil? || metadata_hash.empty? + + metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + end +end diff --git a/test-server/ruby-v2-server/local-ruby-sdk b/test-server/ruby-v2-server/local-ruby-sdk new file mode 160000 index 00000000..e129cf37 --- /dev/null +++ b/test-server/ruby-v2-server/local-ruby-sdk @@ -0,0 +1 @@ +Subproject commit e129cf37c170254ebb631782cae145040cde6d0b diff --git a/test-server/ruby-v3-server/.bundle/config b/test-server/ruby-v3-server/.bundle/config new file mode 100644 index 00000000..23692288 --- /dev/null +++ b/test-server/ruby-v3-server/.bundle/config @@ -0,0 +1,2 @@ +--- +BUNDLE_PATH: "vendor/bundle" diff --git a/test-server/ruby-v3-server/.gitignore b/test-server/ruby-v3-server/.gitignore new file mode 100644 index 00000000..d20a29c0 --- /dev/null +++ b/test-server/ruby-v3-server/.gitignore @@ -0,0 +1,2 @@ +vendor +server.pid \ No newline at end of file diff --git a/test-server/ruby-v3-server/Gemfile b/test-server/ruby-v3-server/Gemfile new file mode 100644 index 00000000..2759a87b --- /dev/null +++ b/test-server/ruby-v3-server/Gemfile @@ -0,0 +1,15 @@ +source 'https://rubygems.org' + +ruby '~> 3.0' + +gem 'sinatra', '~> 3.0' +gem 'puma', '~> 6.0' +gem 'aws-sdk-s3', path: 'local-ruby-sdk/gems/aws-sdk-s3' +gem 'aws-sdk-kms', path: 'local-ruby-sdk/gems/aws-sdk-kms' +gem 'json', '~> 2.0' +gem 'concurrent-ruby', '~> 1.0' +gem 'nokogiri', '~> 1.13' + +group :development do + gem 'rubocop', '~> 1.0' +end diff --git a/test-server/ruby-v3-server/Gemfile.lock b/test-server/ruby-v3-server/Gemfile.lock new file mode 100644 index 00000000..ae04e5fd --- /dev/null +++ b/test-server/ruby-v3-server/Gemfile.lock @@ -0,0 +1,102 @@ +PATH + remote: local-ruby-sdk/gems/aws-sdk-kms + specs: + aws-sdk-kms (1.112.0) + aws-sdk-core (~> 3, >= 3.231.0) + aws-sigv4 (~> 1.5) + +PATH + remote: local-ruby-sdk/gems/aws-sdk-s3 + specs: + aws-sdk-s3 (1.199.0) + aws-sdk-core (~> 3, >= 3.231.0) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.5) + +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + aws-eventstream (1.4.0) + aws-partitions (1.1161.0) + aws-sdk-core (3.232.0) + aws-eventstream (~> 1, >= 1.3.0) + aws-partitions (~> 1, >= 1.992.0) + aws-sigv4 (~> 1.9) + base64 + bigdecimal + jmespath (~> 1, >= 1.6.1) + logger + aws-sigv4 (1.12.1) + aws-eventstream (~> 1, >= 1.0.2) + base64 (0.3.0) + bigdecimal (3.2.3) + concurrent-ruby (1.3.5) + jmespath (1.6.2) + json (2.13.2) + language_server-protocol (3.17.0.5) + lint_roller (1.1.0) + logger (1.7.0) + mustermann (3.0.4) + ruby2_keywords (~> 0.0.1) + nio4r (2.7.4) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + parallel (1.27.0) + parser (3.3.9.0) + ast (~> 2.4.1) + racc + prism (1.5.1) + puma (6.6.1) + nio4r (~> 2.0) + racc (1.8.1) + rack (2.2.17) + rack-protection (3.2.0) + base64 (>= 0.1.0) + rack (~> 2.2, >= 2.2.4) + rainbow (3.1.1) + regexp_parser (2.11.3) + rubocop (1.80.2) + json (~> 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.46.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.46.0) + parser (>= 3.3.7.2) + prism (~> 1.4) + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + sinatra (3.2.0) + mustermann (~> 3.0) + rack (~> 2.2, >= 2.2.4) + rack-protection (= 3.2.0) + tilt (~> 2.0) + tilt (2.6.1) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.1.0) + +PLATFORMS + arm64-darwin-24 + +DEPENDENCIES + aws-sdk-kms! + aws-sdk-s3! + concurrent-ruby (~> 1.0) + json (~> 2.0) + nokogiri (~> 1.13) + puma (~> 6.0) + rubocop (~> 1.0) + sinatra (~> 3.0) + +RUBY VERSION + ruby 3.4.5p51 + +BUNDLED WITH + 2.6.9 diff --git a/test-server/ruby-v3-server/Makefile b/test-server/ruby-v3-server/Makefile new file mode 100644 index 00000000..e4492423 --- /dev/null +++ b/test-server/ruby-v3-server/Makefile @@ -0,0 +1,29 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8092 + +start-server: + @if [ -f $(PID_FILE) ]; then \ + echo "❌ Error: Server already running. Stop before starting."; \ + exit 1; \ + fi; + @echo "Starting Ruby V3 server..." + bundle install + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + bundle exec ruby app.rb & echo $$! > $(PID_FILE) + @echo "Ruby V3 server starting..." + +stop-server: + @if [ -f $(PID_FILE) ]; then \ + kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm $(PID_FILE); \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/ruby-v3-server/README.md b/test-server/ruby-v3-server/README.md new file mode 100644 index 00000000..0c27b3d8 --- /dev/null +++ b/test-server/ruby-v3-server/README.md @@ -0,0 +1,74 @@ +# Ruby S3 Encryption Client Test Server + +This is a Ruby implementation of the S3 Encryption Client test server +that provides an invariant interface around the S3 Encryption Client v3. +It's designed to work alongside other implementations of test servers for cross-language compatibility testing. + +## Overview + +The server provides a REST API that wraps the AWS S3 Encryption Client v3, +allowing tests to verify that all language implementations behave consistently. + +## Endpoints + +- `POST /client` - Create a new S3 encryption client instance +- `PUT /object/{bucket}/{key}` - Encrypt and store an object +- `GET /object/{bucket}/{key}` - Retrieve and decrypt an object +- `GET /health` - Health check endpoint + +## Configuration + +The server runs on port **8092** by default. + +## Setup + +1. Install Ruby 3.x +2. Install dependencies: + + ```bash + cd test-server/ruby-v2-server + bundle install + ``` + +3. Set up AWS credentials (via AWS CLI, environment variables, or IAM roles) + +4. Start the server: + + ```bash + ruby app.rb + # or using Rack + bundle exec rackup -p 8092 + ``` + +## Usage + +The server is designed to be used by the Java test suite in `test-server/java-tests/`. +The tests will automatically discover and use this server for cross-language compatibility testing. + +### Environment Variables + +- `TEST_SERVER_KMS_KEY_ARN` - KMS key ARN for encryption (defaults to test key) +- `TEST_SERVER_S3_BUCKET` - S3 bucket for testing (defaults to test bucket) + +## Architecture + +- `app.rb` - Main Sinatra application +- `lib/client_manager.rb` - Manages S3 encryption client instances +- `lib/metadata_utils.rb` - Handles metadata serialization/deserialization +- `lib/error_handlers.rb` - Smithy-compliant error responses + +## Error Handling + +The server returns errors in the format expected by the Smithy model: + +- `GenericServerError` - Internal server errors +- `S3EncryptionClientError` - Errors from the S3 Encryption Client + +## Compatibility + +This server is compatible with: + +- S3 Encryption Client v3 +- Legacy v1 clients (when `enableLegacyWrappingAlgorithms` is true) +- Legacy v2 clients (when `???` is true) +- Cross-language testing with other implementations diff --git a/test-server/ruby-v3-server/app.rb b/test-server/ruby-v3-server/app.rb new file mode 100644 index 00000000..6d56b8ee --- /dev/null +++ b/test-server/ruby-v3-server/app.rb @@ -0,0 +1,231 @@ +require 'sinatra' +require 'json' +require_relative 'lib/client_manager' +require_relative 'lib/metadata_utils' +require_relative 'lib/error_handlers' +require_relative 'lib/logger' + +class S3ECRubyServer < Sinatra::Base + configure do + set :port, 8092 + set :bind, '0.0.0.0' + set :show_exceptions, false + set :raise_errors, false + end + + def initialize + super + @client_manager = ClientManager.new + S3ECLogger.info("S3EC_SERVER: Ruby server initialized on port #{settings.port}") + end + + # Request logging middleware + before do + @request_id = S3ECLogger.generate_request_id + S3ECLogger.log_request(request.request_method, request.path_info, request.env, @request_id) + end + + # Response logging middleware + after do + S3ECLogger.log_response(response.status, @request_id) + end + + # Health check endpoint + get '/health' do + content_type :json + { status: 'OK', server: 'Ruby S3EC Test Server', port: settings.port.to_i }.to_json + end + + # POST /client - Create S3 encryption client + post '/client' do + begin + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Processing client creation request") + + # Parse request body + request_body = request.body.read + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Request body size: #{request_body.length} bytes") + + parsed_data = JSON.parse(request_body) + config = parsed_data['config'] || {} + + S3ECLogger.debug("CLIENT_ENDPOINT [#{@request_id}]: Parsed config: #{config.inspect}") + + # Create client using client manager + client_id = @client_manager.create_client(config) + + S3ECLogger.info("CLIENT_ENDPOINT [#{@request_id}]: Successfully created client #{client_id}") + + # Return client ID + content_type :json + { clientId: client_id }.to_json + + rescue JSON::ParserError => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'JSON parsing' }, @request_id) + ErrorHandlers.send_generic_server_error(self, "Invalid JSON in request body", 400) + rescue => e + S3ECLogger.log_error(e, { endpoint: '/client', operation: 'client creation', config: config }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, "Failed to create client: #{e.message}") + end + end + + # PUT /object/{bucket}/{key} - Encrypt and put object + put '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Processing PUT request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Get request body + body = request.body.read + S3ECLogger.debug("PUT_ENDPOINT [#{@request_id}]: Request body size: #{body.length} bytes") + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 put_object parameters + put_params = { + bucket: bucket, + key: key, + body: body + } + + # Add encryption context if present + put_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + + # Log S3 operation + S3ECLogger.log_s3_operation('put', bucket, key, encryption_context, "ClientID: #{client_id}, BodySize: #{body.length}") + + # Make the put_object request + response = client.put_object(put_params) + + S3ECLogger.info("PUT_ENDPOINT [#{@request_id}]: Successfully put object s3://#{bucket}/#{key}") + + # Prepare response metadata + response_metadata = MetadataUtils.map_to_array(encryption_context) + S3ECLogger.log_metadata_processing('response', encryption_context, response_metadata) + + # Return response matching Smithy model + content_type :json + { + bucket: bucket, + key: key, + metadata: response_metadata + }.to_json + + rescue Aws::S3::EncryptionV2::Errors::EncryptionError => e + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # GET /object/{bucket}/{key} - Get and decrypt object + get '/object/:bucket/:key' do + bucket = params[:bucket] + key = params[:key] + client_id = request.env['HTTP_CLIENTID'] + + begin + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Processing GET request for s3://#{bucket}/#{key}") + + # Validate client ID + unless client_id + S3ECLogger.log_validation_error('ClientID', 'missing', @request_id) + ErrorHandlers.send_generic_server_error(self, "ClientID header is required", 400) + end + + # Get client from cache + client = @client_manager.get_client(client_id) + unless client + S3ECLogger.log_validation_error('ClientID', client_id, @request_id) + ErrorHandlers.send_generic_server_error(self, "No client found for ClientID: #{client_id}", 404) + end + + # Parse metadata from header + metadata_header = request.env['HTTP_CONTENT_METADATA'] || '' + encryption_context = MetadataUtils.string_to_map(metadata_header) + S3ECLogger.log_metadata_processing('parse', metadata_header, encryption_context) + + # Prepare S3 get_object parameters + get_params = { + bucket: bucket, + key: key + } + + # Add encryption context if present + get_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + + # Log S3 operation + S3ECLogger.log_s3_operation('get', bucket, key, encryption_context, "ClientID: #{client_id}") + + # Make the get_object request + response = client.get_object(get_params) + + # Extract body and metadata + body = response.body.read + metadata = response.metadata || {} + + S3ECLogger.info("GET_ENDPOINT [#{@request_id}]: Successfully got object s3://#{bucket}/#{key}, BodySize: #{body.length}") + + # Set Content-Metadata header in response + metadata_str = MetadataUtils.map_to_string(metadata) + S3ECLogger.log_metadata_processing('response', metadata, metadata_str) + + headers['Content-Metadata'] = metadata_str unless metadata_str.empty? + + # Return the body as response + content_type 'application/octet-stream' + body + + rescue Aws::S3::EncryptionV2::Errors::DecryptionError => e + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) + ErrorHandlers.send_s3_encryption_client_error(self, e.message) + rescue StandardError => e + # Handle generic server errors (return as GenericServerError) + S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'generic_server' }, @request_id) + status_code = e.respond_to?(:code) ? e.code : 500 + ErrorHandlers.send_generic_server_error(self, e.message, status_code) + end + end + + # Global error handler + error do + error = env['sinatra.error'] + context = { + endpoint: request.path_info, + method: request.request_method, + params: params, + error_type: 'global_error_handler' + } + + S3ECLogger.log_error(error, context, @request_id) + ErrorHandlers.send_generic_server_error(self, "Internal server error: #{error.message}") + end + + # Start server when run directly + if __FILE__ == $0 + S3ECLogger.info("S3EC_SERVER: Starting Ruby server on port #{settings.port}...") + run! + end +end diff --git a/test-server/ruby-v3-server/config.ru b/test-server/ruby-v3-server/config.ru new file mode 100644 index 00000000..99d3a689 --- /dev/null +++ b/test-server/ruby-v3-server/config.ru @@ -0,0 +1,3 @@ +require_relative 'app' + +run S3ECRubyServer diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb new file mode 100644 index 00000000..d3b12b23 --- /dev/null +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -0,0 +1,74 @@ +require 'concurrent-ruby' +require 'securerandom' +require 'aws-sdk-s3' +require 'aws-sdk-kms' +require_relative 'logger' + +# Manages S3 Encryption Client instances +class ClientManager + def initialize + @client_cache = Concurrent::Hash.new + @kms_client = Aws::KMS::Client.new(region: 'us-west-2') + S3ECLogger.info("CLIENT_MANAGER: Initialized with KMS client for us-west-2") + end + + # Create a new S3 encryption client and return its ID + def create_client(config) + # Extract configuration + kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + enable_legacy_wrapping = config['enableLegacyWrappingAlgorithms'] || false + + raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? + + # Create S3 encryption client configuration + encryption_config = { + kms_key_id: kms_key_id, + kms_client: @kms_client, + key_wrap_schema: :kms_context, + content_encryption_schema: :aes_gcm_no_padding, + # Set security profile based on legacy wrapping algorithms setting + security_profile: enable_legacy_wrapping ? :v2_and_legacy : :v2 + } + + # Create the S3 encryption client + s3_client = Aws::S3::Client.new(region: 'us-west-2') + encryption_client = Aws::S3::EncryptionV2::Client.new( + client: s3_client, + **encryption_config + ) + + # Generate client ID and store in cache + client_id = SecureRandom.uuid + @client_cache[client_id] = encryption_client + + # Log client creation + S3ECLogger.log_client_creation(config, client_id) + S3ECLogger.log_cache_stats(@client_cache.size) + + client_id + end + + # Get a client by ID + def get_client(client_id) + client = @client_cache[client_id] + if client + S3ECLogger.log_client_cache_hit(client_id) + else + S3ECLogger.log_client_cache_miss(client_id) + end + client + end + + # Remove a client from cache (optional cleanup) + def remove_client(client_id) + removed = @client_cache.delete(client_id) + S3ECLogger.info("CLIENT_CACHE: Removed client #{client_id} from cache") if removed + S3ECLogger.log_cache_stats(@client_cache.size) + removed + end + + # Get cache size (for debugging) + def cache_size + @client_cache.size + end +end diff --git a/test-server/ruby-v3-server/lib/error_handlers.rb b/test-server/ruby-v3-server/lib/error_handlers.rb new file mode 100644 index 00000000..234a9a55 --- /dev/null +++ b/test-server/ruby-v3-server/lib/error_handlers.rb @@ -0,0 +1,42 @@ +# Error handling utilities to match Smithy error types +require 'json' + +class ErrorHandlers + # Create a response that matches the GenericServerError type from the Smithy model + # Used for internal server errors + def self.create_generic_server_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#GenericServerError', + 'message' => message + }.to_json + } + end + + # Create a response that matches the S3EncryptionClientError type from the Smithy model + # Used for errors thrown by the S3 Encryption Client + def self.create_s3_encryption_client_error(message, status_code = 500) + { + status: status_code, + headers: { 'Content-Type' => 'application/json' }, + body: { + '__type' => 'software.amazon.encryption.s3#S3EncryptionClientError', + 'message' => message + }.to_json + } + end + + # Helper method to send error response in Sinatra + def self.send_generic_server_error(app, message, status_code = 500) + error_response = create_generic_server_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end + + # Helper method to send S3EC error response in Sinatra + def self.send_s3_encryption_client_error(app, message, status_code = 500) + error_response = create_s3_encryption_client_error(message, status_code) + app.halt error_response[:status], error_response[:headers], error_response[:body] + end +end diff --git a/test-server/ruby-v3-server/lib/logger.rb b/test-server/ruby-v3-server/lib/logger.rb new file mode 100644 index 00000000..2febcbab --- /dev/null +++ b/test-server/ruby-v3-server/lib/logger.rb @@ -0,0 +1,105 @@ +require 'logger' +require 'securerandom' + +# Centralized logging utility for the S3EC Ruby server +class S3ECLogger + def self.instance + @instance ||= new + end + + def initialize + @logger = Logger.new(STDOUT) + @logger.level = ENV['LOG_LEVEL'] ? Logger.const_get(ENV['LOG_LEVEL'].upcase) : Logger::INFO + @logger.formatter = proc do |severity, datetime, progname, msg| + "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity}: #{msg}\n" + end + end + + # Generate a unique request ID for correlation + def self.generate_request_id + SecureRandom.hex(8) + end + + # Request/Response logging + def self.log_request(method, path, headers = {}, request_id = nil) + client_id = headers['HTTP_CLIENTID'] || headers['ClientID'] || 'none' + content_metadata = headers['HTTP_CONTENT_METADATA'] || headers['Content-Metadata'] || 'none' + + instance.logger.info("REQUEST [#{request_id}] #{method} #{path} | ClientID: #{client_id} | Metadata: #{content_metadata}") + end + + def self.log_response(status, request_id = nil, additional_info = "") + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("RESPONSE [#{request_id}] Status: #{status}#{info_str}") + end + + # Operation-level logging + def self.log_client_creation(config, client_id) + kms_key = config.dig('keyMaterial', 'kmsKeyId') || 'unknown' + legacy_enabled = config['enableLegacyWrappingAlgorithms'] || false + instance.logger.info("CLIENT_CREATION: Created S3EC client #{client_id} | KMS Key: #{kms_key} | Legacy: #{legacy_enabled}") + end + + def self.log_client_cache_hit(client_id) + instance.logger.debug("CACHE_HIT: Found client #{client_id} in cache") + end + + def self.log_client_cache_miss(client_id) + instance.logger.warn("CACHE_MISS: Client #{client_id} not found in cache") + end + + def self.log_cache_stats(cache_size) + instance.logger.debug("CACHE_STATS: Current client cache size: #{cache_size}") + end + + def self.log_s3_operation(operation, bucket, key, encryption_context = {}, additional_info = "") + enc_ctx_str = encryption_context.empty? ? "none" : encryption_context.inspect + info_str = additional_info.empty? ? "" : " | #{additional_info}" + instance.logger.info("S3_OPERATION: #{operation.upcase} s3://#{bucket}/#{key} | EncCtx: #{enc_ctx_str}#{info_str}") + end + + def self.log_metadata_processing(operation, input, output) + instance.logger.debug("METADATA_#{operation.upcase}: Input: #{input.inspect} | Output: #{output.inspect}") + end + + # Enhanced error logging + def self.log_error(error, context = {}, request_id = nil) + error_context = context.empty? ? "" : " | Context: #{context.inspect}" + instance.logger.error("ERROR [#{request_id}] #{error.class}: #{error.message}#{error_context}") + + if error.backtrace && instance.debug? + instance.logger.debug("ERROR_BACKTRACE [#{request_id}]:\n#{error.backtrace.join("\n")}") + end + end + + def self.log_validation_error(field, value, request_id = nil) + instance.logger.warn("VALIDATION_ERROR [#{request_id}] Invalid #{field}: #{value}") + end + + def self.log_aws_error(error, operation, request_id = nil) + instance.logger.error("AWS_ERROR [#{request_id}] #{operation} failed: #{error.class} - #{error.message}") + end + + # Standard logging methods + def self.debug(message) + instance.logger.debug(message) + end + + def self.info(message) + instance.logger.info(message) + end + + def self.warn(message) + instance.logger.warn(message) + end + + def self.error(message) + instance.logger.error(message) + end + + attr_reader :logger + + def debug? + @logger.debug? + end +end diff --git a/test-server/ruby-v3-server/lib/metadata_utils.rb b/test-server/ruby-v3-server/lib/metadata_utils.rb new file mode 100644 index 00000000..72015fcc --- /dev/null +++ b/test-server/ruby-v3-server/lib/metadata_utils.rb @@ -0,0 +1,50 @@ +# Utility class for handling metadata serialization/deserialization +# Matches the format used by Java and Python servers: [key]:[value],[key2]:[value2] +class MetadataUtils + # Convert metadata string to hash + # Input: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + # Output: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + def self.string_to_map(metadata_string) + return {} if metadata_string.nil? || metadata_string.empty? + + metadata = {} + entries = metadata_string.split(',') + + entries.each do |entry| + # Split on "]:[" to separate key and value + parts = entry.split(']:[') + if parts.length == 2 + # Remove remaining brackets from start and end + key = parts[0].delete_prefix("[") # Remove first character '[' + value = parts[1].delete_suffix("]") # Remove last character ']' + metadata[key] = value + else + raise "Malformed metadata list entry: #{entry}" + end + end + + metadata + end + + # Convert hash to metadata string + # Input: {"user-defined-enc-ctx-key" => "user-defined-enc-ctx-value", "user-defined-enc-ctx-key-2" => "user-defined-enc-ctx-value-2"} + # Output: "[user-defined-enc-ctx-key]:[user-defined-enc-ctx-value],[user-defined-enc-ctx-key-2]:[user-defined-enc-ctx-value-2]" + def self.map_to_string(metadata_hash) + return '' if metadata_hash.nil? || metadata_hash.empty? + + entries = metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + + entries.join(',') + end + + # Convert metadata hash to array format (for JSON responses) + def self.map_to_array(metadata_hash) + return [] if metadata_hash.nil? || metadata_hash.empty? + + metadata_hash.map do |key, value| + "[#{key}]:[#{value}]" + end + end +end diff --git a/test-server/ruby-v3-server/local-ruby-sdk b/test-server/ruby-v3-server/local-ruby-sdk new file mode 160000 index 00000000..e129cf37 --- /dev/null +++ b/test-server/ruby-v3-server/local-ruby-sdk @@ -0,0 +1 @@ +Subproject commit e129cf37c170254ebb631782cae145040cde6d0b From f759c668ea60cc257a707a06601946e3892ec4b7 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 19 Sep 2025 16:56:02 -0700 Subject: [PATCH 2/5] like this? --- test-server/Makefile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test-server/Makefile b/test-server/Makefile index 7008565c..66500260 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -18,7 +18,10 @@ start-servers: @echo "Starting all servers..." $(MAKE) start-all-servers @echo "Waiting for servers to start..." - $(MAKE) wait-all-servers + @for dir in $(SERVER_DIRS); do \ + echo "Waiting for server in $$dir..."; \ + $(MAKE) -C $$dir wait-for-server; \ + done start-all-servers: $(START_SERVER_TARGETS) @@ -105,3 +108,13 @@ wait-for-port: echo "Waiting for $$PORT to start ($$i/$(TIMEOUT))..."; \ sleep 1; \ done + +# Here are some helpful curl commands +# that you can use to test specific test servers: +test-create-client: + @echo $(PORT) + @curl -X POST \ + -H "Content-Type: application/json" \ + -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ + -d '{"config":{"enableLegacyUnauthenticatedModes":false,"enableDelayedAuthenticationMode":false,"enableLegacyWrappingAlgorithms":false,"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}}}' \ + http://localhost:$(PORT)/client \ No newline at end of file From dabe62dad32b5474777f7885711b788be659bdd4 Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 19 Sep 2025 20:58:05 -0700 Subject: [PATCH 3/5] change --- .../amazon/encryption/s3/RoundTripTests.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 23219611..9b950340 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -63,8 +63,6 @@ public class RoundTripTests { private static final String PHP_V3 = "PHP-V3"; private static final String RUBY_V2 = "Ruby-V2"; private static final String RUBY_V3 = "Ruby-V3"; - - private static final List serverList; private static final Map serverMap; @@ -345,13 +343,11 @@ public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLa .build()); fail("Expected exception!"); } catch (S3EncryptionClientError e) { - assertTrue( - e.getMessage().contains("Provided encryption context does not match information retrieved from S3") || - // Ruby error message - (e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context") - && decLang.languageName.startsWith("ruby-v") - ) - ); + if (decLang.languageName.equals(RUBY_V3) || decLang.languageName.equals(RUBY_V2)) { + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); + } else { + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + } } } From 52ef9aa8c9b05d18eeca4fe16725bd7fd942a1ce Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 19 Sep 2025 21:26:59 -0700 Subject: [PATCH 4/5] small change --- .../amazon/encryption/s3/RoundTripTests.java | 23 +++++-------------- test-server/ruby-v2-server/app.rb | 4 ++-- test-server/ruby-v3-server/app.rb | 4 ++-- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 9b950340..eff9157c 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -64,7 +64,6 @@ public class RoundTripTests { private static final String RUBY_V2 = "Ruby-V2"; private static final String RUBY_V3 = "Ruby-V3"; - private static final List serverList; private static final Map serverMap; private static final String KMS_KEY_ARN = System.getenv("TEST_SERVER_KMS_KEY_ARN") != null ? @@ -74,18 +73,8 @@ public class RoundTripTests { System.getenv("TEST_SERVER_S3_BUCKET") : "s3ec-test-server-github-bucket"; static { - serverList = new ArrayList<>(14); - serverList.add(new LanguageServerTarget(JAVA_V3, "8080")); - serverList.add(new LanguageServerTarget(PYTHON_V3, "8081")); - serverList.add(new LanguageServerTarget(GO_V3, "8082")); - serverList.add(new LanguageServerTarget(NET_V2, "8083")); - serverList.add(new LanguageServerTarget(NET_V3, "8084")); - serverList.add(new LanguageServerTarget(PHP_V2, "8087")); - serverList.add(new LanguageServerTarget(PHP_V3, "8093")); - serverList.add(new LanguageServerTarget(RUBY_V2, "8086")); - serverList.add(new LanguageServerTarget(RUBY_V3, "8092")); - - serverMap = new HashMap<>(14); + + serverMap = new HashMap<>(); serverMap.put(JAVA_V3, new LanguageServerTarget(JAVA_V3, "8080")); serverMap.put(PYTHON_V3, new LanguageServerTarget(PYTHON_V3, "8081")); serverMap.put(GO_V3, new LanguageServerTarget(GO_V3, "8082")); @@ -155,7 +144,7 @@ public String toString() { @BeforeAll public static void setup() { // Wait for servers to start - for (LanguageServerTarget server : serverList) { + for (LanguageServerTarget server : serverMap.values()) { if (!serverListening(server.getServerURI())) { throw new RuntimeException(String.format("Test Server for %s is not running at endpoint: %s", server.getLanguageName(), server.getServerURI())); } @@ -185,14 +174,14 @@ static S3ECTestServerClient testServerClientFor(LanguageServerTarget server) { } static Stream clientsForTest() { - return serverList.stream() + return serverMap.values().stream() .map(LanguageServerTarget::getLanguageName) .map(Arguments::of); } static Stream crossLanguageClients() { - return serverList.stream() - .flatMap(t1 -> serverList.stream() + return serverMap.values().stream() + .flatMap(t1 -> serverMap.values().stream() .flatMap(t2 -> Stream.of( Arguments.of(t1, t2) ))); diff --git a/test-server/ruby-v2-server/app.rb b/test-server/ruby-v2-server/app.rb index 755379d1..6096a5ea 100644 --- a/test-server/ruby-v2-server/app.rb +++ b/test-server/ruby-v2-server/app.rb @@ -16,7 +16,7 @@ class S3ECRubyServer < Sinatra::Base def initialize super @client_manager = ClientManager.new - S3ECLogger.info("S3EC_SERVER: Ruby server initialized on port #{settings.port}") + S3ECLogger.info("S3EC_SERVER: Ruby V2 server initialized on port #{settings.port}") end # Request logging middleware @@ -33,7 +33,7 @@ def initialize # Health check endpoint get '/health' do content_type :json - { status: 'OK', server: 'Ruby S3EC Test Server', port: settings.port.to_i }.to_json + { status: 'OK', server: 'Ruby V2 S3EC Test Server', port: settings.port.to_i }.to_json end # POST /client - Create S3 encryption client diff --git a/test-server/ruby-v3-server/app.rb b/test-server/ruby-v3-server/app.rb index 6d56b8ee..abc25932 100644 --- a/test-server/ruby-v3-server/app.rb +++ b/test-server/ruby-v3-server/app.rb @@ -16,7 +16,7 @@ class S3ECRubyServer < Sinatra::Base def initialize super @client_manager = ClientManager.new - S3ECLogger.info("S3EC_SERVER: Ruby server initialized on port #{settings.port}") + S3ECLogger.info("S3EC_SERVER: Ruby V3 server initialized on port #{settings.port}") end # Request logging middleware @@ -33,7 +33,7 @@ def initialize # Health check endpoint get '/health' do content_type :json - { status: 'OK', server: 'Ruby S3EC Test Server', port: settings.port.to_i }.to_json + { status: 'OK', server: 'Ruby V3 S3EC Test Server', port: settings.port.to_i }.to_json end # POST /client - Create S3 encryption client From 52dda01463b127f2a1cf1b5560d1d24e77e5446a Mon Sep 17 00:00:00 2001 From: Ryan Emery Date: Fri, 19 Sep 2025 22:11:01 -0700 Subject: [PATCH 5/5] Add filtering --- test-server/Makefile | 2 +- test-server/java-tests/build.gradle.kts | 2 + .../amazon/encryption/s3/RoundTripTests.java | 51 +++++++++++++++---- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 66500260..f23b738d 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -56,7 +56,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel integ + ./gradlew --build-cache --parallel integ -Dtest.filter.servers="$(FILTER)" @echo "Tests completed successfully" # Stop the servers diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 3a00e348..bc37514f 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -48,6 +48,8 @@ tasks { classpath = sourceSets["it"].runtimeClasspath outputs.upToDateWhen { false } outputs.cacheIf { false } + // Passing information from Gradle into the tests so that we can filter our servers + systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) // For debugging // // Enable System.out output // testLogging { diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index eff9157c..459bbbd3 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -14,11 +14,14 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; @@ -73,19 +76,45 @@ public class RoundTripTests { System.getenv("TEST_SERVER_S3_BUCKET") : "s3ec-test-server-github-bucket"; static { - - serverMap = new HashMap<>(); - serverMap.put(JAVA_V3, new LanguageServerTarget(JAVA_V3, "8080")); - serverMap.put(PYTHON_V3, new LanguageServerTarget(PYTHON_V3, "8081")); - serverMap.put(GO_V3, new LanguageServerTarget(GO_V3, "8082")); - serverMap.put(NET_V2, new LanguageServerTarget(NET_V2, "8083")); - serverMap.put(NET_V3, new LanguageServerTarget(NET_V3, "8084")); - serverMap.put(PHP_V2, new LanguageServerTarget(PHP_V2, "8087")); - serverMap.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); - serverMap.put(RUBY_V2, new LanguageServerTarget(RUBY_V2, "8086")); - serverMap.put(RUBY_V3, new LanguageServerTarget(RUBY_V3, "8092")); + final Map servers = new LinkedHashMap<>(); + servers.put(JAVA_V3, new LanguageServerTarget(JAVA_V3, "8080")); + servers.put(PYTHON_V3, new LanguageServerTarget(PYTHON_V3, "8081")); + servers.put(GO_V3, new LanguageServerTarget(GO_V3, "8082")); + servers.put(NET_V2, new LanguageServerTarget(NET_V2, "8083")); + servers.put(NET_V3, new LanguageServerTarget(NET_V3, "8084")); + servers.put(PHP_V2, new LanguageServerTarget(PHP_V2, "8087")); + servers.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); + servers.put(RUBY_V2, new LanguageServerTarget(RUBY_V2, "8086")); + servers.put(RUBY_V3, new LanguageServerTarget(RUBY_V3, "8092")); + + serverMap = filterServers(servers); } + private static Map filterServers(Map allServers) { + + final String maybeFilter = System.getProperty("test.filter.servers"); + if (maybeFilter == null || maybeFilter.trim().isEmpty()) { + return allServers; // No filtering - use all servers + } + + final String[] filters = Arrays.stream(maybeFilter.split(",")) + .map(String::trim) + .map(String::toLowerCase) + .toArray(String[]::new); + + return allServers.entrySet().stream() + .filter(entry -> { + String key = entry.getKey().toLowerCase(); + return Arrays.stream(filters).anyMatch(key::contains); + }) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (e1, e2) -> e1, // merge function (not really needed) + LinkedHashMap::new // preserve order + )); + } + // Encryption context validation behavior varies by implementation: // - Go: Does not validate encryption context on decrypt operations // - .NET: Only validates against encryption context stored in the object metadata