diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 880ca3f4..182b7f47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,16 +41,6 @@ jobs: git config --global credential.helper store echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials - - name: Cache git submodules - uses: actions/cache@v4 - with: - path: | - .git/modules - test-server/*/.git - key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }} - restore-keys: | - ${{ runner.os }}-submodules- - - name: Optimize git for performance run: | git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }} @@ -59,13 +49,12 @@ jobs: - name: Checkout submodules with --jobs run: | - git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }} + git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }} - name: Update cpp submodules recursively with --jobs run: | git submodule update --init --recursive \ - --depth 1 \ - --filter=blob:none \ + --depth 1 --single-branch \ --jobs ${{ steps.cpu-count.outputs.count }} \ --force \ test-server/cpp-v2-transition-server/aws-sdk-cpp \ @@ -171,9 +160,7 @@ jobs: - name: Wait for servers to start run: cd test-server && make wait-all-servers env: - AWS_REGION: us-west-2 - TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }} - TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }} + MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }} - name: Run run-tests run: cd test-server && make run-tests diff --git a/test-server/Makefile b/test-server/Makefile index 9b18b857..94a76b3f 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -2,9 +2,6 @@ .PHONY: all start-servers run-tests stop-servers clean ci check-env help -# Default target -all: start-all-servers wait-all-servers run-tests - # CI target for GitHub Actions ci: $(MAKE) build-all-servers @@ -20,13 +17,19 @@ START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) # Build all servers in parallel -build-all-servers: export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1) -build-all-servers: $(BUILD_SERVER_TARGETS) +build-all-servers: + @echo "[`date +%H:%M:%S`] Building all servers..." + @$(MAKE) $(BUILD_SERVER_TARGETS) + @echo "[`date +%H:%M:%S`] All servers built." + @echo "Stopping dotnet servers... this is here to speed up CI because if this target is run with -j it will stay open until it shutdown." + @dotnet build-server shutdown + @echo "[`date +%H:%M:%S`] Dotnet build servers stopped" $(BUILD_SERVER_TARGETS): build-%: @if [ -f $*/Makefile ]; then \ - echo "Building server in $*..."; \ - $(MAKE) -C $* build-server; \ + echo "[`date +%H:%M:%S`] Building server in $*..." && \ + $(MAKE) -C $* build-server && \ + echo "[`date +%H:%M:%S`] Server $* built successfully"; \ else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ @@ -44,18 +47,17 @@ start-servers: $(MAKE) -C $$dir wait-for-server; \ done -# Start servers sequentially (no parallel execution) start-all-servers: - @$(MAKE) MAKEFLAGS= $(START_SERVER_TARGETS) + @$(MAKE) $(START_SERVER_TARGETS) $(START_SERVER_TARGETS): start-%: @if [ -f $*/Makefile ]; then \ - echo "Starting server in $*..."; \ + echo "Starting server in $*..." && \ $(MAKE) -C $* start-server; \ else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ - fi; + fi wait-all-servers: @echo "Waiting for all servers to be ready..." @@ -64,12 +66,12 @@ wait-all-servers: $(WAIT_SERVER_TARGETS): wait-%: @if [ -f $*/Makefile ]; then \ - echo "Waiting server in $*..."; \ + echo "Waiting server in $*..." && \ $(MAKE) -C $* wait-for-server; \ else \ echo "❌ Error: no Makefile found in $*"; \ exit 1; \ - fi; + fi # Run the Java tests @@ -82,7 +84,9 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --info --parallel integ -Dtest.filter.servers="$(FILTER)" + ./gradlew --build-cache --info --parallel --no-daemon integ \ + $(if $(TEST),--tests "$(TEST)",) \ + -Dtest.filter.servers="$(FILTER)" @echo "Tests completed successfully" # Stop the servers diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 77357c37..2d0a4b55 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -11,7 +11,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp V2 server..." - cd build && make + cd build && $(MAKE) start-server: @echo "Starting Cpp V2 server..." @@ -20,7 +20,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V2 server starting..." stop-server: @@ -31,7 +31,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f build/server.log + @rm -f server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index a2b05810..e8ffe770 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -1,8 +1,53 @@ +/* + * S3 Encryption Test Server - C++ V2 + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + #include +#include #include #include #include #include +#include +#include #include #include #include @@ -10,14 +55,33 @@ #include #include +#include #include #include +#include +#include +#include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> - client_cache; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() std::string generate_uuid() { uuid_t uuid; @@ -27,6 +91,55 @@ std::string generate_uuid() { return std::string(uuid_str); } +std::shared_ptr get_client(const std::string &client_id) +{ + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V2] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V2] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); +} + std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -52,6 +165,9 @@ std::string make_error(const std::string &message, int status_code) { MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller + try { json request = json::parse(body); std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; @@ -68,17 +184,32 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, CryptoConfigurationV2 config(materials); if (legacy1 || legacy2) config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (legacy2) + config.SetUnAuthenticatedRangeGet(RangeGetMode::ALL); if (inst_put) config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - auto encryption_client = std::make_shared(config); + // Each client gets a large connection pool since we cannot share HTTP clients + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + // Disable automatic checksum calculation for encrypted streams + // The ChecksumInterceptor cannot handle non-seekable SymmetricCryptoStream + // which causes intermittent "BadDigest: CRC64NVME you specified did not match" errors + // when the stream gets consumed during checksum calculation and can't be rewound + clientConfig.checksumConfig.requestChecksumCalculation = + Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED; + + auto encryption_client = std::make_shared(config, clientConfig); std::string client_id = generate_uuid(); - client_cache[client_id] = encryption_client; + set_client(client_id, encryption_client); json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2] handle_create_client exception: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"An exception was thrown.\"}"); } catch (...) { @@ -89,13 +220,18 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: metadata is empty\n"); return; } + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; + int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -128,6 +264,9 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + // Add to map map.emplace(key, value); @@ -138,14 +277,25 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } + + fprintf(stderr, "[CPP-V2] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + std::string bucket, + std::string key, + std::string client_id, + std::string metadata, + std::string range) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu, range=%s\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length(), range.c_str()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V2] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -153,46 +303,107 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(key); - - // S3EncryptionGetObjectOutcome outcome ; - // if (metadata.empty()) { - // outcome = it->second->GetObject(request); - // } else { - // Aws::Map kmsContextMap; - // fill_context(kmsContextMap, metadata); - // outcome = it->second->GetObject(request, kmsContextMap); - // } + + // Add range header if provided + if (!range.empty()) { + request.SetRange(range); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: Setting range=%s\n", range.c_str()); + } Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - auto outcome = it->second->GetObject(request, kmsContextMap); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid + auto outcome = client->GetObject(request, kmsContextMap); + + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); - return send_response(connection, 200, content); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V2] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V2] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V2] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V2] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V2] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V2] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - auto msg = make_error("An exception was thrown", 500); + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V2] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &body, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V2] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V2] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -200,85 +411,249 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - auto stream = std::make_shared(body); + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); request.SetBody(stream); - auto outcome = it->second->PutObject(request, kmsContextMap); + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation + auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V2] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V2] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; // Safe to delete now - all synchronous operations are complete + *con_cls = nullptr; + } +} + MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - std::string method_str(method); - bool is_push = method_str == "POST" || method_str == "PUT"; - static int dummy; - if (*con_cls == nullptr) { - if (is_push) { - *con_cls = new std::string(); - } else { - *con_cls = &dummy; + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V2] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); } + + // Initialize request context on first call + if (*con_cls == nullptr) { + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); + fprintf(stderr, "[CPP-V2] REQUEST INIT: allocated new request context for %s %s\n", method, url); return MHD_YES; } - if (is_push && *upload_data_size) { + + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V2] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V2] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + + // Accumulate request body data for POST/PUT requests + if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - std::string url_str(url); - + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V2] PROCESSING: %s %s\n", method, url); + + // Handle client creation endpoint if (is_push && url_str == "/client") { - std::unique_ptr body(static_cast(*con_cls)); - return handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2] Handling /client endpoint\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2] /client handler returned: %d\n", result); + return result; } + // Handle object operations if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V2] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); - std::string metadata = get_header_value(connection, "content-metadata"); + + fprintf(stderr, "[CPP-V2] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + if (method_str == "GET") { - return handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V2] Dispatching to handle_get_object\n"); + std::string range = get_header_value(connection, "Range"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata, range); + fprintf(stderr, "[CPP-V2] handle_get_object returned: %d\n", result); + return result; } else if (method_str == "PUT") { - std::unique_ptr body(static_cast(*con_cls)); - *upload_data_size = 0; - return handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2] Dispatching to handle_put_object\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2] handle_put_object returned: %d\n", result); + return result; + } else { + fprintf(stderr, "[CPP-V2] Method not allowed: %s\n", method); + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V2] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V2] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V2] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V2] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V2] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; } int main() { Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; + fprintf(stderr, "[CPP-V2] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + int port = 8085; + struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, - &request_handler, NULL, MHD_OPTION_END); + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, + MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "Failed to start server on port %d\n", port); + fprintf(stderr, "[CPP-V2] Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile index 16b70796..0383b4d8 100644 --- a/test-server/cpp-v2-transition-server/Makefile +++ b/test-server/cpp-v2-transition-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp transition server..." - cd build && make + cd build && $(MAKE) start-server: @echo "Starting Cpp transition server..." @@ -19,7 +19,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp transition server starting..." stop-server: @@ -30,7 +30,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f build/server.log + @rm -f server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v2-transition-server/aws-sdk-cpp b/test-server/cpp-v2-transition-server/aws-sdk-cpp index 87402c99..cec1f193 160000 --- a/test-server/cpp-v2-transition-server/aws-sdk-cpp +++ b/test-server/cpp-v2-transition-server/aws-sdk-cpp @@ -1 +1 @@ -Subproject commit 87402c99fd3c9107c6ccc6edf545fd4b05b2b551 +Subproject commit cec1f1933be672f65627c11ff2d853e07c5b3ff4 diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp index 1fcedc3c..9e9f942d 100644 --- a/test-server/cpp-v2-transition-server/main.cpp +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -1,8 +1,55 @@ +/* + * S3 Encryption Test Server - C++ V2 Transition + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + #include +#include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -10,14 +57,33 @@ #include #include +#include #include #include +#include +#include +#include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> - client_cache; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() std::string generate_uuid() { uuid_t uuid; @@ -27,6 +93,55 @@ std::string generate_uuid() { return std::string(uuid_str); } +std::shared_ptr get_client(const std::string &client_id) +{ + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); +} + std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -69,6 +184,9 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller + try { json request = json::parse(body); std::string commitmentPolicy = get_config(request, "commitmentPolicy"); @@ -79,7 +197,41 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, return MHD_YES; } - std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + // Extract all key material types + std::string kms_key_id; + std::string rsa_key_blob; + std::string aes_key_blob; + + if (request["config"]["keyMaterial"].contains("kmsKeyId") && + !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { + kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + } + if (request["config"]["keyMaterial"].contains("rsaKey") && + !request["config"]["keyMaterial"]["rsaKey"].is_null()) { + rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; + } + if (request["config"]["keyMaterial"].contains("aesKey") && + !request["config"]["keyMaterial"]["aesKey"].is_null()) { + aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; + } + + // Validate that only one key type is provided + int key_count = 0; + if (!kms_key_id.empty()) key_count++; + if (!rsa_key_blob.empty()) key_count++; + if (!aes_key_blob.empty()) key_count++; + + if (key_count != 1) { + return send_response(connection, 400, + "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); + } + + // RSA is not supported by C++ SDK + if (!rsa_key_blob.empty()) { + return send_response(connection, 501, + "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); + } + bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; bool inst_put = false; @@ -88,22 +240,56 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - auto materials = - std::make_shared(kms_key_id); - CryptoConfigurationV2 config(materials); + // Create CryptoConfigurationV2 based on key type + std::shared_ptr config; + + if (!aes_key_blob.empty()) { + // Base64 decode the AES key + Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); + if (decoded.GetLength() == 0) { + return send_response(connection, 400, + "{\"error\":\"Failed to decode AES key\"}"); + } + + Aws::Utils::CryptoBuffer key_buffer( + decoded.GetUnderlyingData(), + decoded.GetLength() + ); + + auto materials = std::make_shared< + Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( + key_buffer + ); + config = std::make_shared(materials); + } else if (!kms_key_id.empty()) { + auto materials = std::make_shared(kms_key_id); + config = std::make_shared(materials); + } else { + return send_response(connection, 400, + "{\"error\":\"No valid key material provided\"}"); + } + + // Apply common configuration settings (applies to both AES and KMS) if (legacy1 || legacy2) - config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + config->SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (legacy2) + config->SetUnAuthenticatedRangeGet(RangeGetMode::ALL); if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - - auto encryption_client = std::make_shared(config); + config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Create S3EncryptionClientV2 with standard configuration + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + auto encryption_client = std::make_shared(*config, clientConfig); std::string client_id = generate_uuid(); - client_cache[client_id] = encryption_client; + set_client(client_id, encryption_client); json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] handle_create_client exception: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"An exception was thrown.\"}"); } catch (...) { @@ -114,13 +300,18 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: metadata is empty\n"); return; } + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; + int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -153,6 +344,9 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + // Add to map map.emplace(key, value); @@ -163,14 +357,25 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + std::string bucket, + std::string key, + std::string client_id, + std::string metadata, + std::string range) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu, range=%s\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length(), range.c_str()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -178,46 +383,107 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(key); - - // S3EncryptionGetObjectOutcome outcome ; - // if (metadata.empty()) { - // outcome = it->second->GetObject(request); - // } else { - // Aws::Map kmsContextMap; - // fill_context(kmsContextMap, metadata); - // outcome = it->second->GetObject(request, kmsContextMap); - // } + + // Add range header if provided + if (!range.empty()) { + request.SetRange(range); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: Setting range=%s\n", range.c_str()); + } Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - auto outcome = it->second->GetObject(request, kmsContextMap); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid + auto outcome = client->GetObject(request, kmsContextMap); + + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); - return send_response(connection, 200, content); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2-TRANSITION] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - auto msg = make_error("An exception was thrown", 500); + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &body, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -225,86 +491,249 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - auto stream = std::make_shared(body); + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); request.SetBody(stream); - auto outcome = it->second->PutObject(request, kmsContextMap); + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation + auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V2-TRANSITION] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; // Safe to delete now - all synchronous operations are complete + *con_cls = nullptr; + } +} + MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - std::string method_str(method); - bool is_push = method_str == "POST" || method_str == "PUT"; - static int dummy; - if (*con_cls == nullptr) { - if (is_push) { - *con_cls = new std::string(); - } else { - *con_cls = &dummy; + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); } + + // Initialize request context on first call + if (*con_cls == nullptr) { + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST INIT: allocated new request context for %s %s\n", method, url); return MHD_YES; } - if (is_push && *upload_data_size) { + + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + + // Accumulate request body data for POST/PUT requests + if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } - - std::string url_str(url); - + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V2-TRANSITION] PROCESSING: %s %s\n", method, url); + + // Handle client creation endpoint if (is_push && url_str == "/client") { - std::unique_ptr body(static_cast(*con_cls)); - return handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2-TRANSITION] Handling /client endpoint\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V2-TRANSITION] /client handler returned: %d\n", result); + return result; } + // Handle object operations if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V2-TRANSITION] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); - std::string metadata = get_header_value(connection, "content-metadata"); + + fprintf(stderr, "[CPP-V2-TRANSITION] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + if (method_str == "GET") { - return handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_get_object\n"); + std::string range = get_header_value(connection, "Range"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata, range); + fprintf(stderr, "[CPP-V2-TRANSITION] handle_get_object returned: %d\n", result); + return result; } else if (method_str == "PUT") { - std::unique_ptr body(static_cast(*con_cls)); - *upload_data_size = 0; - return handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] Dispatching to handle_put_object\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V2-TRANSITION] handle_put_object returned: %d\n", result); + return result; + } else { + fprintf(stderr, "[CPP-V2-TRANSITION] Method not allowed: %s\n", method); + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V2-TRANSITION] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V2-TRANSITION] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V2-TRANSITION] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; } int main() { Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; + fprintf(stderr, "[CPP-V2-TRANSITION] [WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + int port = 8097; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, - &request_handler, NULL, MHD_OPTION_END); + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, + MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "Failed to start server on port %d\n", port); + fprintf(stderr, "[CPP-V2-TRANSITION] Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } diff --git a/test-server/cpp-v3-server/CMakeLists.txt b/test-server/cpp-v3-server/CMakeLists.txt index b282dbc4..0faac5f0 100644 --- a/test-server/cpp-v3-server/CMakeLists.txt +++ b/test-server/cpp-v3-server/CMakeLists.txt @@ -7,6 +7,7 @@ set(CMAKE_CXX_STANDARD 17) set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") +set(ENABLE_ADDRESS_SANITIZER ON CACHE BOOL "Enable Address Sanitizer") # Add AWS SDK as subdirectory add_subdirectory(aws-sdk-cpp) @@ -18,12 +19,21 @@ find_package(nlohmann_json REQUIRED) add_executable(s3ec-server main.cpp) -target_include_directories(s3ec-server PRIVATE +# Enable Address Sanitizer for the executable +target_compile_options(s3ec-server PRIVATE -fsanitize=address -fno-omit-frame-pointer) +target_link_options(s3ec-server PRIVATE -fsanitize=address) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_include_directories(s3ec-server PRIVATE ${LIBMICROHTTPD_INCLUDE_DIRS} /opt/homebrew/include ) -target_link_directories(s3ec-server PRIVATE +target_link_directories(s3ec-server PRIVATE ${LIBMICROHTTPD_LIBRARY_DIRS} /opt/homebrew/lib ) @@ -36,4 +46,4 @@ target_link_libraries(s3ec-server aws-cpp-sdk-s3-encryption nlohmann_json::nlohmann_json uuid -) \ No newline at end of file +) diff --git a/test-server/cpp-v3-server/Makefile b/test-server/cpp-v3-server/Makefile index 46f0c9db..e90c8d73 100644 --- a/test-server/cpp-v3-server/Makefile +++ b/test-server/cpp-v3-server/Makefile @@ -10,7 +10,7 @@ build/s3ec-server: build-server: | build/s3ec-server @echo "Building Cpp V3 server..." - cd build && make + cd build && $(MAKE) start-server: @echo "Starting Cpp V3 server..." @@ -19,7 +19,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE) + ./s3ec-server > ../server.log 2>&1 & echo $$! > ../$(PID_FILE) @echo "Cpp V3 server starting..." stop-server: @@ -30,7 +30,7 @@ stop-server: kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \ rm -f $(PID_FILE); \ fi - @rm -f build/server.log + @rm -f server.log @echo "Server stopped" wait-for-server: diff --git a/test-server/cpp-v3-server/aws-sdk-cpp b/test-server/cpp-v3-server/aws-sdk-cpp index 87402c99..cec1f193 160000 --- a/test-server/cpp-v3-server/aws-sdk-cpp +++ b/test-server/cpp-v3-server/aws-sdk-cpp @@ -1 +1 @@ -Subproject commit 87402c99fd3c9107c6ccc6edf545fd4b05b2b551 +Subproject commit cec1f1933be672f65627c11ff2d853e07c5b3ff4 diff --git a/test-server/cpp-v3-server/main.cpp b/test-server/cpp-v3-server/main.cpp index 1f74974c..4e7227df 100644 --- a/test-server/cpp-v3-server/main.cpp +++ b/test-server/cpp-v3-server/main.cpp @@ -1,8 +1,56 @@ +/* + * S3 Encryption Test Server - C++ V3 + * + * CONCURRENCY AND SYNCHRONIZATION DESIGN: + * + * 1. Threading Model: + * - Uses MHD_USE_POLL_INTERNALLY with fixed thread pool + * - Thread pool size = CPU cores * 2 (auto-detected at startup) + * - Threads are reused across connections for efficiency + * - I/O multiplexing (poll) distributes connections across thread pool + * - All S3 operations are SYNCHRONOUS - server waits for S3 completion before responding + * - POLL mechanism avoids FD_SETSIZE=1024 limitation of select() + * + * 2. Resource Scaling: + * - All limits automatically scale with detected CPU count: + * * Thread pool size = num_cores * 2 + * * Connection limit = num_cores * 2 + * * S3 client maxConnections = num_cores * 2 + * - Multiplier of 2 accounts for I/O blocking without starving throughput + * - Ensures optimal resource usage on any hardware configuration + * + * 3. Client Cache (client_cache_secret): + * - Protected by std::shared_mutex for efficient concurrent access + * - get_client() uses shared_lock (multiple threads can read simultaneously) + * - set_client() uses unique_lock (exclusive write access) + * - This allows concurrent GET/PUT operations without serialization + * - UUID-based keys guarantee uniqueness (always insert, never update) + * + * 4. Memory Management: + * - Request body allocated in request_handler (*con_cls = new std::string()) + * - Body lifetime managed by libmicrohttpd - valid until request_completed() + * - All handler functions complete synchronously before returning + * - request_completed() safely deletes body after response sent + * - No memory leaks under sustained concurrent load + * + * 5. Synchronous Operation Guarantees: + * - GetObject: Waits for S3, reads full response stream, then returns + * - PutObject: Waits for S3 operation to complete, then returns + * - No async callbacks or background operations + * - Client receives response only after S3 operation completes + */ + #include +#include #include +#include #include #include #include +#include +#include +#include +#include #include #include #include @@ -12,12 +60,31 @@ #include #include #include +#include +#include +#include +#include +#include using json = nlohmann::json; using namespace Aws::S3Encryption; using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; -std::unordered_map> - client_cache; + +// LRU cache for S3 encryption clients +// Limits memory and connection pool growth by evicting least recently used clients +const size_t MAX_CACHED_CLIENTS = 100; // Reasonable limit for concurrent test operations + +struct ClientCacheEntry { + std::shared_ptr client; + std::list::iterator lru_iter; +}; + +std::unordered_map client_cache_secret; +std::list lru_order; // Most recently used at front +std::shared_timed_mutex client_mutex; // Using shared_timed_mutex (C++14 compatible) for concurrent reads + +// Threading configuration - set at startup based on CPU cores +unsigned int g_thread_pool_size = 8; // Default, will be overwritten in main() std::string generate_uuid() { uuid_t uuid; @@ -27,6 +94,55 @@ std::string generate_uuid() { return std::string(uuid_str); } +std::shared_ptr get_client(const std::string &client_id) +{ + // Need unique_lock to update LRU order even on reads + std::unique_lock lock(client_mutex); + auto it = client_cache_secret.find(client_id); + if (it == client_cache_secret.end()) { + return std::shared_ptr(); + } else { + // Move to front of LRU list (mark as most recently used) + lru_order.erase(it->second.lru_iter); + lru_order.push_front(client_id); + it->second.lru_iter = lru_order.begin(); + + return it->second.client; + } +} + +void set_client(const std::string &client_id, std::shared_ptr client) +{ + // UUID guarantees unique keys - always insert, never update + // Still need exclusive lock because std::unordered_map isn't thread-safe for concurrent inserts + std::unique_lock lock(client_mutex); + + // Add to front of LRU list (most recently used) + lru_order.push_front(client_id); + + ClientCacheEntry entry; + entry.client = client; + entry.lru_iter = lru_order.begin(); + + client_cache_secret.emplace(client_id, entry); + + // Evict least recently used clients if we exceed the limit + while (client_cache_secret.size() > MAX_CACHED_CLIENTS) { + std::string lru_client_id = lru_order.back(); + lru_order.pop_back(); + + auto evict_it = client_cache_secret.find(lru_client_id); + if (evict_it != client_cache_secret.end()) { + fprintf(stderr, "[CPP-V3] [CACHE-EVICT] Evicting client %s (cache size was %zu)\n", + lru_client_id.c_str(), client_cache_secret.size()); + client_cache_secret.erase(evict_it); + } + } + + fprintf(stderr, "[CPP-V3] [CACHE-ADD] Added client %s (cache size now %zu)\n", + client_id.c_str(), client_cache_secret.size()); +} + std::string get_header_value(struct MHD_Connection *connection, const char *key) { const char *value = @@ -67,9 +183,47 @@ std::string get_config(json & request, const char * x) MHD_Result handle_create_client(struct MHD_Connection *connection, const std::string &body) { + // Body is kept alive by *con_cls until request_completed fires, so it's safe to use directly + // All operations here are synchronous and complete before returning to caller + try { json request = json::parse(body); - std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + + // Extract all key material types + std::string kms_key_id; + std::string rsa_key_blob; + std::string aes_key_blob; + + if (request["config"]["keyMaterial"].contains("kmsKeyId") && + !request["config"]["keyMaterial"]["kmsKeyId"].is_null()) { + kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + } + if (request["config"]["keyMaterial"].contains("rsaKey") && + !request["config"]["keyMaterial"]["rsaKey"].is_null()) { + rsa_key_blob = request["config"]["keyMaterial"]["rsaKey"]; + } + if (request["config"]["keyMaterial"].contains("aesKey") && + !request["config"]["keyMaterial"]["aesKey"].is_null()) { + aes_key_blob = request["config"]["keyMaterial"]["aesKey"]; + } + + // Validate that only one key type is provided + int key_count = 0; + if (!kms_key_id.empty()) key_count++; + if (!rsa_key_blob.empty()) key_count++; + if (!aes_key_blob.empty()) key_count++; + + if (key_count != 1) { + return send_response(connection, 400, + "{\"error\":\"KeyMaterial must contain exactly one non-null key type\"}"); + } + + // RSA is not supported by C++ SDK + if (!rsa_key_blob.empty()) { + return send_response(connection, 501, + "{\"error\":\"RSA key wrapping is not supported in C++ S3 Encryption Client\"}"); + } + bool legacy1 = request["config"]["enableLegacyWrappingAlgorithms"]; bool legacy2 = request["config"]["enableLegacyUnauthenticatedModes"]; bool inst_put = false; @@ -78,33 +232,81 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, inst_put = request["config"]["instructionFileConfig"]["enableInstructionFilePutObject"]; } - auto materials = - std::make_shared(kms_key_id); - CryptoConfigurationV3 config(materials); - if (legacy1 || legacy2) - config.AllowLegacy(); - if (inst_put) - config.SetStorageMethod(StorageMethod::INSTRUCTION_FILE); - std::string commitmentPolicy = get_config(request, "commitmentPolicy"); std::string encryptionAlgorithm = get_config(request, "encryptionAlgorithm"); - + + // Create CryptoConfigurationV3 based on key type + std::optional config; + + if (!aes_key_blob.empty()) { + // Base64 decode the AES key + Aws::Utils::ByteBuffer decoded = Aws::Utils::HashingUtils::Base64Decode(aes_key_blob); + if (decoded.GetLength() == 0) { + return send_response(connection, 400, + "{\"error\":\"Failed to decode AES key\"}"); + } + + Aws::Utils::CryptoBuffer key_buffer( + decoded.GetUnderlyingData(), + decoded.GetLength() + ); + + auto materials = std::make_shared< + Aws::S3Encryption::Materials::SimpleEncryptionMaterialsWithGCMAAD>( + key_buffer + ); + config.emplace(materials); + } else if (!kms_key_id.empty()) { + auto materials = std::make_shared(kms_key_id); + config.emplace(materials); + } else { + return send_response(connection, 400, + "{\"error\":\"No valid key material provided\"}"); + } + + // Apply common configuration settings (applies to both AES and KMS) + if (legacy1 || legacy2) + config->AllowLegacy(); + if (legacy2) + config->SetUnAuthenticatedRangeGet(RangeGetMode::ALL); + if (inst_put) + config->SetStorageMethod(StorageMethod::INSTRUCTION_FILE); + + // Configure commitment policy (applies to both AES and KMS) if (commitmentPolicy == "REQUIRE_ENCRYPT_REQUIRE_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - if (encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF" || + encryptionAlgorithm == "ALG_AES_256_CBC_IV16_NO_KDF") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_REQUIRE_DECRYPT); } else if (commitmentPolicy == "REQUIRE_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); + if (encryptionAlgorithm == "ALG_AES_256_GCM_IV12_TAG16_NO_KDF") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config->SetCommitmentPolicy(CommitmentPolicy::REQUIRE_ENCRYPT_ALLOW_DECRYPT); } else if (commitmentPolicy == "FORBID_ENCRYPT_ALLOW_DECRYPT") { - if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") return unsupported(connection, commitmentPolicy, encryptionAlgorithm); - config.SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); + if (encryptionAlgorithm == "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY") { + return unsupported(connection, commitmentPolicy, encryptionAlgorithm); + } + config->SetCommitmentPolicy(CommitmentPolicy::FORBID_ENCRYPT_ALLOW_DECRYPT); } - - auto encryption_client = std::make_shared(config); + + // Create S3EncryptionClientV3 with standard configuration + Aws::Client::ClientConfiguration clientConfig; + clientConfig.maxConnections = 512; // Large pool per client + clientConfig.retryStrategy = Aws::Client::InitRetryStrategy("standard"); + + // Disable automatic checksum calculation for encrypted streams + // The ChecksumInterceptor cannot handle non-seekable SymmetricCryptoStream + // which causes intermittent "BadDigest: CRC64NVME you specified did not match" errors + // when the stream gets consumed during checksum calculation and can't be rewound + clientConfig.checksumConfig.requestChecksumCalculation = + Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED; + + auto encryption_client = std::make_shared(*config, clientConfig); std::string client_id = generate_uuid(); - client_cache[client_id] = encryption_client; + set_client(client_id, encryption_client); json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); @@ -120,13 +322,18 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, void fill_context(Aws::Map &map, const std::string &metadata) { if (metadata.empty()) { + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: metadata is empty\n"); return; } + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: raw metadata='%s' (length=%zu)\n", + metadata.c_str(), metadata.length()); + // Parse metadata format: [key1]:[value1],[key2]:[value2],... // or single pair: [key]:[value] std::string current = metadata; size_t pos = 0; + int pair_count = 0; while (pos < current.length()) { // Find opening bracket for key @@ -159,6 +366,9 @@ void fill_context(Aws::Map &map, std::string value = current.substr(value_start + 1, value_end - value_start - 1); + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: parsed pair #%d: key='%s', value='%s'\n", + ++pair_count, key.c_str(), value.c_str()); + // Add to map map.emplace(key, value); @@ -169,14 +379,25 @@ void fill_context(Aws::Map &map, pos = comma + 1; } } + + fprintf(stderr, "[CPP-V3] [DEBUG] fill_context: completed, parsed %d pairs into map\n", pair_count); } MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + std::string bucket, + std::string key, + std::string client_id, + std::string metadata, + std::string range) { + // Get thread ID for debugging concurrent operations + std::thread::id thread_id = std::this_thread::get_id(); + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject START: thread=%lu, bucket=%s, key=%s, client_id=%s, metadata_length=%zu, range=%s\n", + (unsigned long)std::hash{}(thread_id), bucket.c_str(), key.c_str(), client_id.c_str(), metadata.length(), range.c_str()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V3] GetObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } @@ -184,48 +405,107 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(key); - - // S3EncryptionGetObjectOutcome outcome ; - // if (metadata.empty()) { - // outcome = it->second->GetObject(request); - // } else { - // Aws::Map kmsContextMap; - // fill_context(kmsContextMap, metadata); - // outcome = it->second->GetObject(request, kmsContextMap); - // } + + // Add range header if provided + if (!range.empty()) { + request.SetRange(range); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: Setting range=%s\n", range.c_str()); + } Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); - auto outcome = it->second->GetObject(request, kmsContextMap); + + // Log the encryption context map size and contents + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: encryption context map size=%zu\n", kmsContextMap.size()); + for (const auto& pair : kmsContextMap) { + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: context['%s']='%s'\n", + pair.first.c_str(), pair.second.c_str()); + } + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: calling client->GetObject() for key=%s\n", key.c_str()); + + // Keep outcome alive to ensure stream remains valid + auto outcome = client->GetObject(request, kmsContextMap); + + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject: client->GetObject() returned for key=%s\n", key.c_str()); if (outcome.IsSuccess()) { + // Read the stream completely before outcome goes out of scope auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); - return send_response(connection, 200, content); + std::stringstream buffer; + buffer << stream.rdbuf(); + std::string content = buffer.str(); + + // Validate we read something + if (content.empty() && stream.fail()) { + fprintf(stderr, "[CPP-V3] GetObject error: Failed to read stream for bucket=%s, key=%s\n", + bucket.c_str(), key.c_str()); + auto msg = make_error("Failed to read response stream", 500); + return send_response(connection, 500, msg); + } + + fprintf(stderr, "[CPP-V3] GetObject success: bucket=%s, key=%s, size=%zu bytes\n", + bucket.c_str(), key.c_str(), content.length()); + + // Create and send response + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + + // Add keep-alive header + MHD_add_response_header(response, "Connection", "keep-alive"); + MHD_add_response_header(response, "Keep-Alive", "timeout=30, max=100"); + + MHD_Result ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + + return ret; } else { + // Enhanced error logging with thread info + auto error = outcome.GetError(); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject FAILED: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject error details:\n"); + fprintf(stderr, "[CPP-V3] [DEBUG] - Message: %s\n", error.GetMessage().c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ExceptionName: %s\n", error.GetExceptionName().c_str()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ResponseCode: %d\n", (int)error.GetResponseCode()); + fprintf(stderr, "[CPP-V3] [DEBUG] - ShouldRetry: %s\n", error.ShouldRetry() ? "true" : "false"); + auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "handle_get_object error %s\n", msg.c_str()); + fprintf(stderr, "[CPP-V3] GetObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "handle_get_object exception %s\n", e.what()); - auto msg = make_error("An exception was thrown", 500); + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject EXCEPTION: thread=%lu, key=%s, what=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str(), e.what()); + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } catch (...) { + fprintf(stderr, "[CPP-V3] [DEBUG] GetObject UNKNOWN EXCEPTION: thread=%lu, key=%s\n", + (unsigned long)std::hash{}(thread_id), key.c_str()); + auto msg = make_error("Unknown error in GetObject", 500); return send_response(connection, 500, msg); } } MHD_Result handle_put_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &body, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + std::string bucket, + std::string key, + std::string client_id, + std::string body, + std::string metadata) { + fprintf(stderr, "[CPP-V3] PutObject request: bucket=%s, key=%s, client_id=%s, body_size=%zu\n", + bucket.c_str(), key.c_str(), client_id.c_str(), body.length()); + + auto client = get_client(client_id); + if (!client) { + fprintf(stderr, "[CPP-V3] PutObject error: Client not found for client_id=%s\n", client_id.c_str()); return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } try { + // Create owned copy of body data to ensure it lives through the S3 operation + auto body_ptr = std::make_shared(body); + Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -233,85 +513,247 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBucket(bucket); request.SetKey(key); - auto stream = std::make_shared(body); + // Create stream from owned body data + auto stream = std::make_shared(*body_ptr); request.SetBody(stream); - auto outcome = it->second->PutObject(request, kmsContextMap); + // Synchronous call - waits for S3 operation to complete + // body_ptr keeps the data alive through this entire operation + auto outcome = client->PutObject(request, kmsContextMap); if (outcome.IsSuccess()) { + fprintf(stderr, "[CPP-V3] PutObject success: bucket=%s, key=%s\n", bucket.c_str(), key.c_str()); json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); } else { auto msg = make_error(outcome.GetError().GetMessage(), 500); - fprintf(stderr, "handle_put_object error %s\n", msg.c_str()); + fprintf(stderr, "[CPP-V3] PutObject AWS error: %s\n", msg.c_str()); return send_response(connection, 500, msg); } } catch (const std::exception &e) { - fprintf(stderr, "handle_put_object exception %s\n", e.what()); + fprintf(stderr, "[CPP-V3] PutObject exception: %s\n", e.what()); auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } +void request_completed(void *cls, struct MHD_Connection *connection, + void **con_cls, enum MHD_RequestTerminationCode toe) { + // Clean up the request-specific context when request is truly complete + // This is called AFTER all handlers have returned and the response has been sent + + // Log why the request was terminated + const char* reason = "UNKNOWN"; + switch (toe) { + case MHD_REQUEST_TERMINATED_COMPLETED_OK: + reason = "COMPLETED_OK"; + break; + case MHD_REQUEST_TERMINATED_WITH_ERROR: + reason = "WITH_ERROR"; + break; + case MHD_REQUEST_TERMINATED_TIMEOUT_REACHED: + reason = "TIMEOUT_REACHED"; + break; + case MHD_REQUEST_TERMINATED_DAEMON_SHUTDOWN: + reason = "DAEMON_SHUTDOWN"; + break; + case MHD_REQUEST_TERMINATED_READ_ERROR: + reason = "READ_ERROR"; + break; + case MHD_REQUEST_TERMINATED_CLIENT_ABORT: + reason = "CLIENT_ABORT"; + break; + } + fprintf(stderr, "[CPP-V3] request_completed called, reason=%s, con_cls=%p\n", + reason, *con_cls); + + if (*con_cls != nullptr) { + std::string *body = static_cast(*con_cls); + delete body; // Safe to delete now - all synchronous operations are complete + *con_cls = nullptr; + } +} + MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **con_cls) { - std::string method_str(method); - bool is_push = method_str == "POST" || method_str == "PUT"; - static int dummy; - if (*con_cls == nullptr) { - if (is_push) { - *con_cls = new std::string(); - } else { - *con_cls = &dummy; + try { + std::string method_str(method); + std::string url_str(url); + bool is_push = method_str == "POST" || method_str == "PUT"; + + // LOG: Every request entry (even first-time calls) + if (*con_cls == nullptr) { + fprintf(stderr, "[CPP-V3] REQUEST START: method=%s, url=%s, version=%s, con_cls=NULL, upload_data_size=%zu\n", + method, url, version, *upload_data_size); } + + // Initialize request context on first call + if (*con_cls == nullptr) { + // Allocate unique state for each request to avoid race conditions + *con_cls = new std::string(); + fprintf(stderr, "[CPP-V3] REQUEST INIT: allocated new request context for %s %s\n", method, url); return MHD_YES; } - if (is_push && *upload_data_size) { + + // LOG: Subsequent calls + if (is_push && *upload_data_size > 0) { + fprintf(stderr, "[CPP-V3] REQUEST DATA: %s %s receiving %zu bytes\n", method, url, *upload_data_size); + } else if (*upload_data_size == 0) { + fprintf(stderr, "[CPP-V3] REQUEST COMPLETE: %s %s ready for processing\n", method, url); + } + + // Accumulate request body data for POST/PUT requests + if (is_push && *upload_data_size > 0) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; return MHD_YES; } + + // At this point, *upload_data_size == 0, meaning we have all the data + // Now we can safely process the request + + // LOG: About to process request + fprintf(stderr, "[CPP-V3] PROCESSING: %s %s\n", method, url); - std::string url_str(url); - + // Handle client creation endpoint if (is_push && url_str == "/client") { - std::unique_ptr body(static_cast(*con_cls)); - return handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V3] Handling /client endpoint\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_create_client(connection, *body); + fprintf(stderr, "[CPP-V3] /client handler returned: %d\n", result); + return result; } + // Handle object operations if (url_str.find("/object/") == 0) { + fprintf(stderr, "[CPP-V3] Handling /object/ endpoint\n"); std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); - std::string metadata = get_header_value(connection, "content-metadata"); + + fprintf(stderr, "[CPP-V3] Object operation: bucket=%s, key=%s, client_id=%s, method=%s\n", + bucket.c_str(), key.c_str(), client_id.c_str(), method); + if (method_str == "GET") { - return handle_get_object(connection, bucket, key, client_id, metadata); + fprintf(stderr, "[CPP-V3] Dispatching to handle_get_object\n"); + std::string range = get_header_value(connection, "Range"); + MHD_Result result = handle_get_object(connection, bucket, key, client_id, metadata, range); + fprintf(stderr, "[CPP-V3] handle_get_object returned: %d\n", result); + return result; } else if (method_str == "PUT") { - std::unique_ptr body(static_cast(*con_cls)); - *upload_data_size = 0; - return handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V3] Dispatching to handle_put_object\n"); + std::string *body = static_cast(*con_cls); + MHD_Result result = handle_put_object(connection, bucket, key, client_id, *body, metadata); + fprintf(stderr, "[CPP-V3] handle_put_object returned: %d\n", result); + return result; + } else { + fprintf(stderr, "[CPP-V3] Method not allowed: %s\n", method); + return send_response(connection, 405, "{\"error\":\"Method not allowed\"}"); } } } - return send_response(connection, 404, - "{\"error\":\"Not idea what is happening\"}"); + // Return error for unrecognized endpoints + fprintf(stderr, "[CPP-V3] ERROR: Unrecognized endpoint: %s %s\n", method, url); + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); + } catch (const std::exception &e) { + fprintf(stderr, "[CPP-V3] FATAL: Unhandled exception in request_handler: %s (method=%s, url=%s)\n", + e.what(), method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unhandled exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Unknown exception in request_handler (method=%s, url=%s)\n", + method, url); + // Try to send error response, but connection might already be broken + try { + return send_response(connection, 500, + "{\"error\":\"Internal server error: unknown exception\"}"); + } catch (...) { + fprintf(stderr, "[CPP-V3] FATAL: Failed to send error response\n"); + return MHD_NO; + } + } +} + +// Error log callback for libmicrohttpd +void log_mhd_error(void* cls, const char* fmt, va_list ap) { + fprintf(stderr, "[CPP-V3] [MHD-ERROR] "); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); +} + +// Connection notification callback - called when a client connects +MHD_Result notify_connection(void *cls, + struct MHD_Connection *connection, + void **socket_context, + enum MHD_ConnectionNotificationCode toe) { + if (toe == MHD_CONNECTION_NOTIFY_STARTED) { + fprintf(stderr, "[CPP-V3] [MHD-CONNECT] New connection started\n"); + } else if (toe == MHD_CONNECTION_NOTIFY_CLOSED) { + fprintf(stderr, "[CPP-V3] [MHD-DISCONNECT] Connection closed\n"); + } + return MHD_YES; } int main() { Aws::SDKOptions options; + + // Configure AWS SDK logging to output to stderr (which goes to server.log) + // Using Debug level to capture all SDK activity including CryptoModule errors + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; + options.loggingOptions.logger_create_fn = []() { + return std::make_shared( + Aws::Utils::Logging::LogLevel::Debug + ); + }; + + fprintf(stderr, "[CONFIG] AWS SDK logging enabled at Debug level\n"); + Aws::InitAPI(options); + + // Detect CPU core count and configure threading + unsigned int num_cores = std::thread::hardware_concurrency(); + if (num_cores == 0) { + num_cores = 4; // Fallback if detection fails + fprintf(stderr, "[WARNING] CPU core detection failed, defaulting to %u cores\n", num_cores); + } + + // Thread pool size = num_cores * 2 (allows for I/O blocking without starving throughput) + g_thread_pool_size = num_cores * 2; + unsigned int connection_limit = g_thread_pool_size; + + // Log configuration + fprintf(stderr, "[CONFIG] Detected CPU cores: %u\n", num_cores); + fprintf(stderr, "[CONFIG] Thread pool size: %u\n", g_thread_pool_size); + fprintf(stderr, "[CONFIG] Connection limit: %u\n", connection_limit); + fprintf(stderr, "[CONFIG] Each S3 client will use 512 max connections\n"); + int port = 8091; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, - &request_handler, NULL, MHD_OPTION_END); + MHD_start_daemon(MHD_USE_POLL_INTERNALLY | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG, + port, NULL, NULL, + &request_handler, NULL, + MHD_OPTION_EXTERNAL_LOGGER, log_mhd_error, NULL, + MHD_OPTION_NOTIFY_CONNECTION, notify_connection, NULL, + MHD_OPTION_NOTIFY_COMPLETED, request_completed, NULL, + MHD_OPTION_THREAD_POOL_SIZE, g_thread_pool_size, + MHD_OPTION_CONNECTION_LIMIT, connection_limit, + MHD_OPTION_CONNECTION_TIMEOUT, 10, + MHD_OPTION_END); if (!daemon) { fprintf(stderr, "Failed to start server on port %d\n", port); diff --git a/test-server/go-v3-server/main.go b/test-server/go-v3-server/main.go index d201ffe2..0384c5ff 100644 --- a/test-server/go-v3-server/main.go +++ b/test-server/go-v3-server/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" @@ -23,6 +24,7 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client + mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -66,7 +68,12 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -141,7 +148,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return @@ -172,8 +184,10 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache + // Store client in cache (protected by mutex) + s.mu.Lock() s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -194,8 +208,10 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -264,8 +280,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/go-v3-transition-server/local-go-s3ec b/test-server/go-v3-transition-server/local-go-s3ec index f51a4402..912914ad 160000 --- a/test-server/go-v3-transition-server/local-go-s3ec +++ b/test-server/go-v3-transition-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit f51a4402c741cd989c7984336de560e9c54baf17 +Subproject commit 912914ad1c14942c3ea8638fb37f9e2c46445a84 diff --git a/test-server/go-v3-transition-server/main.go b/test-server/go-v3-transition-server/main.go index 799a9668..64556f12 100644 --- a/test-server/go-v3-transition-server/main.go +++ b/test-server/go-v3-transition-server/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "github.com/aws/amazon-s3-encryption-client-go/v3/client" "github.com/aws/amazon-s3-encryption-client-go/v3/materials" @@ -24,6 +25,7 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV3 kmsClient *kms.Client + mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -68,7 +70,12 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -143,7 +150,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return @@ -189,8 +201,10 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache + // Store client in cache (protected by mutex) + s.mu.Lock() s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -211,8 +225,10 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -281,8 +297,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/go-v4-server/local-go-s3ec b/test-server/go-v4-server/local-go-s3ec index f51a4402..912914ad 160000 --- a/test-server/go-v4-server/local-go-s3ec +++ b/test-server/go-v4-server/local-go-s3ec @@ -1 +1 @@ -Subproject commit f51a4402c741cd989c7984336de560e9c54baf17 +Subproject commit 912914ad1c14942c3ea8638fb37f9e2c46445a84 diff --git a/test-server/go-v4-server/main.go b/test-server/go-v4-server/main.go index 672236ac..50999e95 100644 --- a/test-server/go-v4-server/main.go +++ b/test-server/go-v4-server/main.go @@ -8,6 +8,7 @@ import ( "log" "net/http" "strings" + "sync" "github.com/aws/amazon-s3-encryption-client-go/v4/client" "github.com/aws/amazon-s3-encryption-client-go/v4/materials" @@ -24,6 +25,7 @@ import ( type Server struct { clientCache map[string]*client.S3EncryptionClientV4 kmsClient *kms.Client + mu sync.RWMutex } // CreateClientInput represents the input for creating a client @@ -68,7 +70,12 @@ type ErrorResponse struct { // NewServer creates a new server instance func NewServer() (*Server, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { return nil, fmt.Errorf("failed to load AWS config: %w", err) } @@ -143,7 +150,12 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-west-2")) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-west-2"), + config.WithRetryMaxAttempts(5), + config.WithRetryMode(aws.RetryModeAdaptive), + ) if err != nil { s.createS3EncryptionClientError(w, fmt.Sprintf("Failed to load AWS config: %v", err), http.StatusInternalServerError) return @@ -189,8 +201,10 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { // Generate client ID clientID := uuid.New().String() - // Store client in cache + // Store client in cache (protected by mutex) + s.mu.Lock() s.clientCache[clientID] = s3EncryptionClient + s.mu.Unlock() // Return response w.Header().Set("Content-Type", "application/json") @@ -211,8 +225,10 @@ func (s *Server) putObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) @@ -281,8 +297,10 @@ func (s *Server) getObject(w http.ResponseWriter, r *http.Request) { return } - // Get client from cache + // Get client from cache (protected by mutex) + s.mu.RLock() client, exists := s.clientCache[clientID] + s.mu.RUnlock() if !exists { s.createGenericServerError(w, fmt.Sprintf("No client found for ClientID: %s", clientID), http.StatusNotFound) diff --git a/test-server/java-tests/build.gradle.kts b/test-server/java-tests/build.gradle.kts index 106f82ef..2d1cbdeb 100644 --- a/test-server/java-tests/build.gradle.kts +++ b/test-server/java-tests/build.gradle.kts @@ -16,6 +16,9 @@ dependencies { // Test dependencies testImplementation("org.junit.jupiter:junit-jupiter:5.13.0") testRuntimeOnly("org.junit.platform:junit-platform-launcher") + // JUnit Suite support for test ordering + testImplementation("org.junit.platform:junit-platform-suite-api:1.10.0") + testRuntimeOnly("org.junit.platform:junit-platform-suite-engine:1.10.0") testImplementation("com.amazonaws:aws-java-sdk:1.12.788") testImplementation("software.amazon.awssdk:s3:2.37.1") testImplementation("org.bouncycastle:bcprov-jdk15on:1.70") @@ -49,6 +52,17 @@ tasks { classpath = sourceSets["it"].runtimeClasspath outputs.upToDateWhen { false } outputs.cacheIf { false } + + // Enable parallel test execution + systemProperty("junit.jupiter.execution.parallel.enabled", "true") + systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") + systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") + // Configure thread pool size - adjust based on I/O-bound nature of tests + systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") + maxParallelForks = 1 // One JVM + systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", + Math.max(1, Runtime.getRuntime().availableProcessors() - 3).toString()) // Scale with CPU, reserve 3 cores + // Passing information from Gradle into the tests so that we can filter our servers systemProperty("test.filter.servers", System.getProperty("test.filter.servers")) // For debugging diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java index 100925a9..b161feb6 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ExhaustiveRoundTripTests1_25.java @@ -177,10 +177,10 @@ public void GIVEN_GCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncrypt @MethodSource("software.amazon.encryption.s3.TestUtils#encryptImprovedDecryptImproved") public void GIVEN_KCGCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncryptAllowDecrypt_WHEN_Decrypt_THEN_Pass( TestUtils.LanguageServerTarget encLang, TestUtils.LanguageServerTarget decLang - ) { + ) throws Exception { S3ECTestServerClient encClient = TestUtils.testServerClientFor(encLang); - final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang; + final String objectKey = "encrypt-kc-gcm-decrypt-improved-test-key-" + encLang + "-" + decLang; final String input = "simple-test-input"; KeyMaterial kmsKeyArn = KeyMaterial.builder() .kmsKeyId(TestUtils.KMS_KEY_ARN) @@ -211,6 +211,13 @@ public void GIVEN_KCGCMEncryptedData_AND_ImprovedClientDecryptingWithForbidEncry .build()); String decS3ECId = decClientOutput.getClientId(); + // At high concurrency, this test tends to get: + // BadDigest Message: The CRC64NVME you specified did not match the calculated checksum. + // I think this is a read after write issue. + // A better fix, would be to break this tests suite up into encrypt/decrypt + // rather than having a test for many pairs and doing encrypt/decrypt on each pair + Thread.sleep(100); + // When: decrypt KC-GCM object with an improved version client with ForbidEncryptAllowDecrypt policy GetObjectOutput output = decClient.getObject(GetObjectInput.builder() .clientID(decS3ECId) diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java new file mode 100644 index 00000000..ca495f56 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTestSuite.java @@ -0,0 +1,258 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * GCM Test Suite + * + * This suite enforces execution order between GCM encrypt and decrypt phases: + * 1. EncryptTests - All encrypt tests run in parallel (within this phase) + * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel + * + * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion + * and DecryptTests awaits before proceeding. + */ +public class GCMTestSuite { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * GCM Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using GCM (without key commitment) encryption algorithm. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + @DisplayName("GCMTestSuite - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBase = "test-gcm-kms"; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe list for storing encrypted object keys + private static final List crossLanguageObjects = + Collections.synchronizedList(new ArrayList<>()); + + /** + * Public accessor for decrypt tests to retrieve encrypted object keys + */ + static List getCrossLanguageObjects() { + return new ArrayList<>(crossLanguageObjects); // Return defensive copy + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), + crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @AfterAll + static void signalEncryptionComplete() { + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * GCM Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first (enforced by @Order). + */ + @Nested + @DisplayName("GCMTestSuite - Decrypt") + class DecryptTests { + private static List crossLanguageObjects; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects from the encrypt phase + crossLanguageObjects = EncryptTests.getCrossLanguageObjects(); + + // Verify we have objects to decrypt + if (crossLanguageObjects.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java deleted file mode 100644 index 6eef0b5f..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java +++ /dev/null @@ -1,203 +0,0 @@ -/* -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0 -*/ - -package software.amazon.encryption.s3; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static software.amazon.encryption.s3.TestUtils.*; - -import java.lang.annotation.ElementType; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import com.amazonaws.services.s3.model.KMSEncryptionMaterials; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.GetObjectInput; -import software.amazon.encryption.s3.model.GetObjectOutput; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.PutObjectInput; -import software.amazon.encryption.s3.model.S3ECConfig; -import software.amazon.encryption.s3.model.S3EncryptionClientError; - -import com.amazonaws.services.s3.AmazonS3Encryption; -import com.amazonaws.services.s3.AmazonS3EncryptionClient; -import com.amazonaws.services.s3.model.CryptoConfiguration; -import com.amazonaws.services.s3.model.CryptoMode; -import com.amazonaws.services.s3.model.CryptoStorageMode; -import software.amazon.encryption.s3.TestUtils.*; -import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; -import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; - -/** -* Exhaustive tests for S3 Encryption Client round-trip operations. -* These tests cover various combinations of client versions, commitment policies, and encryption modes. -* -* Tests are based on the exhaustive test matrix defined at: -* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 -* -*/ - -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class GCMTests { - private static String sharedObjectKeyBase = "test-gcm-kms"; - private static KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - private static List crossLanguageObjects = new ArrayList<>(); - - @Order(1) - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Transition configured with the default should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(3) - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should encrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_encrypt_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBase + language.getLanguageName()), crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(10) - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(11) - @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(12) - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(13) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - - @Order(14) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should fail to decrypt GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_fail_to_decrypt_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt_fails(client, S3ECId, crossLanguageObjects, EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); - } - -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java new file mode 100644 index 00000000..6460fbad --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/InstructionFileFailures.java @@ -0,0 +1,788 @@ +/* +* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +* SPDX-License-Identifier: Apache-2.0 +*/ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.TestAbortedException; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** +* Instruction File Failures Test Suite +* +* This suite enforces execution order between encrypt and decrypt phases: +* 1. EncryptTests - Encrypts objects with various key materials and creates test copies +* 2. DecryptTests - Waits for encrypt phase to complete, then tests decryption scenarios +* +* Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion +* and DecryptTests awaits before proceeding. +* +*/ +public class InstructionFileFailures { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + // Object key suffixes for test copies + private static final String SUFFIX_GOOD_COPY = "-good-copy"; + private static final String SUFFIX_BAD_BOTH_META_AND_INSTRUCTION = "-bad-both-meta-and-instruction"; + private static final String SUFFIX_BAD_ONLY_INSTRUCTION = "-bad-only-instruction"; + + /** + * Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using various key materials (KMS, RSA, AES) with instruction files. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + @DisplayName("InstructionFileFailures - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-instruction-files-cases"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List crossLanguageObjectsKms = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsRsa = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsAes = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + RSA_KEY = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) + .build(); + + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesSecretKey = keyGen.generateKey(); + + AES_KEY = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) + .build(); + } + + /** + * Public accessors for decrypt tests to retrieve encrypted object keys and key materials + */ + static List getCrossLanguageObjectsKms() { + return new ArrayList<>(crossLanguageObjectsKms); + } + + static List getCrossLanguageObjectsRsa() { + return new ArrayList<>(crossLanguageObjectsRsa); + } + + static List getCrossLanguageObjectsAes() { + return new ArrayList<>(crossLanguageObjectsAes); + } + + static KeyMaterial getRsaKey() { + return RSA_KEY; + } + + static KeyMaterial getAesKey() { + return AES_KEY; + } + + static KeyMaterial getKmsKeyArn() { + return kmsKeyArn; + } + + public static Stream improvedClientsCanPutKMSWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawRSAWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawAESWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: Encrypt KMS KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encryptWithInstructionFilesKmsKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-kms" + language.getLanguageName()), + crossLanguageObjectsKms, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptWithInstructionFilesRsaKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-rsa" + language.getLanguageName()), + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Encrypt AES KC-GCM with instruction files") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptWithInstructionFilesAesKcGcm(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + "-aes" + language.getLanguageName()), + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + static void makeCopiesToVerifyThings() throws Exception { + // Create a plaintext S3 client to copy objects with instruction files + try (S3Client ptS3Client = S3Client.create()) { + List allCrossLanguageObjects = Stream.of( + crossLanguageObjectsKms.stream(), + crossLanguageObjectsRsa.stream(), + crossLanguageObjectsAes.stream() + ).flatMap(s -> s).collect(Collectors.toList()); + for (String objectKey : allCrossLanguageObjects) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + // Get the instruction file + String instructionFileKey = objectKey + ".instruction"; + ResponseBytes instructionFile = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(instructionFileKey) + .build()); + + String instructionFileJson = instructionFile.asUtf8String(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Put a strict copy, to verify that we know how to do this + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_GOOD_COPY, + encryptedObject.asByteArray(), + objectMetadata, + instructionFileJson + ); + + ObjectMapper mapper = new ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + instructionFileMap.put("x-amz-c", objectMetadata.get("x-amz-c")); + instructionFileMap.put("x-amz-d", objectMetadata.get("x-amz-d")); + instructionFileMap.put("x-amz-i", objectMetadata.get("x-amz-i")); + + String instructionFileWithCommitmentValues = mapper.writeValueAsString(instructionFileMap); + + // Put instruction files that should fail: + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION, + encryptedObject.asByteArray(), + objectMetadata, + instructionFileWithCommitmentValues + ); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_ONLY_INSTRUCTION, + encryptedObject.asByteArray(), + Map.of(), + instructionFileWithCommitmentValues + ); + + } + } + } + + static void putObjectWithInstructionFile( + S3Client ptS3Client, + String newObjectKey, + byte[] objectData, + Map objectMetadata, + String instructionFileJson + ) { + + // Put the encrypted object copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + + // Put the instruction file copy + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(newObjectKey + ".instruction") + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(instructionFileJson.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + + @AfterAll + static void signalEncryptionComplete() throws Exception { + makeCopiesToVerifyThings(); + + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first. + */ + @Nested + @DisplayName("InstructionFileFailures - Decrypt") + class DecryptTests { + private static List crossLanguageObjectsKms; + private static List crossLanguageObjectsRsa; + private static List crossLanguageObjectsAes; + private static KeyMaterial kmsKeyArn; + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects and key materials from the encrypt phase + crossLanguageObjectsKms = EncryptTests.getCrossLanguageObjectsKms(); + crossLanguageObjectsRsa = EncryptTests.getCrossLanguageObjectsRsa(); + crossLanguageObjectsAes = EncryptTests.getCrossLanguageObjectsAes(); + kmsKeyArn = EncryptTests.getKmsKeyArn(); + RSA_KEY = EncryptTests.getRsaKey(); + AES_KEY = EncryptTests.getAesKey(); + + // Verify we have objects to decrypt + if (crossLanguageObjectsKms.isEmpty() && crossLanguageObjectsRsa.isEmpty() && crossLanguageObjectsAes.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + public static Stream clientsCanGetKMSWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream clientsCanGetRawRSAWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream clientsCanGetRawAESWithInstructionFile() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + // KMS instruction files decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt KMS encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsOriginalAndGoodCopyObjectsSucceeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsKms, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_GOOD_COPY) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsKms + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithDuplicateCommitmentInMetadataAndInstructionFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithCommitmentOnlyInInstructionFileFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithDuplicateCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt KMS instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetKMSWithInstructionFile") + void decryptKmsWithInstructionFileCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsKms + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // RSA instruction file decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt RSA encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaOriginalAndGoodCopyObjectsSucceeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_GOOD_COPY) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsRsa + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithDuplicateCommitmentInMetadataAndInstructionFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithCommitmentOnlyInInstructionFileFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithDuplicateCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt RSA instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawRSAWithInstructionFile") + void decryptRsaWithInstructionFileCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(RSA_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsRsa + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // AES instruction file decrypt + + @ParameterizedTest(name = "{0}: Successfully decrypt AES encrypted original and good-copy objects") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesOriginalAndGoodCopyObjectsSucceeds(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.Decrypt( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_GOOD_COPY) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + crossLanguageObjectsAes + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is duplicated in metadata and instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithDuplicateCommitmentInMetadataAndInstructionFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES when commitment is only in instruction file") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithCommitmentOnlyInInstructionFileFails(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES duplicate commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithDuplicateCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_BOTH_META_AND_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to decrypt AES instruction file commitment with FORBID_ENCRYPT_ALLOW_DECRYPT policy") + @MethodSource("software.amazon.encryption.s3.InstructionFileFailures$DecryptTests#clientsCanGetRawAESWithInstructionFile") + void decryptAesWithInstructionFileCommitmentFailsWithForbidPolicy(TestUtils.LanguageServerTarget language) { + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(AES_KEY) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt_fails( + client, + S3ECId, + crossLanguageObjectsAes + .stream() + .map(key -> key + SUFFIX_BAD_ONLY_INSTRUCTION) + .collect(Collectors.toList()), + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java new file mode 100644 index 00000000..d256f909 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTestSuite.java @@ -0,0 +1,391 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opentest4j.TestAbortedException; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * KC-GCM Test Suite + * + * This suite enforces execution order between KC-GCM encrypt and decrypt phases: + * 1. EncryptTests - All encrypt tests run in parallel (within this phase) + * 2. DecryptTests - Waits for encrypt phase to complete, then all decrypt tests run in parallel + * + * Coordination is achieved using a CountDownLatch that EncryptTests signals upon completion + * and DecryptTests awaits before proceeding. + */ +public class KC_GCMTestSuite { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + /** + * KC-GCM Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using Key Commitment GCM encryption algorithm. + * All tests in this class can run in parallel with each other. + * The encrypted objects are stored in thread-safe lists for use by DecryptTests. + */ + @Nested + @DisplayName("KC_GCMTestSuite - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; + private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-rsa-instruction-file"; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List crossLanguageObjectsMetaDataMode = + Collections.synchronizedList(new ArrayList<>()); + private static final List crossLanguageObjectsInstructionFiles = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyPair RSA_KEY_PAIR_1; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); + } + + /** + * Public accessors for decrypt tests to retrieve encrypted object keys and RSA key + */ + static List getCrossLanguageObjectsMetaDataMode() { + return new ArrayList<>(crossLanguageObjectsMetaDataMode); + } + + static List getCrossLanguageObjectsInstructionFiles() { + return new ArrayList<>(crossLanguageObjectsInstructionFiles); + } + + static KeyPair getRsaKeyPair() { + return RSA_KEY_PAIR_1; + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm_kms( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file_rsa( + TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), + crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_kms( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_encrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), + crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @AfterAll + static void signalEncryptionComplete() { + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * KC-GCM Decryption Tests - Decrypt Phase + * + * These tests decrypt objects that were encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first (enforced by @Order). + */ + @Nested + @DisplayName("KC_GCMTestSuite - Decrypt") + class DecryptTests { + private static List crossLanguageObjectsMetaDataMode; + private static List crossLanguageObjectsInstructionFiles; + private static KeyPair RSA_KEY_PAIR_1; + private static final KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects and RSA key from the encrypt phase + crossLanguageObjectsMetaDataMode = EncryptTests.getCrossLanguageObjectsMetaDataMode(); + crossLanguageObjectsInstructionFiles = EncryptTests.getCrossLanguageObjectsInstructionFiles(); + RSA_KEY_PAIR_1 = EncryptTests.getRsaKeyPair(); + + // Verify we have objects to decrypt + if (crossLanguageObjectsMetaDataMode.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_the_default_should_decrypt_kc_gcm_kms( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_the_default_should_decrypt_kc_gcm( + TestUtils.LanguageServerTarget language + ) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") + void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file( + final TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Transition configured with default should decrypt KC-GCM (instruction file)") + @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") + void transition_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file_rsa( + final TestUtils.LanguageServerTarget language + ) { + if (!RAW_SUPPORTED.contains(language.getLanguageName())) { + throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); + } + + KeyMaterial rsaKey = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) + .build(); + + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .keyMaterial(rsaKey).build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java deleted file mode 100644 index ee4279d6..00000000 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java +++ /dev/null @@ -1,264 +0,0 @@ -/* -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -* SPDX-License-Identifier: Apache-2.0 -*/ - -package software.amazon.encryption.s3; - -import static software.amazon.encryption.s3.TestUtils.*; - -import java.nio.ByteBuffer; -import java.security.KeyPair; -import java.security.KeyPairGenerator; -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.opentest4j.TestAbortedException; -import software.amazon.encryption.s3.client.S3ECTestServerClient; -import software.amazon.encryption.s3.model.CommitmentPolicy; -import software.amazon.encryption.s3.model.CreateClientInput; -import software.amazon.encryption.s3.model.CreateClientOutput; -import software.amazon.encryption.s3.model.EncryptionAlgorithm; -import software.amazon.encryption.s3.model.InstructionFileConfig; -import software.amazon.encryption.s3.model.KeyMaterial; -import software.amazon.encryption.s3.model.S3ECConfig; - -/** -* Exhaustive tests for S3 Encryption Client round-trip operations. -* These tests cover various combinations of client versions, commitment policies, and encryption modes. -* -* Tests are based on the exhaustive test matrix defined at: -* https://tiny.amazon.com/3xnzwczl/loopcloumicrpeyJ3 -* -*/ - -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class KC_GCMTests { - private static final String sharedObjectKeyBaseMetaDataMode = "test-kc-gcm-kms"; - private static final String sharedObjectKeyBaseInsFileMode = "test-kc-gcm-kms-instruction-file"; - private static KeyMaterial kmsKeyArn = KeyMaterial.builder() - .kmsKeyId(TestUtils.KMS_KEY_ARN) - .build(); - private static final List crossLanguageObjectsMetaDataMode = new ArrayList<>(); - private static final List crossLanguageObjectsInstructionFiles = new ArrayList<>(); - private static KeyPair RSA_KEY_PAIR_1; - - @BeforeAll - static void setupKeys() throws Exception { - KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); - keyPairGen.initialize(2048); - RSA_KEY_PAIR_1 = keyPairGen.generateKeyPair(); - } - - @Order(1) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm_ins_file(TestUtils.LanguageServerTarget language) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseInsFileMode + language.getLanguageName()), crossLanguageObjectsInstructionFiles, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(2) - @ParameterizedTest(name = "{0}: Improved configured with the default should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_the_default_should_encrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Encrypt(client, S3ECId, appendTestSuffix(sharedObjectKeyBaseMetaDataMode + language.getLanguageName()), crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(10) - @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_the_default_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(11) - @ParameterizedTest(name = "{0}: Transition configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#transitionClientsForTest") - void transition_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(12) - @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_forbid_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(13) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_allow_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_ALLOW_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(14) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(15) - @ParameterizedTest(name = "{0}: Improved configured with the default should decrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_the_default_should_decrypt_kc_gcm(TestUtils.LanguageServerTarget language) { - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .keyMaterial(kmsKeyArn) - // .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsMetaDataMode, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - - @Order(16) - @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptRequireDecrypt should encrypt KC-GCM") - @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest") - void improved_configured_with_require_encrypt_require_decrypt_should_decrypt_kc_gcm_ins_file(final TestUtils.LanguageServerTarget language) { - if (!RAW_SUPPORTED.contains(language.getLanguageName())) { - throw new TestAbortedException("Not encrypting raw keyring with: " + language.getLanguageName()); - } - - KeyMaterial rsaKey = KeyMaterial.builder() - .rsaKey(ByteBuffer.wrap(RSA_KEY_PAIR_1.getPrivate().getEncoded())) - .build(); - - S3ECTestServerClient client = TestUtils.testServerClientFor(language); - CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() - .config(S3ECConfig.builder() - .instructionFileConfig(InstructionFileConfig.builder() - .enableInstructionFilePutObject(true) - .build()) - .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) - .commitmentPolicy(CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT) - .keyMaterial(rsaKey).build()) - .build()); - String S3ECId = clientOutput.getClientId(); - - TestUtils.Decrypt(client, S3ECId, crossLanguageObjectsInstructionFiles, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); - } - -} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java new file mode 100644 index 00000000..5a954e1f --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RangedGetTests.java @@ -0,0 +1,1687 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.GetObjectInput; +import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.S3ECConfig; + +/** + * Ranged Get Tests - S3 Encryption Client Cross-Language Compatibility + * + * PURPOSE: + * This test suite validates that ranged get operations (partial object reads) work correctly + * across all three encryption algorithms (CBC, GCM, KC-GCM) and that commitment validation + * occurs properly during ranged gets for KC-GCM encrypted objects. + * + * WHAT IS BEING TESTED: + * 1. Ranged gets successfully retrieve partial content from encrypted objects across all algorithms + * 2. Commitment validation is enforced during ranged gets for KC-GCM encrypted objects + * 3. Corrupted commitment metadata (removed, moved, or mutated) causes ranged gets to fail + * 4. Various byte ranges work correctly: start, end, middle, whole file, and auth tag only + * + * WHY THIS IS IMPORTANT: + * - Ranged gets are a critical S3 feature that must work with encrypted objects + * - KC-GCM's commitment mechanism must be validated even for partial reads to prevent + * commitment-based issues where an actor control the encryption keys + * - Cross-language compatibility ensures all SDKs handle ranged gets consistently + * - Edge cases (first/last bytes, auth tags) verify boundary condition handling + * + * TEST STRUCTURE: + * This suite uses a two-phase approach with enforced ordering: + * 1. EncryptTests - Encrypts objects with CBC, GCM, and KC-GCM algorithms + * - Creates corrupted KC-GCM test cases with manipulated commitment metadata + * - All encrypt tests can run in parallel within this phase + * 2. RangedGetTests - Waits for encryption to complete, then tests ranged gets + * - Tests successful ranged gets on valid objects + * - Tests failed ranged gets on corrupted commitment objects + * - All ranged get tests can run in parallel within this phase + * + * Coordination uses a CountDownLatch to ensure all encryption completes before ranged gets begin. + * + * INPUT DIMENSIONS: + * - Encryption Algorithm: CBC, GCM, KC-GCM + * - Language Implementation: All languages supporting RANGED_GETS_SUPPORTED + * - Byte Range Types: + * * Start (bytes 0-99) + * * End (last 100 bytes) + * * Middle (100 bytes centered in file) + * * Whole file (all bytes) + * * Auth tag only (last 16 bytes for authenticated algorithms) + * - Storage Mode (KC-GCM only): + * * Object Metadata Storage (all metadata in object, no instruction file) + * * Instruction File Storage (c/d/i in metadata, x-amz-3/w/m/t in instruction file) + * - Commitment State (KC-GCM only): + * * Valid - Object Metadata Storage (original and good-copy) + * * Valid - Instruction File Storage (original and good-copy) + * * Corrupted - Object Metadata Storage: + * - Mutated c/d/i: bit flipped in metadata values + * - Invalid c length: c < 28 bytes in metadata + * - Invalid c length: c > 28 bytes in metadata + * * Corrupted - Instruction File Storage: + * - Commitment duplicated: c/d/i in instruction file (already in metadata) + * - Commitment removed: c/d/i removed from metadata + * - Mutated c/d/i in metadata: bit flipped + * - Mutated c/d/i in instruction file: bit flipped + * - Invalid c length: c < 28 bytes in metadata + * - Invalid c length: c > 28 bytes in metadata + * + * EXPECTED RESULTS: + * - Positive: Ranged gets on valid CBC, GCM, KC-GCM objects return correct partial content + * - Negative: Ranged gets on corrupted KC-GCM objects fail with commitment validation errors + * + * REPRESENTATIVE VALUES: + * - Bit flip position: Randomly selected per test run, included in object key name + * - File size: Object keys themselves (short strings) serve as representative small files + * - Byte ranges: Fixed patterns covering important boundary conditions + * + * SCOPE: + * - Languages in RANGED_GETS_SUPPORTED set are tested, + * the encrypt tests are to create values that are then tested. + * - CBC and GCM tests validate ranged get functionality works + * - KC-GCM tests focus on commitment validation during ranged gets + */ +public class RangedGetTests { + // Synchronization latch - released when encrypt phase completes + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + + // Random number generator for bit flipping (seeded for reproducibility) + private static final Random random = new Random(System.currentTimeMillis()); + + // Object key suffixes for test copies + private static final String SUFFIX_GOOD_COPY = "-good-copy"; + private static final String SUFFIX_BAD_MUTATED_C = "-bad-mutated-c-bit-"; + private static final String SUFFIX_BAD_MUTATED_D = "-bad-mutated-d-bit-"; + private static final String SUFFIX_BAD_MUTATED_I = "-bad-mutated-i-bit-"; + private static final String SUFFIX_BAD_INVALID_D_LENGTH_SHORT = "-bad-invalid-d-length-short"; + private static final String SUFFIX_BAD_INVALID_D_LENGTH_LONG = "-bad-invalid-d-length-long"; + private static final String SUFFIX_BAD_COMMITMENT_IN_INSTRUCTION = "-bad-commitment-in-instruction"; + + /** + * Encryption Tests - Encrypt Phase + * + * These tests encrypt objects using CBC, GCM, and KC-GCM algorithms, then create + * corrupted copies for failure testing. All tests in this class can run in parallel. + */ + @Nested + @DisplayName("RangedGetTests - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBase = "test-ranged-get"; + private static KeyMaterial kmsKeyArn = KeyMaterial.builder() + .kmsKeyId(TestUtils.KMS_KEY_ARN) + .build(); + + // Thread-safe lists for storing encrypted object keys + private static final List cbcObjects = + Collections.synchronizedList(new ArrayList<>()); + private static final List gcmObjects = + Collections.synchronizedList(new ArrayList<>()); + // KC-GCM with Object Metadata Storage (all metadata in object) + private static final List kcGcmObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + // KC-GCM with Instruction File Storage (c/d/i in metadata, rest in instruction file) + private static final List kcGcmObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + // Corruption test lists for metadata storage mode + private static final List mutatedCObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedDObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedIObjectsMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthShortMetadata = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthLongMetadata = + Collections.synchronizedList(new ArrayList<>()); + // Corruption test lists for instruction file storage mode + private static final List mutatedCObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedDObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List mutatedIObjectsInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthShortInstruction = + Collections.synchronizedList(new ArrayList<>()); + private static final List invalidDLengthLongInstruction = + Collections.synchronizedList(new ArrayList<>()); + + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setupKeys() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair keyPair = keyPairGen.generateKeyPair(); + + RSA_KEY = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(keyPair.getPrivate().getEncoded())) + .build(); + + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesSecretKey = keyGen.generateKey(); + + AES_KEY = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesSecretKey.getEncoded())) + .build(); + } + + /** + * Public accessors for ranged get tests to retrieve encrypted object keys + */ + static List getCbcObjects() { + return new ArrayList<>(cbcObjects); + } + + static List getGcmObjects() { + return new ArrayList<>(gcmObjects); + } + + static List getKcGcmObjectsMetadata() { + return new ArrayList<>(kcGcmObjectsMetadata); + } + + static List getKcGcmObjectsInstruction() { + return new ArrayList<>(kcGcmObjectsInstruction); + } + + static List getMutatedCObjectsMetadata() { + return new ArrayList<>(mutatedCObjectsMetadata); + } + + static List getMutatedDObjectsMetadata() { + return new ArrayList<>(mutatedDObjectsMetadata); + } + + static List getMutatedIObjectsMetadata() { + return new ArrayList<>(mutatedIObjectsMetadata); + } + + static List getInvalidDLengthShortMetadata() { + return new ArrayList<>(invalidDLengthShortMetadata); + } + + static List getInvalidDLengthLongMetadata() { + return new ArrayList<>(invalidDLengthLongMetadata); + } + + static List getMutatedCObjectsInstruction() { + return new ArrayList<>(mutatedCObjectsInstruction); + } + + static List getMutatedDObjectsInstruction() { + return new ArrayList<>(mutatedDObjectsInstruction); + } + + static List getMutatedIObjectsInstruction() { + return new ArrayList<>(mutatedIObjectsInstruction); + } + + static List getInvalidDLengthShortInstruction() { + return new ArrayList<>(invalidDLengthShortInstruction); + } + + static List getInvalidDLengthLongInstruction() { + return new ArrayList<>(invalidDLengthLongInstruction); + } + + static KeyMaterial getKmsKeyArn() { + return kmsKeyArn; + } + + static KeyMaterial getRsaKey() { + return RSA_KEY; + } + + static KeyMaterial getAesKey() { + return AES_KEY; + } + + // GCM can be encrypted by transition and improved clients + public static Stream transitionAndImprovedForGCM() { + return Stream.concat( + transitionClientsForTest(), + improvedClientsForTest() + ); + } + + // KC-GCM can be encrypted by improved clients only + public static Stream improvedClientsForKCGCM() { + return improvedClientsForTest(); + } + + public static Stream improvedClientsCanPutKMSWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> !KMS_INSTRUCTION_FILE_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @org.junit.jupiter.api.Test + void encryptCbcForRangedGets() { + // Use old V1 client for CBC encryption (legacy algorithm) + // Only Java V1 client is available - no V1 test servers for other languages + EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(TestUtils.KMS_KEY_ARN); + + CryptoConfiguration v1Config = + new CryptoConfiguration(CryptoMode.EncryptionOnly) + .withStorageMode(CryptoStorageMode.ObjectMetadata) + .withAwsKmsRegion(TestUtils.KMS_REGION); + + AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() + .withCryptoConfiguration(v1Config) + .withEncryptionMaterials(materialsProvider) + .build(); + + String objectKey = appendTestSuffix(sharedObjectKeyBase + "-cbc-java"); + v1Client.putObject(TestUtils.BUCKET, objectKey, objectKey); + cbcObjects.add(objectKey); + } + + @ParameterizedTest(name = "{0}: Encrypt GCM for ranged get testing") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$EncryptTests#transitionAndImprovedForGCM") + void encryptGcmForRangedGets(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-gcm-" + language.getLanguageName()), + gcmObjects, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Encrypt KC-GCM with Object Metadata Storage for ranged get testing") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$EncryptTests#improvedClientsForKCGCM") + void encryptKcGcmMetadataForRangedGets(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-kc-gcm-metadata-" + language.getLanguageName()), + kcGcmObjectsMetadata, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + + @ParameterizedTest(name = "{0}: Encrypt KC-GCM with Instruction file Storage for ranged get testing") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$EncryptTests#improvedClientsCanPutKMSWithInstructionFile") + void encryptKcGcmInstructionFileForRangedGets(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .instructionFileConfig( + InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build() + ) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt( + client, + S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-kc-gcm-instruction-java" + language.getLanguageName()), + kcGcmObjectsInstruction, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + /** + * Flips a random bit in the given byte array + * @param data The byte array to modify + * @return The bit position that was flipped + */ + static int flipRandomBit(byte[] data) { + if (data.length == 0) { + return -1; + } + int bitPosition = random.nextInt(data.length * 8); + int byteIndex = bitPosition / 8; + int bitIndex = bitPosition % 8; + data[byteIndex] ^= (1 << bitIndex); + return bitPosition; + } + + /** + * Creates corrupted copies of KC-GCM objects for failure testing + * Handles both object metadata storage and instruction file storage modes + */ + static void createCorruptedCopies() throws Exception { + try (S3Client ptS3Client = S3Client.create()) { + ObjectMapper mapper = new ObjectMapper(); + + // Process metadata storage mode objects (all V3 keys in metadata, no instruction file) + for (String objectKey : kcGcmObjectsMetadata) { + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + byte[] objectData = encryptedObject.asByteArray(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Create good copy + putObjectWithMetadata(ptS3Client, objectKey + SUFFIX_GOOD_COPY, objectData, objectMetadata); + + // Extract commitment values from metadata + String commitC = objectMetadata.get("x-amz-c"); + String commitD = objectMetadata.get("x-amz-d"); + String commitI = objectMetadata.get("x-amz-i"); + + // Create mutated commitment copies in metadata + if (commitC != null) { + byte[] commitCBytes = Base64.getDecoder().decode(commitC); + int bitPos = flipRandomBit(commitCBytes); + String mutatedC = Base64.getEncoder().encodeToString(commitCBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-c", mutatedC); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_C + bitPos; + putObjectWithMetadata(ptS3Client, mutatedKey, objectData, mutatedMetadata); + mutatedCObjectsMetadata.add(mutatedKey); + } + + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + int bitPos = flipRandomBit(commitDBytes); + String mutatedD = Base64.getEncoder().encodeToString(commitDBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-d", mutatedD); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_D + bitPos; + putObjectWithMetadata(ptS3Client, mutatedKey, objectData, mutatedMetadata); + mutatedDObjectsMetadata.add(mutatedKey); + } + + if (commitI != null) { + byte[] commitIBytes = Base64.getDecoder().decode(commitI); + int bitPos = flipRandomBit(commitIBytes); + String mutatedI = Base64.getEncoder().encodeToString(commitIBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-i", mutatedI); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_I + bitPos; + putObjectWithMetadata(ptS3Client, mutatedKey, objectData, mutatedMetadata); + mutatedIObjectsMetadata.add(mutatedKey); + } + + // Create invalid D length copies (metadata storage) + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + + // Short D (< 28 bytes) - truncate to 20 bytes + int shortLength = Math.min(20, commitDBytes.length); + byte[] shortDBytes = new byte[shortLength]; + System.arraycopy(commitDBytes, 0, shortDBytes, 0, shortLength); + String shortD = Base64.getEncoder().encodeToString(shortDBytes); + Map shortDMetadata = new java.util.HashMap<>(objectMetadata); + shortDMetadata.put("x-amz-d", shortD); + String shortDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_SHORT; + putObjectWithMetadata(ptS3Client, shortDKey, objectData, shortDMetadata); + invalidDLengthShortMetadata.add(shortDKey); + + // Long D (> 28 bytes) - extend to 40 bytes + byte[] longDBytes = new byte[40]; + System.arraycopy(commitDBytes, 0, longDBytes, 0, commitDBytes.length); + // Fill remaining bytes with zeros + for (int i = commitDBytes.length; i < 40; i++) { + longDBytes[i] = 0; + } + String longD = Base64.getEncoder().encodeToString(longDBytes); + Map longDMetadata = new java.util.HashMap<>(objectMetadata); + longDMetadata.put("x-amz-d", longD); + String longDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_LONG; + putObjectWithMetadata(ptS3Client, longDKey, objectData, longDMetadata); + invalidDLengthLongMetadata.add(longDKey); + } + } + + // Process instruction file storage mode objects (c/d/i in metadata, x-amz-3/w/m/t in instruction file) + for (String objectKey : kcGcmObjectsInstruction) { + // Get the encrypted object + ResponseBytes encryptedObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + + byte[] objectData = encryptedObject.asByteArray(); + Map objectMetadata = encryptedObject.response().metadata(); + + // Get the instruction file + ResponseBytes instructionObject = ptS3Client.getObjectAsBytes(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey + ".instruction") + .build()); + + String originalInstructionFileJson = new String(instructionObject.asByteArray(), StandardCharsets.UTF_8); + + // Create good copy (both object and instruction file) + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_GOOD_COPY, + objectData, + objectMetadata, + originalInstructionFileJson + ); + + // Extract commitment values from metadata + String commitC = objectMetadata.get("x-amz-c"); + String commitD = objectMetadata.get("x-amz-d"); + String commitI = objectMetadata.get("x-amz-i"); + + // Corruption: Add c/d/i to instruction file (duplication - should fail) + Map corruptedInstructionMap = mapper.readValue(originalInstructionFileJson, Map.class); + corruptedInstructionMap.put("x-amz-c", commitC); + corruptedInstructionMap.put("x-amz-d", commitD); + corruptedInstructionMap.put("x-amz-i", commitI); + String corruptedInstructionJson = mapper.writeValueAsString(corruptedInstructionMap); + + putObjectWithInstructionFile( + ptS3Client, + objectKey + SUFFIX_BAD_COMMITMENT_IN_INSTRUCTION, + objectData, + objectMetadata, + corruptedInstructionJson + ); + + // Create mutated commitment copies in metadata + if (commitC != null) { + byte[] commitCBytes = Base64.getDecoder().decode(commitC); + int bitPos = flipRandomBit(commitCBytes); + String mutatedC = Base64.getEncoder().encodeToString(commitCBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-c", mutatedC); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_C + bitPos; + putObjectWithInstructionFile(ptS3Client, mutatedKey, objectData, mutatedMetadata, originalInstructionFileJson); + mutatedCObjectsInstruction.add(mutatedKey); + } + + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + int bitPos = flipRandomBit(commitDBytes); + String mutatedD = Base64.getEncoder().encodeToString(commitDBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-d", mutatedD); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_D + bitPos; + putObjectWithInstructionFile(ptS3Client, mutatedKey, objectData, mutatedMetadata, originalInstructionFileJson); + mutatedDObjectsInstruction.add(mutatedKey); + } + + if (commitI != null) { + byte[] commitIBytes = Base64.getDecoder().decode(commitI); + int bitPos = flipRandomBit(commitIBytes); + String mutatedI = Base64.getEncoder().encodeToString(commitIBytes); + Map mutatedMetadata = new java.util.HashMap<>(objectMetadata); + mutatedMetadata.put("x-amz-i", mutatedI); + String mutatedKey = objectKey + SUFFIX_BAD_MUTATED_I + bitPos; + putObjectWithInstructionFile(ptS3Client, mutatedKey, objectData, mutatedMetadata, originalInstructionFileJson); + mutatedIObjectsInstruction.add(mutatedKey); + } + + // Create invalid D length copies (instruction file storage) + if (commitD != null) { + byte[] commitDBytes = Base64.getDecoder().decode(commitD); + + // Short D (< 28 bytes) - truncate to 20 bytes + int shortLength = Math.min(20, commitDBytes.length); + byte[] shortDBytes = new byte[shortLength]; + System.arraycopy(commitDBytes, 0, shortDBytes, 0, shortLength); + String shortD = Base64.getEncoder().encodeToString(shortDBytes); + Map shortDMetadata = new java.util.HashMap<>(objectMetadata); + shortDMetadata.put("x-amz-d", shortD); + String shortDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_SHORT; + putObjectWithInstructionFile(ptS3Client, shortDKey, objectData, shortDMetadata, originalInstructionFileJson); + invalidDLengthShortInstruction.add(shortDKey); + + // Long D (> 28 bytes) - extend to 40 bytes + byte[] longDBytes = new byte[40]; + System.arraycopy(commitDBytes, 0, longDBytes, 0, commitDBytes.length); + // Fill remaining bytes with zeros + for (int i = commitDBytes.length; i < 40; i++) { + longDBytes[i] = 0; + } + String longD = Base64.getEncoder().encodeToString(longDBytes); + Map longDMetadata = new java.util.HashMap<>(objectMetadata); + longDMetadata.put("x-amz-d", longD); + String longDKey = objectKey + SUFFIX_BAD_INVALID_D_LENGTH_LONG; + putObjectWithInstructionFile(ptS3Client, longDKey, objectData, longDMetadata, originalInstructionFileJson); + invalidDLengthLongInstruction.add(longDKey); + } + } + } + } + + static void putObjectWithMetadata( + S3Client ptS3Client, + String objectKey, + byte[] objectData, + Map objectMetadata + ) { + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + } + + static void putObjectWithInstructionFile( + S3Client ptS3Client, + String objectKey, + byte[] objectData, + Map objectMetadata, + String instructionFileJson + ) { + // Put the encrypted object + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey) + .metadata(objectMetadata) + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes(objectData)); + + // Put the instruction file + ptS3Client.putObject(builder -> builder + .bucket(TestUtils.BUCKET) + .key(objectKey + ".instruction") + .build(), + software.amazon.awssdk.core.sync.RequestBody.fromBytes( + instructionFileJson.getBytes(StandardCharsets.UTF_8))); + } + + @AfterAll + static void signalEncryptionComplete() throws Exception { + createCorruptedCopies(); + + // Signal that all encryption tests have completed + encryptPhaseComplete.countDown(); + } + } + + /** + * Ranged Get Tests - Test Phase + * + * These tests perform ranged get operations on objects encrypted by EncryptTests. + * All tests in this class can run fully in parallel with each other. + * They depend on EncryptTests completing first. + */ + @Nested + @DisplayName("RangedGetTests - RangedGet") + class RangedGetTestsNested { + private static List cbcObjects; + private static List gcmObjects; + private static List kcGcmObjects; + private static List kcGcmObjectsInstruction; + private static List mutatedCObjects; + private static List mutatedDObjects; + private static List mutatedIObjects; + private static List mutatedCObjectsInstruction; + private static List mutatedDObjectsInstruction; + private static List mutatedIObjectsInstruction; + private static KeyMaterial kmsKeyArn; + private static KeyMaterial RSA_KEY; + private static KeyMaterial AES_KEY; + + @BeforeAll + static void setup() throws InterruptedException { + // Wait for all encryption tests to complete + encryptPhaseComplete.await(); + + // Import encrypted objects from the encrypt phase + cbcObjects = EncryptTests.getCbcObjects(); + gcmObjects = EncryptTests.getGcmObjects(); + // Import KC-GCM objects for both storage modes + kcGcmObjects = EncryptTests.getKcGcmObjectsMetadata(); + kcGcmObjectsInstruction = EncryptTests.getKcGcmObjectsInstruction(); + // Import corrupted objects for metadata storage mode + mutatedCObjects = EncryptTests.getMutatedCObjectsMetadata(); + mutatedDObjects = EncryptTests.getMutatedDObjectsMetadata(); + mutatedIObjects = EncryptTests.getMutatedIObjectsMetadata(); + // Import corrupted objects for instruction file storage mode + mutatedCObjectsInstruction = EncryptTests.getMutatedCObjectsInstruction(); + mutatedDObjectsInstruction = EncryptTests.getMutatedDObjectsInstruction(); + mutatedIObjectsInstruction = EncryptTests.getMutatedIObjectsInstruction(); + kmsKeyArn = EncryptTests.getKmsKeyArn(); + RSA_KEY = EncryptTests.getRsaKey(); + AES_KEY = EncryptTests.getAesKey(); + + // Verify we have objects to test + if (cbcObjects.isEmpty() && gcmObjects.isEmpty() && kcGcmObjects.isEmpty()) { + throw new IllegalStateException( + "No encrypted objects found. Ensure EncryptTests runs first."); + } + } + + public static Stream rangedGetSupportedClients() { + Stream improved = improvedClientsForTest() + .filter(target -> RANGED_GETS_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + Stream transition = transitionClientsForTest() + .filter(target -> RANGED_GETS_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + + return Stream.concat(improved, transition); + } + + public static Stream rangedGetCBCSupportedClients() { + return rangedGetSupportedClients() + // This is just a quick hack. Perhaps it would be good to have an equivalent group for languages. + .filter(target -> !((LanguageServerTarget) target.get()[0]).getLanguageName().startsWith("CPP")); + } + + // CBC Ranged Get Tests + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + cbcObjects, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the last 5 bytes + for (String objectKey : cbcObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + cbcObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get CBC objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetCBCSupportedClients") + void rangedGetCbcWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .enableLegacyWrappingAlgorithms(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the whole file using range + for (String objectKey : cbcObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF + ); + } + } + + // // GCM Ranged Get Tests + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + gcmObjects, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the last 5 bytes + for (String objectKey : gcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + gcmObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // For each object, get its length and test the whole file using range + for (String objectKey : gcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get GCM objects - Include tag") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetGcmTagOnlySucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .commitmentPolicy(CommitmentPolicy.FORBID_ENCRYPT_ALLOW_DECRYPT) + .encryptionAlgorithm(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + gcmObjects, + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF + ); + } + + // KC-GCM Ranged Get Tests - Valid Objects + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - Include tag") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmTagOnlySucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects, + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjects + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjects) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjects) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjects) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + // KC-GCM Instruction File Storage - Valid Object Tests + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - start range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionStartSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction, + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 0, + 5, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - middle range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMiddleSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - Include tag") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionTagOnlySucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction, + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + + TestUtils.RangedGet( + client, + S3ECId, + kcGcmObjectsInstruction + .stream() + .map(key -> key + "-good-copy") + .collect(Collectors.toList()), + 10, + 1000, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - end range") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionEndSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjectsInstruction) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjectsInstruction) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + long rangeStart = Math.max(0, objectLength - 5); + long rangeEnd = objectLength - 1; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + rangeStart, + rangeEnd, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + @ParameterizedTest(name = "{0}: Successfully ranged get KC-GCM Instruction File objects - whole file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionWholeFileSucceeds(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test original objects + for (String objectKey : kcGcmObjectsInstruction) { + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(objectKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + // Test good-copy objects + for (String objectKey : kcGcmObjectsInstruction) { + String goodCopyKey = objectKey + "-good-copy"; + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(goodCopyKey) + .build()); + long objectLength = fullOutput.getBody().array().length; + + TestUtils.RangedGet( + client, + S3ECId, + java.util.Collections.singletonList(goodCopyKey), + 0, + objectLength - 1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } + + // KC-GCM Ranged Get Tests - Failure Cases + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with commitment duplicated in instruction file") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionCommitmentInInstructionFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + // Test instruction file storage mode objects with c/d/i duplicated into instruction file + TestUtils.RangedGet_fails( + client, + S3ECId, + kcGcmObjectsInstruction.stream() + .map(key -> key + "-bad-commitment-in-instruction") + .collect(Collectors.toList()), + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with mutated commitment C") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMutatedCFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedCObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with mutated commitment D") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMutatedDFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedDObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with mutated commitment I") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMutatedIFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedIObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with mutated commitment C in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMutatedCFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedCObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with mutated commitment D in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMutatedDFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedDObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with mutated commitment I in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionMutatedIFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + mutatedIObjectsInstruction, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with invalid C length (too short) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMetadataInvalidCLengthShortFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthShortObjects = EncryptTests.getInvalidDLengthShortMetadata(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthShortObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM with invalid D length (too long) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmMetadataInvalidDLengthLongFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthLongObjects = EncryptTests.getInvalidDLengthLongMetadata(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthLongObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with invalid D length (too short) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionInvalidDLengthShortFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthShortObjects = EncryptTests.getInvalidDLengthShortInstruction(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthShortObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + + @ParameterizedTest(name = "{0}: Fail to ranged get KC-GCM Instruction File with invalid D length (too long) in metadata") + @MethodSource("software.amazon.encryption.s3.RangedGetTests$RangedGetTestsNested#rangedGetSupportedClients") + void rangedGetKcGcmInstructionInvalidDLengthLongFails(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(kmsKeyArn) + .enableLegacyUnauthenticatedModes(true) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + List invalidDLengthLongObjects = EncryptTests.getInvalidDLengthLongInstruction(); + + TestUtils.RangedGet_fails( + client, + S3ECId, + invalidDLengthLongObjects, + 5, + 10, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY + ); + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java new file mode 100644 index 00000000..3053afb6 --- /dev/null +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/ReEncryptTests.java @@ -0,0 +1,648 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.encryption.s3; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.TestUtils.*; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.amazonaws.services.s3.AmazonS3Encryption; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; + +import software.amazon.encryption.s3.TestUtils.LanguageServerTarget; +import software.amazon.encryption.s3.client.S3ECTestServerClient; +import software.amazon.encryption.s3.model.CommitmentPolicy; +import software.amazon.encryption.s3.model.CreateClientInput; +import software.amazon.encryption.s3.model.CreateClientOutput; +import software.amazon.encryption.s3.model.EncryptionAlgorithm; +import software.amazon.encryption.s3.model.InstructionFileConfig; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3ECConfig; +import software.amazon.encryption.s3.model.S3EncryptionClientError; + +/** + * ReEncrypt Instruction File Tests - S3 Encryption Client Cross-Language Compatibility + * + * PURPOSE: + * This test suite validates that instruction file re-encryption enables key rotation without + * re-uploading encrypted objects, and that re-encrypted objects maintain cross-language + * compatibility and commitment validation guarantees. + * + * WHAT IS BEING TESTED: + * 1. Instruction file re-encryption for KC-GCM algorithm with raw keyrings + * 2. Re-encryption across different raw keyring types (AES, RSA) + * 3. Same-type keyring rotation (AES => AES, RSA => RSA) + * 4. Cross-type keyring rotation (AES => RSA, RSA => AES) + * 5. Default instruction file suffix (.instruction) and custom suffixes (.instruction-rsa, .instruction-aes) + * 6. Cross-language compatibility: all languages can decrypt after re-encryption + * 7. Rotation enforcement to prevent re-encryption with the same key + * + * WHY THIS IS IMPORTANT: + * - Key rotation is a critical security operation that should not require expensive object re-uploads + * - ReEncryptInstructionFile enables updating the encrypted data key without touching the ciphertext + * - Raw keyrings (AES, RSA) provide direct key material access required for re-encryption + * - Cross-type rotation (e.g., AES to RSA) enables flexibility in key management strategies + * - Commitment validation must be maintained even when instruction files are re-encrypted + * - Cross-language compatibility ensures key rotation doesn't break existing clients + * - Rotation enforcement prevents accidental re-encryption with the same key material + * - Custom instruction file suffixes enable sharing encrypted objects with partners + * + * TEST STRUCTURE: + * This suite uses a three-phase approach with enforced ordering: + * 1. EncryptTests - Encrypts objects with instruction files using AES and RSA keyrings + * - All encrypt tests can run in parallel within this phase + * - Signals encryptPhaseComplete latch when done + * 2. ReEncryptTests - Waits for encryption to complete, then re-encrypts instruction files + * - Tests same-type rotations (AES => AES, RSA => RSA) + * - Tests cross-type rotations (AES => RSA with .instruction-rsa suffix, RSA => AES with .instruction-aes suffix) + * - Tests rotation enforcement (same key rejection) + * - All re-encrypt tests can run in parallel within this phase + * - Tracks which objects were re-encrypted to which keys to prevent conflicts + * - Signals reEncryptPhaseComplete latch when done + * 3. DecryptReEncryptedTests - Waits for re-encryption to complete, then tests decryption + * - Tests cross-language decryption compatibility after re-encryption + * - Uses tracked object lists to decrypt with correct keys and custom instruction file suffixes + * - All decrypt tests can run in parallel within this phase + * + * Coordination uses two CountDownLatches: + * - encryptPhaseComplete: Ensures all encryption completes before re-encryption begins + * - reEncryptPhaseComplete: Ensures all re-encryption completes before decryption begins + * + * INPUT DIMENSIONS: + * - Source Key Material: AES (256-bit), RSA (2048-bit key pairs) + * - Destination Key Material: Different AES or RSA keys (raw keyrings) + * - Encryption Algorithm: KC-GCM (ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY) + * - Instruction File Suffix: default (.instruction), custom (.instruction-rsa, .instruction-aes) + * - Language for Re-encryption: Java V3-Transition, Java V4 (RE_ENCRYPT_SUPPORTED) + * - Language for Decryption: All languages supporting instruction files + * - Rotation Enforcement: enforceRotation flag (true/false) + * + * EXPECTED RESULTS: + * - Positive: Re-encryption succeeds with different key material, all languages can decrypt + * - Negative: Re-encryption fails when enforceRotation detects same key material + * + * REPRESENTATIVE VALUES: + * - Object keys themselves (short strings) serve as representative small plaintext files + * - Instruction file suffix: ".instruction" (default), ".instruction-rsa", ".instruction-aes" + * - Key materials: Generated once per type and reused across tests + * + * FILTERING: + * - Only languages in RE_ENCRYPT_SUPPORTED can perform re-encryption operations + * - Languages in INSTRUCTION_FILE_GET_UNSUPPORTED cannot decrypt with instruction files + * + * NOTE: KMS keyrings are NOT supported for re-encryption as the reEncryptInstructionFile + * method requires RawKeyring instances (AES or RSA) which provide direct access to key material. + * + */ +public class ReEncryptTests { + // Synchronization latches for three-phase coordination + private static final CountDownLatch encryptPhaseComplete = new CountDownLatch(1); + private static final CountDownLatch reEncryptPhaseComplete = new CountDownLatch(1); + + // Tracking lists for re-encrypted objects - shared across nested test classes + private static final List reEncryptedAesToAes = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedRsaToRsa = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedAesToRsa = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedRsaToAesDefault = Collections.synchronizedList(new ArrayList<>()); + private static final List reEncryptedAesToRsaDefault = Collections.synchronizedList(new ArrayList<>()); + + @Nested + @DisplayName("ReEncryptTests - Encrypt") + class EncryptTests { + private static final String sharedObjectKeyBase = "test-reencrypt"; + + private static SecretKey aesKey1, aesKey2; + private static KeyMaterial aesKeyMaterial1, aesKeyMaterial2; + private static KeyPair rsaKeyPair1, rsaKeyPair2; + private static KeyMaterial rsaKeyMaterial1, rsaKeyMaterial2; + + // Separate object lists for each re-encryption path to avoid conflicts + private static final List kcGcmObjectsAesToAes = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsAesToRsaCustom = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsAesToRsaDefault = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsRsaToRsa = Collections.synchronizedList(new ArrayList<>()); + private static final List kcGcmObjectsRsaToAesDefault = Collections.synchronizedList(new ArrayList<>()); + + @BeforeAll + static void generateKeys() throws Exception { + KeyGenerator aesKeyGen = KeyGenerator.getInstance("AES"); + aesKeyGen.init(256); + aesKey1 = aesKeyGen.generateKey(); + aesKey2 = aesKeyGen.generateKey(); + + Map aesMatDesc1 = new HashMap<>(); + aesMatDesc1.put("keyId", "aes-key-1"); + aesKeyMaterial1 = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesKey1.getEncoded())) + .materialsDescription(aesMatDesc1) + .build(); + + Map aesMatDesc2 = new HashMap<>(); + aesMatDesc2.put("keyId", "aes-key-2"); + aesKeyMaterial2 = KeyMaterial.builder() + .aesKey(ByteBuffer.wrap(aesKey2.getEncoded())) + .materialsDescription(aesMatDesc2) + .build(); + + KeyPairGenerator rsaKeyGen = KeyPairGenerator.getInstance("RSA"); + rsaKeyGen.initialize(2048); + rsaKeyPair1 = rsaKeyGen.generateKeyPair(); + rsaKeyPair2 = rsaKeyGen.generateKeyPair(); + + Map rsaMatDesc1 = new HashMap<>(); + rsaMatDesc1.put("keyId", "rsa-key-1"); + rsaKeyMaterial1 = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(rsaKeyPair1.getPrivate().getEncoded())) + .materialsDescription(rsaMatDesc1) + .build(); + + Map rsaMatDesc2 = new HashMap<>(); + rsaMatDesc2.put("keyId", "rsa-key-2"); + rsaKeyMaterial2 = KeyMaterial.builder() + .rsaKey(ByteBuffer.wrap(rsaKeyPair2.getPrivate().getEncoded())) + .materialsDescription(rsaMatDesc2) + .build(); + } + + static List getKcGcmObjectsAesToAes() { return new ArrayList<>(kcGcmObjectsAesToAes); } + static List getKcGcmObjectsAesToRsaCustom() { return new ArrayList<>(kcGcmObjectsAesToRsaCustom); } + static List getKcGcmObjectsAesToRsaDefault() { return new ArrayList<>(kcGcmObjectsAesToRsaDefault); } + static List getKcGcmObjectsRsaToRsa() { return new ArrayList<>(kcGcmObjectsRsaToRsa); } + static List getKcGcmObjectsRsaToAesDefault() { return new ArrayList<>(kcGcmObjectsRsaToAesDefault); } + static KeyMaterial getAesKeyMaterial1() { return aesKeyMaterial1; } + static KeyMaterial getAesKeyMaterial2() { return aesKeyMaterial2; } + static KeyMaterial getRsaKeyMaterial1() { return rsaKeyMaterial1; } + static KeyMaterial getRsaKeyMaterial2() { return rsaKeyMaterial2; } + + public static Stream improvedClientsCanPutRawRSAWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + public static Stream improvedClientsCanPutRawAESWithInstructionFile() { + return improvedClientsForTest() + .filter(target -> !INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: Encrypt AES objects for AES => AES re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptAesForAesToAesReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-aes-to-aes-" + language.getLanguageName()), + kcGcmObjectsAesToAes, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt AES objects for AES => RSA custom suffix re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptAesForAesToRsaCustomReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-aes-to-rsa-custom-" + language.getLanguageName()), + kcGcmObjectsAesToRsaCustom, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt AES objects for AES => RSA default suffix re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawAESWithInstructionFile") + void encryptAesForAesToRsaDefaultReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-aes-to-rsa-default-" + language.getLanguageName()), + kcGcmObjectsAesToRsaDefault, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA objects for RSA => RSA re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptRsaForRsaToRsaReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-rsa-to-rsa-" + language.getLanguageName()), + kcGcmObjectsRsaToRsa, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @ParameterizedTest(name = "{0}: Encrypt RSA objects for RSA => AES default suffix re-encryption") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$EncryptTests#improvedClientsCanPutRawRSAWithInstructionFile") + void encryptRsaForRsaToAesDefaultReencrypt(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + TestUtils.Encrypt(client, S3ECId, + appendTestSuffix(sharedObjectKeyBase + "-rsa-to-aes-default-" + language.getLanguageName()), + kcGcmObjectsRsaToAesDefault, EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + + @AfterAll + static void signalEncryptionComplete() { + encryptPhaseComplete.countDown(); + } + } + + @Nested + @DisplayName("ReEncryptTests - ReEncrypt") + class ReEncryptTestsNested { + private static List kcGcmObjectsAesToAes, kcGcmObjectsAesToRsaCustom, kcGcmObjectsAesToRsaDefault; + private static List kcGcmObjectsRsaToRsa, kcGcmObjectsRsaToAesDefault; + private static KeyMaterial aesKeyMaterial1, aesKeyMaterial2, rsaKeyMaterial1, rsaKeyMaterial2; + + @BeforeAll + static void setup() throws InterruptedException { + encryptPhaseComplete.await(); + kcGcmObjectsAesToAes = EncryptTests.getKcGcmObjectsAesToAes(); + kcGcmObjectsAesToRsaCustom = EncryptTests.getKcGcmObjectsAesToRsaCustom(); + kcGcmObjectsAesToRsaDefault = EncryptTests.getKcGcmObjectsAesToRsaDefault(); + kcGcmObjectsRsaToRsa = EncryptTests.getKcGcmObjectsRsaToRsa(); + kcGcmObjectsRsaToAesDefault = EncryptTests.getKcGcmObjectsRsaToAesDefault(); + aesKeyMaterial1 = EncryptTests.getAesKeyMaterial1(); + aesKeyMaterial2 = EncryptTests.getAesKeyMaterial2(); + rsaKeyMaterial1 = EncryptTests.getRsaKeyMaterial1(); + rsaKeyMaterial2 = EncryptTests.getRsaKeyMaterial2(); + } + + public static Stream reencryptSupportedClients() { + return improvedClientsForTest() + .filter(target -> RE_ENCRYPT_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + } + + @ParameterizedTest(name = "{0}: ReEncrypt AES => AES instruction file") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptAesToAesInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsAesToAes.size(); i++) { + String objectKey = kcGcmObjectsAesToAes.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(aesKeyMaterial2).build()); + + assertNotNull(response); + reEncryptedAesToAes.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt RSA => RSA instruction file") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptRsaToRsaInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsRsaToRsa.size(); i++) { + String objectKey = kcGcmObjectsRsaToRsa.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(rsaKeyMaterial2).build()); + + assertNotNull(response); + reEncryptedRsaToRsa.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt AES => RSA instruction file with custom suffix") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptAesToRsaInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsAesToRsaCustom.size(); i++) { + String objectKey = kcGcmObjectsAesToRsaCustom.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(rsaKeyMaterial1) + // Java always prepends a `.` + .instructionFileSuffix("instruction-rsa") + .build()); + + assertNotNull(response); + reEncryptedAesToRsa.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt RSA => AES instruction file (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptRsaToAesDefaultInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsRsaToAesDefault.size(); i++) { + String objectKey = kcGcmObjectsRsaToAesDefault.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(aesKeyMaterial1) + .build()); + + assertNotNull(response); + reEncryptedRsaToAesDefault.add(objectKey); + } + } + + @ParameterizedTest(name = "{0}: ReEncrypt AES => RSA instruction file (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$ReEncryptTestsNested#reencryptSupportedClients") + void reencryptAesToRsaDefaultInstructionFile(TestUtils.LanguageServerTarget language) { + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + + for (int i = 0; i < kcGcmObjectsAesToRsaDefault.size(); i++) { + String objectKey = kcGcmObjectsAesToRsaDefault.get(i); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .instructionFileConfig(InstructionFileConfig.builder() + .enableInstructionFilePutObject(true) + .build()) + .build()) + .build()); + String S3ECId = clientOutput.getClientId(); + + ReEncryptOutput response = client.reEncrypt(ReEncryptInput.builder() + .bucket(TestUtils.BUCKET).key(objectKey).clientID(S3ECId) + .newKeyMaterial(rsaKeyMaterial1) + .build()); + + assertNotNull(response); + reEncryptedAesToRsaDefault.add(objectKey); + } + } + + @AfterAll + static void signalReEncryptionComplete() { + reEncryptPhaseComplete.countDown(); + } + } + + @Nested + @DisplayName("ReEncryptTests - DecryptReEncrypted") + class DecryptReEncryptedTests { + private static KeyMaterial aesKeyMaterial1, aesKeyMaterial2, rsaKeyMaterial1, rsaKeyMaterial2; + + @BeforeAll + static void setup() throws InterruptedException { + reEncryptPhaseComplete.await(); + aesKeyMaterial1 = EncryptTests.getAesKeyMaterial1(); + aesKeyMaterial2 = EncryptTests.getAesKeyMaterial2(); + rsaKeyMaterial1 = EncryptTests.getRsaKeyMaterial1(); + rsaKeyMaterial2 = EncryptTests.getRsaKeyMaterial2(); + } + + public static Stream clientsCanGetRawRSAWithInstructionFile() { + return Stream.concat( + improvedClientsForTest().filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest().filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + public static Stream clientsCanGetRawAESWithInstructionFile() { + return Stream.concat( + improvedClientsForTest().filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest().filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + public static Stream clientsCanGetRawRSAWithInstructionFileAndCustomSuffix() { + return Stream.concat( + improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + public static Stream clientsCanGetRawAESWithInstructionFileAndCustomSuffix() { + return Stream.concat( + improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())), + transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + .filter(target -> CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())) + ); + } + + @ParameterizedTest(name = "{0}: Decrypt AES => AES re-encrypted objects") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawAESWithInstructionFile") + void decryptReencryptedAesToAesObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedAesToAes.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder().keyMaterial(aesKeyMaterial2).build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedAesToAes, aesKeyMaterial2, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedAesToAes, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + + @ParameterizedTest(name = "{0}: Decrypt RSA => RSA re-encrypted objects") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawRSAWithInstructionFile") + void decryptReencryptedRsaToRsaObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedRsaToRsa.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder().keyMaterial(rsaKeyMaterial2).build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedRsaToRsa, rsaKeyMaterial2, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedRsaToRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + + @ParameterizedTest(name = "{0}: Decrypt AES => RSA re-encrypted objects with custom suffix") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawRSAWithInstructionFileAndCustomSuffix") + void decryptReencryptedAesToRsaObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedAesToRsa.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedAesToRsa, rsaKeyMaterial1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedAesToRsa, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY, + reEncryptedAesToRsa, ".instruction-rsa"); + } + } + + @ParameterizedTest(name = "{0}: Decrypt RSA => AES re-encrypted objects (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawAESWithInstructionFile") + void decryptReencryptedRsaToAesDefaultObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedRsaToAesDefault.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(aesKeyMaterial1) + .build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedRsaToAesDefault, aesKeyMaterial1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedRsaToAesDefault, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + + @ParameterizedTest(name = "{0}: Decrypt AES => RSA re-encrypted objects (default suffix)") + @MethodSource("software.amazon.encryption.s3.ReEncryptTests$DecryptReEncryptedTests#clientsCanGetRawRSAWithInstructionFile") + void decryptReencryptedAesToRsaDefaultObjects(TestUtils.LanguageServerTarget language) { + if (reEncryptedAesToRsaDefault.isEmpty()) return; + S3ECTestServerClient client = TestUtils.testServerClientFor(language); + CreateClientOutput clientOutput = client.createClient(CreateClientInput.builder() + .config(S3ECConfig.builder() + .keyMaterial(rsaKeyMaterial1) + .build()) + .build()); + + // C++ clients require materials description to be passed per-operation + if (language.getLanguageName().startsWith("CPP")) { + TestUtils.DecryptWithMaterialsDescription(client, clientOutput.getClientId(), + reEncryptedAesToRsaDefault, rsaKeyMaterial1, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else { + TestUtils.Decrypt(client, clientOutput.getClientId(), reEncryptedAesToRsaDefault, + EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } + } + } +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 468fc708..cf45006e 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -214,9 +214,9 @@ public void crossLanguageTestKmsWithSubsetEncCtxFails(LanguageServerTarget encLa fail("Expected exception!"); } catch (S3EncryptionClientError e) { if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_CURRENT) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { - assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); } else { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); } } } @@ -278,9 +278,9 @@ public void crossLanguageTestKmsWithIncorrectEncCtxFails(LanguageServerTarget en fail("Expected exception!"); } catch (S3EncryptionClientError e) { if (decLang.getLanguageName().equals(RUBY_V3) || decLang.getLanguageName().equals(RUBY_V2_CURRENT) || decLang.getLanguageName().equals(RUBY_V2_TRANSITION)) { - assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context")); + assertTrue(e.getMessage().contains("Value of encryption context from envelope does not match the provided encryption context"), "Actual error: " + e.getMessage()); } else { - assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3")); + assertTrue(e.getMessage().contains("Provided encryption context does not match information retrieved from S3"), "Actual error: " + e.getMessage()); } } } @@ -427,15 +427,15 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(TestUtils.LanguageServerTarget la || language.getLanguageName().equals(CPP_V2_CURRENT) || language.getLanguageName().equals(CPP_V2_TRANSITION) || language.getLanguageName().equals(CPP_V3)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" - )); + ), "Actual error:" + e.getMessage()); } else if (language.getLanguageName().equals(RUBY_V3) || language.getLanguageName().equals(RUBY_V2_CURRENT) || language.getLanguageName().equals(RUBY_V2_TRANSITION)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object." ), "Actual error:" + e.getMessage()); } else if (language.getLanguageName().equals(PHP_V3)) { - assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."));; + assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration @SecurityProfile=V3. Retry with V3_AND_LEGACY enabled or reencrypt the object."), "Actual error: " + e.getMessage()); } else { - assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); + assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"), "Actual error: " + e.getMessage()); } } } @@ -536,7 +536,7 @@ public void instructionFileReadV2Format(TestUtils.LanguageServerTarget language) @ParameterizedTest(name = "{displayName} for Encrypt: {0}, Decrypt: {1}") @MethodSource("software.amazon.encryption.s3.TestUtils#crossLanguageClients") - public void instructionFileWriteAndRead(LanguageServerTarget encLang, LanguageServerTarget decLang) { + public void instructionFileWriteAndRead(LanguageServerTarget encLang, LanguageServerTarget decLang) throws Exception { if (INSTRUCTION_FILE_PUT_UNSUPPORTED.contains(encLang.getLanguageName())) { throw new TestAbortedException("not testing " + encLang.getLanguageName()); } @@ -601,6 +601,14 @@ public void instructionFileWriteAndRead(LanguageServerTarget encLang, LanguageSe // Ruby and PHP do not include it :( assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); } + + // At high concurrency, this test tends to get: + // BadDigest Message: The CRC64NVME you specified did not match the calculated checksum. + // I think this is a read after write issue. + // A better fix, would be to break this tests suite up into encrypt/decrypt + // rather than having a test for many pairs and doing encrypt/decrypt on each pair + Thread.sleep(100); + assertFalse(ptInstFile.asUtf8String().isEmpty()); // Read should be enabled by default GetObjectOutput output = decClient.getObject(GetObjectInput.builder() @@ -665,7 +673,7 @@ public void instructionFileWriteAndReadWithRSA(LanguageServerTarget encLang, Lan .key(objectKey + ".instruction") .build()); } - assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); + // assertTrue(ptInstFile.response().metadata().containsKey("x-amz-crypto-instr-file")); assertFalse(ptInstFile.asUtf8String().isEmpty()); // Read should be enabled by default GetObjectOutput output = decClient.getObject(GetObjectInput.builder() @@ -676,4 +684,4 @@ public void instructionFileWriteAndReadWithRSA(LanguageServerTarget encLang, Lan assertEquals(input, new String(output.getBody().array())); } -} \ No newline at end of file +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index d3d25d58..f2065115 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -6,6 +6,8 @@ package software.amazon.encryption.s3; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import java.net.Socket; import java.net.URI; @@ -17,10 +19,14 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.amazonaws.services.s3.model.S3Object; +import com.fasterxml.jackson.databind.ObjectMapper; + import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -32,6 +38,7 @@ import software.amazon.encryption.s3.model.EncryptionAlgorithm; import software.amazon.encryption.s3.model.GetObjectInput; import software.amazon.encryption.s3.model.GetObjectOutput; +import software.amazon.encryption.s3.model.KeyMaterial; import software.amazon.encryption.s3.model.PutObjectInput; import software.amazon.encryption.s3.model.PutObjectOutput; import software.amazon.encryption.s3.model.S3EncryptionClientError; @@ -95,12 +102,35 @@ public class TestUtils { public static final Set ENCRYPTION_CONTEXT_ON_ENCRYPT_UNSUPPORTED = Set.of(NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4); - // For now, only .NET and Java have RSA support - public static final Set RAW_SUPPORTED = + public static final Set RE_ENCRYPT_SUPPORTED = + Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4); + + public static final Set RANGED_GETS_SUPPORTED = + Set.of( + JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 + , CPP_V2_CURRENT, CPP_V2_TRANSITION, CPP_V3 + ); + + // Cpp only supports Raw AES + public static final Set RAW_AES_SUPPORTED = Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 + , RUBY_V2_TRANSITION, RUBY_V3 + , CPP_V2_CURRENT, CPP_V2_TRANSITION, CPP_V3 ); + public static final Set RAW_RSA_SUPPORTED = + Set.of(JAVA_V3_CURRENT, JAVA_V3_TRANSITION, JAVA_V4 + , NET_V2_CURRENT, NET_V3_CURRENT, NET_V3_TRANSITION, NET_V4 + , RUBY_V2_TRANSITION, RUBY_V3 + ); + + // Intersection of RAW_AES_SUPPORTED and RAW_RSA_SUPPORTED + public static final Set RAW_SUPPORTED = + RAW_AES_SUPPORTED.stream() + .filter(RAW_RSA_SUPPORTED::contains) + .collect(Collectors.toSet()); + // .NET only supports decrypting instruction files using AES and RSA. // Python MUST support decrypting KMS instruction files, but does not yet. public static final Set KMS_INSTRUCTION_FILE_UNSUPPORTED = @@ -116,6 +146,19 @@ public class TestUtils { public static final Set INSTRUCTION_FILE_GET_UNSUPPORTED = Set.of(PYTHON_V3); + // Languages that support custom instruction file suffix on GetObject + // Only Java, Ruby, and PHP servers have been updated with this feature + // This is a current gap. + public static final Set CUSTOM_INSTRUCTION_SUFFIX_GET_SUPPORTED = + Set.of( + JAVA_V3_TRANSITION, + JAVA_V4, + RUBY_V2_TRANSITION, + RUBY_V3, + PHP_V2_TRANSITION, + PHP_V3 + ); + public static final Set CURRENT_VERSIONS = Set.of( JAVA_V3_CURRENT, @@ -361,6 +404,28 @@ public static Stream improvedClientsForTest() { .map(Arguments::of); } + /** + * Get stream of arguments for clients that support RAW AES (includes CPP). + */ + public static Stream clientsRawAesForTest() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + Stream transition = transitionClientsForTest() + .filter(target -> RAW_AES_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + return Stream.concat(improved, transition); + } + + /** + * Get stream of arguments for clients that support RAW RSA (excludes CPP). + */ + public static Stream clientsRawRsaForTest() { + Stream improved = improvedClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + Stream transition = transitionClientsForTest() + .filter(target -> RAW_RSA_SUPPORTED.contains(((LanguageServerTarget) target.get()[0]).getLanguageName())); + return Stream.concat(improved, transition); + } + /** * These functions provide a stream of arguments for parameterized tests. * @return Stream of Arguments containing pairs of LanguageServerTarget for encryption and decryption @@ -449,22 +514,58 @@ public static String appendTestSuffix(final String s) { private static AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient(); public static EncryptionAlgorithm GetEncryptionAlgorithm(String objectKey) { + // Lambda to determine encryption algorithm from a metadata map + java.util.function.Function, Optional> getAlgorithmFromMap = (map) -> { + if (map.containsKey("x-amz-c")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY); + } else if (map.containsKey("x-amz-cek-alg")) { + String cek = (String) map.get("x-amz-cek-alg"); + if (cek.contains("CBC")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF); + } else if (cek.contains("GCM")) { + return Optional.of(EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF); + } + } + return Optional.empty(); + }; + ObjectMetadata metadata = s3Client.getObjectMetadata(TestUtils.BUCKET, objectKey); Map userMetadata = metadata.getUserMetadata(); - // This is optimized to not need to go to the instruction files for commit_key - if (userMetadata.containsKey("x-amz-c")) { - return EncryptionAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY; - } else if (userMetadata.containsKey("x-amz-cek-alg")) { - String cek = userMetadata.get("x-amz-cek-alg"); - if (cek.contains("CBC")) { - return EncryptionAlgorithm.ALG_AES_256_CBC_IV16_NO_KDF; - } else if (cek.contains("GCM")) { - return EncryptionAlgorithm.ALG_AES_256_GCM_IV12_TAG16_NO_KDF; + // Try to get algorithm from object metadata + Optional algorithm = getAlgorithmFromMap.apply(userMetadata); + if (algorithm.isPresent()) { + return algorithm.get(); + } + + // Check instruction file + try { + String instructionFileKey = objectKey + ".instruction"; + com.amazonaws.services.s3.model.S3Object instructionFileObject = + s3Client.getObject(TestUtils.BUCKET, instructionFileKey); + + // Read instruction file content + java.io.InputStream inputStream = instructionFileObject.getObjectContent(); + String instructionFileJson = new String( + inputStream.readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8 + ); + inputStream.close(); + + // Parse JSON to get metadata + com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + Map instructionFileMap = mapper.readValue(instructionFileJson, Map.class); + + // Try to get algorithm from instruction file + algorithm = getAlgorithmFromMap.apply(instructionFileMap); + if (algorithm.isPresent()) { + return algorithm.get(); } + } catch (Exception e) { + // Instruction file doesn't exist or couldn't be read } - throw new RuntimeException("Need to support instruction files!"); + throw new RuntimeException("Could not determine encryption algorithm from object metadata or instruction file!"); } public static void Encrypt( @@ -492,23 +593,154 @@ public static void Encrypt( public static void Decrypt( S3ECTestServerClient client, - String S3ECId, List crossLanguageObjects, + String S3ECId, + List crossLanguageObjects, EncryptionAlgorithm expectedEncryptionAlgorithm ) { - for (String objectKey : crossLanguageObjects) { - GetObjectOutput output = client.getObject(GetObjectInput.builder() - .clientID(S3ECId) - .bucket(TestUtils.BUCKET) - .key(objectKey) - .build()); - - // Then: Pass - assertEquals(objectKey, new String(output.getBody().array())); - assertEquals( - expectedEncryptionAlgorithm, - GetEncryptionAlgorithm(objectKey), - "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm - ); + // Call 5-parameter version with crossLanguageObjects as expectedPlaintexts + Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, crossLanguageObjects); + } + + public static void Decrypt( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts + ) { + Decrypt(client, S3ECId, crossLanguageObjects, expectedEncryptionAlgorithm, expectedPlaintexts, null); + } + + public static void Decrypt( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts, + String instructionFileSuffix + ) { + if (crossLanguageObjects.isEmpty()) { + fail("There is nothing to decrypt"); + } + + List failures = new ArrayList<>(); + for (int i = 0; i < crossLanguageObjects.size(); i++) { + try { + String objectKey = crossLanguageObjects.get(i); + String expectedPlaintext = expectedPlaintexts.get(i); + + GetObjectInput.Builder builder = GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey); + + // Add custom instruction file suffix if provided + if (instructionFileSuffix != null && !instructionFileSuffix.isEmpty()) { + builder.instructionFileSuffix(instructionFileSuffix); + } + + GetObjectOutput output = client.getObject(builder.build()); + + // Then: Pass + assertEquals(expectedPlaintext, new String(output.getBody().array())); + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + } catch (Exception e) { + failures.add(String.format( + "Failed to decrypt object '%s' (index %d): %s - %s", + crossLanguageObjects.get(i), i, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Decryption failed for %d out of %d objects:\n%s", + failures.size(), crossLanguageObjects.size(), + String.join("\n", failures) + )); + } + } + + /** + * Decrypt helper for C++ clients that require materials description per-operation. + * + * C++ SDK Design: Unlike Java/. NET/etc where materials description is embedded in the + * keyring during client creation, the C++ SDK requires passing materials description + * as a contextMap parameter to each GetObject/PutObject operation. + * + * This helper extracts materials description from KeyMaterial and passes it via the + * Content-Metadata header on each GetObject call, which the C++ server converts to + * the contextMap parameter required by the C++ SDK. + */ + public static void DecryptWithMaterialsDescription( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + KeyMaterial keyMaterial, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + DecryptWithMaterialsDescription(client, S3ECId, crossLanguageObjects, keyMaterial, + expectedEncryptionAlgorithm, crossLanguageObjects); + } + + /** + * Decrypt helper for C++ clients with custom expected plaintexts. + */ + public static void DecryptWithMaterialsDescription( + S3ECTestServerClient client, + String S3ECId, + List crossLanguageObjects, + KeyMaterial keyMaterial, + EncryptionAlgorithm expectedEncryptionAlgorithm, + List expectedPlaintexts + ) { + if (crossLanguageObjects.isEmpty()) { + throw new AssertionError("There is nothing to decrypt"); + } + + // Extract materials description from KeyMaterial + List metadata = (keyMaterial.getMaterialsDescription() != null) + ? metadataMapToList(keyMaterial.getMaterialsDescription()) + : new ArrayList<>(); + + List failures = new ArrayList<>(); + for (int i = 0; i < crossLanguageObjects.size(); i++) { + try { + String objectKey = crossLanguageObjects.get(i); + String expectedPlaintext = expectedPlaintexts.get(i); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .metadata(metadata) // Pass materials description for C++ + .build()); + + // Then: Pass + assertEquals(expectedPlaintext, new String(output.getBody().array())); + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "When decrypting the EncryptionAlgorithm does not match the expected value: " + expectedEncryptionAlgorithm + ); + } catch (Exception e) { + failures.add(String.format( + "Failed to decrypt object '%s' (index %d): %s - %s", + crossLanguageObjects.get(i), i, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Decryption failed for %d out of %d objects:\n%s", + failures.size(), crossLanguageObjects.size(), + String.join("\n", failures) + )); } } @@ -517,6 +749,11 @@ public static void Decrypt_fails( String S3ECId, List crossLanguageObjects, EncryptionAlgorithm expectedEncryptionAlgorithm ) { + + if (crossLanguageObjects.isEmpty()) { + throw new AssertionError("There is nothing to decrypt"); + } + List successfulDecrypt = new ArrayList<>(); for (String objectKey : crossLanguageObjects) { try { @@ -541,4 +778,113 @@ public static void Decrypt_fails( assertEquals(successfulDecrypt.size(), 0, "Decryption should have failed:" + String.join(",", successfulDecrypt)); } + + /** + * Perform ranged get operation with specified byte range + */ + public static void RangedGet( + S3ECTestServerClient client, + String S3ECId, + List objectKeys, + long rangeStart, + long rangeEnd, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + + if (objectKeys.isEmpty()) { + throw new AssertionError("There is nothing to get"); + } + + List failures = new ArrayList<>(); + for (String objectKey : objectKeys) { + try { + // Get the full object first to know expected content + GetObjectOutput fullOutput = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .build()); + byte[] fullContent = fullOutput.getBody().array(); + + // Perform ranged get + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .range("bytes=" + rangeStart + "-" + rangeEnd) + .build()); + + // Verify the ranged content matches expected slice + byte[] rangedContent = output.getBody().array(); + int startIndex = (int) rangeStart; + int endIndex = (int) Math.min(rangeEnd + 1, fullContent.length); // +1 because HTTP ranges are inclusive + byte[] expectedContent = Arrays.copyOfRange(fullContent, startIndex, endIndex); + assertArrayEquals(expectedContent, rangedContent, + "Ranged get returned unexpected data for:" + objectKey); + + // Verify encryption algorithm + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "Encryption algorithm mismatch for " + objectKey + ); + } catch (Exception e) { + failures.add(String.format( + "Failed ranged get on '%s': %s - %s", + objectKey, e.getClass().getSimpleName(), e.getMessage() + )); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError(String.format( + "Ranged get failed for %d out of %d objects:\n%s", + failures.size(), objectKeys.size(), + String.join("\n", failures) + )); + } + } + + /** + * Perform ranged get operations that are expected to fail + */ + public static void RangedGet_fails( + S3ECTestServerClient client, + String S3ECId, + List objectKeys, + long rangeStart, + long rangeEnd, + EncryptionAlgorithm expectedEncryptionAlgorithm + ) { + + if (objectKeys.isEmpty()) { + throw new AssertionError("There is nothing to get"); + } + + List successfulGets = new ArrayList<>(); + for (String objectKey : objectKeys) { + try { + assertEquals( + expectedEncryptionAlgorithm, + GetEncryptionAlgorithm(objectKey), + "Encryption algorithm mismatch for " + objectKey + ); + + GetObjectOutput output = client.getObject(GetObjectInput.builder() + .clientID(S3ECId) + .bucket(TestUtils.BUCKET) + .key(objectKey) + .range("bytes=" + rangeStart + "-" + rangeEnd) + .build()); + + // Should have failed but didn't + successfulGets.add(objectKey); + } catch (S3EncryptionClientError e) { + // This is expected - the ranged get should fail + } + } + + assertEquals(0, successfulGets.size(), + "Ranged get should have failed for: " + String.join(", ", successfulGets)); + } } diff --git a/test-server/java-v3-server/Makefile b/test-server/java-v3-server/Makefile index 692e80b3..59dcdff5 100644 --- a/test-server/java-v3-server/Makefile +++ b/test-server/java-v3-server/Makefile @@ -7,7 +7,7 @@ PORT := 8080 build-server: @echo "Building Java V3 server..." - ./gradlew --build-cache --parallel build + ./gradlew --build-cache --parallel --no-daemon build start-server: @echo "Starting Java V3 server..." diff --git a/test-server/java-v3-server/gradle.properties b/test-server/java-v3-server/gradle.properties index 08afce82..483cd315 100644 --- a/test-server/java-v3-server/gradle.properties +++ b/test-server/java-v3-server/gradle.properties @@ -4,8 +4,21 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index 1d198590..6a3da066 100644 --- a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,5 +1,8 @@ package software.amazon.encryption.s3; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.core.traits.Trait; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.S3EncryptionClient; @@ -34,9 +37,11 @@ public class CreateClientOperationImpl implements CreateClientOperation { private Map clientCache_; + private Map keyringCache_; - public CreateClientOperationImpl(Map clientCache) { + public CreateClientOperationImpl(Map clientCache, Map keyringCache) { clientCache_ = clientCache; + keyringCache_ = keyringCache; } // Copied from S3EC. @@ -106,12 +111,25 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + // Client Creation boolean instFilePut = false; if (input.getConfig().getInstructionFileConfig() != null) { instFilePut = input.getConfig().getInstructionFileConfig().isEnableInstructionFilePutObject(); } S3Client s3Client = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) .instructionFileConfig(InstructionFileConfig.builder() .instructionFileClient(S3Client.create()) .enableInstructionFilePutObject(instFilePut) @@ -121,6 +139,7 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c UUID uuid = UUID.randomUUID(); String uuidString = uuid.toString(); clientCache_.put(uuidString, s3Client); + keyringCache_.put(uuidString, keyring); return CreateClientOutput.builder() .clientId(uuidString) .build(); diff --git a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java index e7c5493f..fbccd458 100644 --- a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java +++ b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -34,10 +34,16 @@ public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { Map ecMap = metadataListToMap(input.getMetadata()); try { - ResponseBytes resp = s3Client.getObjectAsBytes(builder -> builder - .bucket(input.getBucket()) - .key(input.getKey()) - .overrideConfiguration(withAdditionalConfiguration(ecMap))); + ResponseBytes resp = s3Client.getObjectAsBytes(builder -> { + builder.bucket(input.getBucket()) + .key(input.getKey()) + .overrideConfiguration(withAdditionalConfiguration(ecMap)); + + // Add range header if provided + if (input.getRange() != null && !input.getRange().isEmpty()) { + builder.range(input.getRange()); + } + }); List mdAsList = metadataMapToList(resp.response().metadata()); // Can't use asBB else it gets mad bc cant access backing array diff --git a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java new file mode 100644 index 00000000..dd376429 --- /dev/null +++ b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java @@ -0,0 +1,185 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RawKeyring; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.ReEncryptOperation; +import software.amazon.smithy.java.server.RequestContext; + +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; +import java.util.HashMap; +import java.util.Map; + +public class ReEncryptOperationImpl implements ReEncryptOperation { + private final Map clientCache_; + private final Map keyringCache_; + + public ReEncryptOperationImpl(Map clientCache, Map keyringCache) { + clientCache_ = clientCache; + keyringCache_ = keyringCache; + } + + @Override + public ReEncryptOutput reEncrypt(ReEncryptInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + + // Ensure we have an S3EncryptionClient, not just a plain S3Client + if (!(s3Client instanceof S3EncryptionClient)) { + throw new IllegalStateException( + "Client " + input.getClientID() + " is not an S3EncryptionClient"); + } + + S3EncryptionClient s3EncryptionClient = (S3EncryptionClient) s3Client; + + // Create a new keyring from the provided newKeyMaterial + KeyMaterial newKeyMaterial = input.getNewKeyMaterial(); + if (newKeyMaterial == null) { + throw new IllegalStateException( + "newKeyMaterial is required for ReEncrypt operation"); + } + + RawKeyring newKeyring = createKeyringFromMaterial(newKeyMaterial); + + try { + // Build the ReEncryptInstructionFileRequest + ReEncryptInstructionFileRequest.Builder requestBuilder = + ReEncryptInstructionFileRequest.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .newKeyring(newKeyring); + + // Add optional instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + requestBuilder.instructionFileSuffix(input.getInstructionFileSuffix()); + } + + // Add optional enforceRotation if provided + if (input.isEnforceRotation() != null) { + requestBuilder.enforceRotation(input.isEnforceRotation()); + } + + ReEncryptInstructionFileRequest reEncryptRequest = requestBuilder.build(); + + // Perform the re-encryption + ReEncryptInstructionFileResponse response = + s3EncryptionClient.reEncryptInstructionFile(reEncryptRequest); + + // Build and return the output + return ReEncryptOutput.builder() + .bucket(response.bucket()) + .key(response.key()) + .instructionFileSuffix(response.instructionFileSuffix()) + .enforceRotation(response.enforceRotation()) + .build(); + + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + /** + * Creates a RawKeyring from KeyMaterial. + * The KeyMaterial should have exactly one of: aesKey, rsaKey, or kmsKeyId set. + */ + private RawKeyring createKeyringFromMaterial(KeyMaterial keyMaterial) { + try { + // Get materials description from KeyMaterial if provided + MaterialsDescription materialsDescription = null; + if (keyMaterial.getMaterialsDescription() != null && !keyMaterial.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder builder = MaterialsDescription.builder(); + for (Map.Entry entry : keyMaterial.getMaterialsDescription().entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + materialsDescription = builder.build(); + } + + // Check for AES key + if (keyMaterial.getAesKey() != null) { + byte[] aesKeyBytes = new byte[keyMaterial.getAesKey().remaining()]; + keyMaterial.getAesKey().get(aesKeyBytes); + SecretKey secretKey = new SecretKeySpec(aesKeyBytes, "AES"); + + AesKeyring.Builder keyringBuilder = AesKeyring.builder() + .wrappingKey(secretKey); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + // Check for RSA key + if (keyMaterial.getRsaKey() != null) { + byte[] rsaKeyBytes = new byte[keyMaterial.getRsaKey().remaining()]; + keyMaterial.getRsaKey().get(rsaKeyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(rsaKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + + // Derive the public key from the private key + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + PartialRsaKeyPair keyPair = PartialRsaKeyPair.builder() + .privateKey(privateKey) + .publicKey(publicKey) + .build(); + + RsaKeyring.Builder keyringBuilder = RsaKeyring.builder() + .wrappingKeyPair(keyPair); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + throw new IllegalStateException( + "KeyMaterial must have either aesKey or rsaKey set"); + } catch (Exception e) { + throw new IllegalStateException("Failed to create keyring from KeyMaterial: " + e.getMessage(), e); + } + } +} diff --git a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java index 8ad437f4..be53f20c 100644 --- a/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java +++ b/test-server/java-v3-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -27,14 +27,16 @@ public void run() { // Obviously this can get messy in a real service. // Assume that the tests behave and don't induce weird race conditions. Map clientCache = new ConcurrentHashMap<>(); + Map keyringCache = new ConcurrentHashMap<>(); Server server = Server.builder() .endpoints(endpoint) .addService( S3ECTestServer.builder() - .addCreateClientOperation(new CreateClientOperationImpl(clientCache)) + .addCreateClientOperation(new CreateClientOperationImpl(clientCache, keyringCache)) .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) + .addReEncryptOperation(new ReEncryptOperationImpl(clientCache, keyringCache)) .build()) .build(); System.out.println("Starting server..."); diff --git a/test-server/java-v3-transition-server/Makefile b/test-server/java-v3-transition-server/Makefile index 5a25a8aa..81726b59 100644 --- a/test-server/java-v3-transition-server/Makefile +++ b/test-server/java-v3-transition-server/Makefile @@ -10,7 +10,7 @@ build-server: cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." @echo "Building Java V3 Transition server..." - ./gradlew --build-cache --parallel build + ./gradlew --build-cache --parallel --no-daemon build start-server: @echo "Starting Java V3 Transition server..." diff --git a/test-server/java-v3-transition-server/gradle.properties b/test-server/java-v3-transition-server/gradle.properties index 08afce82..483cd315 100644 --- a/test-server/java-v3-transition-server/gradle.properties +++ b/test-server/java-v3-transition-server/gradle.properties @@ -4,8 +4,21 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v3-transition-server/s3ec-staging b/test-server/java-v3-transition-server/s3ec-staging index 6413811b..183f1984 160000 --- a/test-server/java-v3-transition-server/s3ec-staging +++ b/test-server/java-v3-transition-server/s3ec-staging @@ -1 +1 @@ -Subproject commit 6413811bb81037999b8238e02047e0e403f78c1f +Subproject commit 183f1984ed1679e8aa4cb368aeda66f2131a2061 diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index 425c0334..956f454b 100644 --- a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,5 +1,8 @@ package software.amazon.encryption.s3; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.algorithms.AlgorithmSuite; @@ -35,9 +38,11 @@ public class CreateClientOperationImpl implements CreateClientOperation { private final Map clientCache_; + private final Map keyringCache_; - public CreateClientOperationImpl(Map clientCache) { + public CreateClientOperationImpl(Map clientCache, Map keyringCache) { clientCache_ = clientCache; + keyringCache_ = keyringCache; } // Copied from S3EC. @@ -106,9 +111,22 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + // V3 Transition server configuration // Existing Builder defaults to FORBID_ENCRYPT and ALG_AES_256_GCM_IV12_TAG16_NO_KDF S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) .keyring(keyring) .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); @@ -140,6 +158,7 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c UUID uuid = UUID.randomUUID(); String uuidString = uuid.toString(); clientCache_.put(uuidString, s3Client); + keyringCache_.put(uuidString, keyring); return CreateClientOutput.builder() .clientId(uuidString) .build(); diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java index 86749489..d3ab8289 100644 --- a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -35,11 +35,25 @@ public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { S3Client s3Client = clientCache_.get(input.getClientID()); Map ecMap = metadataListToMap(input.getMetadata()); - try { - ResponseBytes resp = s3Client.getObjectAsBytes(builder -> builder - .bucket(input.getBucket()) - .key(input.getKey()) - .overrideConfiguration(withAdditionalConfiguration(ecMap))); + try { + ResponseBytes resp = s3Client.getObjectAsBytes(builder -> { + builder.bucket(input.getBucket()) + .key(input.getKey()); + + // Add custom instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + builder.overrideConfiguration(config -> config + .putExecutionAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX, input.getInstructionFileSuffix()) + .applyMutation(c -> withAdditionalConfiguration(ecMap).accept(c))); + } else { + builder.overrideConfiguration(withAdditionalConfiguration(ecMap)); + } + + // Add range header if provided + if (input.getRange() != null && !input.getRange().isEmpty()) { + builder.range(input.getRange()); + } + }); List mdAsList = metadataMapToList(resp.response().metadata()); // Can't use asBB else it gets mad bc cant access backing array diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java new file mode 100644 index 00000000..7a809761 --- /dev/null +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java @@ -0,0 +1,183 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RawKeyring; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.ReEncryptOperation; +import software.amazon.smithy.java.server.RequestContext; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; +import java.util.Map; + +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +public class ReEncryptOperationImpl implements ReEncryptOperation { + private final Map clientCache_; + + public ReEncryptOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public ReEncryptOutput reEncrypt(ReEncryptInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + + // Ensure we have an S3EncryptionClient, not just a plain S3Client + if (!(s3Client instanceof S3EncryptionClient)) { + throw new IllegalStateException( + "Client " + input.getClientID() + " is not an S3EncryptionClient"); + } + + S3EncryptionClient s3EncryptionClient = (S3EncryptionClient) s3Client; + + // Create a new keyring from the provided newKeyMaterial + KeyMaterial newKeyMaterial = input.getNewKeyMaterial(); + if (newKeyMaterial == null) { + throw new IllegalStateException( + "newKeyMaterial is required for ReEncrypt operation"); + } + + RawKeyring newKeyring = createKeyringFromMaterial(newKeyMaterial); + + try { + // Build the ReEncryptInstructionFileRequest + ReEncryptInstructionFileRequest.Builder requestBuilder = + ReEncryptInstructionFileRequest.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .newKeyring(newKeyring); + + // Add optional instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + requestBuilder.instructionFileSuffix(input.getInstructionFileSuffix()); + } + + // Add optional enforceRotation if provided + if (input.isEnforceRotation() != null) { + requestBuilder.enforceRotation(input.isEnforceRotation()); + } + + ReEncryptInstructionFileRequest reEncryptRequest = requestBuilder.build(); + + // Perform the re-encryption + ReEncryptInstructionFileResponse response = + s3EncryptionClient.reEncryptInstructionFile(reEncryptRequest); + + // Build and return the output + return ReEncryptOutput.builder() + .bucket(response.bucket()) + .key(response.key()) + .instructionFileSuffix(response.instructionFileSuffix()) + .enforceRotation(response.enforceRotation()) + .build(); + + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + /** + * Creates a RawKeyring from KeyMaterial. + * The KeyMaterial should have exactly one of: aesKey, rsaKey, or kmsKeyId set. + */ + private RawKeyring createKeyringFromMaterial(KeyMaterial keyMaterial) { + try { + // Get materials description from KeyMaterial if provided + MaterialsDescription materialsDescription = null; + if (keyMaterial.getMaterialsDescription() != null && !keyMaterial.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder builder = MaterialsDescription.builder(); + for (Map.Entry entry : keyMaterial.getMaterialsDescription().entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + materialsDescription = builder.build(); + } + + // Check for AES key + if (keyMaterial.getAesKey() != null) { + byte[] aesKeyBytes = new byte[keyMaterial.getAesKey().remaining()]; + keyMaterial.getAesKey().get(aesKeyBytes); + SecretKey secretKey = new SecretKeySpec(aesKeyBytes, "AES"); + + AesKeyring.Builder keyringBuilder = AesKeyring.builder() + .wrappingKey(secretKey); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + // Check for RSA key + if (keyMaterial.getRsaKey() != null) { + byte[] rsaKeyBytes = new byte[keyMaterial.getRsaKey().remaining()]; + keyMaterial.getRsaKey().get(rsaKeyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(rsaKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + + // Derive the public key from the private key + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + PartialRsaKeyPair keyPair = PartialRsaKeyPair.builder() + .privateKey(privateKey) + .publicKey(publicKey) + .build(); + + RsaKeyring.Builder keyringBuilder = RsaKeyring.builder() + .wrappingKeyPair(keyPair); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + throw new IllegalStateException( + "KeyMaterial must have either aesKey or rsaKey set"); + } catch (Exception e) { + throw new IllegalStateException("Failed to create keyring from KeyMaterial: " + e.getMessage(), e); + } + } +} diff --git a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java index a992cabd..78c84dff 100644 --- a/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java +++ b/test-server/java-v3-transition-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -28,14 +28,16 @@ public void run() { // Obviously this can get messy in a real service. // Assume that the tests behave and don't induce weird race conditions. Map clientCache = new ConcurrentHashMap<>(); + Map keyringCache = new ConcurrentHashMap<>(); Server server = Server.builder() .endpoints(endpoint) .addService( S3ECTestServer.builder() - .addCreateClientOperation(new CreateClientOperationImpl(clientCache)) + .addCreateClientOperation(new CreateClientOperationImpl(clientCache, keyringCache)) .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) + .addReEncryptOperation(new ReEncryptOperationImpl(clientCache)) .build()) .build(); System.out.println("Starting server..."); diff --git a/test-server/java-v4-server/Makefile b/test-server/java-v4-server/Makefile index 418e0127..3d1aae2a 100644 --- a/test-server/java-v4-server/Makefile +++ b/test-server/java-v4-server/Makefile @@ -10,7 +10,7 @@ build-server: cd s3ec-staging && mvn --batch-mode -no-transfer-progress clean compile && mvn -B -ntp install -DskipTests @echo "S3EC build completed." @echo "Building Java V4 server..." - ./gradlew --build-cache --parallel build + ./gradlew --build-cache --parallel --no-daemon build start-server: @echo "Starting Java V4 server..." diff --git a/test-server/java-v4-server/gradle.properties b/test-server/java-v4-server/gradle.properties index 08afce82..483cd315 100644 --- a/test-server/java-v4-server/gradle.properties +++ b/test-server/java-v4-server/gradle.properties @@ -4,8 +4,21 @@ smithyGradleVersion=1.1.0 smithyVersion=[1,2] # Performance optimization settings + +# Force no-daemon mode - ensures Gradle doesn't try to keep a daemon alive +org.gradle.daemon=false + +# Set minimal idle timeout for any daemon-like behavior (1 second) +org.gradle.daemon.idletimeout=1000 + +# JVM arguments to prevent forking a separate JVM process +# By matching the JVM args here with what Gradle expects, we avoid the +# "single-use Daemon process will be forked" behavior +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC + +# Keep builds fast with parallel execution and caching org.gradle.parallel=true org.gradle.caching=true -org.gradle.daemon=true -org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -org.gradle.workers.max=4 + +# Configure on demand to reduce startup time +org.gradle.configureondemand=true diff --git a/test-server/java-v4-server/s3ec-staging b/test-server/java-v4-server/s3ec-staging index db0c743e..7a1899bb 160000 --- a/test-server/java-v4-server/s3ec-staging +++ b/test-server/java-v4-server/s3ec-staging @@ -1 +1 @@ -Subproject commit db0c743eec335d16e6dceaf2b09d84becb0f74f8 +Subproject commit 7a1899bb8be6f137a3031ff76f2a1bf3f278e98d diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java index cb20d5ac..23f3a11d 100644 --- a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/CreateClientOperationImpl.java @@ -1,11 +1,15 @@ package software.amazon.encryption.s3; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.algorithms.AlgorithmSuite; import software.amazon.encryption.s3.materials.AesKeyring; import software.amazon.encryption.s3.materials.Keyring; import software.amazon.encryption.s3.materials.KmsKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; import software.amazon.encryption.s3.materials.PartialRsaKeyPair; import software.amazon.encryption.s3.materials.RsaKeyring; import software.amazon.encryption.s3.model.CreateClientInput; @@ -35,9 +39,11 @@ public class CreateClientOperationImpl implements CreateClientOperation { private final Map clientCache_; + private final Map keyringCache_; - public CreateClientOperationImpl(Map clientCache) { + public CreateClientOperationImpl(Map clientCache, Map keyringCache) { clientCache_ = clientCache; + keyringCache_ = keyringCache; } // Copied from S3EC. @@ -67,10 +73,21 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c if (key.getAesKey() != null) { byte[] keyBytes = new byte[key.getAesKey().remaining()]; key.getAesKey().get(keyBytes); - keyring = AesKeyring.builder() + + AesKeyring.Builder aesBuilder = AesKeyring.builder() .wrappingKey(new SecretKeySpec(keyBytes, "AES")) - .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) - .build(); + .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()); + + // Add materials description if provided + if (key.getMaterialsDescription() != null && !key.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder matDescBuilder = MaterialsDescription.builder(); + for (Map.Entry entry : key.getMaterialsDescription().entrySet()) { + matDescBuilder.put(entry.getKey(), entry.getValue()); + } + aesBuilder.materialsDescription(matDescBuilder.build()); + } + + keyring = aesBuilder.build(); } else if (key.getRsaKey() != null) { try { byte[] keyBytes = new byte[key.getRsaKey().remaining()]; @@ -86,12 +103,22 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c // Generate public key PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); - keyring = RsaKeyring.builder() + RsaKeyring.Builder rsaBuilder = RsaKeyring.builder() .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .wrappingKeyPair(PartialRsaKeyPair.builder() .publicKey(publicKey) - .privateKey(privateKey).build()) - .build(); + .privateKey(privateKey).build()); + + // Add materials description if provided + if (key.getMaterialsDescription() != null && !key.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder matDescBuilder = MaterialsDescription.builder(); + for (Map.Entry entry : key.getMaterialsDescription().entrySet()) { + matDescBuilder.put(entry.getKey(), entry.getValue()); + } + rsaBuilder.materialsDescription(matDescBuilder.build()); + } + + keyring = rsaBuilder.build(); } catch (NoSuchAlgorithmException | InvalidKeySpecException nse) { throw GenericServerError.builder() .message(nse.getMessage()) @@ -106,8 +133,21 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c throw new RuntimeException("No KeyMaterial found!"); } + // Configure S3 client with adaptive retry for throttling + RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(5) + .throttlingBackoffStrategy(BackoffStrategy.defaultThrottlingStrategy()) + .build(); + + S3Client wrappedClient = S3Client.builder() + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build()) + .build(); + // V4-Improved server configuration S3EncryptionClient.Builder s3ClientBuilder = S3EncryptionClient.builderV4() + .wrappedClient(wrappedClient) .keyring(keyring) .enableLegacyWrappingAlgorithms(input.getConfig().isEnableLegacyWrappingAlgorithms()) .enableLegacyUnauthenticatedModes(input.getConfig().isEnableLegacyUnauthenticatedModes()); @@ -139,6 +179,7 @@ public CreateClientOutput createClient(CreateClientInput input, RequestContext c UUID uuid = UUID.randomUUID(); String uuidString = uuid.toString(); clientCache_.put(uuidString, s3Client); + keyringCache_.put(uuidString, keyring); return CreateClientOutput.builder() .clientId(uuidString) .build(); diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java index 17e9a8ee..a1964085 100644 --- a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/GetObjectOperationImpl.java @@ -34,10 +34,24 @@ public GetObjectOutput getObject(GetObjectInput input, RequestContext context) { Map ecMap = metadataListToMap(input.getMetadata()); try { - ResponseBytes resp = s3Client.getObjectAsBytes(builder -> builder - .bucket(input.getBucket()) - .key(input.getKey()) - .overrideConfiguration(withAdditionalConfiguration(ecMap))); + ResponseBytes resp = s3Client.getObjectAsBytes(builder -> { + builder.bucket(input.getBucket()) + .key(input.getKey()); + + // Add custom instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + builder.overrideConfiguration(config -> config + .putExecutionAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX, input.getInstructionFileSuffix()) + .applyMutation(c -> withAdditionalConfiguration(ecMap).accept(c))); + } else { + builder.overrideConfiguration(withAdditionalConfiguration(ecMap)); + } + + // Add range header if provided + if (input.getRange() != null && !input.getRange().isEmpty()) { + builder.range(input.getRange()); + } + }); List mdAsList = metadataMapToList(resp.response().metadata()); // Can't use asBB else it gets mad bc cant access backing array diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java new file mode 100644 index 00000000..6a7cd5b6 --- /dev/null +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/ReEncryptOperationImpl.java @@ -0,0 +1,183 @@ +package software.amazon.encryption.s3; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest; +import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse; +import software.amazon.encryption.s3.materials.AesKeyring; +import software.amazon.encryption.s3.materials.MaterialsDescription; +import software.amazon.encryption.s3.materials.PartialRsaKeyPair; +import software.amazon.encryption.s3.materials.RawKeyring; +import software.amazon.encryption.s3.materials.RsaKeyring; +import software.amazon.encryption.s3.model.GenericServerError; +import software.amazon.encryption.s3.model.KeyMaterial; +import software.amazon.encryption.s3.model.ReEncryptInput; +import software.amazon.encryption.s3.model.ReEncryptOutput; +import software.amazon.encryption.s3.model.S3EncryptionClientError; +import software.amazon.encryption.s3.service.ReEncryptOperation; +import software.amazon.smithy.java.server.RequestContext; + +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateCrtKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.RSAPublicKeySpec; +import java.util.HashMap; +import java.util.Map; + +public class ReEncryptOperationImpl implements ReEncryptOperation { + private final Map clientCache_; + + public ReEncryptOperationImpl(Map clientCache) { + clientCache_ = clientCache; + } + + @Override + public ReEncryptOutput reEncrypt(ReEncryptInput input, RequestContext context) { + try { + S3Client s3Client = clientCache_.get(input.getClientID()); + + // Ensure we have an S3EncryptionClient, not just a plain S3Client + if (!(s3Client instanceof S3EncryptionClient)) { + throw new IllegalStateException( + "Client " + input.getClientID() + " is not an S3EncryptionClient"); + } + + S3EncryptionClient s3EncryptionClient = (S3EncryptionClient) s3Client; + + // Create a new keyring from the provided newKeyMaterial + KeyMaterial newKeyMaterial = input.getNewKeyMaterial(); + if (newKeyMaterial == null) { + throw new IllegalStateException( + "newKeyMaterial is required for ReEncrypt operation"); + } + + RawKeyring newKeyring = createKeyringFromMaterial(newKeyMaterial); + + try { + // Build the ReEncryptInstructionFileRequest + ReEncryptInstructionFileRequest.Builder requestBuilder = + ReEncryptInstructionFileRequest.builder() + .bucket(input.getBucket()) + .key(input.getKey()) + .newKeyring(newKeyring); + + // Add optional instruction file suffix if provided + if (input.getInstructionFileSuffix() != null && !input.getInstructionFileSuffix().isEmpty()) { + requestBuilder.instructionFileSuffix(input.getInstructionFileSuffix()); + } + + // Add optional enforceRotation if provided + if (input.isEnforceRotation() != null) { + requestBuilder.enforceRotation(input.isEnforceRotation()); + } + + ReEncryptInstructionFileRequest reEncryptRequest = requestBuilder.build(); + + // Perform the re-encryption + ReEncryptInstructionFileResponse response = + s3EncryptionClient.reEncryptInstructionFile(reEncryptRequest); + + // Build and return the output + return ReEncryptOutput.builder() + .bucket(response.bucket()) + .key(response.key()) + .instructionFileSuffix(response.instructionFileSuffix()) + .enforceRotation(response.enforceRotation()) + .build(); + + } catch (S3EncryptionClientException s3EncryptionClientException) { + // Modeled exceptions MUST be returned as such + StringWriter sw = new StringWriter(); + s3EncryptionClientException.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw S3EncryptionClientError.builder() + .message(stackTrace) + .build(); + } + } catch (Exception e) { + // Don't wrap modeled errors + if (e instanceof S3EncryptionClientError) { + throw e; + } + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString(); + throw GenericServerError.builder() + .message(stackTrace) + .build(); + } + } + + /** + * Creates a RawKeyring from KeyMaterial. + * The KeyMaterial should have exactly one of: aesKey, rsaKey, or kmsKeyId set. + */ + private RawKeyring createKeyringFromMaterial(KeyMaterial keyMaterial) { + try { + // Get materials description from KeyMaterial if provided + MaterialsDescription materialsDescription = null; + if (keyMaterial.getMaterialsDescription() != null && !keyMaterial.getMaterialsDescription().isEmpty()) { + MaterialsDescription.Builder builder = MaterialsDescription.builder(); + for (Map.Entry entry : keyMaterial.getMaterialsDescription().entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + materialsDescription = builder.build(); + } + + // Check for AES key + if (keyMaterial.getAesKey() != null) { + byte[] aesKeyBytes = new byte[keyMaterial.getAesKey().remaining()]; + keyMaterial.getAesKey().get(aesKeyBytes); + SecretKey secretKey = new SecretKeySpec(aesKeyBytes, "AES"); + + AesKeyring.Builder keyringBuilder = AesKeyring.builder() + .wrappingKey(secretKey); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + // Check for RSA key + if (keyMaterial.getRsaKey() != null) { + byte[] rsaKeyBytes = new byte[keyMaterial.getRsaKey().remaining()]; + keyMaterial.getRsaKey().get(rsaKeyBytes); + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(rsaKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + RSAPrivateCrtKey privateKey = (RSAPrivateCrtKey) keyFactory.generatePrivate(keySpec); + + // Derive the public key from the private key + RSAPublicKeySpec publicKeySpec = new RSAPublicKeySpec( + privateKey.getModulus(), + privateKey.getPublicExponent() + ); + PublicKey publicKey = keyFactory.generatePublic(publicKeySpec); + + PartialRsaKeyPair keyPair = PartialRsaKeyPair.builder() + .privateKey(privateKey) + .publicKey(publicKey) + .build(); + + RsaKeyring.Builder keyringBuilder = RsaKeyring.builder() + .wrappingKeyPair(keyPair); + + if (materialsDescription != null) { + keyringBuilder.materialsDescription(materialsDescription); + } + + return keyringBuilder.build(); + } + + throw new IllegalStateException( + "KeyMaterial must have either aesKey or rsaKey set"); + } catch (Exception e) { + throw new IllegalStateException("Failed to create keyring from KeyMaterial: " + e.getMessage(), e); + } + } +} diff --git a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java index d394b72b..88d5b981 100644 --- a/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java +++ b/test-server/java-v4-server/src/main/java/software/amazon/encryption/s3/S3ECJavaTestServer.java @@ -27,14 +27,16 @@ public void run() { // Obviously this can get messy in a real service. // Assume that the tests behave and don't induce weird race conditions. Map clientCache = new ConcurrentHashMap<>(); + Map keyringCache = new ConcurrentHashMap<>(); Server server = Server.builder() .endpoints(endpoint) .addService( S3ECTestServer.builder() - .addCreateClientOperation(new CreateClientOperationImpl(clientCache)) + .addCreateClientOperation(new CreateClientOperationImpl(clientCache, keyringCache)) .addGetObjectOperation(new GetObjectOperationImpl(clientCache)) .addPutObjectOperation(new PutObjectOperationImpl(clientCache)) + .addReEncryptOperation(new ReEncryptOperationImpl(clientCache)) .build()) .build(); System.out.println("Starting server..."); diff --git a/test-server/model/client.smithy b/test-server/model/client.smithy index bba19b62..5772e8c2 100644 --- a/test-server/model/client.smithy +++ b/test-server/model/client.smithy @@ -25,7 +25,16 @@ structure CreateClientOutput { structure KeyMaterial { rsaKey: Blob, aesKey: Blob, - kmsKeyId: String + kmsKeyId: String, + /// Optional materials description for keyring differentiation + /// Used to distinguish between different key materials for rotation enforcement + materialsDescription: MaterialsDescriptionMap +} + +/// Map of materials description key-value pairs +map MaterialsDescriptionMap { + key: String, + value: String } enum CommitmentPolicy { diff --git a/test-server/model/object.smithy b/test-server/model/object.smithy index 6d793353..93e78370 100644 --- a/test-server/model/object.smithy +++ b/test-server/model/object.smithy @@ -21,6 +21,7 @@ resource Object { } read: GetObject put: PutObject + operations: [ReEncrypt] } @idempotent @@ -35,6 +36,14 @@ operation PutObject { @required $key + /// Encryption context/materials description to use for this operation. + /// For most SDKs (Java, .NET, etc.), materials description is embedded in the keyring/materials + /// during client creation and this parameter is typically empty/unused. + /// + /// For C++ SDK: Materials description MUST be passed per-operation via this parameter + /// because the C++ SDK's EncryptionMaterials constructor does not accept materials description. + /// Instead, GetObject/PutObject operations accept a contextMap parameter that becomes the + /// materials description. @httpHeader("Content-Metadata") $metadata @@ -72,7 +81,14 @@ operation GetObject { @required $key - /// Should probably be renamed to be EC specific + /// Encryption context/materials description to use for this operation. + /// For most SDKs (Java, .NET, etc.), materials description is embedded in the keyring/materials + /// during client creation and this parameter is typically empty/unused. + /// + /// For C++ SDK: Materials description MUST be passed per-operation via this parameter + /// because the C++ SDK's EncryptionMaterials constructor does not accept materials description. + /// Instead, GetObject/PutObject operations accept a contextMap parameter that becomes the + /// materials description. @httpHeader("Content-Metadata") $metadata @@ -80,7 +96,16 @@ operation GetObject { @required @notProperty clientID: String - } + + @httpHeader("Range") + @notProperty + range: String + + /// Custom instruction file suffix to use when reading instruction files + @httpHeader("InstructionFileSuffix") + @notProperty + instructionFileSuffix: String + } output := for Object { @httpHeader("Content-Metadata") @@ -93,8 +118,7 @@ operation GetObject { } } -@readonly -@http(method: "GET", uri: "/object/{bucket}/{key}") +@http(method: "POST", uri: "/object/{bucket}/{key}/reencrypt") operation ReEncrypt { input := for Object { @httpLabel @@ -104,30 +128,41 @@ operation ReEncrypt { @httpLabel @required $key - - /// Should probably be renamed to be EC specific - @httpHeader("Content-Metadata") - $metadata @httpHeader("ClientID") @required @notProperty clientID: String - /// Custom instruction file suffix + /// New key material to use for re-encryption + @httpPayload + @required + @notProperty + newKeyMaterial: KeyMaterial + + /// Custom instruction file suffix for RSA keyring re-encryption @httpHeader("InstructionFileSuffix") @notProperty instructionFileSuffix: String - } - output := for Object { - @httpHeader("Content-Metadata") + /// Whether to enforce rotation by verifying the key has changed + @httpHeader("EnforceRotation") + @notProperty + enforceRotation: Boolean + } + + output := { @required - $metadata + bucket: String @required - @httpPayload - $body + key: String + + @notProperty + instructionFileSuffix: String + + @notProperty + enforceRotation: Boolean } } diff --git a/test-server/net-v2-v3-server/Controllers/ClientController.cs b/test-server/net-v2-v3-server/Controllers/ClientController.cs index e33a58e6..437233a8 100644 --- a/test-server/net-v2-v3-server/Controllers/ClientController.cs +++ b/test-server/net-v2-v3-server/Controllers/ClientController.cs @@ -21,8 +21,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "[NET-current] EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "[NET-current] SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "[NET-current] AesKey not supported" }); try { @@ -47,6 +45,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "Created EncryptionMaterialsV2: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-current] Created EncryptionMaterialsV2: AES"); } else { return StatusCode(501, new GenericServerError { Message = "[NET-current] Unknown or missing key material!" }); @@ -62,6 +69,11 @@ public IActionResult CreateClient([FromBody] ClientRequest request) logger.LogInformation("[NET-current] Created securityProfile= {securityProfile}", securityProfile.ToString()); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile); + + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -91,4 +103,4 @@ public IActionResult CreateClient([FromBody] ClientRequest request) }); } } -} \ No newline at end of file +} diff --git a/test-server/net-v3-transition-server/Controllers/ClientController.cs b/test-server/net-v3-transition-server/Controllers/ClientController.cs index a66fb342..3deeff61 100644 --- a/test-server/net-v3-transition-server/Controllers/ClientController.cs +++ b/test-server/net-v3-transition-server/Controllers/ClientController.cs @@ -21,8 +21,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "AesKey not supported" }); try { @@ -47,6 +45,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV2(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "Created EncryptionMaterialsV2: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV2(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V3-Transitional] Created EncryptionMaterialsV2: AES"); } else { return StatusCode(501, new GenericServerError { Message = "Unknown or missing key material!" }); @@ -67,6 +74,11 @@ public IActionResult CreateClient([FromBody] ClientRequest request) logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm); var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm); + + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -118,4 +130,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti _ => ContentEncryptionAlgorithm.AesGcm }; } -} \ No newline at end of file +} diff --git a/test-server/net-v3-transition-server/s3ec-v3-transition-branch b/test-server/net-v3-transition-server/s3ec-v3-transition-branch index ad825917..c3bf38b9 160000 --- a/test-server/net-v3-transition-server/s3ec-v3-transition-branch +++ b/test-server/net-v3-transition-server/s3ec-v3-transition-branch @@ -1 +1 @@ -Subproject commit ad8259173de365a13e8b3932ee02493f599f597f +Subproject commit c3bf38b93c25f7169982073b1ffd1f3d00f59073 diff --git a/test-server/net-v4-server/Controllers/ClientController.cs b/test-server/net-v4-server/Controllers/ClientController.cs index b9fbe3f9..2ef8b921 100644 --- a/test-server/net-v4-server/Controllers/ClientController.cs +++ b/test-server/net-v4-server/Controllers/ClientController.cs @@ -20,8 +20,6 @@ public IActionResult CreateClient([FromBody] ClientRequest request) return StatusCode(501, new GenericServerError { Message = "[NET-V4] EnableDelayedAuthenticationMode not supported" }); if (request.Config.SetBufferSize.HasValue) return StatusCode(501, new GenericServerError { Message = "[NET-V4] SetBufferSize not supported" }); - if (request.Config.KeyMaterial.AesKey != null) - return StatusCode(501, new GenericServerError { Message = "[NET-V4] AesKey not supported" }); try { @@ -46,6 +44,15 @@ public IActionResult CreateClient([FromBody] ClientRequest request) encryptionMaterial = new EncryptionMaterialsV4(rsaKey, AsymmetricAlgorithmType.RsaOaepSha1); logger.LogInformation( "[NET-V4] Created EncryptionMaterialsV4: RSA"); + } + else if (request.Config.KeyMaterial.AesKey != null) + { + var aesKeyBytes = request.Config.KeyMaterial.AesKey; + var aes = Aes.Create(); + aes.Key = aesKeyBytes; + encryptionMaterial = new EncryptionMaterialsV4(aes, SymmetricAlgorithmType.AesGcm); + logger.LogInformation( + "[NET-V4] Created EncryptionMaterialsV4: AES"); } else { return StatusCode(501, new GenericServerError { Message = "[NET-V4] Unknown or missing key material!" }); @@ -79,6 +86,10 @@ public IActionResult CreateClient([FromBody] ClientRequest request) ? new AmazonS3CryptoConfigurationV4() : new AmazonS3CryptoConfigurationV4(securityProfile, commitmentPolicy, encryptionAlgorithm); + // Add retry configuration for throttling + configuration.RetryMode = Amazon.Runtime.RequestRetryMode.Adaptive; + configuration.MaxErrorRetry = 5; + if (request.Config.InstructionFileConfig?.EnableInstructionFilePutObject == true) { configuration.StorageMode = CryptoStorageMode.InstructionFile; @@ -130,4 +141,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti _ => ContentEncryptionAlgorithm.AesGcmWithCommitment }; } -} \ No newline at end of file +} diff --git a/test-server/net-v4-server/Makefile b/test-server/net-v4-server/Makefile index e2df658a..b52bbd49 100644 --- a/test-server/net-v4-server/Makefile +++ b/test-server/net-v4-server/Makefile @@ -32,7 +32,7 @@ start-net-V4-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - dotnet run --no-build & echo $! > $(PID_FILE_NET_V4) + dotnet run --no-build > server.log 2>&1 & echo $! > $(PID_FILE_NET_V4) @echo ".NET V4 server starting..." wait-for-server: diff --git a/test-server/net-v4-server/s3ec-net-v4-improved b/test-server/net-v4-server/s3ec-net-v4-improved index 1c0a458c..04f70c8b 160000 --- a/test-server/net-v4-server/s3ec-net-v4-improved +++ b/test-server/net-v4-server/s3ec-net-v4-improved @@ -1 +1 @@ -Subproject commit 1c0a458c19b351c266199c72072de746362c5326 +Subproject commit 04f70c8b70e25c7a1a36fcd5a420c40806157c66 diff --git a/test-server/php-v2-server/Makefile b/test-server/php-v2-server/Makefile index a9d04134..719ea238 100644 --- a/test-server/php-v2-server/Makefile +++ b/test-server/php-v2-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 server starting..." stop-server: diff --git a/test-server/php-v2-transition-server/Makefile b/test-server/php-v2-transition-server/Makefile index 61eb3a84..a3d038de 100644 --- a/test-server/php-v2-transition-server/Makefile +++ b/test-server/php-v2-transition-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V2 Transition server starting..." stop-server: diff --git a/test-server/php-v2-transition-server/src/get_object.php b/test-server/php-v2-transition-server/src/get_object.php index 5800e850..847fa9e3 100644 --- a/test-server/php-v2-transition-server/src/get_object.php +++ b/test-server/php-v2-transition-server/src/get_object.php @@ -20,6 +20,9 @@ function handleGetObject($params) $metadata = $_SERVER['HTTP_CONTENT_METADATA'] ?? ''; $encryptionContext = metadataStringToMap($metadata); + // Get custom instruction file suffix if provided + $instructionFileSuffix = $_SERVER['HTTP_INSTRUCTIONFILESUFFIX'] ?? null; + // Extract bucket and key from URL parameters $bucket = $params['bucket'] ?? null; $key = $params['key'] ?? null; @@ -44,14 +47,21 @@ function handleGetObject($params) // Start output buffering before the AWS call to capture any unwanted output ob_start(); - $result = $s3ec->getObject([ + $getObjectParams = [ '@SecurityProfile' => $legacy, '@MaterialsProvider' => $materialProvider, '@KmsEncryptionContext' => $encryptionContext, '@CommitmentPolicy' => $commitmentPolicy, 'Bucket' => $bucket, 'Key' => $key, - ]); + ]; + + // Add custom instruction file suffix if provided + if (!is_null($instructionFileSuffix) && !empty($instructionFileSuffix)) { + $getObjectParams['@InstructionFileSuffix'] = $instructionFileSuffix; + } + + $result = $s3ec->getObject($getObjectParams); // Capture and discard any unwanted output from AWS SDK $unwantedOutput = ob_get_clean(); @@ -80,6 +90,10 @@ function handleGetObject($params) } if (strpos($e->getMessage(), "@SecurityProfile=V2") !== false) { return S3EncryptionClientError($e->getMessage() . " " . "Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms"); + } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { + return S3EncryptionClientError($e->getMessage()); } else { error_log("This is the error: " . $e->getMessage()); return GenericServerError("Server error: " . $e->getMessage(), 500); diff --git a/test-server/php-v3-server/Makefile b/test-server/php-v3-server/Makefile index 2b9661f2..9460d4ed 100644 --- a/test-server/php-v3-server/Makefile +++ b/test-server/php-v3-server/Makefile @@ -15,7 +15,7 @@ start-server: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - composer run start > server.log 2>&1 & echo $$! > $(PID_FILE) + composer run start --timeout=0 > server.log 2>&1 & echo $$! > $(PID_FILE) @echo "PHP V3 server starting..." stop-server: diff --git a/test-server/php-v3-server/local-php-sdk b/test-server/php-v3-server/local-php-sdk index e32c9f2b..3acb3ad4 160000 --- a/test-server/php-v3-server/local-php-sdk +++ b/test-server/php-v3-server/local-php-sdk @@ -1 +1 @@ -Subproject commit e32c9f2b009a43cf88f2ab35e1e532114c8390c9 +Subproject commit 3acb3ad4d98debcfc2148290cd6fcea83962fe08 diff --git a/test-server/php-v3-server/src/get_object.php b/test-server/php-v3-server/src/get_object.php index 3de7f779..25a88a02 100644 --- a/test-server/php-v3-server/src/get_object.php +++ b/test-server/php-v3-server/src/get_object.php @@ -20,6 +20,9 @@ function handleGetObject($params) $metadata = $_SERVER['HTTP_CONTENT_METADATA'] ?? ''; $encryptionContext = metadataStringToMap($metadata); + // Get custom instruction file suffix if provided + $instructionFileSuffix = $_SERVER['HTTP_INSTRUCTIONFILESUFFIX'] ?? null; + // Extract bucket and key from URL parameters $bucket = $params['bucket'] ?? null; $key = $params['key'] ?? null; @@ -44,14 +47,21 @@ function handleGetObject($params) // Start output buffering before the AWS call to capture any unwanted output ob_start(); - $result = $s3ec->getObject([ + $getObjectParams = [ '@SecurityProfile' => $legacy, '@MaterialsProvider' => $materialProvider, '@KmsEncryptionContext' => $encryptionContext, '@CommitmentPolicy' => $commitmentPolicy, 'Bucket' => $bucket, 'Key' => $key, - ]); + ]; + + // Add custom instruction file suffix if provided + if (!is_null($instructionFileSuffix) && !empty($instructionFileSuffix)) { + $getObjectParams['@InstructionFileSuffix'] = $instructionFileSuffix; + } + + $result = $s3ec->getObject($getObjectParams); // Capture and discard any unwanted output from AWS SDK $unwantedOutput = ob_get_clean(); @@ -84,6 +94,10 @@ function handleGetObject($params) return S3EncryptionClientError($e->getMessage()); } elseif (strpos($e->getMessage(), "Message is encrypted with a non commiting algorithm but commitment policy is set to REQUIRE_ENCRYPT_REQUIRE_DECRYPT. Select a valid commitment policy to decrypt this object.") !== false) { return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "One or more reserved keys found in Instruction file when they should not be present.") !== false) { + return S3EncryptionClientError($e->getMessage()); + } elseif (strpos($e->getMessage(), "Expected a V3 envelope but was unable to constuct one.") !== false) { + return S3EncryptionClientError($e->getMessage()); } else { error_log("This is the error: " . $e->getMessage()); return GenericServerError("Server argument: " . $e->getMessage(), 500); diff --git a/test-server/ruby-v2-server/app.rb b/test-server/ruby-v2-server/app.rb index cde757a3..96e55c8a 100644 --- a/test-server/ruby-v2-server/app.rb +++ b/test-server/ruby-v2-server/app.rb @@ -132,7 +132,7 @@ def initialize metadata: response_metadata }.to_json - rescue Aws::S3::EncryptionV2::Errors::EncryptionError => e + rescue Aws::S3::EncryptionV2::Errors::EncryptionError, Aws::S3::EncryptionV3::Errors::EncryptionError => e S3ECLogger.log_error(e, { endpoint: '/put', error_category: 'EncryptionError' }, @request_id) ErrorHandlers.send_s3_encryption_client_error(self, e.message) rescue StandardError => e @@ -176,8 +176,15 @@ def initialize key: key } - # Add encryption context if present - get_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + # Add custom instruction file suffix if present + instruction_file_suffix = request.env['HTTP_INSTRUCTIONFILESUFFIX'] + if instruction_file_suffix && !instruction_file_suffix.empty? + get_params[:envelope_location] = :instruction_file + get_params[:instruction_file_suffix] = instruction_file_suffix + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Using custom instruction file suffix: #{instruction_file_suffix}") + elsif !encryption_context.empty? + get_params[:kms_encryption_context] = encryption_context + end # Log S3 operation S3ECLogger.log_s3_operation('get', bucket, key, encryption_context, "ClientID: #{client_id}") @@ -201,7 +208,7 @@ def initialize content_type 'application/octet-stream' body - rescue Aws::S3::EncryptionV2::Errors::DecryptionError => e + rescue Aws::S3::EncryptionV2::Errors::DecryptionError, Aws::S3::EncryptionV3::Errors::DecryptionError => e S3ECLogger.log_error(e, { endpoint: '/get', error_category: 'DecryptionError' }, @request_id) ErrorHandlers.send_s3_encryption_client_error(self, e.message) rescue StandardError => e diff --git a/test-server/ruby-v2-server/lib/client_manager.rb b/test-server/ruby-v2-server/lib/client_manager.rb index 717003bf..3da62b45 100644 --- a/test-server/ruby-v2-server/lib/client_manager.rb +++ b/test-server/ruby-v2-server/lib/client_manager.rb @@ -2,6 +2,8 @@ require 'securerandom' require 'aws-sdk-s3' require 'aws-sdk-kms' +require 'openssl' +require 'base64' require_relative 'logger' # Manages S3 Encryption Client instances @@ -14,20 +16,42 @@ def initialize # Create a new S3 encryption client and return its ID def create_client(config) - # Extract configuration + # Extract all key material types kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + rsa_key_blob = config.dig('keyMaterial', 'rsaKey') + aes_key_blob = config.dig('keyMaterial', 'aesKey') inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') - raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? + # Validate that only one key type is provided + key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count + raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 # Create S3 encryption client configuration encryption_config = { - kms_key_id: kms_key_id, - kms_client: @kms_client, - key_wrap_schema: :kms_context, content_encryption_schema: :aes_gcm_no_padding, envelope_location: inst_file_put ? :instruction_file : :metadata - }.tap do |hash| + } + + # Configure based on key type + if kms_key_id + encryption_config[:kms_key_id] = kms_key_id + encryption_config[:kms_client] = @kms_client + encryption_config[:key_wrap_schema] = :kms_context + elsif rsa_key_blob + # Parse RSA private key from PKCS8 format + key_bytes = Base64.decode64(rsa_key_blob) + rsa_key = OpenSSL::PKey::RSA.new(key_bytes) + encryption_config[:encryption_key] = rsa_key + encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 + elsif aes_key_blob + # Extract AES key bytes + key_bytes = Base64.decode64(aes_key_blob) + encryption_config[:encryption_key] = key_bytes + encryption_config[:key_wrap_schema] = :aes_gcm + end + + # Apply legacy settings + encryption_config.tap do |hash| if !config['enableLegacyWrappingAlgorithms'].nil? || !config['enableLegacyUnauthenticatedModes'].nil? legacy_modes = config['enableLegacyWrappingAlgorithms'] || config['enableLegacyUnauthenticatedModes'] # Set security profile based on legacy wrapping algorithms setting @@ -36,7 +60,13 @@ def create_client(config) end # Create the S3 encryption client - s3_client = Aws::S3::Client.new(region: 'us-west-2') + # Create the S3 encryption client with retry configuration for throttling + s3_client = Aws::S3::Client.new( + region: 'us-west-2', + retry_mode: 'adaptive', + retry_limit: 5, + retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } + ) encryption_client = Aws::S3::EncryptionV2::Client.new( client: s3_client, **encryption_config diff --git a/test-server/ruby-v3-server/app.rb b/test-server/ruby-v3-server/app.rb index 8ebceac0..80ac972f 100644 --- a/test-server/ruby-v3-server/app.rb +++ b/test-server/ruby-v3-server/app.rb @@ -176,8 +176,15 @@ def initialize key: key } - # Add encryption context if present - get_params[:kms_encryption_context] = encryption_context unless encryption_context.empty? + # Add custom instruction file suffix if present + instruction_file_suffix = request.env['HTTP_INSTRUCTIONFILESUFFIX'] + if instruction_file_suffix && !instruction_file_suffix.empty? + get_params[:envelope_location] = :instruction_file + get_params[:instruction_file_suffix] = instruction_file_suffix + S3ECLogger.debug("GET_ENDPOINT [#{@request_id}]: Using custom instruction file suffix: #{instruction_file_suffix}") + elsif !encryption_context.empty? + get_params[:kms_encryption_context] = encryption_context + end # Log S3 operation S3ECLogger.log_s3_operation('get', bucket, key, encryption_context, "ClientID: #{client_id}") diff --git a/test-server/ruby-v3-server/lib/client_manager.rb b/test-server/ruby-v3-server/lib/client_manager.rb index a6fb551f..5ee3f1ec 100644 --- a/test-server/ruby-v3-server/lib/client_manager.rb +++ b/test-server/ruby-v3-server/lib/client_manager.rb @@ -2,6 +2,8 @@ require 'securerandom' require 'aws-sdk-s3' require 'aws-sdk-kms' +require 'openssl' +require 'base64' require_relative 'logger' # Manages S3 Encryption Client instances @@ -14,11 +16,17 @@ def initialize # Create a new S3 encryption client and return its ID def create_client(config) - # Extract configuration + # Extract all key material types kms_key_id = config.dig('keyMaterial', 'kmsKeyId') + rsa_key_blob = config.dig('keyMaterial', 'rsaKey') + aes_key_blob = config.dig('keyMaterial', 'aesKey') inst_file_put = config.dig('instructionFileConfig', 'enableInstructionFilePutObject') content_alg = config.dig('encryptionAlgorithm') + # Validate that only one key type is provided + key_count = [kms_key_id, rsa_key_blob, aes_key_blob].compact.count + raise 'KeyMaterial must contain exactly one non-null key type' unless key_count == 1 + # translate between canonical AlgSuite and Ruby symbols if content_alg.nil? || content_alg == 'ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY' content_alg = :alg_aes_256_gcm_hkdf_sha512_commit_key @@ -28,16 +36,32 @@ def create_client(config) raise 'Unknown content encryption algorithm provided: ' + content_alg end - raise 'KMS Key ID is required' if kms_key_id.nil? || kms_key_id.empty? - # Create S3 encryption client configuration encryption_config = { - kms_key_id: kms_key_id, - kms_client: @kms_client, - key_wrap_schema: :kms_context, envelope_location: inst_file_put ? :instruction_file : :metadata, content_encryption_schema: content_alg - }.tap do |hash| + } + + # Configure based on key type + if kms_key_id + encryption_config[:kms_key_id] = kms_key_id + encryption_config[:kms_client] = @kms_client + encryption_config[:key_wrap_schema] = :kms_context + elsif rsa_key_blob + # Parse RSA private key from PKCS8 format + key_bytes = Base64.decode64(rsa_key_blob) + rsa_key = OpenSSL::PKey::RSA.new(key_bytes) + encryption_config[:encryption_key] = rsa_key + encryption_config[:key_wrap_schema] = :rsa_oaep_sha1 + elsif aes_key_blob + # Extract AES key bytes + key_bytes = Base64.decode64(aes_key_blob) + encryption_config[:encryption_key] = key_bytes + encryption_config[:key_wrap_schema] = :aes_gcm + end + + # Apply additional configuration + encryption_config.tap do |hash| if !config['commitmentPolicy'].nil? hash[:commitment_policy] = case config['commitmentPolicy'] when 'FORBID_ENCRYPT_ALLOW_DECRYPT' @@ -61,7 +85,13 @@ def create_client(config) end # Create the S3 encryption client - s3_client = Aws::S3::Client.new(region: 'us-west-2') + # Create the S3 encryption client with retry configuration for throttling + s3_client = Aws::S3::Client.new( + region: 'us-west-2', + retry_mode: 'adaptive', + retry_limit: 5, + retry_backoff: lambda { |c| sleep(2 ** c.retries * 0.3 * rand) } + ) encryption_client = Aws::S3::EncryptionV3::Client.new( client: s3_client, **encryption_config