From 167cdb79407d794f00b8ed063ad4dc18e9bb8e6d Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Thu, 18 Sep 2025 11:01:39 -0400 Subject: [PATCH 01/13] implement test server for c++ V2 --- test-server/cpp-v2-server/CMakeLists.txt | 29 +++ test-server/cpp-v2-server/README.md | 37 +++ test-server/cpp-v2-server/main.cpp | 283 +++++++++++++++++++++++ 3 files changed, 349 insertions(+) create mode 100644 test-server/cpp-v2-server/CMakeLists.txt create mode 100644 test-server/cpp-v2-server/README.md create mode 100644 test-server/cpp-v2-server/main.cpp diff --git a/test-server/cpp-v2-server/CMakeLists.txt b/test-server/cpp-v2-server/CMakeLists.txt new file mode 100644 index 00000000..16a5fd72 --- /dev/null +++ b/test-server/cpp-v2-server/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.16) +project(s3ec-cpp-server) + +set(CMAKE_CXX_STANDARD 17) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) + +find_package(AWSSDK REQUIRED COMPONENTS s3 s3-encryption kms) +find_package(nlohmann_json REQUIRED) + +add_executable(s3ec-server main.cpp) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_link_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_LIBRARY_DIRS} + /opt/homebrew/lib +) + +target_link_libraries(s3ec-server + ${LIBMICROHTTPD_LIBRARIES} + ${AWSSDK_LINK_LIBRARIES} + nlohmann_json::nlohmann_json + uuid +) \ No newline at end of file diff --git a/test-server/cpp-v2-server/README.md b/test-server/cpp-v2-server/README.md new file mode 100644 index 00000000..06d55be1 --- /dev/null +++ b/test-server/cpp-v2-server/README.md @@ -0,0 +1,37 @@ +# C++ S3 Encryption Test Server + +Minimal C++ implementation of the S3 Encryption test server. + +## Dependencies + +- libmicrohttpd +- AWS SDK for C++ +- nlohmann/json +- uuid + +On MacOS you can +```bash +brew install libmicrohttpd nlohmann-json ossp-uuid +``` + +## Build + +```bash +mkdir build && cd build +cmake .. +make +``` + +## Run + +```bash +./s3ec-server +``` + +Server runs on localhost:8081 + +## API Endpoints + +- `POST /client` - Create S3 encryption client +- `GET /object/{bucket}/{key}` - Get encrypted object +- `PUT /object/{bucket}/{key}` - Put encrypted object \ No newline at end of file diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp new file mode 100644 index 00000000..dc53f38f --- /dev/null +++ b/test-server/cpp-v2-server/main.cpp @@ -0,0 +1,283 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using json = nlohmann::json; +using namespace Aws::S3Encryption; +using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; +std::unordered_map> + client_cache; + +std::string generate_uuid() { + uuid_t uuid; + uuid_generate(uuid); + char uuid_str[37]; + uuid_unparse(uuid, uuid_str); + return std::string(uuid_str); +} + +MHD_Result print_key(void *cls, enum MHD_ValueKind kind, const char *key, + const char *value) { + fprintf(stderr, "%s: %s\n", key, value); + return MHD_YES; +} + +std::string get_header_value(struct MHD_Connection *connection, + const char *key) { + const char *value = + MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); + return value ? std::string(value) : ""; +} + +MHD_Result send_response(struct MHD_Connection *connection, int status_code, + const std::string &content) { + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.c_str(), MHD_RESPMEM_MUST_COPY); + MHD_Result ret = MHD_queue_response(connection, status_code, response); + if (ret != MHD_YES) + fprintf(stderr, "MHD_queue_response returned %d\n", ret); + MHD_destroy_response(response); + return ret; +} + +MHD_Result handle_create_client(struct MHD_Connection *connection, + const std::string &body) { + try { + json request = json::parse(body); + std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + + Aws::KMS::KMSClient kms_client; + auto materials = + std::make_shared(kms_key_id); + CryptoConfigurationV2 config(materials); + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + + auto encryption_client = std::make_shared(config); + + std::string client_id = generate_uuid(); + client_cache[client_id] = encryption_client; + + json response = {{"clientId", client_id}}; + return send_response(connection, 200, response.dump()); + } catch (const std::exception &e) { + fprintf(stderr, "Error: %s\n", e.what()); + return send_response(connection, 500, + "{\"error\":\"" + std::string(e.what()) + "\"}"); + } catch (...) { + fprintf(stderr, "Super secret error"); + return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); + } +} + +MHD_Result handle_get_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + fprintf(stderr, "handle_get_object <%s>\n", metadata.c_str()); + try { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + auto outcome = it->second->GetObject(request); + if (outcome.IsSuccess()) { + auto &stream = outcome.GetResult().GetBody(); + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); + } else { + fprintf(stderr, "GetObject Failed : %s\n", + outcome.GetError().GetMessage().c_str()); + return send_response(connection, 500, "{\"error\":\"GetObject failed\"}"); + } + } catch (const std::exception &e) { + return send_response(connection, 500, + "{\"error\":\"" + std::string(e.what()) + "\"}"); + } +} + +void fill_context(Aws::Map &map, + const std::string &metadata) { + if (metadata.empty()) { + return; + } + + // Parse metadata format: [key1]:[value1],[key2]:[value2],... + // or single pair: [key]:[value] + std::string current = metadata; + size_t pos = 0; + + while (pos < current.length()) { + // Find opening bracket for key + size_t key_start = current.find('[', pos); + if (key_start == std::string::npos) + break; + + // Find closing bracket for key + size_t key_end = current.find(']', key_start); + if (key_end == std::string::npos) + break; + + // Find colon separator + size_t colon = current.find(':', key_end); + if (colon == std::string::npos) + break; + + // Find opening bracket for value + size_t value_start = current.find('[', colon); + if (value_start == std::string::npos) + break; + + // Find closing bracket for value + size_t value_end = current.find(']', value_start); + if (value_end == std::string::npos) + break; + + // Extract key and value + std::string key = current.substr(key_start + 1, key_end - key_start - 1); + std::string value = + current.substr(value_start + 1, value_end - value_start - 1); + + // Add to map + map.emplace(key, value); + + // Move to next pair (look for comma or next opening bracket) + pos = value_end + 1; + size_t comma = current.find(',', pos); + if (comma != std::string::npos) { + pos = comma + 1; + } + } +} + +MHD_Result handle_put_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &body, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + fprintf(stderr, "handle_put_object <%s>\n", metadata.c_str()); + try { + Aws::Map kmsContextMap; + // Parse metadata and populate the context map + fill_context(kmsContextMap, metadata); + + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + auto stream = std::make_shared(body); + request.SetBody(stream); + + auto outcome = it->second->PutObject(request, kmsContextMap); + if (outcome.IsSuccess()) { + json response = {{"bucket", bucket}, {"key", key}}; + return send_response(connection, 200, response.dump()); + } else { + return send_response(connection, 500, "{\"error\":\"PutObject failed\"}"); + } + } catch (const std::exception &e) { + return send_response(connection, 500, + "{\"error\":\"" + std::string(e.what()) + "\"}"); + } +} + +MHD_Result request_handler(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) { + std::string method_str(method); + bool is_push = method_str == "POST" || method_str == "PUT"; + static int dummy; + if (*con_cls == nullptr) { + if (is_push) { + *con_cls = new std::string(); + } else { + *con_cls = &dummy; + } + return MHD_YES; + } + if (is_push && *upload_data_size) { + std::string *body = static_cast(*con_cls); + body->append(upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + + std::string url_str(url); + + if (is_push && url_str == "/client") { + std::string *body = static_cast(*con_cls); + auto foo = handle_create_client(connection, *body); + delete body; + return foo; + } + + // fprintf(stderr, "request_handler <%s> <%s> <%s>\n", url, method, + // upload_data); fprintf(stderr, "keys<<\n"); MHD_get_connection_values + // (connection, MHD_HEADER_KIND, &print_key, NULL); fprintf(stderr, ">>\n"); + + if (url_str.find("/object/") == 0) { + std::string path = url_str.substr(8); // Remove "/object/" + size_t slash_pos = path.find('/'); + if (slash_pos != std::string::npos) { + std::string bucket = path.substr(0, slash_pos); + std::string key = path.substr(slash_pos + 1); + std::string client_id = get_header_value(connection, "clientid"); + + std::string metadata = get_header_value(connection, "content-metadata"); + if (method_str == "GET") { + return handle_get_object(connection, bucket, key, client_id, metadata); + } else if (method_str == "PUT") { + std::string *body = static_cast(*con_cls); + *upload_data_size = 0; + auto foo = handle_put_object(connection, bucket, key, client_id, *body, + metadata); + delete body; + return foo; + } + } + } + + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); +} + +int main() { + Aws::SDKOptions options; + Aws::InitAPI(options); + + struct MHD_Daemon *daemon = + MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION, 8085, NULL, NULL, + &request_handler, NULL, MHD_OPTION_END); + + if (!daemon) { + return 1; + } + + printf("Server running on port 8085\n"); + getchar(); + + MHD_stop_daemon(daemon); + Aws::ShutdownAPI(options); + fprintf(stderr, "Ending server\n"); + return 0; +} From 3000858b045ff47d890afcd0b1b00720a609a698 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Thu, 18 Sep 2025 12:08:06 -0400 Subject: [PATCH 02/13] m --- test-server/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test-server/Makefile b/test-server/Makefile index 4831d68d..bfb92180 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -98,6 +98,12 @@ stop-servers: fi @echo "Servers stopped" +install-cpp-dependencies: + brew install libmicrohttpd nlohmann-json ossp-uuid aws-sdk-cpp + +start-cpp-v2-server: + cd cpp-v2-server && mkdir -p build && cd build && cmake .. && make && ./s3ec-server + # Clean up logs and pid files clean: stop-servers @echo "Cleaning up..." @@ -113,6 +119,8 @@ help: @echo " start-python-v3-server : Start only the Python V3 server" @echo " start-java-v3-server : Start only the Java V3 server" @echo " start-go-v3-server : Start only the Go V3 server" + @echo " start-cpp-v2-server : Start only the C++ V2 server" + @echo " install-cpp-dependencies: use brew to install things necessary for start-cpp-v2-server" @echo " run-tests : Run Java tests" @echo " stop-servers : Stop running servers" @echo " clean : Stop servers and clean up logs" From 9ad923c7d42e07119c0d05522c083a7f875bf066 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Thu, 18 Sep 2025 12:08:52 -0400 Subject: [PATCH 03/13] m --- test-server/cpp-v2-server/CMakeLists.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test-server/cpp-v2-server/CMakeLists.txt b/test-server/cpp-v2-server/CMakeLists.txt index 16a5fd72..d9fd9d1b 100644 --- a/test-server/cpp-v2-server/CMakeLists.txt +++ b/test-server/cpp-v2-server/CMakeLists.txt @@ -3,10 +3,17 @@ project(s3ec-cpp-server) set(CMAKE_CXX_STANDARD 17) +# Configure AWS SDK build options +set(BUILD_ONLY "kms;s3" CACHE STRING "Build only KMS and S3 components") +set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") + +# Add AWS SDK as subdirectory +add_subdirectory(/Users/ajewell/head/aws-sdk-cpp aws-sdk-cpp) + find_package(PkgConfig REQUIRED) pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) -find_package(AWSSDK REQUIRED COMPONENTS s3 s3-encryption kms) find_package(nlohmann_json REQUIRED) add_executable(s3ec-server main.cpp) @@ -23,7 +30,10 @@ target_link_directories(s3ec-server PRIVATE target_link_libraries(s3ec-server ${LIBMICROHTTPD_LIBRARIES} - ${AWSSDK_LINK_LIBRARIES} + aws-cpp-sdk-core + aws-cpp-sdk-kms + aws-cpp-sdk-s3 + aws-cpp-sdk-s3-encryption nlohmann_json::nlohmann_json uuid ) \ No newline at end of file From 35cc7e222cc31c9914c799ff87a02cc4311b3b82 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 09:53:43 -0400 Subject: [PATCH 04/13] m --- test-server/Makefile | 4 +- test-server/cpp-v2-server/CMakeLists.txt | 4 +- test-server/cpp-v2-server/main.cpp | 205 ++++++++++++------ .../amazon/encryption/s3/RoundTripTests.java | 5 +- 4 files changed, 148 insertions(+), 70 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index bfb92180..3c415cb6 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -99,7 +99,9 @@ stop-servers: @echo "Servers stopped" install-cpp-dependencies: - brew install libmicrohttpd nlohmann-json ossp-uuid aws-sdk-cpp + brew install libmicrohttpd nlohmann-json ossp-uuid + cd cpp-v2-server && git clone --recurse-submodules git@github.com:aws/aws-sdk-cpp.git + cd cpp-v2-server/aws-sdk-cpp && git checkout --track remotes/origin/ajewell/ec-for-get-object start-cpp-v2-server: cd cpp-v2-server && mkdir -p build && cd build && cmake .. && make && ./s3ec-server diff --git a/test-server/cpp-v2-server/CMakeLists.txt b/test-server/cpp-v2-server/CMakeLists.txt index d9fd9d1b..4b5bd3b8 100644 --- a/test-server/cpp-v2-server/CMakeLists.txt +++ b/test-server/cpp-v2-server/CMakeLists.txt @@ -4,12 +4,12 @@ project(s3ec-cpp-server) set(CMAKE_CXX_STANDARD 17) # Configure AWS SDK build options -set(BUILD_ONLY "kms;s3" CACHE STRING "Build only KMS and S3 components") +set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") # Add AWS SDK as subdirectory -add_subdirectory(/Users/ajewell/head/aws-sdk-cpp aws-sdk-cpp) +add_subdirectory(aws-sdk-cpp) find_package(PkgConfig REQUIRED) pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index dc53f38f..b3fa281c 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -19,7 +19,8 @@ using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; std::unordered_map> client_cache; -std::string generate_uuid() { +std::string generate_uuid() +{ uuid_t uuid; uuid_generate(uuid); char uuid_str[37]; @@ -28,20 +29,23 @@ std::string generate_uuid() { } MHD_Result print_key(void *cls, enum MHD_ValueKind kind, const char *key, - const char *value) { + const char *value) +{ fprintf(stderr, "%s: %s\n", key, value); return MHD_YES; } std::string get_header_value(struct MHD_Connection *connection, - const char *key) { + const char *key) +{ const char *value = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); return value ? std::string(value) : ""; } MHD_Result send_response(struct MHD_Connection *connection, int status_code, - const std::string &content) { + const std::string &content) +{ struct MHD_Response *response = MHD_create_response_from_buffer( content.length(), (void *)content.c_str(), MHD_RESPMEM_MUST_COPY); MHD_Result ret = MHD_queue_response(connection, status_code, response); @@ -51,17 +55,33 @@ MHD_Result send_response(struct MHD_Connection *connection, int status_code, return ret; } +std::string make_error(const std::string &message, int status_code) +{ + std::string inner = "{\"__type\": \"software.amazon.encryption.s3#S3EncryptionClientError\", \"message\": \"" + message + "\"}"; + return inner; +} + MHD_Result handle_create_client(struct MHD_Connection *connection, - const std::string &body) { - try { + const std::string &body) +{ + try + { json request = json::parse(body); std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + bool legacy = request["config"]["enableLegacyWrappingAlgorithms"]; Aws::KMS::KMSClient kms_client; auto materials = std::make_shared(kms_key_id); CryptoConfigurationV2 config(materials); - config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + if (legacy) + { + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + } + else + { + config.SetSecurityProfile(SecurityProfile::V2); + } auto encryption_client = std::make_shared(config); @@ -70,50 +90,25 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); - } catch (const std::exception &e) { + } + catch (const std::exception &e) + { fprintf(stderr, "Error: %s\n", e.what()); return send_response(connection, 500, "{\"error\":\"" + std::string(e.what()) + "\"}"); - } catch (...) { + } + catch (...) + { fprintf(stderr, "Super secret error"); return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); } } -MHD_Result handle_get_object(struct MHD_Connection *connection, - const std::string &bucket, const std::string &key, - const std::string &client_id, - const std::string &metadata) { - auto it = client_cache.find(client_id); - if (it == client_cache.end()) { - return send_response(connection, 404, "{\"error\":\"Client not found\"}"); - } - fprintf(stderr, "handle_get_object <%s>\n", metadata.c_str()); - try { - Aws::S3::Model::GetObjectRequest request; - request.SetBucket(bucket); - request.SetKey(key); - - auto outcome = it->second->GetObject(request); - if (outcome.IsSuccess()) { - auto &stream = outcome.GetResult().GetBody(); - std::string content((std::istreambuf_iterator(stream)), - std::istreambuf_iterator()); - return send_response(connection, 200, content); - } else { - fprintf(stderr, "GetObject Failed : %s\n", - outcome.GetError().GetMessage().c_str()); - return send_response(connection, 500, "{\"error\":\"GetObject failed\"}"); - } - } catch (const std::exception &e) { - return send_response(connection, 500, - "{\"error\":\"" + std::string(e.what()) + "\"}"); - } -} - void fill_context(Aws::Map &map, - const std::string &metadata) { - if (metadata.empty()) { + const std::string &metadata) +{ + if (metadata.empty()) + { return; } @@ -122,7 +117,8 @@ void fill_context(Aws::Map &map, std::string current = metadata; size_t pos = 0; - while (pos < current.length()) { + while (pos < current.length()) + { // Find opening bracket for key size_t key_start = current.find('[', pos); if (key_start == std::string::npos) @@ -159,25 +155,80 @@ void fill_context(Aws::Map &map, // Move to next pair (look for comma or next opening bracket) pos = value_end + 1; size_t comma = current.find(',', pos); - if (comma != std::string::npos) { + if (comma != std::string::npos) + { pos = comma + 1; } } } +MHD_Result handle_get_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) +{ + auto it = client_cache.find(client_id); + if (it == client_cache.end()) + { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + fprintf(stderr, "handle_get_object <%s>\n", metadata.c_str()); + try + { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // S3EncryptionGetObjectOutcome outcome ; + // if (metadata.empty()) { + // outcome = it->second->GetObject(request); + // } else { + // Aws::Map kmsContextMap; + // fill_context(kmsContextMap, metadata); + // outcome = it->second->GetObject(request, kmsContextMap); + // } + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + auto outcome = it->second->GetObject(request, kmsContextMap); + + if (outcome.IsSuccess()) + { + auto &stream = outcome.GetResult().GetBody(); + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); + } + else + { + auto msg = make_error( outcome.GetError().GetMessage(), 500); + fprintf(stderr, "GetObject Failed : %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } + } + catch (const std::exception &e) + { + auto msg = make_error( e.what(), 500); + fprintf(stderr, "GetObject Threw : %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } +} + MHD_Result handle_put_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, const std::string &body, - const std::string &metadata) { + const std::string &metadata) +{ auto it = client_cache.find(client_id); - if (it == client_cache.end()) { + if (it == client_cache.end()) + { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } fprintf(stderr, "handle_put_object <%s>\n", metadata.c_str()); - try { + try + { Aws::Map kmsContextMap; - // Parse metadata and populate the context map fill_context(kmsContextMap, metadata); Aws::S3::Model::PutObjectRequest request; @@ -188,34 +239,48 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBody(stream); auto outcome = it->second->PutObject(request, kmsContextMap); - if (outcome.IsSuccess()) { + if (outcome.IsSuccess()) + { json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); - } else { - return send_response(connection, 500, "{\"error\":\"PutObject failed\"}"); } - } catch (const std::exception &e) { - return send_response(connection, 500, - "{\"error\":\"" + std::string(e.what()) + "\"}"); + else + { + auto msg = make_error( outcome.GetError().GetMessage(), 500); + fprintf(stderr, "PutObject Failed : %s\n", msg.c_str()); + return send_response(connection, 500, msg); + } + } + catch (const std::exception &e) + { + auto msg = make_error( e.what(), 500); + fprintf(stderr, "PutObject Threw : %s\n", msg.c_str()); + return send_response(connection, 500, msg); } } MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, - size_t *upload_data_size, void **con_cls) { + size_t *upload_data_size, void **con_cls) +{ std::string method_str(method); bool is_push = method_str == "POST" || method_str == "PUT"; static int dummy; - if (*con_cls == nullptr) { - if (is_push) { + if (*con_cls == nullptr) + { + if (is_push) + { *con_cls = new std::string(); - } else { + } + else + { *con_cls = &dummy; } return MHD_YES; } - if (is_push && *upload_data_size) { + if (is_push && *upload_data_size) + { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; @@ -224,7 +289,8 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string url_str(url); - if (is_push && url_str == "/client") { + if (is_push && url_str == "/client") + { std::string *body = static_cast(*con_cls); auto foo = handle_create_client(connection, *body); delete body; @@ -235,18 +301,23 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, // upload_data); fprintf(stderr, "keys<<\n"); MHD_get_connection_values // (connection, MHD_HEADER_KIND, &print_key, NULL); fprintf(stderr, ">>\n"); - if (url_str.find("/object/") == 0) { + if (url_str.find("/object/") == 0) + { std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); - if (slash_pos != std::string::npos) { + if (slash_pos != std::string::npos) + { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); std::string metadata = get_header_value(connection, "content-metadata"); - if (method_str == "GET") { + if (method_str == "GET") + { return handle_get_object(connection, bucket, key, client_id, metadata); - } else if (method_str == "PUT") { + } + else if (method_str == "PUT") + { std::string *body = static_cast(*con_cls); *upload_data_size = 0; auto foo = handle_put_object(connection, bucket, key, client_id, *body, @@ -261,7 +332,8 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, "{\"error\":\"Not idea what is happening\"}"); } -int main() { +int main() +{ Aws::SDKOptions options; Aws::InitAPI(options); @@ -269,7 +341,8 @@ int main() { MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION, 8085, NULL, NULL, &request_handler, NULL, MHD_OPTION_END); - if (!daemon) { + if (!daemon) + { return 1; } diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 535e4d8d..2ec03e73 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -494,7 +494,10 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { .build()); fail("Expected Exception"); } catch (S3EncryptionClientError e) { - assertTrue(e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms")); + assertTrue( + e.getMessage().contains("Enable legacy wrapping algorithms to use legacy key wrapping algorithm: kms") + || e.getMessage().contains("Retry with V2_AND_LEGACY enabled") + ); } } From c8cdae48b58cd4337484b0225564af849d507094 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 09:57:54 -0400 Subject: [PATCH 05/13] m --- test-server/cpp-v2-server/CMakeLists.txt | 2 +- test-server/cpp-v2-server/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test-server/cpp-v2-server/CMakeLists.txt b/test-server/cpp-v2-server/CMakeLists.txt index 4b5bd3b8..b282dbc4 100644 --- a/test-server/cpp-v2-server/CMakeLists.txt +++ b/test-server/cpp-v2-server/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.16) -project(s3ec-cpp-server) +project(s3ec-cpp-v2-server) set(CMAKE_CXX_STANDARD 17) diff --git a/test-server/cpp-v2-server/README.md b/test-server/cpp-v2-server/README.md index 06d55be1..8e77feda 100644 --- a/test-server/cpp-v2-server/README.md +++ b/test-server/cpp-v2-server/README.md @@ -28,7 +28,7 @@ make ./s3ec-server ``` -Server runs on localhost:8081 +Server runs on localhost:8085 ## API Endpoints From c9224b5dddb259ee0a693a9f8e9e1c467a95717e Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 10:00:46 -0400 Subject: [PATCH 06/13] m --- test-server/Makefile | 132 +++++++++++++++---------------------------- 1 file changed, 46 insertions(+), 86 deletions(-) diff --git a/test-server/Makefile b/test-server/Makefile index 3c415cb6..90d15648 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -1,6 +1,6 @@ # Makefile for S3 Encryption Client Testing -.PHONY: all start-servers start-python-v3-server start-java-v3-server start-go-v3-server run-tests stop-servers clean ci check-env help +.PHONY: all start-servers run-tests stop-servers clean ci check-env help # Default target all: start-servers run-tests @@ -8,65 +8,31 @@ all: start-servers run-tests # CI target for GitHub Actions ci: start-servers run-tests stop-servers +SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | sort) -# Start Python server in background -start-python-v3-server: - @echo "Starting Python V3 server..." - cd python-v3-server && \ - python -m venv .venv && \ - .venv/bin/python -m ensurepip && \ - .venv/bin/python -m pip install -e . && \ - .venv/bin/python -m pip install -e ../.. && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - .venv/bin/python src/main.py & echo $$! > ../python-v3-server.pid - @echo "Python server starting..." - -# Start Java server in background -start-java-v3-server: - @echo "Starting Java V3 server..." - cd java-v3-server && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel run & echo $$! > ../java-v3-server.pid - @echo "Java server starting..." - -# Start Go server in background -start-go-v3-server: - @echo "Starting Go V3 server..." - cd go-v3-server && \ - go mod tidy && \ - AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ - AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ - AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ - AWS_REGION="us-west-2" \ - go run . & echo $$! > ../go-v3-server.pid - @echo "Go server starting..." +SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) # Start all servers in parallel start-servers: - @echo "Starting servers in parallel..." - @$(MAKE) -j3 start-python-v3-server start-java-v3-server start-go-v3-server - @echo "Waiting for servers to be ready..." - @for i in $$(seq 1 360); do \ - if nc -z localhost 8080 && nc -z localhost 8081 && nc -z localhost 8082; then \ - echo "Ports are open, waiting for servers to initialize..."; \ - sleep 5; \ - echo "All servers are ready!"; \ - break; \ - fi; \ - if [ $$i -eq 360 ]; then \ - echo "Timeout waiting for servers to start"; \ - exit 1; \ - fi; \ - echo "Waiting for servers to start ($$i/360)..."; \ - sleep 1; \ + @echo "Starting all servers..." + $(MAKE) start-all-servers + @echo "Waiting for servers to start..." + @for dir in $(SERVER_DIRS); do \ + echo "Waiting for server in $$dir..."; \ + $(MAKE) -C $$dir wait-for-server; \ done +start-all-servers: $(SERVER_TARGETS) + +$(SERVER_TARGETS): start-%: + @if [ -f $*/Makefile ]; then \ + echo "Starting server in $*..."; \ + $(MAKE) -C $* start-server; \ + else \ + echo "❌ Error: no Makefile found in $$dir"; \ + exit 1; \ + fi; \ + # Run the Java tests run-tests: @@ -84,48 +50,20 @@ run-tests: # Stop the servers stop-servers: @echo "Stopping servers..." - @if [ -f python-v3-server.pid ]; then \ - kill $$(cat python-v3-server.pid) 2>/dev/null || true; \ - rm python-v3-server.pid; \ - fi - @if [ -f java-v3-server.pid ]; then \ - kill $$(cat java-v3-server.pid) 2>/dev/null || true; \ - rm java-v3-server.pid; \ - fi - @if [ -f go-v3-server.pid ]; then \ - kill $$(cat go-v3-server.pid) 2>/dev/null || true; \ - rm go-v3-server.pid; \ - fi + @for dir in $(SERVER_DIRS); do \ + echo "Starting server in $$dir..."; \ + $(MAKE) -C $$dir stop-server; \ + done @echo "Servers stopped" -install-cpp-dependencies: - brew install libmicrohttpd nlohmann-json ossp-uuid - cd cpp-v2-server && git clone --recurse-submodules git@github.com:aws/aws-sdk-cpp.git - cd cpp-v2-server/aws-sdk-cpp && git checkout --track remotes/origin/ajewell/ec-for-get-object - -start-cpp-v2-server: - cd cpp-v2-server && mkdir -p build && cd build && cmake .. && make && ./s3ec-server - -# Clean up logs and pid files -clean: stop-servers - @echo "Cleaning up..." - @rm -f python-v3-server.log java-v3-server.log go-v3-server.log - @echo "Cleanup complete" - # Help target help: @echo "Available targets:" @echo " all : Start servers and run tests (default, output to stdout)" @echo " ci : Run in CI mode (start servers, run tests, stop servers)" @echo " start-servers : Start all servers in parallel" - @echo " start-python-v3-server : Start only the Python V3 server" - @echo " start-java-v3-server : Start only the Java V3 server" - @echo " start-go-v3-server : Start only the Go V3 server" - @echo " start-cpp-v2-server : Start only the C++ V2 server" - @echo " install-cpp-dependencies: use brew to install things necessary for start-cpp-v2-server" @echo " run-tests : Run Java tests" @echo " stop-servers : Stop running servers" - @echo " clean : Stop servers and clean up logs" @echo " check-env : Check if required environment variables are set" @echo " help : Show this help message" @@ -136,3 +74,25 @@ check-env: @if [ -z "$$AWS_SECRET_ACCESS_KEY" ]; then echo "AWS_SECRET_ACCESS_KEY is not set"; else echo "AWS_SECRET_ACCESS_KEY is set"; fi @if [ -z "$$AWS_SESSION_TOKEN" ]; then echo "AWS_SESSION_TOKEN is not set"; else echo "AWS_SESSION_TOKEN is set"; fi @if [ -z "$$AWS_REGION" ]; then echo "AWS_REGION is not set (will use us-west-2 as default)"; else echo "AWS_REGION is set to $$AWS_REGION"; fi + +TIMEOUT := 360 + +wait-for-port: + @if [ -z "$(PORT)" ]; then \ + echo "❌ Error: PORT is required"; \ + exit 1; \ + fi + @for i in $$(seq 1 $(TIMEOUT)); do \ + if nc -z localhost $$PORT; then \ + echo "Ports are open, waiting for servers to initialize..."; \ + sleep 5; \ + echo "Server at $$PORT is ready!"; \ + break; \ + fi; \ + if [ $$i -eq $(TIMEOUT) ]; then \ + echo "Timeout waiting for $$PORT start"; \ + exit 1; \ + fi; \ + echo "Waiting for $$PORT to start ($$i/$(TIMEOUT))..."; \ + sleep 1; \ + done From 7cdf7e69f881a70449ba0de874f045a3036c3a5c Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 11:04:31 -0400 Subject: [PATCH 07/13] m --- test-server/cpp-v2-server/Makefile | 33 ++++ test-server/cpp-v2-server/main.cpp | 167 ++++++------------ .../amazon/encryption/s3/RoundTripTests.java | 2 + 3 files changed, 87 insertions(+), 115 deletions(-) create mode 100644 test-server/cpp-v2-server/Makefile diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile new file mode 100644 index 00000000..7238b99d --- /dev/null +++ b/test-server/cpp-v2-server/Makefile @@ -0,0 +1,33 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8085 + +build/s3ec-server: + pwd + ls aws-sdk-cpp/README.md + brew install libmicrohttpd nlohmann-json ossp-uuid + git clone --recurse-submodules git@github.com:aws/aws-sdk-cpp.git + cd aws-sdk-cpp && git checkout --track remotes/origin/ajewell/ec-for-get-object + mkdir -p build && cd build && cmake .. + +start-server: | build/s3ec-server + @echo "Starting Cpp V2 server..." + cd build && make && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./s3ec-server & echo $$! > $(PID_FILE) + @echo "Go V3 server starting..." + +stop-server: + @if [ -f $(PID_FILE) ]; then \ + kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm $(PID_FILE); \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index b3fa281c..aa790959 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -19,8 +19,7 @@ using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; std::unordered_map> client_cache; -std::string generate_uuid() -{ +std::string generate_uuid() { uuid_t uuid; uuid_generate(uuid); char uuid_str[37]; @@ -28,44 +27,32 @@ std::string generate_uuid() return std::string(uuid_str); } -MHD_Result print_key(void *cls, enum MHD_ValueKind kind, const char *key, - const char *value) -{ - fprintf(stderr, "%s: %s\n", key, value); - return MHD_YES; -} - std::string get_header_value(struct MHD_Connection *connection, - const char *key) -{ + const char *key) { const char *value = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); return value ? std::string(value) : ""; } MHD_Result send_response(struct MHD_Connection *connection, int status_code, - const std::string &content) -{ + const std::string &content) { struct MHD_Response *response = MHD_create_response_from_buffer( content.length(), (void *)content.c_str(), MHD_RESPMEM_MUST_COPY); MHD_Result ret = MHD_queue_response(connection, status_code, response); - if (ret != MHD_YES) - fprintf(stderr, "MHD_queue_response returned %d\n", ret); MHD_destroy_response(response); return ret; } -std::string make_error(const std::string &message, int status_code) -{ - std::string inner = "{\"__type\": \"software.amazon.encryption.s3#S3EncryptionClientError\", \"message\": \"" + message + "\"}"; - return inner; +std::string make_error(const std::string &message, int status_code) { + return "{\"__type\": " + "\"software.amazon.encryption.s3#S3EncryptionClientError\", " + "\"message\": \"" + + message + "\"}"; } MHD_Result handle_create_client(struct MHD_Connection *connection, - const std::string &body) -{ - try - { + const std::string &body) { + try { json request = json::parse(body); std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; bool legacy = request["config"]["enableLegacyWrappingAlgorithms"]; @@ -74,12 +61,9 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, auto materials = std::make_shared(kms_key_id); CryptoConfigurationV2 config(materials); - if (legacy) - { + if (legacy) { config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); - } - else - { + } else { config.SetSecurityProfile(SecurityProfile::V2); } @@ -90,25 +74,17 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, json response = {{"clientId", client_id}}; return send_response(connection, 200, response.dump()); - } - catch (const std::exception &e) - { - fprintf(stderr, "Error: %s\n", e.what()); + } catch (const std::exception &e) { return send_response(connection, 500, "{\"error\":\"" + std::string(e.what()) + "\"}"); - } - catch (...) - { - fprintf(stderr, "Super secret error"); + } catch (...) { return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); } } void fill_context(Aws::Map &map, - const std::string &metadata) -{ - if (metadata.empty()) - { + const std::string &metadata) { + if (metadata.empty()) { return; } @@ -117,8 +93,7 @@ void fill_context(Aws::Map &map, std::string current = metadata; size_t pos = 0; - while (pos < current.length()) - { + while (pos < current.length()) { // Find opening bracket for key size_t key_start = current.find('[', pos); if (key_start == std::string::npos) @@ -155,8 +130,7 @@ void fill_context(Aws::Map &map, // Move to next pair (look for comma or next opening bracket) pos = value_end + 1; size_t comma = current.find(',', pos); - if (comma != std::string::npos) - { + if (comma != std::string::npos) { pos = comma + 1; } } @@ -165,16 +139,13 @@ void fill_context(Aws::Map &map, MHD_Result handle_get_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, - const std::string &metadata) -{ + const std::string &metadata) { auto it = client_cache.find(client_id); - if (it == client_cache.end()) - { + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } - fprintf(stderr, "handle_get_object <%s>\n", metadata.c_str()); - try - { + + try { Aws::S3::Model::GetObjectRequest request; request.SetBucket(bucket); request.SetKey(key); @@ -192,24 +163,17 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, fill_context(kmsContextMap, metadata); auto outcome = it->second->GetObject(request, kmsContextMap); - if (outcome.IsSuccess()) - { + if (outcome.IsSuccess()) { auto &stream = outcome.GetResult().GetBody(); std::string content((std::istreambuf_iterator(stream)), std::istreambuf_iterator()); return send_response(connection, 200, content); - } - else - { - auto msg = make_error( outcome.GetError().GetMessage(), 500); - fprintf(stderr, "GetObject Failed : %s\n", msg.c_str()); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); return send_response(connection, 500, msg); } - } - catch (const std::exception &e) - { - auto msg = make_error( e.what(), 500); - fprintf(stderr, "GetObject Threw : %s\n", msg.c_str()); + } catch (const std::exception &e) { + auto msg = make_error(e.what(), 500); return send_response(connection, 500, msg); } } @@ -218,16 +182,13 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, const std::string &bucket, const std::string &key, const std::string &client_id, const std::string &body, - const std::string &metadata) -{ + const std::string &metadata) { auto it = client_cache.find(client_id); - if (it == client_cache.end()) - { + if (it == client_cache.end()) { return send_response(connection, 404, "{\"error\":\"Client not found\"}"); } - fprintf(stderr, "handle_put_object <%s>\n", metadata.c_str()); - try - { + + try { Aws::Map kmsContextMap; fill_context(kmsContextMap, metadata); @@ -239,48 +200,35 @@ MHD_Result handle_put_object(struct MHD_Connection *connection, request.SetBody(stream); auto outcome = it->second->PutObject(request, kmsContextMap); - if (outcome.IsSuccess()) - { + if (outcome.IsSuccess()) { json response = {{"bucket", bucket}, {"key", key}}; return send_response(connection, 200, response.dump()); - } - else - { - auto msg = make_error( outcome.GetError().GetMessage(), 500); - fprintf(stderr, "PutObject Failed : %s\n", msg.c_str()); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); return send_response(connection, 500, msg); } - } - catch (const std::exception &e) - { - auto msg = make_error( e.what(), 500); - fprintf(stderr, "PutObject Threw : %s\n", msg.c_str()); - return send_response(connection, 500, msg); + } catch (const std::exception &e) { + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); } } MHD_Result request_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, - size_t *upload_data_size, void **con_cls) -{ + size_t *upload_data_size, void **con_cls) { std::string method_str(method); bool is_push = method_str == "POST" || method_str == "PUT"; static int dummy; - if (*con_cls == nullptr) - { - if (is_push) - { + if (*con_cls == nullptr) { + if (is_push) { *con_cls = new std::string(); - } - else - { + } else { *con_cls = &dummy; } return MHD_YES; } - if (is_push && *upload_data_size) - { + if (is_push && *upload_data_size) { std::string *body = static_cast(*con_cls); body->append(upload_data, *upload_data_size); *upload_data_size = 0; @@ -289,35 +237,25 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string url_str(url); - if (is_push && url_str == "/client") - { + if (is_push && url_str == "/client") { std::string *body = static_cast(*con_cls); auto foo = handle_create_client(connection, *body); delete body; return foo; } - // fprintf(stderr, "request_handler <%s> <%s> <%s>\n", url, method, - // upload_data); fprintf(stderr, "keys<<\n"); MHD_get_connection_values - // (connection, MHD_HEADER_KIND, &print_key, NULL); fprintf(stderr, ">>\n"); - - if (url_str.find("/object/") == 0) - { + if (url_str.find("/object/") == 0) { std::string path = url_str.substr(8); // Remove "/object/" size_t slash_pos = path.find('/'); - if (slash_pos != std::string::npos) - { + if (slash_pos != std::string::npos) { std::string bucket = path.substr(0, slash_pos); std::string key = path.substr(slash_pos + 1); std::string client_id = get_header_value(connection, "clientid"); std::string metadata = get_header_value(connection, "content-metadata"); - if (method_str == "GET") - { + if (method_str == "GET") { return handle_get_object(connection, bucket, key, client_id, metadata); - } - else if (method_str == "PUT") - { + } else if (method_str == "PUT") { std::string *body = static_cast(*con_cls); *upload_data_size = 0; auto foo = handle_put_object(connection, bucket, key, client_id, *body, @@ -332,8 +270,7 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, "{\"error\":\"Not idea what is happening\"}"); } -int main() -{ +int main() { Aws::SDKOptions options; Aws::InitAPI(options); @@ -341,13 +278,13 @@ int main() MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION, 8085, NULL, NULL, &request_handler, NULL, MHD_OPTION_END); - if (!daemon) - { + if (!daemon) { + fprintf(stderr, "Failed to start server on port 8085\n"); return 1; } - printf("Server running on port 8085\n"); - getchar(); + fprintf(stderr, "Server running on port 8085\n"); + sleep(10000); MHD_stop_daemon(daemon); Aws::ShutdownAPI(options); diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 2ec03e73..20381fa8 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -68,11 +68,13 @@ public class RoundTripTests { serverList.add(new LanguageServerTarget("Java-V3", "8080")); serverList.add(new LanguageServerTarget("Python-V3", "8081")); serverList.add(new LanguageServerTarget("Go-V3", "8082")); + serverList.add(new LanguageServerTarget("Cpp-V2", "8085")); serverMap = new HashMap<>(14); serverMap.put("Java-V3", new LanguageServerTarget("Java-V3", "8080")); serverMap.put("Python-V3", new LanguageServerTarget("Python-V3", "8081")); serverMap.put("Go-V3", new LanguageServerTarget("Go-V3", "8082")); + serverMap.put("Cpp-V2", new LanguageServerTarget("Cpp-V2", "8085")); } // These S3EC implementations do not validate encryption context provided to getObject (i.e. on decrypt). From 6b6bcd77b73424875bd176fe20af61ad869df14c Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 12:42:18 -0400 Subject: [PATCH 08/13] m --- test-server/cpp-v2-server/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 7238b99d..e8145593 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -6,8 +6,6 @@ PID_FILE := server.pid PORT := 8085 build/s3ec-server: - pwd - ls aws-sdk-cpp/README.md brew install libmicrohttpd nlohmann-json ossp-uuid git clone --recurse-submodules git@github.com:aws/aws-sdk-cpp.git cd aws-sdk-cpp && git checkout --track remotes/origin/ajewell/ec-for-get-object From f133f014bf36884c34912cee3c8c7265adc0c90d Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 12:59:17 -0400 Subject: [PATCH 09/13] m --- .github/workflows/lint.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 16057711..637003f3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,7 +8,7 @@ on: jobs: lint: - runs-on: ubuntu-latest + runs-on: macos-13 steps: - name: Checkout code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8025246..e636855e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: macos-13 permissions: id-token: write contents: read From a2ed8254542ecf7a5837a96d2120a0c6c7f07c59 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 13:04:04 -0400 Subject: [PATCH 10/13] m --- test-server/cpp-v2-server/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index e8145593..2d6328e8 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -7,7 +7,7 @@ PORT := 8085 build/s3ec-server: brew install libmicrohttpd nlohmann-json ossp-uuid - git clone --recurse-submodules git@github.com:aws/aws-sdk-cpp.git + git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git cd aws-sdk-cpp && git checkout --track remotes/origin/ajewell/ec-for-get-object mkdir -p build && cd build && cmake .. From 715785045dfd88b1f09dcae781c34219984a5626 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 13:18:09 -0400 Subject: [PATCH 11/13] m --- .github/workflows/test.yml | 5 +++++ test-server/cpp-v2-server/Makefile | 2 +- test-server/cpp-v2-server/main.cpp | 9 +++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e636855e..2b1fe8a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,11 @@ jobs: with: python-version: ${{ inputs.python-version || '3.11' }} + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25 + # Cache uv dependencies - name: Cache uv dependencies uses: actions/cache@v3 diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index 2d6328e8..e9156d64 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -19,7 +19,7 @@ start-server: | build/s3ec-server AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ ./s3ec-server & echo $$! > $(PID_FILE) - @echo "Go V3 server starting..." + @echo "Cpp V2 server starting..." stop-server: @if [ -f $(PID_FILE) ]; then \ diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index aa790959..51cfd65a 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -37,7 +37,7 @@ std::string get_header_value(struct MHD_Connection *connection, MHD_Result send_response(struct MHD_Connection *connection, int status_code, const std::string &content) { struct MHD_Response *response = MHD_create_response_from_buffer( - content.length(), (void *)content.c_str(), MHD_RESPMEM_MUST_COPY); + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); MHD_Result ret = MHD_queue_response(connection, status_code, response); MHD_destroy_response(response); return ret; @@ -76,7 +76,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, return send_response(connection, 200, response.dump()); } catch (const std::exception &e) { return send_response(connection, 500, - "{\"error\":\"" + std::string(e.what()) + "\"}"); + "{\"error\":\"An exception was thrown.\"}"); } catch (...) { return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); } @@ -173,7 +173,7 @@ MHD_Result handle_get_object(struct MHD_Connection *connection, return send_response(connection, 500, msg); } } catch (const std::exception &e) { - auto msg = make_error(e.what(), 500); + auto msg = make_error("An exception was thrown", 500); return send_response(connection, 500, msg); } } @@ -275,11 +275,12 @@ int main() { Aws::InitAPI(options); struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION, 8085, NULL, NULL, + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, 8085, NULL, NULL, &request_handler, NULL, MHD_OPTION_END); if (!daemon) { fprintf(stderr, "Failed to start server on port 8085\n"); + Aws::ShutdownAPI(options); return 1; } From a21bc8ec965dcdf9b1008b653f2b83fd3e7e5d62 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Fri, 19 Sep 2025 14:28:41 -0400 Subject: [PATCH 12/13] m --- test-server/cpp-v2-server/main.cpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index 51cfd65a..ccc72d76 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -238,10 +238,8 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, std::string url_str(url); if (is_push && url_str == "/client") { - std::string *body = static_cast(*con_cls); - auto foo = handle_create_client(connection, *body); - delete body; - return foo; + std::unique_ptr body(static_cast(*con_cls)); + return handle_create_client(connection, *body); } if (url_str.find("/object/") == 0) { @@ -256,12 +254,9 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, if (method_str == "GET") { return handle_get_object(connection, bucket, key, client_id, metadata); } else if (method_str == "PUT") { - std::string *body = static_cast(*con_cls); + std::unique_ptr body(static_cast(*con_cls)); *upload_data_size = 0; - auto foo = handle_put_object(connection, bucket, key, client_id, *body, - metadata); - delete body; - return foo; + return handle_put_object(connection, bucket, key, client_id, *body, metadata); } } } From e2bab004e554d3a4c11335bf629b34ead2662f9f Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Mon, 22 Sep 2025 10:36:04 -0400 Subject: [PATCH 13/13] m --- test-server/cpp-v2-server/main.cpp | 1 - .../it/java/software/amazon/encryption/s3/RoundTripTests.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index ccc72d76..869e617b 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -57,7 +57,6 @@ MHD_Result handle_create_client(struct MHD_Connection *connection, std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; bool legacy = request["config"]["enableLegacyWrappingAlgorithms"]; - Aws::KMS::KMSClient kms_client; auto materials = std::make_shared(kms_key_id); CryptoConfigurationV2 config(materials); diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index ec834c70..cd67f2e0 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -561,7 +561,7 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { } catch (S3EncryptionClientError e) { if (language.equals(NET_V3) || language.equals(NET_V2) || language.equals(CPP_V2)) { assertTrue(e.getMessage().contains( - "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration V2." + "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" )); } else if (language.equals(RUBY_V3) || language.equals(RUBY_V2)) { assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object."));