diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9fa8a8f8..a07c9a96 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,17 +12,26 @@ on: jobs: test: - runs-on: macos-13 + runs-on: macos-14 permissions: id-token: write contents: read steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: submodules: true token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }} + + - name: Checkout CPP code + uses: actions/checkout@v5 + with: + submodules: recursive + token: ${{ secrets.PAT_FOR_CPP }} + repository: awslabs/aws-sdk-cpp-staging + ref: fire-egg-dev + path: test-server/cpp-v2-transition-server/aws-sdk-cpp/ - name: Set up Python uses: actions/setup-python@v5 @@ -35,9 +44,9 @@ jobs: ruby-version: '3.4' - name: Set up PHP with Composer - uses: shivammathur/setup-php@v2 + uses: shivammathur/setup-php@verbose with: - php-version: '8.1' + php-version: '8.1' - name: Install PHP V2 dependencies working-directory: ./test-server/php-v2-server @@ -68,14 +77,14 @@ jobs: # Cache Gradle dependencies and build outputs - name: Cache Gradle packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.gradle/caches ~/.gradle/wrapper test-server/java-v3-server/.gradle test-server/java-tests/.gradle - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + key: ${{ runner.os }}-gradle-${{ hashFiles('test-server/java-v3-server/**/*.gradle*', 'test-server/java-tests/**/gradle-wrapper.properties', 'test-server/java-tests/**/*.gradle*', 'test-server/java-v3-server/**/gradle-wrapper.properties') }} restore-keys: | ${{ runner.os }}-gradle- diff --git a/test-server/Makefile b/test-server/Makefile index f23b738d..7fd66285 100644 --- a/test-server/Makefile +++ b/test-server/Makefile @@ -9,6 +9,7 @@ all: start-servers run-tests ci: start-servers run-tests stop-servers SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | sort) +# SERVER_DIRS := cpp-v3-server START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS)) WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS)) @@ -56,7 +57,7 @@ run-tests: AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ AWS_REGION="us-west-2" \ - ./gradlew --build-cache --parallel integ -Dtest.filter.servers="$(FILTER)" + ./gradlew --build-cache --info --parallel integ -Dtest.filter.servers="$(FILTER)" @echo "Tests completed successfully" # Stop the servers @@ -106,7 +107,7 @@ wait-for-port: exit 1; \ fi; \ echo "Waiting for $$PORT to start ($$i/$(TIMEOUT))..."; \ - sleep 1; \ + sleep 3; \ done # Here are some helpful curl commands @@ -117,4 +118,4 @@ test-create-client: -H "Content-Type: application/json" \ -H "User-Agent: smithy-java/0.0.3 ua/2.1 os/macos#15.5 lang/java#23.0.2" \ -d '{"config":{"enableLegacyUnauthenticatedModes":false,"enableDelayedAuthenticationMode":false,"enableLegacyWrappingAlgorithms":false,"keyMaterial":{"kmsKeyId":"arn:aws:kms:us-west-2:370957321024:alias/S3EC-Test-Server-Github-KMS-Key"}}}' \ - http://localhost:$(PORT)/client \ No newline at end of file + http://localhost:$(PORT)/client diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile index ad5c951e..9e0f04b1 100644 --- a/test-server/cpp-v2-server/Makefile +++ b/test-server/cpp-v2-server/Makefile @@ -8,7 +8,7 @@ PORT := 8085 build/s3ec-server: brew install libmicrohttpd nlohmann-json ossp-uuid git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git - cd aws-sdk-cpp + cd aws-sdk-cpp mkdir -p build && cd build && cmake .. start-server: | build/s3ec-server diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp index 869e617b..9be401f8 100644 --- a/test-server/cpp-v2-server/main.cpp +++ b/test-server/cpp-v2-server/main.cpp @@ -267,18 +267,18 @@ MHD_Result request_handler(void *cls, struct MHD_Connection *connection, int main() { Aws::SDKOptions options; Aws::InitAPI(options); - + int port = 8085; struct MHD_Daemon *daemon = - MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, 8085, NULL, NULL, + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, &request_handler, NULL, MHD_OPTION_END); if (!daemon) { - fprintf(stderr, "Failed to start server on port 8085\n"); + fprintf(stderr, "Failed to start server on port %d\n", port); Aws::ShutdownAPI(options); return 1; } - fprintf(stderr, "Server running on port 8085\n"); + fprintf(stderr, "Server running on port %d\n", port); sleep(10000); MHD_stop_daemon(daemon); diff --git a/test-server/cpp-v2-transition-server/CMakeLists.txt b/test-server/cpp-v2-transition-server/CMakeLists.txt new file mode 100644 index 00000000..b282dbc4 --- /dev/null +++ b/test-server/cpp-v2-transition-server/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 3.16) +project(s3ec-cpp-v2-server) + +set(CMAKE_CXX_STANDARD 17) + +# Configure AWS SDK build options +set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") +set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") + +# Add AWS SDK as subdirectory +add_subdirectory(aws-sdk-cpp) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) + +find_package(nlohmann_json REQUIRED) + +add_executable(s3ec-server main.cpp) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_link_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_LIBRARY_DIRS} + /opt/homebrew/lib +) + +target_link_libraries(s3ec-server + ${LIBMICROHTTPD_LIBRARIES} + aws-cpp-sdk-core + aws-cpp-sdk-kms + aws-cpp-sdk-s3 + aws-cpp-sdk-s3-encryption + nlohmann_json::nlohmann_json + uuid +) \ No newline at end of file diff --git a/test-server/cpp-v2-transition-server/Makefile b/test-server/cpp-v2-transition-server/Makefile new file mode 100644 index 00000000..05803c78 --- /dev/null +++ b/test-server/cpp-v2-transition-server/Makefile @@ -0,0 +1,29 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8097 + +build/s3ec-server: + brew install libmicrohttpd nlohmann-json ossp-uuid + mkdir -p build && cd build && cmake .. + +start-server: | build/s3ec-server + @echo "Starting Cpp V2 server..." + cd build && make && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./s3ec-server & echo $$! > $(PID_FILE) + @echo "Cpp V2 server starting..." + +stop-server: + @if [ -f $(PID_FILE) ]; then \ + kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm $(PID_FILE); \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/cpp-v2-transition-server/README.md b/test-server/cpp-v2-transition-server/README.md new file mode 100644 index 00000000..8e77feda --- /dev/null +++ b/test-server/cpp-v2-transition-server/README.md @@ -0,0 +1,37 @@ +# C++ S3 Encryption Test Server + +Minimal C++ implementation of the S3 Encryption test server. + +## Dependencies + +- libmicrohttpd +- AWS SDK for C++ +- nlohmann/json +- uuid + +On MacOS you can +```bash +brew install libmicrohttpd nlohmann-json ossp-uuid +``` + +## Build + +```bash +mkdir build && cd build +cmake .. +make +``` + +## Run + +```bash +./s3ec-server +``` + +Server runs on localhost:8085 + +## API Endpoints + +- `POST /client` - Create S3 encryption client +- `GET /object/{bucket}/{key}` - Get encrypted object +- `PUT /object/{bucket}/{key}` - Put encrypted object \ No newline at end of file diff --git a/test-server/cpp-v2-transition-server/main.cpp b/test-server/cpp-v2-transition-server/main.cpp new file mode 100644 index 00000000..32735d60 --- /dev/null +++ b/test-server/cpp-v2-transition-server/main.cpp @@ -0,0 +1,289 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using json = nlohmann::json; +using namespace Aws::S3Encryption; +using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; +std::unordered_map> + client_cache; + +std::string generate_uuid() { + uuid_t uuid; + uuid_generate(uuid); + char uuid_str[37]; + uuid_unparse(uuid, uuid_str); + return std::string(uuid_str); +} + +std::string get_header_value(struct MHD_Connection *connection, + const char *key) { + const char *value = + MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); + return value ? std::string(value) : ""; +} + +MHD_Result send_response(struct MHD_Connection *connection, int status_code, + const std::string &content) { + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + MHD_Result ret = MHD_queue_response(connection, status_code, response); + MHD_destroy_response(response); + return ret; +} + +std::string make_error(const std::string &message, int status_code) { + return "{\"__type\": " + "\"software.amazon.encryption.s3#S3EncryptionClientError\", " + "\"message\": \"" + + message + "\"}"; +} + +MHD_Result handle_create_client(struct MHD_Connection *connection, + const std::string &body) { + try { + json request = json::parse(body); + std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + bool legacy = request["config"]["enableLegacyWrappingAlgorithms"]; + + auto materials = + std::make_shared(kms_key_id); + CryptoConfigurationV2 config(materials); + if (legacy) { + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + } else { + config.SetSecurityProfile(SecurityProfile::V2); + } + + auto encryption_client = std::make_shared(config); + + std::string client_id = generate_uuid(); + client_cache[client_id] = encryption_client; + + json response = {{"clientId", client_id}}; + return send_response(connection, 200, response.dump()); + } catch (const std::exception &e) { + return send_response(connection, 500, + "{\"error\":\"An exception was thrown.\"}"); + } catch (...) { + return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); + } +} + +void fill_context(Aws::Map &map, + const std::string &metadata) { + if (metadata.empty()) { + return; + } + + // Parse metadata format: [key1]:[value1],[key2]:[value2],... + // or single pair: [key]:[value] + std::string current = metadata; + size_t pos = 0; + + while (pos < current.length()) { + // Find opening bracket for key + size_t key_start = current.find('[', pos); + if (key_start == std::string::npos) + break; + + // Find closing bracket for key + size_t key_end = current.find(']', key_start); + if (key_end == std::string::npos) + break; + + // Find colon separator + size_t colon = current.find(':', key_end); + if (colon == std::string::npos) + break; + + // Find opening bracket for value + size_t value_start = current.find('[', colon); + if (value_start == std::string::npos) + break; + + // Find closing bracket for value + size_t value_end = current.find(']', value_start); + if (value_end == std::string::npos) + break; + + // Extract key and value + std::string key = current.substr(key_start + 1, key_end - key_start - 1); + std::string value = + current.substr(value_start + 1, value_end - value_start - 1); + + // Add to map + map.emplace(key, value); + + // Move to next pair (look for comma or next opening bracket) + pos = value_end + 1; + size_t comma = current.find(',', pos); + if (comma != std::string::npos) { + pos = comma + 1; + } + } +} + +MHD_Result handle_get_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // S3EncryptionGetObjectOutcome outcome ; + // if (metadata.empty()) { + // outcome = it->second->GetObject(request); + // } else { + // Aws::Map kmsContextMap; + // fill_context(kmsContextMap, metadata); + // outcome = it->second->GetObject(request, kmsContextMap); + // } + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + auto outcome = it->second->GetObject(request, kmsContextMap); + + if (outcome.IsSuccess()) { + auto &stream = outcome.GetResult().GetBody(); + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + auto msg = make_error("An exception was thrown", 500); + return send_response(connection, 500, msg); + } +} + +MHD_Result handle_put_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &body, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + auto stream = std::make_shared(body); + request.SetBody(stream); + + auto outcome = it->second->PutObject(request, kmsContextMap); + if (outcome.IsSuccess()) { + json response = {{"bucket", bucket}, {"key", key}}; + return send_response(connection, 200, response.dump()); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } +} + +MHD_Result request_handler(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) { + std::string method_str(method); + bool is_push = method_str == "POST" || method_str == "PUT"; + static int dummy; + if (*con_cls == nullptr) { + if (is_push) { + *con_cls = new std::string(); + } else { + *con_cls = &dummy; + } + return MHD_YES; + } + if (is_push && *upload_data_size) { + std::string *body = static_cast(*con_cls); + body->append(upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + + std::string url_str(url); + + if (is_push && url_str == "/client") { + std::unique_ptr body(static_cast(*con_cls)); + return handle_create_client(connection, *body); + } + + if (url_str.find("/object/") == 0) { + std::string path = url_str.substr(8); // Remove "/object/" + size_t slash_pos = path.find('/'); + if (slash_pos != std::string::npos) { + std::string bucket = path.substr(0, slash_pos); + std::string key = path.substr(slash_pos + 1); + std::string client_id = get_header_value(connection, "clientid"); + + std::string metadata = get_header_value(connection, "content-metadata"); + if (method_str == "GET") { + return handle_get_object(connection, bucket, key, client_id, metadata); + } else if (method_str == "PUT") { + std::unique_ptr body(static_cast(*con_cls)); + *upload_data_size = 0; + return handle_put_object(connection, bucket, key, client_id, *body, metadata); + } + } + } + + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); +} + +int main() { + Aws::SDKOptions options; + Aws::InitAPI(options); + int port = 8097; + + struct MHD_Daemon *daemon = + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, port, NULL, NULL, + &request_handler, NULL, MHD_OPTION_END); + + if (!daemon) { + fprintf(stderr, "Failed to start server on port %d\n", port); + Aws::ShutdownAPI(options); + return 1; + } + + fprintf(stderr, "Server running on port %d\n", port); + sleep(10000); + + MHD_stop_daemon(daemon); + Aws::ShutdownAPI(options); + fprintf(stderr, "Ending server\n"); + return 0; +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 1c8f839e..6a16ba7b 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -374,7 +374,8 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { .build()); fail("Expected Exception"); } catch (S3EncryptionClientError e) { - if (language.equals(NET_V3) || language.equals(NET_V2_CURRENT) || language.equals(CPP_V2_CURRENT)) { + if (language.equals(NET_V3) || language.equals(NET_V2_CURRENT) + || language.equals(CPP_V2_CURRENT) || language.equals(CPP_V2_TRANSITION) || language.equals(CPP_V3)) { assertTrue(e.getMessage().contains( "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" )); diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java index 395d11c7..80939d0f 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/TestUtils.java @@ -127,7 +127,7 @@ public class TestUtils { // servers.put(JAVA_V3_TRANSITION, new LanguageServerTarget(JAVA_V3_TRANSITION, "8094")); // servers.put(GO_V3_TRANSITION, new LanguageServerTarget(GO_V3_TRANSITION, "8095")); // servers.put(NET_V2_TRANSITION, new LanguageServerTarget(NET_V2_TRANSITION, "8096")); - // servers.put(CPP_V2_TRANSITION, new LanguageServerTarget(CPP_V2_TRANSITION, "8097")); + servers.put(CPP_V2_TRANSITION, new LanguageServerTarget(CPP_V2_TRANSITION, "8097")); // servers.put(RUBY_V2_TRANSITION, new LanguageServerTarget(RUBY_V2_TRANSITION, "8098")); // servers.put(PHP_V2_TRANSITION, new LanguageServerTarget(PHP_V2_TRANSITION, "8099"));