diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 16057711..637003f3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,7 +8,7 @@ on: jobs: lint: - runs-on: ubuntu-latest + runs-on: macos-13 steps: - name: Checkout code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 180ed6de..49b125ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: macos-13 permissions: id-token: write contents: read @@ -50,6 +50,11 @@ jobs: shell: bash run: composer install + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.25 + # Cache uv dependencies - name: Cache uv dependencies uses: actions/cache@v3 diff --git a/test-server/cpp-v2-server/CMakeLists.txt b/test-server/cpp-v2-server/CMakeLists.txt new file mode 100644 index 00000000..b282dbc4 --- /dev/null +++ b/test-server/cpp-v2-server/CMakeLists.txt @@ -0,0 +1,39 @@ +cmake_minimum_required(VERSION 3.16) +project(s3ec-cpp-v2-server) + +set(CMAKE_CXX_STANDARD 17) + +# Configure AWS SDK build options +set(BUILD_ONLY "kms;s3;s3-encryption" CACHE STRING "Build only KMS, S3, and S3-encryption components") +set(ENABLE_TESTING OFF CACHE BOOL "Disable testing") +set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build static libraries") + +# Add AWS SDK as subdirectory +add_subdirectory(aws-sdk-cpp) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(LIBMICROHTTPD REQUIRED libmicrohttpd) + +find_package(nlohmann_json REQUIRED) + +add_executable(s3ec-server main.cpp) + +target_include_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_INCLUDE_DIRS} + /opt/homebrew/include +) + +target_link_directories(s3ec-server PRIVATE + ${LIBMICROHTTPD_LIBRARY_DIRS} + /opt/homebrew/lib +) + +target_link_libraries(s3ec-server + ${LIBMICROHTTPD_LIBRARIES} + aws-cpp-sdk-core + aws-cpp-sdk-kms + aws-cpp-sdk-s3 + aws-cpp-sdk-s3-encryption + nlohmann_json::nlohmann_json + uuid +) \ No newline at end of file diff --git a/test-server/cpp-v2-server/Makefile b/test-server/cpp-v2-server/Makefile new file mode 100644 index 00000000..e9156d64 --- /dev/null +++ b/test-server/cpp-v2-server/Makefile @@ -0,0 +1,31 @@ +# Makefile for S3 Encryption Client Testing + +.PHONY: start-server stop-server wait-for-server + +PID_FILE := server.pid +PORT := 8085 + +build/s3ec-server: + brew install libmicrohttpd nlohmann-json ossp-uuid + git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git + cd aws-sdk-cpp && git checkout --track remotes/origin/ajewell/ec-for-get-object + mkdir -p build && cd build && cmake .. + +start-server: | build/s3ec-server + @echo "Starting Cpp V2 server..." + cd build && make && \ + AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \ + AWS_REGION="us-west-2" \ + ./s3ec-server & echo $$! > $(PID_FILE) + @echo "Cpp V2 server starting..." + +stop-server: + @if [ -f $(PID_FILE) ]; then \ + kill $$(cat $(PID_FILE)) 2>/dev/null || true; \ + rm $(PID_FILE); \ + fi + +wait-for-server: + $(MAKE) -C .. wait-for-port PORT=$(PORT) diff --git a/test-server/cpp-v2-server/README.md b/test-server/cpp-v2-server/README.md new file mode 100644 index 00000000..8e77feda --- /dev/null +++ b/test-server/cpp-v2-server/README.md @@ -0,0 +1,37 @@ +# C++ S3 Encryption Test Server + +Minimal C++ implementation of the S3 Encryption test server. + +## Dependencies + +- libmicrohttpd +- AWS SDK for C++ +- nlohmann/json +- uuid + +On MacOS you can +```bash +brew install libmicrohttpd nlohmann-json ossp-uuid +``` + +## Build + +```bash +mkdir build && cd build +cmake .. +make +``` + +## Run + +```bash +./s3ec-server +``` + +Server runs on localhost:8085 + +## API Endpoints + +- `POST /client` - Create S3 encryption client +- `GET /object/{bucket}/{key}` - Get encrypted object +- `PUT /object/{bucket}/{key}` - Put encrypted object \ No newline at end of file diff --git a/test-server/cpp-v2-server/main.cpp b/test-server/cpp-v2-server/main.cpp new file mode 100644 index 00000000..869e617b --- /dev/null +++ b/test-server/cpp-v2-server/main.cpp @@ -0,0 +1,288 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using json = nlohmann::json; +using namespace Aws::S3Encryption; +using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials; +std::unordered_map> + client_cache; + +std::string generate_uuid() { + uuid_t uuid; + uuid_generate(uuid); + char uuid_str[37]; + uuid_unparse(uuid, uuid_str); + return std::string(uuid_str); +} + +std::string get_header_value(struct MHD_Connection *connection, + const char *key) { + const char *value = + MHD_lookup_connection_value(connection, MHD_HEADER_KIND, key); + return value ? std::string(value) : ""; +} + +MHD_Result send_response(struct MHD_Connection *connection, int status_code, + const std::string &content) { + struct MHD_Response *response = MHD_create_response_from_buffer( + content.length(), (void *)content.data(), MHD_RESPMEM_MUST_COPY); + MHD_Result ret = MHD_queue_response(connection, status_code, response); + MHD_destroy_response(response); + return ret; +} + +std::string make_error(const std::string &message, int status_code) { + return "{\"__type\": " + "\"software.amazon.encryption.s3#S3EncryptionClientError\", " + "\"message\": \"" + + message + "\"}"; +} + +MHD_Result handle_create_client(struct MHD_Connection *connection, + const std::string &body) { + try { + json request = json::parse(body); + std::string kms_key_id = request["config"]["keyMaterial"]["kmsKeyId"]; + bool legacy = request["config"]["enableLegacyWrappingAlgorithms"]; + + auto materials = + std::make_shared(kms_key_id); + CryptoConfigurationV2 config(materials); + if (legacy) { + config.SetSecurityProfile(SecurityProfile::V2_AND_LEGACY); + } else { + config.SetSecurityProfile(SecurityProfile::V2); + } + + auto encryption_client = std::make_shared(config); + + std::string client_id = generate_uuid(); + client_cache[client_id] = encryption_client; + + json response = {{"clientId", client_id}}; + return send_response(connection, 200, response.dump()); + } catch (const std::exception &e) { + return send_response(connection, 500, + "{\"error\":\"An exception was thrown.\"}"); + } catch (...) { + return send_response(connection, 500, "{\"error\":\"Unknown error\"}"); + } +} + +void fill_context(Aws::Map &map, + const std::string &metadata) { + if (metadata.empty()) { + return; + } + + // Parse metadata format: [key1]:[value1],[key2]:[value2],... + // or single pair: [key]:[value] + std::string current = metadata; + size_t pos = 0; + + while (pos < current.length()) { + // Find opening bracket for key + size_t key_start = current.find('[', pos); + if (key_start == std::string::npos) + break; + + // Find closing bracket for key + size_t key_end = current.find(']', key_start); + if (key_end == std::string::npos) + break; + + // Find colon separator + size_t colon = current.find(':', key_end); + if (colon == std::string::npos) + break; + + // Find opening bracket for value + size_t value_start = current.find('[', colon); + if (value_start == std::string::npos) + break; + + // Find closing bracket for value + size_t value_end = current.find(']', value_start); + if (value_end == std::string::npos) + break; + + // Extract key and value + std::string key = current.substr(key_start + 1, key_end - key_start - 1); + std::string value = + current.substr(value_start + 1, value_end - value_start - 1); + + // Add to map + map.emplace(key, value); + + // Move to next pair (look for comma or next opening bracket) + pos = value_end + 1; + size_t comma = current.find(',', pos); + if (comma != std::string::npos) { + pos = comma + 1; + } + } +} + +MHD_Result handle_get_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + // S3EncryptionGetObjectOutcome outcome ; + // if (metadata.empty()) { + // outcome = it->second->GetObject(request); + // } else { + // Aws::Map kmsContextMap; + // fill_context(kmsContextMap, metadata); + // outcome = it->second->GetObject(request, kmsContextMap); + // } + + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + auto outcome = it->second->GetObject(request, kmsContextMap); + + if (outcome.IsSuccess()) { + auto &stream = outcome.GetResult().GetBody(); + std::string content((std::istreambuf_iterator(stream)), + std::istreambuf_iterator()); + return send_response(connection, 200, content); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + auto msg = make_error("An exception was thrown", 500); + return send_response(connection, 500, msg); + } +} + +MHD_Result handle_put_object(struct MHD_Connection *connection, + const std::string &bucket, const std::string &key, + const std::string &client_id, + const std::string &body, + const std::string &metadata) { + auto it = client_cache.find(client_id); + if (it == client_cache.end()) { + return send_response(connection, 404, "{\"error\":\"Client not found\"}"); + } + + try { + Aws::Map kmsContextMap; + fill_context(kmsContextMap, metadata); + + Aws::S3::Model::PutObjectRequest request; + request.SetBucket(bucket); + request.SetKey(key); + + auto stream = std::make_shared(body); + request.SetBody(stream); + + auto outcome = it->second->PutObject(request, kmsContextMap); + if (outcome.IsSuccess()) { + json response = {{"bucket", bucket}, {"key", key}}; + return send_response(connection, 200, response.dump()); + } else { + auto msg = make_error(outcome.GetError().GetMessage(), 500); + return send_response(connection, 500, msg); + } + } catch (const std::exception &e) { + auto msg = make_error(e.what(), 500); + return send_response(connection, 500, msg); + } +} + +MHD_Result request_handler(void *cls, struct MHD_Connection *connection, + const char *url, const char *method, + const char *version, const char *upload_data, + size_t *upload_data_size, void **con_cls) { + std::string method_str(method); + bool is_push = method_str == "POST" || method_str == "PUT"; + static int dummy; + if (*con_cls == nullptr) { + if (is_push) { + *con_cls = new std::string(); + } else { + *con_cls = &dummy; + } + return MHD_YES; + } + if (is_push && *upload_data_size) { + std::string *body = static_cast(*con_cls); + body->append(upload_data, *upload_data_size); + *upload_data_size = 0; + return MHD_YES; + } + + std::string url_str(url); + + if (is_push && url_str == "/client") { + std::unique_ptr body(static_cast(*con_cls)); + return handle_create_client(connection, *body); + } + + if (url_str.find("/object/") == 0) { + std::string path = url_str.substr(8); // Remove "/object/" + size_t slash_pos = path.find('/'); + if (slash_pos != std::string::npos) { + std::string bucket = path.substr(0, slash_pos); + std::string key = path.substr(slash_pos + 1); + std::string client_id = get_header_value(connection, "clientid"); + + std::string metadata = get_header_value(connection, "content-metadata"); + if (method_str == "GET") { + return handle_get_object(connection, bucket, key, client_id, metadata); + } else if (method_str == "PUT") { + std::unique_ptr body(static_cast(*con_cls)); + *upload_data_size = 0; + return handle_put_object(connection, bucket, key, client_id, *body, metadata); + } + } + } + + return send_response(connection, 404, + "{\"error\":\"Not idea what is happening\"}"); +} + +int main() { + Aws::SDKOptions options; + Aws::InitAPI(options); + + struct MHD_Daemon *daemon = + MHD_start_daemon(MHD_USE_SELECT_INTERNALLY, 8085, NULL, NULL, + &request_handler, NULL, MHD_OPTION_END); + + if (!daemon) { + fprintf(stderr, "Failed to start server on port 8085\n"); + Aws::ShutdownAPI(options); + return 1; + } + + fprintf(stderr, "Server running on port 8085\n"); + sleep(10000); + + MHD_stop_daemon(daemon); + Aws::ShutdownAPI(options); + fprintf(stderr, "Ending server\n"); + return 0; +} diff --git a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java index 459bbbd3..cd67f2e0 100644 --- a/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java +++ b/test-server/java-tests/src/it/java/software/amazon/encryption/s3/RoundTripTests.java @@ -60,6 +60,7 @@ public class RoundTripTests { private static final String JAVA_V3 = "Java-V3"; private static final String PYTHON_V3 = "Python-V3"; private static final String GO_V3 = "Go-V3"; + private static final String CPP_V2 = "CPP-V2"; private static final String NET_V2 = "NET-V2"; private static final String NET_V3 = "NET-V3"; private static final String PHP_V2 = "PHP-V2"; @@ -82,6 +83,7 @@ public class RoundTripTests { servers.put(GO_V3, new LanguageServerTarget(GO_V3, "8082")); servers.put(NET_V2, new LanguageServerTarget(NET_V2, "8083")); servers.put(NET_V3, new LanguageServerTarget(NET_V3, "8084")); + servers.put(CPP_V2, new LanguageServerTarget(CPP_V2, "8085")); servers.put(PHP_V2, new LanguageServerTarget(PHP_V2, "8087")); servers.put(PHP_V3, new LanguageServerTarget(PHP_V3, "8093")); servers.put(RUBY_V2, new LanguageServerTarget(RUBY_V2, "8086")); @@ -557,9 +559,9 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { .build()); fail("Expected Exception"); } catch (S3EncryptionClientError e) { - if (language.equals(NET_V3) || language.equals(NET_V2)) { + if (language.equals(NET_V3) || language.equals(NET_V2) || language.equals(CPP_V2)) { assertTrue(e.getMessage().contains( - "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration V2." + "The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration" )); } else if (language.equals(RUBY_V3) || language.equals(RUBY_V2)) { assertTrue(e.getMessage().contains("The requested object is encrypted with V1 encryption schemas that have been disabled by client configuration security_profile = :v2. Retry with :v2_and_legacy or re-encrypt the object.")); @@ -568,5 +570,4 @@ public void kmsV1LegacyFailsWhenLegacyDisabled(String language) { } } } - }