faster parallel tests

.github/workflows/test.yml

@@ -41,15 +41,15 @@ jobs:
           git config --global credential.helper store
           echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials
 
-      - name: Cache git submodules
-        uses: actions/cache@v4
-        with:
-          path: |
-            .git/modules
-            test-server/*/.git
-          key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }}
-          restore-keys: |
-            ${{ runner.os }}-submodules-
+      # - name: Cache git submodules
+      #   uses: actions/cache@v4
+      #   with:
+      #     path: |
+      #       .git/modules
+      #       test-server/*/.git
+      #     key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-submodules-
 
       - name: Optimize git for performance
         run: |
@@ -59,13 +59,12 @@ jobs:
 
       - name: Checkout submodules with --jobs
         run: |
-          git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }}
+          git submodule update --init --depth 1 --single-branch --jobs ${{ steps.cpu-count.outputs.count }}
 
       - name: Update cpp submodules recursively with --jobs
         run: |
           git submodule update --init --recursive \
-            --depth 1 \
-            --filter=blob:none \
+            --depth 1 --single-branch \
             --jobs ${{ steps.cpu-count.outputs.count }} \
             --force \
             test-server/cpp-v2-transition-server/aws-sdk-cpp \
@@ -156,25 +155,25 @@ jobs:
           aws-region: us-west-2
 
       - name: Build the servers
-        run: cd test-server && make build-all-servers
+        run: cd test-server && make build-all-servers FILTER=ruby,java,php,net,go
         env:
           MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }}
           AWS_REGION: us-west-2
 
       - name: Start the servers
-        run: cd test-server && make start-all-servers
+        run: cd test-server && make start-all-servers FILTER=ruby,java,php,net,go
         env:
           AWS_REGION: us-west-2
           TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }}
           TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }}
 
       - name: Wait for servers to start
-        run: cd test-server && make wait-all-servers
+        run: cd test-server && make wait-all-servers FILTER=ruby,java,php,net,go
         env:
           MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }}
 
       - name: Run run-tests
-        run: cd test-server && make run-tests
+        run: cd test-server && make run-tests FILTER=ruby,java,php,net,go
         env:
           AWS_REGION: us-west-2
           TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }}

test-server/Makefile

@@ -2,9 +2,6 @@
 
 .PHONY: all start-servers run-tests stop-servers clean ci check-env help
 
-# Default target
-all: start-all-servers wait-all-servers run-tests
-
 # CI target for GitHub Actions
 ci:
 	$(MAKE) build-all-servers
@@ -20,13 +17,19 @@ START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS))
 WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS))
 
 # Build all servers in parallel
-build-all-servers: export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1)
-build-all-servers: $(BUILD_SERVER_TARGETS)
+build-all-servers:
+	@echo "[`date +%H:%M:%S`] Building all servers..."
+	@$(MAKE) $(BUILD_SERVER_TARGETS)
+	@echo "[`date +%H:%M:%S`] All servers built."
+	@echo "Stopping dotnet servers... this is here to speed up CI because if this target is run with -j it will stay open until it shutdown."
+	@dotnet build-server shutdown
+	@echo "[`date +%H:%M:%S`] Dotnet build servers stopped"
 
 $(BUILD_SERVER_TARGETS): build-%:
 	@if [ -f $*/Makefile ]; then \
-		echo "Building server in $*..."; \
+		echo "[`date +%H:%M:%S`] Building server in $*..."; \
 		$(MAKE) -C $* build-server; \
+		echo "[`date +%H:%M:%S`] Server $* built successfully"; \
 	else \
 		echo "❌ Error: no Makefile found in $*"; \
 		exit 1; \
@@ -44,9 +47,8 @@ start-servers:
 		$(MAKE) -C $$dir wait-for-server; \
 	done
 
-# Start servers sequentially (no parallel execution)
 start-all-servers:
-	@$(MAKE) MAKEFLAGS= $(START_SERVER_TARGETS)
+	@$(MAKE) $(START_SERVER_TARGETS)
 
 $(START_SERVER_TARGETS): start-%:
 	@if [ -f $*/Makefile ]; then \

test-server/cpp-v2-server/Makefile

@@ -11,7 +11,7 @@ build/s3ec-server:
 
 build-server: | build/s3ec-server
 	@echo "Building Cpp V2 server..."
-	cd build && make
+	cd build && $(MAKE)
 
 start-server:
 	@echo "Starting Cpp V2 server..."

test-server/cpp-v2-server/main.cpp

@@ -16,8 +16,8 @@
 using json = nlohmann::json;
 using namespace Aws::S3Encryption;
 using Aws::S3Encryption::Materials::KMSWithContextEncryptionMaterials;
-std::unordered_map<std::string, std::shared_ptr<S3EncryptionClientV2>>
-    client_cache;
+std::unordered_map<std::string, std::shared_ptr<S3EncryptionClientV2>> client_cache_secret;
+std::mutex client_mutex;
 
 std::string generate_uuid() {
   uuid_t uuid;
@@ -27,6 +27,23 @@ std::string generate_uuid() {
   return std::string(uuid_str);
 }
 
+std::shared_ptr<S3EncryptionClientV2> get_client(const std::string &client_id)
+{
+    std::lock_guard<std::mutex> lock(client_mutex);
+    auto it = client_cache_secret.find(client_id);
+    if (it == client_cache_secret.end()) {
+      return std::shared_ptr<S3EncryptionClientV2>();
+    } else {
+      return it->second;
+    }
+}
+
+void set_client(const std::string &client_id, std::shared_ptr<S3EncryptionClientV2> client)
+{
+  std::lock_guard<std::mutex> lock(client_mutex);
+  client_cache_secret[client_id] = client;
+}
+
 std::string get_header_value(struct MHD_Connection *connection,
                              const char *key) {
   const char *value =
@@ -74,7 +91,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection,
     auto encryption_client = std::make_shared<S3EncryptionClientV2>(config);
 
     std::string client_id = generate_uuid();
-    client_cache[client_id] = encryption_client;
+    set_client(client_id, encryption_client);
 
     json response = {{"clientId", client_id}};
     return send_response(connection, 200, response.dump());
@@ -144,8 +161,8 @@ MHD_Result handle_get_object(struct MHD_Connection *connection,
                              const std::string &bucket, const std::string &key,
                              const std::string &client_id,
                              const std::string &metadata) {
-  auto it = client_cache.find(client_id);
-  if (it == client_cache.end()) {
+  auto client = get_client(client_id);
+  if (!client) {
     return send_response(connection, 404, "{\"error\":\"Client not found\"}");
   }
 
@@ -154,18 +171,9 @@ MHD_Result handle_get_object(struct MHD_Connection *connection,
     request.SetBucket(bucket);
     request.SetKey(key);
 
-    // S3EncryptionGetObjectOutcome outcome ;
-    // if (metadata.empty()) {
-    //   outcome = it->second->GetObject(request);
-    // } else {
-    //   Aws::Map<Aws::String, Aws::String> kmsContextMap;
-    //   fill_context(kmsContextMap, metadata);
-    //   outcome = it->second->GetObject(request, kmsContextMap);
-    // }
-
     Aws::Map<Aws::String, Aws::String> kmsContextMap;
     fill_context(kmsContextMap, metadata);
-    auto outcome = it->second->GetObject(request, kmsContextMap);
+    auto outcome = client->GetObject(request, kmsContextMap);
 
     if (outcome.IsSuccess()) {
       auto &stream = outcome.GetResult().GetBody();
@@ -187,8 +195,8 @@ MHD_Result handle_put_object(struct MHD_Connection *connection,
                              const std::string &client_id,
                              const std::string &body,
                              const std::string &metadata) {
-  auto it = client_cache.find(client_id);
-  if (it == client_cache.end()) {
+  auto client = get_client(client_id);
+  if (!client) {
     return send_response(connection, 404, "{\"error\":\"Client not found\"}");
   }
 
@@ -203,7 +211,7 @@ MHD_Result handle_put_object(struct MHD_Connection *connection,
     auto stream = std::make_shared<std::stringstream>(body);
     request.SetBody(stream);
 
-    auto outcome = it->second->PutObject(request, kmsContextMap);
+    auto outcome = client->PutObject(request, kmsContextMap);
     if (outcome.IsSuccess()) {
       json response = {{"bucket", bucket}, {"key", key}};
       return send_response(connection, 200, response.dump());

test-server/cpp-v2-transition-server/Makefile

@@ -10,7 +10,7 @@ build/s3ec-server:
 
 build-server: | build/s3ec-server
 	@echo "Building Cpp transition server..."
-	cd build && make
+	cd build && $(MAKE)
 
 start-server:
 	@echo "Starting Cpp transition server..."