Kessplas/speedup ci

test-server/Makefile

@@ -1,6 +1,6 @@
 # Makefile for S3 Encryption Client Testing
 
-.PHONY: all start-servers run-tests stop-servers clean ci check-env help
+.PHONY: all start-servers run-tests stop-servers clean ci check-env help install-shared-deps
 
 # Default target
 all: start-all-servers wait-all-servers run-tests
@@ -13,8 +13,18 @@ SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./
 START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS))
 WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS))
 
-# Start all servers in parallel
-start-servers:
+# Install shared dependencies to avoid conflicts during parallel startup
+install-shared-deps:
+	@echo "Installing shared dependencies..."
+	@# Install brew packages needed by C++ servers to avoid parallel conflicts
+	@if command -v brew >/dev/null 2>&1; then \
+		echo "Installing C++ dependencies via brew..."; \
+		brew install libmicrohttpd nlohmann-json ossp-uuid || true; \
+	fi
+	@echo "Shared dependencies installed"
+
+# Start all servers sequentially (original approach)
+start-servers: install-shared-deps
 	@echo "Starting all servers..."
 	$(MAKE) start-all-servers
 	@echo "Waiting for servers to start..."
@@ -23,14 +33,15 @@ start-servers:
 		$(MAKE) -C $$dir wait-for-server; \
 	done
 
+# Keep the original sequential method as backup
 start-all-servers: $(START_SERVER_TARGETS)
 
 $(START_SERVER_TARGETS): start-%:
 	@if [ -f $*/Makefile ]; then \
 		echo "Starting server in $*..."; \
 		$(MAKE) -C $* start-server; \
 	else \
-		echo "❌ Error: no Makefile found in $*"; \
+		echo "Error: no Makefile found in $*"; \
 		exit 1; \
 	fi; \
 
@@ -41,11 +52,10 @@ $(WAIT_SERVER_TARGETS): wait-%:
 		echo "Waiting server in $*..."; \
 		$(MAKE) -C $* wait-for-server; \
 	else \
-		echo "❌ Error: no Makefile found in $*"; \
+		echo "Error: no Makefile found in $*"; \
 		exit 1; \
 	fi; \
 
-
 # Run the Java tests
 run-tests:
 	@echo "Running Java tests..."
@@ -63,7 +73,7 @@ run-tests:
 stop-servers:
 	@echo "Stopping servers..."
 	@for dir in $(SERVER_DIRS); do \
-		echo "Starting server in $$dir..."; \
+		echo "Stopping server in $$dir..."; \
 		$(MAKE) -C $$dir stop-server; \
 	done
 	@echo "Servers stopped"
@@ -91,7 +101,7 @@ TIMEOUT := 360
 
 wait-for-port:
 	@if [ -z "$(PORT)" ]; then \
-		echo "❌ Error: PORT is required"; \
+		echo "Error: PORT is required"; \
 		exit 1; \
 	fi
 	@for i in $$(seq 1 $(TIMEOUT)); do \

test-server/cpp-v2-server/Makefile

@@ -6,7 +6,6 @@ PID_FILE := server.pid
 PORT := 8085
 
 build/s3ec-server: 
-	brew install libmicrohttpd nlohmann-json ossp-uuid
 	git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git
 	cd aws-sdk-cpp
 	mkdir -p build && cd build && cmake ..

test-server/cpp-v2-transition-server/Makefile

@@ -6,7 +6,6 @@ PID_FILE := server.pid
 PORT := 8097
 
 build/s3ec-server: 
-	brew install libmicrohttpd nlohmann-json ossp-uuid
 	mkdir -p build && cd build && cmake ..
 
 start-server: | build/s3ec-server

test-server/cpp-v3-server/Makefile

@@ -6,7 +6,6 @@ PID_FILE := server.pid
 PORT := 8091
 
 build/s3ec-server: 
-	brew install libmicrohttpd nlohmann-json ossp-uuid
 	mkdir -p build && cd build && cmake ..
 
 start-server: | build/s3ec-server

test-server/cpp-v3-server/main.cpp

@@ -51,7 +51,7 @@ std::string make_error(const std::string &message, int status_code) {
 }
 
 MHD_Result unsupported(struct MHD_Connection *connection, std::string & commitmentPolicy, std::string & encryptionAlgorithm) {
-    fprintf(stderr, "Unsupported %s %s\n",commitmentPolicy.c_str(), encryptionAlgorithm.c_str() );
+    // fprintf(stderr, "Unsupported %s %s\n",commitmentPolicy.c_str(), encryptionAlgorithm.c_str() );
     send_response(connection, 404, "{\"error\":\"Unsupported Option.\"}");
     return MHD_YES;
 }
@@ -109,7 +109,7 @@ MHD_Result handle_create_client(struct MHD_Connection *connection,
     json response = {{"clientId", client_id}};
     return send_response(connection, 200, response.dump());
   } catch (const std::exception &e) {
-    fprintf(stderr, "handle_create_client exception %s\n", e.what());
+    // fprintf(stderr, "handle_create_client exception %s\n", e.what());
     return send_response(connection, 500,
                         "{\"error\":\"An exception was thrown.\"}");
   } catch (...) {
@@ -205,11 +205,11 @@ MHD_Result handle_get_object(struct MHD_Connection *connection,
       return send_response(connection, 200, content);
     } else {
       auto msg = make_error(outcome.GetError().GetMessage(), 500);
-    fprintf(stderr, "handle_get_object error %s\n", msg.c_str());
+    // fprintf(stderr, "handle_get_object error %s\n", msg.c_str());
       return send_response(connection, 500, msg);
     }
   } catch (const std::exception &e) {
-    fprintf(stderr, "handle_get_object exception %s\n", e.what());
+    // fprintf(stderr, "handle_get_object exception %s\n", e.what());
     auto msg = make_error("An exception was thrown", 500);
     return send_response(connection, 500, msg);
   }
@@ -242,11 +242,11 @@ MHD_Result handle_put_object(struct MHD_Connection *connection,
       return send_response(connection, 200, response.dump());
     } else {
       auto msg = make_error(outcome.GetError().GetMessage(), 500);
-    fprintf(stderr, "handle_put_object error %s\n", msg.c_str());
+    // fprintf(stderr, "handle_put_object error %s\n", msg.c_str());
       return send_response(connection, 500, msg);
     }
   } catch (const std::exception &e) {
-    fprintf(stderr, "handle_put_object exception %s\n", e.what());
+    // fprintf(stderr, "handle_put_object exception %s\n", e.what());
     auto msg = make_error(e.what(), 500);
     return send_response(connection, 500, msg);
   }

test-server/go-v3-server/main.go

@@ -80,7 +80,7 @@ func NewServer() (*Server, error) {
 // createGenericServerError creates a generic server error response
 func (s *Server) createGenericServerError(w http.ResponseWriter, message string, statusCode int) {
 	// Echo error to console
-	log.Printf("[Go V3] GenericServerError: %s (Status: %d)", message, statusCode)
+	// log.Printf("[Go V3] GenericServerError: %s (Status: %d)", message, statusCode)
 
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(statusCode)
@@ -93,7 +93,7 @@ func (s *Server) createGenericServerError(w http.ResponseWriter, message string,
 // createS3EncryptionClientError creates an S3 encryption client error response
 func (s *Server) createS3EncryptionClientError(w http.ResponseWriter, message string, statusCode int) {
 	// Echo error to console
-	log.Printf("[Go V3] S3EncryptionClientError: %s (Status: %d)", message, statusCode)
+	// log.Printf("[Go V3] S3EncryptionClientError: %s (Status: %d)", message, statusCode)
 
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(statusCode)

test-server/java-tests/build.gradle.kts

@@ -16,7 +16,13 @@ dependencies {
     // Test dependencies
     testImplementation("org.junit.jupiter:junit-jupiter:5.13.0")
     testRuntimeOnly("org.junit.platform:junit-platform-launcher")
-    testImplementation("com.amazonaws:aws-java-sdk:1.12.788")
+
+    // AWS SDK v1 - Only include S3 and KMS modules instead of entire SDK
+    testImplementation("com.amazonaws:aws-java-sdk-core:1.12.788")
+    testImplementation("com.amazonaws:aws-java-sdk-s3:1.12.788")
+    testImplementation("com.amazonaws:aws-java-sdk-kms:1.12.788")
+
+    // AWS SDK v2 - S3 module
     testImplementation("software.amazon.awssdk:s3:2.37.1")
     testImplementation("org.bouncycastle:bcprov-jdk15on:1.70")
 }

test-server/java-tests/src/it/java/software/amazon/encryption/s3/CBCDecryptTests.java

@@ -78,6 +78,8 @@ static void encryptCBCObject() {
         .build();
 
         v1Client.putObject(TestUtils.BUCKET, sharedObjectKey, sharedObjectKey);
+
+        validateServersRunning();
     }
 
     @ParameterizedTest(name = "{0}: Transition configured with the default should decrypt CBC")

test-server/java-tests/src/it/java/software/amazon/encryption/s3/GCMTests.java

@@ -64,7 +64,12 @@ class GCMTests {
     .kmsKeyId(TestUtils.KMS_KEY_ARN)
     .build();
     private static List<String> crossLanguageObjects = new ArrayList<>();
-
+
+    @BeforeAll
+    public static void setup() {
+        validateServersRunning();
+    }
+
     @Order(1)
     @ParameterizedTest(name = "{0}: Improved configured with ForbidEncryptAllowDecrypt should encrypt GCM")
     @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest")

test-server/java-tests/src/it/java/software/amazon/encryption/s3/KC_GCMTests.java

@@ -64,7 +64,12 @@ class KC_GCMTests {
     .kmsKeyId(TestUtils.KMS_KEY_ARN)
     .build();
     private static List<String> crossLanguageObjects = new ArrayList<>();
-
+
+    @BeforeAll
+    public static void setup() {
+        validateServersRunning();
+    }
+
     @Order(1)
     @ParameterizedTest(name = "{0}: Improved configured with RequireEncryptAllowDecrypt should encrypt KC-GCM")
     @MethodSource("software.amazon.encryption.s3.TestUtils#improvedClientsForTest")

test-server/net-v2-v3-server/Controllers/ClientController.cs

@@ -35,14 +35,14 @@ public IActionResult CreateClient([FromBody] ClientRequest request)
             // So, we are passing empty dictionary.
             var encryptionContext = new Dictionary<string, string>();
             var encryptionMaterial = new EncryptionMaterialsV2(kmsKeyId, KmsType.KmsContext, encryptionContext);
-            logger.LogInformation(
-                "Created EncryptionMaterialsV2: KMS={KmsKeyId}",
-                kmsKeyId);
+            // logger.LogInformation(
+            //     "Created EncryptionMaterialsV2: KMS={KmsKeyId}",
+            //     kmsKeyId);
             // SecurityProfile V2AndLegacy can decrypt from legacy S3EC but V2 cannot
             var enableLegacyMode = enableLegacyUnauthenticatedModes || enableLegacyWrappingAlgorithms;
             var securityProfile = enableLegacyMode ? SecurityProfile.V2AndLegacy : SecurityProfile.V2;
 
-            logger.LogInformation("Created securityProfile= {securityProfile}", securityProfile.ToString());
+            // logger.LogInformation("Created securityProfile= {securityProfile}", securityProfile.ToString());
 
             var configuration = new AmazonS3CryptoConfigurationV2(securityProfile);
             // Create S3 encryption client
@@ -69,4 +69,4 @@ public IActionResult CreateClient([FromBody] ClientRequest request)
             });
         }
     }
-}
+}

test-server/net-v2-v3-server/Controllers/ObjectController.cs

@@ -13,7 +13,7 @@ public class ObjectController(IClientCacheService clientCacheService, ILogger<Ob
     [HttpPut("{bucket}/{key}")]
     public async Task<IActionResult> PutObject(string bucket, string key)
     {
-        logger.LogInformation("Starting PutObject");
+        // logger.LogInformation("Starting PutObject");
         var clientId = Request.Headers["clientId"].FirstOrDefault();
         if (string.IsNullOrEmpty(clientId))
             return BadRequest(new GenericServerError { Message = "ClientID header is required" });
@@ -62,7 +62,7 @@ public async Task<IActionResult> PutObject(string bucket, string key)
     [HttpGet("{bucket}/{key}")]
     public async Task<IActionResult> GetObject(string bucket, string key)
     {
-        logger.LogInformation("Starting GetObject");
+        // logger.LogInformation("Starting GetObject");
         var clientId = Request.Headers["clientId"].FirstOrDefault();
         if (string.IsNullOrEmpty(clientId))
             return BadRequest(new GenericServerError { Message = "ClientID header is required" });
@@ -102,4 +102,4 @@ public async Task<IActionResult> GetObject(string bucket, string key)
             return StatusCode(500, new S3EncryptionClientError { Message = ex.Message });
         }
     }
-}
+}

test-server/net-v3-transition-server/Controllers/ClientController.cs

@@ -36,19 +36,19 @@ public IActionResult CreateClient([FromBody] ClientRequest request)
             // So, we are passing empty dictionary.
             var encryptionContext = new Dictionary<string, string>();
             var encryptionMaterial = new EncryptionMaterialsV2(kmsKeyId, KmsType.KmsContext, encryptionContext);
-            logger.LogInformation(
-                "[NET-V3-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}",
-                kmsKeyId);
+            // logger.LogInformation(
+            //     "[NET-V3-Transitional] Created EncryptionMaterialsV2: KMS={KmsKeyId}",
+            //     kmsKeyId);
             // SecurityProfile V2AndLegacy can decrypt from legacy S3EC but V2 cannot
             var enableLegacyMode = enableLegacyUnauthenticatedModes || enableLegacyWrappingAlgorithms;
             var securityProfile = enableLegacyMode ? SecurityProfile.V2AndLegacy : SecurityProfile.V2;
 
-            logger.LogInformation("[NET-V3-Transitional] Created securityProfile= {securityProfile}", securityProfile.ToString());
+            // logger.LogInformation("[NET-V3-Transitional] Created securityProfile= {securityProfile}", securityProfile.ToString());
 
             var encryptionAlgorithm = MapEncryptionAlgorithm(request.Config.EncryptionAlgorithm);
             // var encryptionAlgorithm = commitmentPolicy == Amazon.Extensions.S3.Encryption.CommitmentPolicy.ForbidEncryptAllowDecrypt ? ContentEncryptionAlgorithm.AesGcm : ContentEncryptionAlgorithm.AesGcmWithCommitment;
-            logger.LogInformation("[NET-V3-Transitional] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy);
-            logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm);
+            // logger.LogInformation("[NET-V3-Transitional] Created commitmentPolicy= {commitmentPolicy}", commitmentPolicy);
+            // logger.LogInformation("[NET-V3-Transitional] Created encryptionAlgorithm= {encryptionAlgorithm}", encryptionAlgorithm);
 
             var configuration = new AmazonS3CryptoConfigurationV2(securityProfile, commitmentPolicy, encryptionAlgorithm);
             // Create S3 encryption client
@@ -97,4 +97,4 @@ private static ContentEncryptionAlgorithm MapEncryptionAlgorithm(Models.Encrypti
             _ => ContentEncryptionAlgorithm.AesGcm
         };
     }
-}
+}

test-server/net-v3-transition-server/Controllers/ObjectController.cs

@@ -13,7 +13,7 @@ public class ObjectController(IClientCacheService clientCacheService, ILogger<Ob
     [HttpPut("{bucket}/{key}")]
     public async Task<IActionResult> PutObject(string bucket, string key)
     {
-        logger.LogInformation("Starting PutObject");
+        // logger.LogInformation("Starting PutObject");
         var clientId = Request.Headers["clientId"].FirstOrDefault();
         if (string.IsNullOrEmpty(clientId))
             return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" });
@@ -62,7 +62,7 @@ public async Task<IActionResult> PutObject(string bucket, string key)
     [HttpGet("{bucket}/{key}")]
     public async Task<IActionResult> GetObject(string bucket, string key)
     {
-        logger.LogInformation("Starting GetObject");
+        // logger.LogInformation("Starting GetObject");
         var clientId = Request.Headers["clientId"].FirstOrDefault();
         if (string.IsNullOrEmpty(clientId))
             return BadRequest(new GenericServerError { Message = "[NET-V3-Transitional] ClientID header is required" });
@@ -102,4 +102,4 @@ public async Task<IActionResult> GetObject(string bucket, string key)
             return StatusCode(500, new S3EncryptionClientError { Message = ex.Message });
         }
     }
-}
+}

test-server/php-v3-server/src/index.php

@@ -16,14 +16,14 @@ function getSessionIdFromCookiesFile()
     $cookiesFile = __DIR__ . '/../cookies.txt';
 
     if (!file_exists($cookiesFile)) {
-        error_log("cookies.txt file does not exist, will be created when first client is created");
+        // error_log("cookies.txt file does not exist, will be created when first client is created");
         return null;
     }
 
     $lines = file($cookiesFile, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
 
     if ($lines === false) {
-        error_log("Failed to read cookies.txt file");
+        // error_log("Failed to read cookies.txt file");
         return null;
     }
 
@@ -36,12 +36,12 @@ function getSessionIdFromCookiesFile()
         // Parse cookie line: domain, flag, path, secure, expiration, name, value
         $parts = explode("\t", $line);
         if (count($parts) >= 7 && $parts[5] === 'PHPSESSID') {
-            error_log("Found session ID in cookies.txt: " . $parts[6]);
+            // error_log("Found session ID in cookies.txt: " . $parts[6]);
             return $parts[6]; // Return the session ID value
         }
     }
 
-    error_log("No PHPSESSID found in cookies.txt file");
+    // error_log("No PHPSESSID found in cookies.txt file");
     return null;
 }
 
@@ -62,11 +62,11 @@ function writeSessionIdToCookiesFile($sessionId)
     $result = file_put_contents($cookiesFile, $content);
 
     if ($result === false) {
-        error_log("Failed to write session ID to cookies.txt file: $cookiesFile");
+        // error_log("Failed to write session ID to cookies.txt file: $cookiesFile");
         return false;
     }
 
-    error_log("Successfully wrote session ID to cookies.txt: $sessionId");
+    // error_log("Successfully wrote session ID to cookies.txt: $sessionId");
     return true;
 }