all one iv

.github/workflows/test.yml

@@ -21,70 +21,56 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v5
         with:
-          submodules: true
+          submodules: false
           # This is Ryan Emery's (seebees) PAT.
           # To grant this workflow access to a new private repo,
           # ask Ryan to edit this PAT's permissions to add access to a new private repo.
           token: ${{ secrets.PAT_FOR_PRIVATE_RUBY }}
 
-      - name: Checkout CPP code for cpp-v2-transition
-        uses: actions/checkout@v5
-        with:
-          submodules: recursive
-          token: ${{ secrets.PAT_FOR_CPP }}
-          repository: awslabs/aws-sdk-cpp-staging
-          ref: fire-egg-dev
-          path: test-server/cpp-v2-transition-server/aws-sdk-cpp/
-
-      - name: Checkout CPP code cpp-v3
-        uses: actions/checkout@v5
-        with:
-          submodules: recursive
-          token: ${{ secrets.PAT_FOR_CPP }}
-          repository: awslabs/aws-sdk-cpp-staging
-          ref: fire-egg-dev
-          path: test-server/cpp-v3-server/aws-sdk-cpp/
+      # There are a lot of submodules here
+      # This initializes the checkouts in parallel (--jobs)
+      # rather than in series the way actions/checkout@v5 does it.
 
-      - name: Checkout .NET V2 code
-        uses: actions/checkout@v5
-        with:
-          token: ${{ secrets.PAT_FOR_DOTNET }}
-          repository: aws/private-amazon-s3-encryption-client-dotnet-staging
-          # This is the branch for S3EC .NET V2
-          ref: v3sdk-development
-          path: test-server/net-v2-v3-server/s3ec-net-v2/
+      - name: Get CPU count
+        id: cpu-count
+        run: echo "count=$(node -p 'require("os").cpus().length')" >> $GITHUB_OUTPUT
 
-      - name: Checkout .NET V3 code
-        uses: actions/checkout@v5
-        with:
-          token: ${{ secrets.PAT_FOR_DOTNET }}
-          repository: aws/private-amazon-s3-encryption-client-dotnet-staging
-          # This is the branch for S3EC .NET V3
-          ref: s3ec-v3
-          path: test-server/net-v2-v3-server/s3ec-net-v3
+      - name: Setup git submodules with PAT
+        run: |
+          git config --global url."https://github.com/".insteadOf "git@github.com:"
+          git config --global credential.helper store
+          echo "https://x-token-auth:${{ secrets.PAT_FOR_PRIVATE_RUBY }}@github.com" > ~/.git-credentials
 
-      - name: Checkout .NET V3 code (Transition)
-        uses: actions/checkout@v5
-        with:
-          token: ${{ secrets.PAT_FOR_DOTNET }}
-          repository: aws/private-amazon-s3-encryption-client-dotnet-staging
-          # This is the branch for S3EC .NET V3 transition
-          ref: rishav/key-commitment
-          path: test-server/net-v3-transition-server/s3ec-v3-transition-branch
-
-      - name: Checkout .NET V4 (Improved) code
-        uses: actions/checkout@v5
-        with:
-          token: ${{ secrets.PAT_FOR_DOTNET }}
-          repository: aws/private-amazon-s3-encryption-client-dotnet-staging
-          # This is the branch for S3EC .NET V4 (improved)
-          ref: s3ec-v4-WIP
-          path: test-server/net-v4-server/s3ec-net-v4-improved
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: Cache git submodules
+        uses: actions/cache@v4
         with:
-          python-version: ${{ inputs.python-version || '3.11' }}
+          path: |
+            .git/modules
+            test-server/*/.git
+          key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules') }}
+          restore-keys: |
+            ${{ runner.os }}-submodules-
+
+      - name: Optimize git for performance
+        run: |
+          git config --global fetch.parallel ${{ steps.cpu-count.outputs.count }}
+          git config --global submodule.fetchJobs ${{ steps.cpu-count.outputs.count }}
+          git config --global remote.origin.tagOpt --no-tags
+
+      - name: Checkout submodules with --jobs
+        run: |
+          git submodule update --init --depth 1 --jobs ${{ steps.cpu-count.outputs.count }}
+
+      - name: Update cpp submodules recursively with --jobs
+        run: |
+          git submodule update --init --recursive \
+            --depth 1 \
+            --filter=blob:none \
+            --jobs ${{ steps.cpu-count.outputs.count }} \
+            --force \
+            test-server/cpp-v2-transition-server/aws-sdk-cpp \
+            test-server/cpp-v3-server/aws-sdk-cpp \
+            test-server/cpp-v2-server/aws-sdk-cpp
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
@@ -116,17 +102,39 @@ jobs:
         with:
           go-version: 1.25
 
-      # Cache uv dependencies
-      - name: Cache uv dependencies
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-
-
-      - name: Install Uv
-        run: pip install uv
+      - name: Install C++ dependencies
+        run: |
+          brew install libmicrohttpd nlohmann-json ossp-uuid
+
+      # Legacy Python tests:
+      # - name: Set up Python
+      #   uses: actions/setup-python@v5
+      #   with:
+      #     python-version: ${{ inputs.python-version || '3.11' }}
+      # 
+      # # Cache uv dependencies
+      # - name: Cache uv dependencies
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: ~/.cache/uv
+      #     key: ${{ runner.os }}-uv-${{ hashFiles('./test-server/python-v3-server/**/pyproject.toml') }}
+      #     restore-keys: |
+      #       ${{ runner.os }}-uv-
+
+      # - name: Install Uv
+      #   run: pip install uv
+
+      # - name: Install dependencies
+      #   run: make install
+
+      # - name: Run unit tests
+      #   run: make test-unit
+
+      # - name: Run integration tests
+      #   run: make test-integration
+      #   env:
+      #     CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }}
+      #     CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }}
 
       # Cache Gradle dependencies and build outputs
       - name: Cache Gradle packages
@@ -141,32 +149,53 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gradle-
 
-      - name: Install dependencies
-        run: make install
-
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: arn:aws:iam::370957321024:role/S3EC-Python-Github-test-role
           aws-region: us-west-2
 
-      - name: Run unit tests
-        run: make test-unit
+      - name: Build the servers
+        run: cd test-server && make build-all-servers FILTER=ruby,go,cpp
+        env:
+          MAKEFLAGS: -j${{ steps.cpu-count.outputs.count }}
+          AWS_REGION: us-west-2
+
+      - name: Start the servers
+        run: cd test-server && make start-all-servers FILTER=ruby,go,cpp
+        env:
+          AWS_REGION: us-west-2
+          TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }}
+          TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }}
 
-      - name: Run integration tests
-        run: make test-integration
+      - name: Wait for servers to start
+        run: cd test-server && make wait-all-servers FILTER=ruby,go,cpp
         env:
-          CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }}
-          CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }}
+          AWS_REGION: us-west-2
+          TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }}
+          TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }}
 
-      - name: Run test-server tests
-        run: cd test-server && make ci
+      - name: Run run-tests
+        run: cd test-server && make run-tests FILTER=ruby,go,cpp
         env:
           AWS_REGION: us-west-2
           TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }}
           TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }}
           GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true"
 
+      - name: Upload server logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-logs
+          path: |
+            test-server/*/server.log
+            test-server/*/net-v2-server.log
+            test-server/*/net-v3-server.log
+
+      - name: Stop the servers
+        run: cd test-server && make stop-servers FILTER=ruby,go,cpp
+
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@v4

.gitmodules

@@ -24,10 +24,6 @@
 	url = git@github.com:aws/private-amazon-s3-encryption-client-java-staging.git
 	branch = imabhichow/add-kc
 ; 	branch = s3ec/improved
-[submodule "test-server/java-v4-server/specification"]
-	path = test-server/java-v4-server/specification
-	url = git@github.com:awslabs/private-aws-encryption-sdk-specification-staging.git
-	branch = fire-egg-staging
 [submodule "test-server/specification"]
 	path = test-server/specification
 	url = git@github.com:awslabs/private-aws-encryption-sdk-specification-staging.git
@@ -56,6 +52,17 @@
 	path = test-server/net-v3-transition-server/s3ec-v3-transition-branch
 	url = https://github.com/aws/private-amazon-s3-encryption-client-dotnet-staging.git
 	branch = rishav/key-commitment
+[submodule "test-server/cpp-v2-transition-server/aws-sdk-cpp"]
+	path = test-server/cpp-v2-transition-server/aws-sdk-cpp
+	url = git@github.com:awslabs/aws-sdk-cpp-staging.git
+	branch = fire-egg-dev
+[submodule "test-server/cpp-v3-server/aws-sdk-cpp"]
+	path = test-server/cpp-v3-server/aws-sdk-cpp
+	url = git@github.com:awslabs/aws-sdk-cpp-staging.git
+	branch = fire-egg-dev
+[submodule "test-server/cpp-v2-server/aws-sdk-cpp"]
+	path = test-server/cpp-v2-server/aws-sdk-cpp
+	url = git@github.com:awslabs/aws-sdk-cpp-staging.git
 [submodule "all-examples/net/v4/s3ec-v4-local"]
 	path = all-examples/net/v4/s3ec-v4-local
 	url = https://github.com/aws/private-amazon-s3-encryption-client-dotnet-staging.git

test-server/Makefile

@@ -6,15 +6,36 @@
 all: start-all-servers wait-all-servers run-tests
 
 # CI target for GitHub Actions
-ci: start-servers run-tests stop-servers
+ci:
+	$(MAKE) build-all-servers
+	$(MAKE) start-all-servers
+	$(MAKE) wait-all-servers
+	$(MAKE) run-tests
+	$(MAKE) stop-servers
 
 SERVER_DIRS := $(shell find . -maxdepth 1 -type d -name '*-server' | sed 's|^\./||' | $(if $(FILTER),grep -E "$$(echo '$(FILTER)' | sed 's/,/|/g')",cat) | sort)
 
+BUILD_SERVER_TARGETS := $(addprefix build-, $(SERVER_DIRS))
 START_SERVER_TARGETS := $(addprefix start-, $(SERVER_DIRS))
 WAIT_SERVER_TARGETS := $(addprefix wait-, $(SERVER_DIRS))
 
-# Start all servers in parallel
+# Build all servers in parallel
+build-all-servers: export MAKEFLAGS=-j$(shell sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 1)
+build-all-servers: $(BUILD_SERVER_TARGETS)
+
+$(BUILD_SERVER_TARGETS): build-%:
+	@if [ -f $*/Makefile ]; then \
+		echo "Building server in $*..."; \
+		$(MAKE) -C $* build-server; \
+	else \
+		echo "❌ Error: no Makefile found in $*"; \
+		exit 1; \
+	fi
+
+# Build and start all servers
 start-servers:
+	@echo "Building all servers..."
+	$(MAKE) build-all-servers
 	@echo "Starting all servers..."
 	$(MAKE) start-all-servers
 	@echo "Waiting for servers to start..."
@@ -23,7 +44,9 @@ start-servers:
 		$(MAKE) -C $$dir wait-for-server; \
 	done
 
-start-all-servers: $(START_SERVER_TARGETS)
+# Start servers sequentially (no parallel execution)
+start-all-servers:
+	@$(MAKE) MAKEFLAGS= $(START_SERVER_TARGETS)
 
 $(START_SERVER_TARGETS): start-%:
 	@if [ -f $*/Makefile ]; then \
@@ -32,9 +55,12 @@ $(START_SERVER_TARGETS): start-%:
 	else \
 		echo "❌ Error: no Makefile found in $*"; \
 		exit 1; \
-	fi; \
+	fi;
 
-wait-all-servers: $(WAIT_SERVER_TARGETS)
+wait-all-servers:
+	@echo "Waiting for all servers to be ready..."
+	$(MAKE) $(WAIT_SERVER_TARGETS)
+	@echo "All servers are ready!"
 
 $(WAIT_SERVER_TARGETS): wait-%:
 	@if [ -f $*/Makefile ]; then \
@@ -43,7 +69,7 @@ $(WAIT_SERVER_TARGETS): wait-%:
 	else \
 		echo "❌ Error: no Makefile found in $*"; \
 		exit 1; \
-	fi; \
+	fi;
 
 
 # Run the Java tests
@@ -63,7 +89,7 @@ run-tests:
 stop-servers:
 	@echo "Stopping servers..."
 	@for dir in $(SERVER_DIRS); do \
-		echo "Starting server in $$dir..."; \
+		echo "Stopping server in $$dir..."; \
 		$(MAKE) -C $$dir stop-server; \
 	done
 	@echo "Servers stopped"
@@ -94,6 +120,7 @@ wait-for-port:
 		echo "❌ Error: PORT is required"; \
 		exit 1; \
 	fi
+	@echo "Starting to wait for $$PORT to start";
 	@for i in $$(seq 1 $(TIMEOUT)); do \
 		if nc -z localhost $$PORT; then \
 			echo "Ports are open, waiting for servers to initialize..."; \

test-server/cpp-v2-server/Makefile

@@ -1,31 +1,38 @@
 # Makefile for S3 Encryption Client Testing
 
-.PHONY: start-server stop-server wait-for-server
+.PHONY: build-server start-server stop-server wait-for-server
 
 PID_FILE := server.pid
 PORT := 8085
 
-build/s3ec-server: 
-	brew install libmicrohttpd nlohmann-json ossp-uuid
-	git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git
+build/s3ec-server:
 	cd aws-sdk-cpp
 	mkdir -p build && cd build && cmake ..
 
-start-server: | build/s3ec-server
+build-server: | build/s3ec-server
+	@echo "Building Cpp V2 server..."
+	cd build && make
+
+start-server:
 	@echo "Starting Cpp V2 server..."
-	cd build && make && \
+	cd build && \
 	AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \
 	AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \
 	AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \
 	AWS_REGION="us-west-2" \
-	./s3ec-server & echo $$! > $(PID_FILE)
+	./s3ec-server > server.log 2>&1 & echo $$! > ../$(PID_FILE)
 	@echo "Cpp V2 server starting..."
 
 stop-server:
+	@echo "Stopping server on port $(PORT)..."
+	@lsof -ti:$(PORT) | xargs kill -9 2>/dev/null || true
 	@if [ -f $(PID_FILE) ]; then \
-		kill $$(cat $(PID_FILE)) 2>/dev/null || true; \
-		rm $(PID_FILE); \
+		pkill -P $$(cat $(PID_FILE)) 2>/dev/null || true; \
+		kill -9 $$(cat $(PID_FILE)) 2>/dev/null || true; \
+		rm -f $(PID_FILE); \
 	fi
+	@rm -f build/server.log
+	@echo "Server stopped"
 
 wait-for-server:
 	$(MAKE) -C .. wait-for-port PORT=$(PORT)