chore(TestServer): add multilanguage test server, run in CI

.github/workflows/test.yml

@@ -25,9 +25,31 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ inputs.python-version || '3.11' }}
+
+      # Cache uv dependencies
+      - name: Cache uv dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
 
       - name: Install Uv
         run: pip install uv
+
+      # Cache Gradle dependencies and build outputs
+      - name: Cache Gradle packages
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            test-server/java-server/.gradle
+            test-server/java-tests/.gradle
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
 
       - name: Install dependencies
         run: make install
@@ -46,3 +68,11 @@ jobs:
         env:
           CI_S3_BUCKET: ${{ vars.CI_S3_BUCKET }}
           CI_KMS_KEY_ALIAS: ${{ vars.CI_KMS_KEY_ALIAS }}
+
+      - name: Run test-server tests
+        run: cd test-server && make ci
+        env:
+          AWS_REGION: us-west-2
+          TEST_SERVER_S3_BUCKET: ${{ vars.TEST_SERVER_S3_BUCKET }}
+          TEST_SERVER_KMS_KEY_ARN: ${{ vars.TEST_SERVER_KMS_KEY_ARN }}
+          GRADLE_OPTS: "-Dorg.gradle.daemon=true -Dorg.gradle.parallel=true -Dorg.gradle.caching=true"

.gitignore

@@ -1,5 +1,3 @@
-.idea
-.vscode
 # Exclude all pycache directories and bytecode
 __pycache__/
 *.pyc
@@ -14,3 +12,42 @@ build/
 # Uv
 .uv/
 uv.lock
+
+# Gradle
+.gradle/
+gradle-app.setting
+
+# IDE - IntelliJ IDEA
+.idea/
+*.iml
+*.iws
+*.ipr
+
+# IDE - VS Code
+.vscode/
+.settings/
+.project
+.classpath
+
+# Compiled class files
+*.class
+
+# Log files
+*.log
+
+# Package files
+*.jar
+!gradle/wrapper/gradle-wrapper.jar
+!**/gradle/wrapper/gradle-wrapper.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+*.hprof
+.kotlin/
+
+.DS_Store
+smithy-java-core/out

cdk/lib/cdk-stack.ts

@@ -22,7 +22,7 @@ export class S3ECPythonGithub extends cdk.Stack {
   constructor(scope: Construct, id: string, props?: cdk.StackProps) {
     super(scope, id, props);
 
-    // KMS Key - default policy is fine,
+    // KMS Keys - default policy is fine,
     // we use IAM to manage key permissions
     const S3ECGithubKMSKey = new Key(
       this,
@@ -42,8 +42,28 @@ export class S3ECPythonGithub extends cdk.Stack {
         targetKey: S3ECGithubKMSKey
       }
     )
+
+    // KMS Key for test-server
+    const S3ECTestServerKMSKey = new Key(
+      this,
+      "S3ECTestServerKMSKey",
+      {
+        enableKeyRotation: true,
+        description: "KMS Key for Test Server GitHub Action Workflow",
+      }
+    )
 
-    // S3 bucket
+    // KMS alias for test-server
+    const S3ECTestServerKMSKeyAlias = new Alias(
+      this,
+      "S3ECTestServerKMSKeyAlias",
+      {
+        aliasName: "alias/S3EC-Test-Server-Github-KMS-Key",
+        targetKey: S3ECTestServerKMSKey
+      }
+    )
+
+    // S3 buckets
     const AccessConfiguration: BlockPublicAccessOptions = {
       blockPublicAcls: false,
       blockPublicPolicy: false,
@@ -58,6 +78,16 @@ export class S3ECPythonGithub extends cdk.Stack {
         blockPublicAccess: new BlockPublicAccess(AccessConfiguration)
       }
     )
+
+    // New bucket for test-server
+    const S3ECTestServerGithubBucket = new Bucket(
+      this,
+      "S3ECTestServerGithubBucket",
+      {
+        bucketName: "s3ec-test-server-github-bucket",
+        blockPublicAccess: new BlockPublicAccess(AccessConfiguration)
+      }
+    )
 
     // S3 bucket policy
     const S3ECGithubS3BucketPolicy = new ManagedPolicy(
@@ -75,6 +105,7 @@ export class S3ECPythonGithub extends cdk.Stack {
               ],
               resources: [
                 S3ECGithubTestS3Bucket.bucketArn + "/*", // object-level permissions need this extra path
+                S3ECTestServerGithubBucket.bucketArn + "/*", // Add permissions for the new test-server bucket
               ],
             }),
             new PolicyStatement({
@@ -83,7 +114,8 @@ export class S3ECPythonGithub extends cdk.Stack {
                 "s3:ListBucket",
               ],
               resources: [
-                S3ECGithubTestS3Bucket.bucketArn
+                S3ECGithubTestS3Bucket.bucketArn,
+                S3ECTestServerGithubBucket.bucketArn, // Add permissions for the new test-server bucket
               ],
             }),
           ]
@@ -107,6 +139,7 @@ export class S3ECPythonGithub extends cdk.Stack {
               ],
               resources: [
                 S3ECGithubKMSKey.keyArn,
+                S3ECTestServerKMSKey.keyArn, // Add access to the test-server KMS key
               ]
             })
           ]

test-server/Makefile

@@ -0,0 +1,111 @@
+# Makefile for S3 Encryption Client Testing
+
+.PHONY: all start-servers start-python-server start-java-server run-tests stop-servers clean ci check-env help
+
+# Default target
+all: start-servers run-tests
+
+# CI target for GitHub Actions
+ci: start-servers run-tests stop-servers
+
+
+# Start Python server in background
+start-python-server:
+	@echo "Starting Python server..."
+	cd python-server && \
+	python -m venv .venv && \
+	.venv/bin/python -m ensurepip && \
+	.venv/bin/python -m pip install -e . && \
+	.venv/bin/python -m pip install -e ../.. && \
+	AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \
+	AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \
+	AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \
+	AWS_REGION="us-west-2" \
+	.venv/bin/python src/main.py & echo $$! > ../python-server.pid
+	@echo "Python server starting..."
+
+# Start Java server in background
+start-java-server:
+	@echo "Starting Java server..."
+	cd java-server && \
+	AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \
+	AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \
+	AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \
+	AWS_REGION="us-west-2" \
+	./gradlew --build-cache --parallel run & echo $$! > ../java-server.pid
+	@echo "Java server starting..."
+
+# Start both servers in parallel
+start-servers:
+	@echo "Starting servers in parallel..."
+	@$(MAKE) -j2 start-python-server start-java-server
+	@echo "Waiting for servers to be ready..."
+	@for i in $$(seq 1 360); do \
+		if nc -z localhost 8080 && nc -z localhost 8081; then \
+			echo "Ports are open, waiting for servers to initialize..."; \
+			sleep 5; \
+			echo "Both servers are ready!"; \
+			break; \
+		fi; \
+		if [ $$i -eq 360 ]; then \
+			echo "Timeout waiting for servers to start"; \
+			exit 1; \
+		fi; \
+		echo "Waiting for servers to start ($$i/360)..."; \
+		sleep 1; \
+	done
+
+
+# Run the Java tests
+run-tests:
+	@echo "Running Java tests..."
+	@echo "Exporting environment variables from servers to tests..."
+	@# Extract AWS environment variables from the current shell and pass them to the tests
+	cd java-tests && \
+	AWS_ACCESS_KEY_ID="$$AWS_ACCESS_KEY_ID" \
+	AWS_SECRET_ACCESS_KEY="$$AWS_SECRET_ACCESS_KEY" \
+	AWS_SESSION_TOKEN="$$AWS_SESSION_TOKEN" \
+	AWS_REGION="us-west-2" \
+	./gradlew --build-cache --parallel integ
+	@echo "Tests completed successfully"
+
+# Stop the servers
+stop-servers:
+	@echo "Stopping servers..."
+	@if [ -f python-server.pid ]; then \
+		kill $$(cat python-server.pid) 2>/dev/null || true; \
+		rm python-server.pid; \
+	fi
+	@if [ -f java-server.pid ]; then \
+		kill $$(cat java-server.pid) 2>/dev/null || true; \
+		rm java-server.pid; \
+	fi
+	@echo "Servers stopped"
+
+# Clean up logs and pid files
+clean: stop-servers
+	@echo "Cleaning up..."
+	@rm -f python-server.log java-server.log
+	@echo "Cleanup complete"
+
+# Help target
+help:
+	@echo "Available targets:"
+	@echo "  all                : Start servers and run tests (default, output to stdout)"
+	@echo "  ci                 : Run in CI mode (start servers, run tests, stop servers)"
+	@echo "  start-servers      : Start Python and Java servers in parallel (output to stdout)"
+	@echo "  start-python-server: Start only the Python server"
+	@echo "  start-java-server  : Start only the Java server"
+	@echo "  run-tests          : Run Java tests"
+	@echo "  stop-servers       : Stop running servers"
+	@echo "  clean              : Stop servers and clean up logs"
+	@echo "  check-env          : Check if required environment variables are set"
+	@echo "  help               : Show this help message"
+
+# Check if required environment variables are set
+check-env:
+	@echo "Checking required environment variables..."
+	@if [ -z "$$AWS_ACCESS_KEY_ID" ]; then echo "AWS_ACCESS_KEY_ID is not set"; else echo "AWS_ACCESS_KEY_ID is set"; fi
+	@if [ -z "$$AWS_SECRET_ACCESS_KEY" ]; then echo "AWS_SECRET_ACCESS_KEY is not set"; else echo "AWS_SECRET_ACCESS_KEY is set"; fi
+	@if [ -z "$$AWS_SESSION_TOKEN" ]; then echo "AWS_SESSION_TOKEN is not set"; else echo "AWS_SESSION_TOKEN is set"; fi
+	@if [ -z "$$AWS_REGION" ]; then echo "AWS_REGION is not set (will use us-west-2 as default)"; else echo "AWS_REGION is set to $$AWS_REGION"; fi

test-server/README.md

@@ -0,0 +1,61 @@
+# S3EC Generalized Robust Test Framework Machine
+
+Or G-RTFM. Or something. 
+
+## What?
+
+This is a write-once, run-multiple test server. 
+
+## How? 
+
+Use Smithy Java roughly as it is intended. 
+That is, generate a client and a server which share a common model. 
+Then, write more servers, either using the server codegen or parsing the JSON blobject by "hand".
+
+## Running Tests
+
+A Makefile is provided to simplify running the servers and tests. The Makefile handles starting both the Python and Java servers, running the tests, and cleaning up.
+
+### Available Commands
+
+```bash
+# Start servers and run tests (default)
+make
+
+# Run in CI mode (start servers in parallel, run tests, stop servers)
+make ci
+
+# Start Python and Java servers in parallel
+make start-servers
+
+# Start only the Python server
+make start-python-server
+
+# Start only the Java server
+make start-java-server
+
+# Run Java tests
+make run-tests
+
+# Stop running servers
+make stop-servers
+
+# Stop servers and clean up logs
+make clean
+
+# Show help message
+make help
+```
+
+The `ci` target is specifically designed for GitHub Actions workflows, ensuring that servers are properly started in parallel, tests are run, and resources are cleaned up afterward.
+
+## Performance Optimizations
+
+Performance optimizations have been implemented to speed up the test-server CI process, which was previously taking over 5 minutes to run. These optimizations include:
+
+- Parallel server startup
+- Gradle build caching and parallel execution
+- Dependency caching in CI
+- JVM optimizations
+
+For detailed information about the optimizations, see [OPTIMIZATION.md](./OPTIMIZATION.md).